diff --git a/docs/openvino-nightly.md b/docs/openvino-nightly.md new file mode 100644 index 00000000..7e819fd2 --- /dev/null +++ b/docs/openvino-nightly.md @@ -0,0 +1,50 @@ +# OpenVINO nightly builds + +OpenArc pins stable `openvino-genai` (see `dependencies` in `pyproject.toml`). +Some models require a newer nightly build of OpenVINO / GenAI. Example: +`OpenVINO/Qwen3.8-27B-int4-ov` requires OpenVINO 2026.4+ and **segfaults while +loading** on the pinned 2026.3.1 stable. + +## Enabling the nightly (managed by uv) + +The `nightly-ov` dependency group in `pyproject.toml` carries minimum nightly +versions for `openvino`, `openvino-genai`, and `openvino-tokenizers`: + +```bash +uv sync --group nightly-ov \ + --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \ + --index-strategy unsafe-best-match +``` + +Note: a plain `uv sync` afterwards resolves back to the stable pins and will +revert the environment. Either always sync with `--group nightly-ov`, or use +`uv run --no-sync ...` after installing. + +## Manual override (no lockfile changes) + +```bash +uv pip install --pre -U openvino openvino-genai openvino-tokenizers \ + --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \ + --index-strategy unsafe-best-match +uv run --no-sync ... # plain `uv run` / `uv sync` reverts to the stable pins +``` + +## Known-good nightly + +| Date | openvino | openvino-genai / openvino-tokenizers | Used for | +|---|---|---|---| +| 2026-10-07 | 2026.5.0.dev20261007 | 2026.5.0.0.dev20261007 | Qwen3.8-27B-int4-ov (VLM engine) | + +Record the exact nightly in every benchmark result produced while it is +installed. + +## Companion upgrades (2026-10-08) + +- `transformers` was upgraded 5.0.0 → 5.19.0 (5.0.0 does not recognize the + `qwen3_5` architecture; needed for any Qwen3.5/3.8 export work) and + `optimum[openvino]` to the latest release. Note: even with these, + optimum-intel cannot export qwen3_5 for `text-generation-with-past` yet + (see PLAN.md 3.2), so Qwen3.5/3.8 IRs still come from pre-exported hubs. +- `chat_template_kwargs` does **not** reach the Qwen3.5 chat template through + `apply_chat_template` on transformers 5.x — pass `enable_thinking` as a + direct keyword instead (OpenArc's VLM path already splats it correctly). diff --git a/pyproject.toml b/pyproject.toml index 00a108af..9c5e1950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,17 @@ test = [ "pytest-asyncio>=1.4.0", "httpx>=0.27", ] +# OpenVINO nightly wheels (opt-in). Some models require a GenAI nightly newer +# than the pinned stable release. Sync with: +# uv sync --group nightly-ov \ +# --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \ +# --index-strategy unsafe-best-match +# Pin the exact nightly here once validated (see docs/openvino-nightly.md). +nightly-ov = [ + "openvino>=2026.5.0.dev20261007", + "openvino-genai>=2026.5.0.0.dev20261007", + "openvino-tokenizers>=2026.5.0.0.dev20261007", +] [build-system] requires = ["setuptools>=68", "wheel"] diff --git a/src/cli/groups/add.py b/src/cli/groups/add.py index fa45c689..3a357e0a 100644 --- a/src/cli/groups/add.py +++ b/src/cli/groups/add.py @@ -2,6 +2,7 @@ Add command - Add a model configuration to the config file. """ import json +import shutil from pathlib import Path from typing import NoReturn @@ -66,9 +67,9 @@ def _fail(extra: str = "") -> NoReturn: required=True, help='Path to OpenVINO IR converted model.') @click.option('--engine', '--en', - type=click.Choice(['ovgenai', 'openvino', 'optimum']), + type=click.Choice(['ovgenai', 'ovgenai_cb', 'openvino', 'optimum']), required=True, - help='Engine used to load the model (ovgenai, openvino, optimum)') + help='Engine used to load the model (ovgenai, ovgenai_cb, openvino, optimum)') @click.option('--model-type', '--mt', type=click.Choice([ 'llm', 'vlm', 'whisper', 'qwen3_asr', 'kokoro', @@ -153,14 +154,25 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, console.print(f"[red]Error parsing runtime_config JSON:[/red] {e}") console.print('[yellow]Example format: \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\'[/yellow]') ctx.exit(1) - # llm/vlm always get a model cache: compiled blobs live alongside the IR. - # An explicit CACHE_DIR in --runtime-config wins. Resolved to an absolute - # path here because runtime_config values are passed to OpenVINO verbatim, - # unlike model_path they are not resolved against the config file. + # llm/vlm always get a compiled-model cache. OpenVINO keys cache blobs by + # model content, so one shared directory is safe for any number of models + # (and for draft models). An explicit CACHE_DIR in --runtime-config wins. + # Resolved to an absolute path here because runtime_config values are + # passed to OpenVINO verbatim, unlike model_path they are not resolved + # against the config file. Blobs can reach a sizeable fraction of the IR + # size, so warn when the disk looks tight. if model_type in ("llm", "vlm"): - parsed_runtime_config.setdefault( - "CACHE_DIR", str((Path(model_path) / "model_cache").resolve()) - ) + default_cache_dir = Path.home() / ".cache" / "openarc" / "ov_cache" + parsed_runtime_config.setdefault("CACHE_DIR", str(default_cache_dir)) + if parsed_runtime_config["CACHE_DIR"] == str(default_cache_dir): + free_gb = shutil.disk_usage(Path.home()).free / 1e9 + if free_gb < 20: + console.print( + f"[yellow]Warning: only {free_gb:.1f} GB free on the home filesystem; " + f"the OpenVINO compiled-model cache at {default_cache_dir} can take " + f"several GB per model. Free space or set CACHE_DIR via " + f"--runtime-config if loads fail.[/yellow]" + ) # Route the contract-flagged values into their blocks. Validation happens # against the contracts themselves, so a bad value is caught here rather diff --git a/src/cli/groups/status.py b/src/cli/groups/status.py index ba1d167f..ba7e6b2f 100644 --- a/src/cli/groups/status.py +++ b/src/cli/groups/status.py @@ -9,9 +9,21 @@ @cli.command() +@click.option("--doctor", is_flag=True, help="Run local Arc/GPU diagnostics (PCIe link, driver, compute-runtime, render access) instead of querying the server.") @click.pass_context -def status(ctx): +def status(ctx, doctor): """- GET Status of loaded models.""" + if doctor: + from ..modules.doctor import collect_diagnostics, evaluate + console.print("[blue]Running local GPU diagnostics...[/blue]") + findings = evaluate(collect_diagnostics()) + for level, message in findings: + style = "green" if level == "ok" else "yellow" + marker = "OK " if level == "ok" else "WARN" + console.print(f"[{style}]{marker}[/{style}] {message}") + if any(level == "warn" for level, _ in findings): + ctx.exit(1) + return cli_instance = OpenArcCLI(server_config=ctx.obj.server_config) url = f"{cli_instance.base_url}/openarc/status" diff --git a/src/cli/modules/doctor.py b/src/cli/modules/doctor.py new file mode 100644 index 00000000..27c9f4d5 --- /dev/null +++ b/src/cli/modules/doctor.py @@ -0,0 +1,151 @@ +"""Local Intel Arc diagnostics for `openarc status --doctor`. + +Reports the things that silently cost performance or break inference on Arc: +the PCIe link the card actually trained at (vs. what card and slot allow), +the kernel driver in use (xe vs i915), the Intel compute-runtime version, and +render-node group membership. Everything is read from sysfs / the local +package manager; no server or root needed. PCIe link-walking follows +Strata's sycl/serve/xe_telemetry.py. +""" +import glob +import grp +import os +import subprocess + + +def _read(path: str): + try: + with open(path) as f: + return f.read().strip() + except OSError: + return None + + +def _find_intel_gpus(): + """All Intel DRM devices; discrete Arc cards have a tile0 entry (iGPUs don't).""" + gpus = [] + for dev in sorted(glob.glob("/sys/class/drm/card[0-9]*/device")): + if _read(os.path.join(dev, "vendor")) == "0x8086": + real = os.path.realpath(dev) + gpus.append({ + "card": dev.split("/")[-2], + "path": real, + "discrete": os.path.isdir(os.path.join(real, "tile0")), + }) + return gpus + + +def _driver(gpu_path: str): + link = os.path.join(gpu_path, "driver") + if os.path.islink(link): + return os.path.basename(os.path.realpath(link)) + return None + + +def _pcie_link(gpu_path: str): + """(current_gen, max_gen, current_width, max_width) for the card's external link. + + The card's own functions sit behind an internal x1 switch, so the reading + comes from the first upstream port wider than x1; the max is what card AND + slot allow (the root port caps it). + """ + gen = {"2.5": 1, "5.0": 2, "8.0": 3, "16.0": 4, "32.0": 5, "64.0": 6} + d, chain = gpu_path, [] + while d and d.startswith("/sys/devices/pci") and os.path.exists(os.path.join(d, "current_link_speed")): + chain.append(d) + d = os.path.dirname(d) + + def g(path, kind): + return gen.get((_read(os.path.join(path, f"{kind}_link_speed")) or "").split(" ")[0]) + + for port in chain: + width = int(_read(os.path.join(port, "current_link_width")) or 0) + if width > 1: + card_max = g(port, "max") + slot_max = g(chain[-1], "max") + max_gen = min(x for x in (card_max, slot_max) if x) if (card_max or slot_max) else None + max_width_raw = _read(os.path.join(port, "max_link_width")) + max_width = int(max_width_raw) if max_width_raw else None + return g(port, "current"), max_gen, width, max_width + return None, None, None, None + + +def _compute_runtime_version(): + """Intel compute-runtime (Level Zero) version from the local package manager.""" + for cmd in (["pacman", "-Q", "intel-compute-runtime"], + ["dpkg-query", "-W", "-f=${Version}", "intel-level-zero-gpu"]): + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + if out.returncode == 0 and out.stdout.strip(): + return out.stdout.strip().split()[-1] + except (OSError, subprocess.SubprocessError): + continue + return None + + +def _render_access(): + """(node, group, accessible) for each render node.""" + gids = set(os.getgroups()) | {os.getgid()} + rows = [] + for node in sorted(glob.glob("/dev/dri/renderD*")): + try: + st = os.stat(node) + except OSError: + continue + try: + group = grp.getgrgid(st.st_gid).gr_name + except KeyError: + group = str(st.st_gid) + rows.append((node, group, os.access(node, os.R_OK | os.W_OK))) + return rows + + +def collect_diagnostics(): + """Gather all doctor checks. Returns a dict; never raises on missing data.""" + gpus = _find_intel_gpus() + for gpu in gpus: + gpu["driver"] = _driver(gpu["path"]) + cur_gen, max_gen, width, max_width = _pcie_link(gpu["path"]) + gpu["pcie"] = {"gen": cur_gen, "gen_max": max_gen, "width": width, "width_max": max_width} + return { + "gpus": gpus, + "compute_runtime": _compute_runtime_version(), + "render_nodes": _render_access(), + } + + +def evaluate(diag): + """Turn diagnostics into (level, message) findings. level: ok / warn.""" + findings = [] + if not diag["gpus"]: + findings.append(("warn", "no Intel GPU found under /sys/class/drm")) + return findings + for gpu in diag["gpus"]: + label = "discrete Arc" if gpu["discrete"] else "integrated GPU" + driver = gpu["driver"] or "unknown" + if gpu["discrete"] and driver == "i915": + findings.append(("warn", f"{gpu['card']} ({label}) is bound to i915; Arc discrete cards want the xe driver")) + else: + findings.append(("ok", f"{gpu['card']} ({label}) driver: {driver}")) + pcie = gpu["pcie"] + if gpu["discrete"] and pcie["gen"]: + cur = f"Gen{pcie['gen']} x{pcie['width']}" + best = f"Gen{pcie['gen_max']} x{pcie['width_max']}" if pcie["gen_max"] else "unknown" + if (pcie["gen_max"] and pcie["gen"] < pcie["gen_max"]) or \ + (pcie["width_max"] and pcie["width"] < pcie["width_max"]): + findings.append(("warn", f"{gpu['card']} PCIe link trained at {cur}, but card and slot allow {best}; " + "prefill and model loads are link-bound (reseat, riser, or BIOS link settings)")) + else: + findings.append(("ok", f"{gpu['card']} PCIe link: {cur} (max {best})")) + rt = diag["compute_runtime"] + if rt: + findings.append(("ok", f"Intel compute-runtime: {rt}")) + else: + findings.append(("warn", "Intel compute-runtime (Level Zero) not found via pacman/dpkg")) + for node, group, accessible in diag["render_nodes"]: + if accessible: + findings.append(("ok", f"{node}: group '{group}', accessible")) + else: + findings.append(("warn", f"{node}: group '{group}', NOT accessible by this user " + "(add the user to the render group)")) + return findings diff --git a/src/engine/ov_genai/continuous_batch_llm.py b/src/engine/ov_genai/continuous_batch_llm.py index ad493387..c6a695f2 100644 --- a/src/engine/ov_genai/continuous_batch_llm.py +++ b/src/engine/ov_genai/continuous_batch_llm.py @@ -1,269 +1,747 @@ +"""Continuous-batching serving engine over openvino_genai.ContinuousBatchingPipeline. + +Registered as engine ``ovgenai_cb`` (llm and vlm model types) as an opt-in +alternative to ``ovgenai``: requests overlap in OpenVINO's continuous +batching scheduler instead of serializing on the per-model queue worker. + +Requires the ``nightly-ov`` dependency group (see docs/openvino-nightly.md): +the GenerationHandle API used here (add_request / step / can_read / read / +get_perf_metrics) only exists on openvino-genai nightlies, not on the pinned +stable release. +""" +import asyncio +import base64 import gc -import time -from typing import List, Dict, Optional +import itertools +import logging +import os +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from io import BytesIO +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union +import numpy as np import openvino as ov -from transformers import AutoTokenizer, BatchEncoding from openvino_genai import ( - GenerationConfig, ContinuousBatchingPipeline, + GenerationConfig, + GenerationStatus, SchedulerConfig, + StreamerBase, + StreamingStatus, +) +from PIL import Image +from transformers import AutoTokenizer, BatchEncoding + +from src.engine.ov_genai.streamers import ensure_tool_call_parser, select_streamer +from src.engine.ov_genai.utils import ( + apply_temperature, + check_vram_budget, + extract_scheduler_config_from_loader, + format_perf_metrics, + load_draft_model, ) +from src.server.model_registry import ModelRegistry +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.schemas.registration import ModelLoadConfig, ModelType +from src.server.utils.chat import flatten_message_content, flatten_messages +from src.server.utils.resolve_vlm_type import is_qwen3_5_architecture, resolve_vlm_vision_token + +logger = logging.getLogger(__name__) + + +@dataclass +class _RequestState: + """Per-request bookkeeping shared by the step loop and the consumer.""" + request_id: Optional[str] + internal_id: int + gen_config: OVGenAI_GenConfig + streamer: StreamerBase + done: "asyncio.Future[Tuple[Dict[str, Any], str]]" + handle: Optional[Any] = None # openvino_genai GenerationHandle + generated_ids: List[int] = field(default_factory=list) + cancel_requested: bool = False + + +def _retrieve_future_exception(fut: "asyncio.Future") -> None: + # Retrieve terminal exceptions so requests abandoned by their consumer do + # not log "Future exception was never retrieved"; awaiting consumers still + # see the exception. + if not fut.cancelled(): + fut.exception() + + +class OVGenAI_ContinuousBatch: + def __init__(self, load_config: ModelLoadConfig): + self.model = None + self.encoder_tokenizer = None + self.vision_token = None + self.load_config = load_config + self._default_chat_template_kwargs: dict = {} + self.draft_model_loaded = False + self.model_num_assistant_tokens = None + self.model_assistant_confidence_threshold = None -from pydantic import BaseModel, Field + self._id_counter = itertools.count() + self._requests: Dict[int, _RequestState] = {} + self._by_request_id: Dict[str, _RequestState] = {} + self._submit_queue: Optional["asyncio.Queue[_RequestState]"] = None + self._wake_event: Optional[asyncio.Event] = None + self._step_task: Optional[asyncio.Task] = None + self._executor: Optional[ThreadPoolExecutor] = None + self._stopping = False -class ContinuousBatchConfig(BaseModel): - max_num_batched_tokens: int = Field(default=2048, description="Maximum number of tokens to batch together") - max_num_seqs: int = Field(default=48, description="Maximum number of sequences (batch size)") - cache_size: int = Field(default=6, description="KV cache size in GB") - dynamic_split_fuse: bool = Field(default=True, description="Split prompt/generate phases") - enable_prefix_caching: bool = Field(default=True, description="Enable KV-block caching") - use_cache_eviction: bool = Field(default=False, description="Use cache eviction") + @property + def _is_vlm(self) -> bool: + return self.load_config.model_type == ModelType.VLM -class OVGenAI_ContinuousBatchText: - def __init__(self, model_dir: str, device: str, batch_config: ContinuousBatchConfig): + # ------------------------------------------------------------------ + # Input preparation + # ------------------------------------------------------------------ + + def prepare_inputs(self, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]] = None, + chat_template_kwargs: dict = {} + ) -> ov.Tensor: """ - Initialize the continuous batch text generation class. - - Args: - model_dir (str): Path to the OpenVINO model directory - device (str): Device to run inference on (default: "GPU.1") - batch_config (ContinuousBatchConfig): Configuration for batching parameters + Convert a messages (list of {role, content}) into ov.Tensor using the cached AutoTokenizer + and its chat template. """ - self.model_dir = model_dir - self.device = device - self.batch_config = batch_config - self.pipeline = None - self.encoder_tokenizer = None - - def load_model(self): - """Load the model and initialize the continuous batching pipeline.""" - # Initialize HuggingFace AutoTokenizer with chat template - self.encoder_tokenizer = AutoTokenizer.from_pretrained( - self.model_dir, - use_fast=True, + prompt_token_ids = self.encoder_tokenizer.apply_chat_template( + flatten_messages(messages), + tools=tools, + add_generation_prompt=True, + skip_special_tokens=True, + return_tensors="np", + **chat_template_kwargs, + ) + if isinstance(prompt_token_ids, BatchEncoding): + prompt_token_ids = prompt_token_ids['input_ids'] + return ov.Tensor(prompt_token_ids) + + def _vision_token_for_index(self, index: int) -> str: + """ + Return the correctly formatted vision token for the given image index. + Handles templates that may contain an index placeholder like '{i}'. + """ + token_template = self.vision_token if self.vision_token is not None else "" + if "{i}" in token_template: + return token_template.replace("{i}", str(index)) + return token_template + + def prepare_vlm_inputs(self, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]] = None, + chat_template_kwargs: dict = {} + ) -> Tuple[str, List[ov.Tensor]]: + """ + Parse a messages list and prepare text prompt + image tensors for VLM inference. + + Returns: + (tokenized_messages, ov_images) + """ + images: List[Image.Image] = [] + text_messages: List[Dict[str, Any]] = [] + + # Step 1: Extract text and images + for message in messages: + # Multimodal message (list of dict content items) + if isinstance(message.get("content", ""), list): + text_parts: List[str] = [] + + for content_item in message["content"]: + if ( + isinstance(content_item, dict) + and content_item.get("type") == "image_url" + ): + image_url = content_item.get("image_url", {}) + # Check for embedded base64 data + if ( + isinstance(image_url, dict) + and isinstance(image_url.get("url", ""), str) + and image_url["url"].startswith("data:image/") + ): + base64_data = image_url["url"].split(",", 1) + if len(base64_data) > 1: + image_data = base64.b64decode(base64_data[1]) + image = Image.open(BytesIO(image_data)).convert("RGB") + images.append(image) + + # Insert model-specific image token where this image appears + token_str = self._vision_token_for_index(len(images) - 1) + text_parts.append(f" {token_str} ") + + # Handle text segments + elif isinstance(content_item, dict) and content_item.get("type") == "text": + text_parts.append(content_item.get("text", "")) + + # Combine extracted text back into a unified string + text_message = message.copy() + text_message["content"] = flatten_message_content( + " ".join([t for t in text_parts if isinstance(t, str)]) if text_parts else "" + ) + text_messages.append(text_message) + + # Simple text-only message + else: + text_messages.append( + {**message, "content": flatten_message_content(message.get("content"))} + ) + + # Step 2: Build the chat template prompt using cached tokenizer + text_messages = flatten_messages(text_messages) + tokenized_messages: str = self.encoder_tokenizer.apply_chat_template( + text_messages, + tokenize=False, + tools=tools, + add_generation_prompt=True, + **{**self._default_chat_template_kwargs, **chat_template_kwargs}, ) - - # Configure scheduler using the Pydantic model values - # SchedulerConfig does not take kwargs, so we need to set the values manually. Ugh. - scheduler_config = SchedulerConfig() - scheduler_config.max_num_batched_tokens = self.batch_config.max_num_batched_tokens - scheduler_config.max_num_seqs = self.batch_config.max_num_seqs - scheduler_config.cache_size = self.batch_config.cache_size - scheduler_config.dynamic_split_fuse = self.batch_config.dynamic_split_fuse - scheduler_config.enable_prefix_caching = self.batch_config.enable_prefix_caching - scheduler_config.use_cache_eviction = self.batch_config.use_cache_eviction - - # Initialize continuous batching pipeline - self.pipeline = ContinuousBatchingPipeline( - self.model_dir, - device=self.device, - scheduler_config=scheduler_config, + + # Step 3: Convert images to OpenVINO Tensors + ov_images: List[ov.Tensor] = [] + for img in images: + arr = np.array(img, dtype=np.uint8) + tensor = ov.Tensor(arr) + ov_images.append(tensor) + + return tokenized_messages, ov_images + + def _strip_stray_vision_tokens(self, prompt: str, ov_images: List[ov.Tensor]) -> str: + """ + Enforce the vision-tag / image count invariant that OpenVINO's + inputs_embedder checks (vision_sequence.size() == n_visions). + + See OVGenAI_VLM._strip_stray_vision_tokens: a text-only prompt that + happens to mention the model's own vision token would otherwise abort + with "The number of native vision tags must match the number of + provided images/videos". Idempotent: a no-op when images exist or the + token is absent. + """ + if ov_images or not self.vision_token: + return prompt + token_str = self._vision_token_for_index(0) + if not token_str or token_str not in prompt: + return prompt + stray_count = prompt.count(token_str) + logger.warning( + f"[{self.load_config.model_name}] prompt contains " + f"{stray_count} native vision token(s) but no image was provided; " + "stripping token(s) from the input, solves bug found in PR #169" ) - - def unload_model(self): - """Unload the model and clear memory using garbage collection.""" - if self.pipeline: - del self.pipeline - self.pipeline = None - if self.encoder_tokenizer: - del self.encoder_tokenizer - self.encoder_tokenizer = None - gc.collect() - - def prepare_inputs(self, messages: List[Dict[str, str]]) -> ov.Tensor: + return prompt.replace(token_str, " ") + + def _resolve_prompt_and_images( + self, gen_config: OVGenAI_GenConfig + ) -> Tuple[str, List[ov.Tensor]]: """ - Convert a chat-style messages list into an ov.Tensor - that can be consumed by ContinuousBatchingPipeline. + Build (prompt, images) for the pipeline: bench input_ids / raw prompt / chat messages. + """ + if gen_config.input_ids: + prompt = self.encoder_tokenizer.decode(gen_config.input_ids, skip_special_tokens=False) + images: List[ov.Tensor] = [] + elif gen_config.prompt: + prompt = gen_config.prompt + images = [] + else: + prompt, images = self.prepare_vlm_inputs(gen_config.messages, gen_config.tools, gen_config.chat_template_kwargs) + return self._strip_stray_vision_tokens(prompt, images), images + + # ------------------------------------------------------------------ + # Generation consumer protocol (mirrors OVGenAI_LLM) + # ------------------------------------------------------------------ - Uses HuggingFace AutoTokenizer's apply_chat_template. + def generate_type(self, gen_config: OVGenAI_GenConfig): + """ + Unified text generation method that routes to streaming or non-streaming + based on the stream flag in gen_config. Both paths return an async iterator. Args: - messages (List[Dict[str,str]]): conversation turns, - e.g. [{"role": "user", "content": "Hello"}] + gen_config: Configuration containing the stream flag and other parameters Returns: - ov.Tensor: encoded prompt token IDs + - Non-streaming: async iterator yielding [metrics: dict, new_text: str] + - Streaming: async iterator yielding token chunks (str)... then [metrics: dict] """ + if gen_config.stream: + return self.generate_stream(gen_config) + else: + return self.generate_text(gen_config) - prompt_token_ids = self.encoder_tokenizer.apply_chat_template( - messages, - add_generation_prompt=True, # ensures <|assistant|> token is added - skip_special_tokens=True, - return_tensors="np", # returns numpy.ndarray - ) - if isinstance(prompt_token_ids, BatchEncoding): - prompt_token_ids = prompt_token_ids['input_ids'] - return ov.Tensor(prompt_token_ids) - - def generate(self, prompts: List[List[Dict[str, str]]], generation_config: GenerationConfig): + async def generate_text(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[Union[Dict[str, Any], str]]: + """ + Async non-streaming text generation. + Yields in order: metrics (dict), new_text (str). """ - Generate responses for a batch of prompts. - + state = await self._submit(gen_config) + try: + metrics, text = await state.done + except Exception as e: + logger.error(f"[{self.load_config.model_name}] Error during non-streaming generation: {e}", exc_info=True) + raise + logger.info(f"[{self.load_config.model_name}] Generation completed, generated {len(text)} characters") + yield metrics + yield text + + async def generate_stream(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[Union[str, Dict[str, Any]]]: + """ + Async streaming text generation. + Yields token chunks (str or chat_delta dict) as they arrive, then metrics (dict). + """ + state = await self._submit(gen_config) + try: + while True: + chunk = await state.streamer.text_queue.get() + if chunk is None: + break + yield chunk + # Stream fully drained: surface the terminal outcome. Awaiting the + # future re-raises a per-request failure here instead of leaving + # the HTTP client waiting on a stream that already ended. + metrics, _ = await state.done + yield metrics + finally: + if not state.done.done(): + # Consumer abandoned mid-stream (client disconnect, + # cancellation): stop the backend generation so it does not + # run to completion. + self._cancel_state(state) + + async def cancel(self, request_id: str) -> bool: + """ + Cancel an ongoing generation by request_id. + Args: - prompts: List of chat-format message lists - generation_config: Generation configuration - + request_id: The request ID to cancel + Returns: - List of generation results + True if cancellation was triggered, False if request_id is unknown """ + state = self._by_request_id.get(request_id) + if state is None: + return False + self._cancel_state(state) + logger.info(f"[{self.load_config.model_name}] Cancellation triggered for request {request_id}") + return True - # Prepare tokenized inputs as ov.Tensors - encoded_prompts = [self.prepare_inputs(m) for m in prompts] - - # Duplicate configs for batch - generation_configs = [generation_config] * len(encoded_prompts) - - # Run generation - results = self.pipeline.generate(encoded_prompts, generation_configs) - return results - - def collect_metrics(self, results, prompts, start_time, end_time): + def collect_metrics(self, gen_config: OVGenAI_GenConfig, perf_metrics) -> Dict[str, Any]: """ - Collect and display performance metrics from generation results. - - Args: - results: Generation results from pipeline.generate() - prompts: Original prompts for reference - start_time: Generation start time - end_time: Generation end time + Collect and format performance metrics into a dictionary. """ - - total_tokens_generated = 0 - total_ttft = 0 - total_tpot = 0 - total_throughput = 0 - total_generate_duration = 0 - - print("Per-Prompt Performance Metrics:") - print("=" * 80) - - for i, result in enumerate(results): - perf_metrics = result.perf_metrics - - # Extract metrics - load_time = perf_metrics.get_load_time() - ttft = perf_metrics.get_ttft().mean - tpot = perf_metrics.get_tpot().mean - throughput = perf_metrics.get_throughput().mean - generate_duration = perf_metrics.get_generate_duration().mean - num_input_tokens = perf_metrics.get_num_input_tokens() - num_generated_tokens = perf_metrics.get_num_generated_tokens() - - # Aggregate - total_tokens_generated += num_generated_tokens - total_ttft += ttft - total_tpot += tpot - total_throughput += throughput - total_generate_duration += generate_duration - - # Note: result.m_generation_ids contains token IDs, not decoded text - decoded_output = self.encoder_tokenizer.decode( - result.m_generation_ids[0], - skip_special_tokens=True, + return format_perf_metrics(gen_config, perf_metrics) + + # ------------------------------------------------------------------ + # Step loop + # ------------------------------------------------------------------ + + async def _submit(self, gen_config: OVGenAI_GenConfig) -> _RequestState: + """Register a generation request and hand it to the step loop.""" + if self.model is None: + raise RuntimeError(f"[{self.load_config.model_name}] model is not loaded") + ensure_tool_call_parser(gen_config, self.load_config) + self._ensure_step_loop() + streamer = select_streamer(self.model.get_tokenizer(), gen_config) + state = _RequestState( + request_id=gen_config.request_id, + internal_id=next(self._id_counter), + gen_config=gen_config, + streamer=streamer, + done=asyncio.get_running_loop().create_future(), + ) + state.done.add_done_callback(_retrieve_future_exception) + self._requests[state.internal_id] = state + if gen_config.request_id is not None: + self._by_request_id[gen_config.request_id] = state + await self._submit_queue.put(state) + self._wake_event.set() + return state + + def _ensure_step_loop(self) -> None: + """Start the step loop lazily: load_model may run without a running loop.""" + if self._step_task is not None and not self._step_task.done(): + return + self._stopping = False + self._submit_queue = asyncio.Queue() + self._wake_event = asyncio.Event() + if self._executor is None: + self._executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix=f"openarc-cb-{self.load_config.model_name}", + ) + self._step_task = asyncio.create_task(self._step_loop()) + + async def _step_loop(self) -> None: + """Drive the ContinuousBatchingPipeline: admit submissions, step, drain handles.""" + loop = asyncio.get_running_loop() + try: + while True: + if self._stopping: + break + + # Admit newly submitted requests. + while not self._submit_queue.empty(): + state = self._submit_queue.get_nowait() + if state.cancel_requested: + self._fail_request(state, RuntimeError( + f"request {state.request_id} was cancelled before it could start" + )) + continue + try: + self._admit(state) + except Exception as e: + logger.error( + f"[{self.load_config.model_name}] failed to admit request {state.request_id}: {e}", + exc_info=True, + ) + self._fail_request(state, e) + + active = [s for s in self._requests.values() if s.handle is not None] + if not active: + # Park until the next submission wakes us. Re-check the + # queue after clearing so a submission racing the clear + # is not missed. + self._wake_event.clear() + if self._submit_queue.empty() and not self._stopping: + await self._wake_event.wait() + continue + + await loop.run_in_executor(self._executor, self.model.step) + for state in active: + try: + self._drain_handle(state) + except Exception as e: + logger.error( + f"[{self.load_config.model_name}] failed to drain request {state.request_id}: {e}", + exc_info=True, + ) + self._fail_request(state, e) + except asyncio.CancelledError: + raise + except Exception as e: + # step() itself raising is model-fatal: fail every active request + # (the worker layer unloads the model on hard inference errors). + logger.error( + f"[{self.load_config.model_name}] continuous batching step loop failed: {e}", + exc_info=True, + ) + finally: + for state in list(self._requests.values()): + self._fail_request(state, RuntimeError( + f"[{self.load_config.model_name}] generation stopped: " + "model is unloading or the batching loop failed" + )) + + def _admit(self, state: _RequestState) -> None: + """Tokenize and add one request to the pipeline.""" + gen_config = state.gen_config + generation_config = self.create_generation_config(gen_config) + + if self._is_vlm: + prompt, ov_images = self._resolve_prompt_and_images(gen_config) + if ov_images: + state.handle = self.model.add_request( + state.internal_id, prompt, ov_images, generation_config + ) + else: + state.handle = self.model.add_request( + state.internal_id, prompt, generation_config + ) + return + + # LLM: pre-encoded input_ids, raw prompts, and chat messages + if gen_config.input_ids: + # Pre-encoded input IDs (used by /openarc/bench endpoint for benchmarking) + prompt_token_ids = ov.Tensor(np.array(gen_config.input_ids, dtype=np.int64).reshape(1, -1)) + state.handle = self.model.add_request(state.internal_id, prompt_token_ids, generation_config) + elif gen_config.prompt: + # Raw text (used by /v1/completions endpoint); the pipeline tokenizes it + state.handle = self.model.add_request(state.internal_id, gen_config.prompt, generation_config) + else: + # Chat template tokenization for messages (used by /v1/chat/completions endpoint) + prompt_token_ids = self.prepare_inputs(gen_config.messages, gen_config.tools, gen_config.chat_template_kwargs) + state.handle = self.model.add_request(state.internal_id, prompt_token_ids, generation_config) + + def _drain_handle(self, state: _RequestState) -> None: + """Read available tokens from a handle and feed the request's streamer.""" + handle = state.handle + # read() blocks until tokens are available, so only call it while + # can_read() reports pending output. + while handle.can_read(): + outputs = handle.read() + for output in outputs.values(): + ids = list(output.generated_ids) + if not ids: + continue + state.generated_ids.extend(ids) + status = state.streamer.write(ids) + if status == StreamingStatus.CANCEL: + self._cancel_handle(state) + break + gen_status = handle.get_status() + if gen_status != GenerationStatus.RUNNING: + self._finish_request(state, gen_status) + + def _finish_request(self, state: _RequestState, status: GenerationStatus) -> None: + """Terminate a request whose handle left RUNNING.""" + try: + state.streamer.end() + except Exception as e: + logger.warning( + f"[{self.load_config.model_name}] streamer end failed for request {state.request_id}: {e}" + ) + + if status == GenerationStatus.IGNORED: + # The scheduler dropped the request (per-request KV cache OOM). + # Fail this request; the rest of the batch keeps serving. + self._fail_request(state, RuntimeError( + f"[{self.load_config.model_name}] request was ignored by the continuous " + "batching scheduler (KV cache exhausted); retry with fewer concurrent " + "requests or a smaller cache_size" + )) + return + + # Perf metrics are only available once the handle leaves RUNNING, and + # may throw for CANCEL/IGNORED handles. + try: + perf_metrics = state.handle.get_perf_metrics() + metrics = format_perf_metrics(state.gen_config, perf_metrics) + except Exception as e: + logger.warning( + f"[{self.load_config.model_name}] perf metrics unavailable for request " + f"{state.request_id} (status {status}): {e}" + ) + metrics = {"stream": state.gen_config.stream} + + text = "" if state.gen_config.stream else self._assemble_text(state) + if not state.done.done(): + state.done.set_result((metrics, text)) + self._drop_request(state) + + def _assemble_text(self, state: _RequestState) -> str: + """Decode the full generated text for a non-streaming request.""" + # gemma4/museglimmer protocol tags are special=True: their tool + # streamers keep the raw tagged text so the route-level + # parse_generation can split reasoning/tool calls. + raw_text = getattr(state.streamer, "raw_text", None) + if raw_text is not None: + return raw_text + if not state.generated_ids: + return "" + decoder_tokenizer = self.model.get_tokenizer() + return decoder_tokenizer.decode([state.generated_ids], skip_special_tokens=True)[0] + + def _fail_request(self, state: _RequestState, exc: BaseException) -> None: + """Fail one request without disturbing the rest of the batch.""" + try: + # Wake a streaming consumer blocked on the stream queue. + state.streamer.text_queue.put_nowait(None) + except Exception: + pass + if not state.done.done(): + state.done.set_exception(exc) + if state.handle is not None: + self._cancel_handle(state) + self._drop_request(state) + + def _drop_request(self, state: _RequestState) -> None: + self._requests.pop(state.internal_id, None) + if state.request_id is not None: + self._by_request_id.pop(state.request_id, None) + + def _cancel_state(self, state: _RequestState) -> None: + state.cancel_requested = True + state.streamer.cancel() + if state.handle is not None: + self._cancel_handle(state) + + def _cancel_handle(self, state: _RequestState) -> None: + handle = state.handle + if handle is None: + return + try: + handle.cancel() + except Exception as e: + # Upstream bug openvinotoolkit/model_server#4428: + # GenerationHandle.cancel() can SIGSEGV the whole process when it + # races the scheduler under load. If cancel() raises, fall back to + # stop(), which ends the request cleanly at the next step boundary + # instead of tearing it down mid-step. + logger.warning( + f"[{self.load_config.model_name}] handle.cancel() failed for request " + f"{state.request_id} ({e}); falling back to handle.stop()" ) + try: + handle.stop() + except Exception: + logger.warning( + f"[{self.load_config.model_name}] handle.stop() also failed for " + f"request {state.request_id}", + exc_info=True, + ) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def load_model(self, loader: ModelLoadConfig): + """Load the ContinuousBatchingPipeline and cache the tokenizer. + + Args: + loader: ModelLoadConfig containing model_path, device, engine, and runtime_config. + """ + logger.info(f"{loader.model_name} loading...") + logger.info(f"{loader.model_type} on {loader.device} with {loader.runtime_config}") + + check_vram_budget(loader) + + # Load draft model for speculative decoding if provided + ( + draft_model, + self.draft_model_loaded, + self.model_num_assistant_tokens, + self.model_assistant_confidence_threshold, + ) = load_draft_model(loader) + + scheduler_config = extract_scheduler_config_from_loader(loader) + pipeline_kwargs = {**(loader.runtime_config or {})} + if loader.cache_dir: + pipeline_kwargs['CACHE_DIR'] = loader.cache_dir + if draft_model is not None: + pipeline_kwargs['draft_model'] = draft_model + + sched = scheduler_config.get("scheduler_config") + if sched is None: + sched = SchedulerConfig() + + self.model = ContinuousBatchingPipeline( + loader.model_path, + scheduler_config=sched, + device=loader.device, + properties=pipeline_kwargs, + ) + + self.encoder_tokenizer = AutoTokenizer.from_pretrained(loader.model_path) + + if self._is_vlm: + self.vision_token = resolve_vlm_vision_token(loader.model_path) + # Auto-detect Qwen3.5 architecture and inject enable_thinking + self._detect_chat_template_defaults(loader) + + logger.info(f"{loader.model_name} loaded successfully") + + # Warm up: one short generation so OpenVINO compiles its GPU kernels + # at load time instead of on the first request. OPENARC_WARMUP=0 opts out. + if os.getenv('OPENARC_WARMUP', '1') != '0': + try: + warm_cfg = GenerationConfig() + warm_cfg.max_new_tokens = 8 + warm_cfg.do_sample = False + warm_cfg.apply_chat_template = False + if hasattr(self.model, "generate"): + self.model.generate("Hello", warm_cfg) + else: + handle = self.model.add_request(-1, "Hello", warm_cfg) + while handle.get_status() == GenerationStatus.RUNNING: + self.model.step() + logger.info(f"{loader.model_name} warm-up generation complete") + except Exception as e: + logger.warning(f"{loader.model_name} warm-up generation failed: {e}") + + async def unload_model(self, registry: ModelRegistry, model_name: str) -> bool: + """Unregister model from registry and free memory resources. + + Args: + registry: ModelRegistry to unregister from + model_name: Public model name to unload + + Returns: + True if the model was found and unregistered, else False. + """ + removed = await registry.register_unload(model_name) + + # Stop the batching loop: cancel every in-flight handle, then wake the + # loop so it exits at the top of its next iteration (its finally + # block fails any remaining requests, terminating their consumers). + self._stopping = True + for state in list(self._requests.values()): + self._cancel_state(state) + if self._wake_event is not None: + self._wake_event.set() + if self._step_task is not None: + try: + await self._step_task + except asyncio.CancelledError: + pass + self._step_task = None + if self._executor is not None: + self._executor.shutdown(wait=False) + self._executor = None + self._submit_queue = None + self._wake_event = None + + if self.model is not None: + del self.model + self.model = None + + if self.encoder_tokenizer is not None: + del self.encoder_tokenizer + self.encoder_tokenizer = None + + if self.vision_token is not None: + del self.vision_token + self.vision_token = None + + gc.collect() + logger.info(f"[{self.load_config.model_name}] unloaded successfully") + return removed + + def _detect_chat_template_defaults(self, loader: ModelLoadConfig) -> None: + """Read config.json and set default chat_template_kwargs for known architectures.""" + import json + config_path = os.path.join(loader.model_path, "config.json") + try: + with open(config_path, "r") as f: + config = json.load(f) + architectures = config.get("architectures", []) + if isinstance(architectures, list) and is_qwen3_5_architecture(architectures): + self._default_chat_template_kwargs = {"enable_thinking": True} + logger.info(f"{loader.model_name}: detected Qwen3.5 architecture, enabling thinking") + except Exception as e: + logger.debug(f"{loader.model_name}: could not detect architecture defaults: {e}") + + def create_generation_config(self, config: OVGenAI_GenConfig) -> GenerationConfig: + """ + Converts the config received by the API to the OpenVino-compatible config. + """ + generation_kwargs = self.model.get_config() if self.model else GenerationConfig() + generation_kwargs.max_new_tokens = config.max_tokens + apply_temperature(generation_kwargs, config.temperature) + generation_kwargs.top_k = config.top_k + generation_kwargs.top_p = config.top_p + generation_kwargs.repetition_penalty = config.repetition_penalty + # Prompts arrive fully templated (messages) or raw (prompt/input_ids); + # the pipeline must not apply a chat template on top. + generation_kwargs.apply_chat_template = False + + if config.seed: + generation_kwargs.rng_seed = config.seed + if config.frequency_penalty: + generation_kwargs.frequency_penalty = config.frequency_penalty + if config.presence_penalty: + generation_kwargs.presence_penalty = config.presence_penalty - print(f"Prompt {i+1}: {prompts[i][0]['content'][:50]}...") - print(f" Load time: {load_time / 1000:.2f} s") - print(f" TTFT: {ttft / 1000:.2f} s") - print(f" TPOT: {tpot:.2f} ms/token") - print(f" Throughput: {throughput:.2f} tokens/s") - print(f" Generate duration: {generate_duration / 1000:.2f} s") - print(f" Input tokens: {num_input_tokens}") - print(f" Generated tokens: {num_generated_tokens}") - print(f" Response: {decoded_output[:200]}...") - print("-" * 80) - - # ------------------------------------------------------------------- - # Aggregate metrics - # ------------------------------------------------------------------- - num_prompts = len(results) - print("\nAggregate Performance Metrics:") - print("=" * 50) - print(f"Total processing time: {end_time - start_time:.2f} seconds") - print(f"Average time per prompt: {(end_time - start_time) / num_prompts:.2f} seconds") - print(f"Total tokens generated: {total_tokens_generated}") - print(f"Average TTFT: {total_ttft / num_prompts / 1000:.2f} s") - print(f"Average TPOT: {total_tpot / num_prompts:.2f} ms/token") - print(f"Average throughput: {total_throughput / num_prompts:.2f} tokens/s") - print(f"Average generate duration: {total_generate_duration / num_prompts / 1000:.2f} s") - print(f"Overall throughput: {total_tokens_generated / (end_time - start_time):.2f} tokens/s") - - # ------------------------------------------------------------------- - # Pipeline metrics - # ------------------------------------------------------------------- - metrics = self.pipeline.get_metrics() - print("\nPipeline System Metrics:") - print("=" * 50) - print(f"Requests processed: {metrics.requests}") - print(f"Scheduled requests: {metrics.scheduled_requests}") - print(f"Cache usage: {metrics.cache_usage:.2f}%") - print(f"Max cache usage: {metrics.max_cache_usage:.2f}%") - print(f"Average cache usage: {metrics.avg_cache_usage:.2f}%") - -def main(): - """Main entry point for the continuous batch text generation example.""" - # ------------------------------------------------------------------- - # Model directory (OpenVINO-IR export of quantized Llama 3.2 model) - # ------------------------------------------------------------------- - model_dir = "/mnt/Ironwolf-4TB/Models/OpenVINO/Qwen/Qwen3-1.7B-int8_asym-ov" - - # Create batch configuration - batch_config = ContinuousBatchConfig() - - # Initialize the continuous batch text generator - generator = OVGenAI_ContinuousBatchText( - model_dir=model_dir, - device="GPU.1", - batch_config=batch_config - ) - - # Load the model - generator.load_model() - - # ------------------------------------------------------------------- - # Generation config (applies per-prompt) - # ------------------------------------------------------------------- - generation_config = GenerationConfig( - max_new_tokens=1024, - temperature=0.7, - top_p=0.9, - do_sample=True, - ) - - # ------------------------------------------------------------------- - # Example prompts rewritten into chat-format messages - # ------------------------------------------------------------------- - prompts = [ - [{"role": "user", "content": "You're the fastest Llama this side of the equator"}], - [{"role": "user", "content": "What is the capital of France?"}], - [{"role": "user", "content": "Explain machine learning in simple terms"}], - [{"role": "user", "content": "Write a short story about a robot"}], - [{"role": "user", "content": "What is the capital of France?"}], - [{"role": "user", "content": "Explain machine learning in simple terms"}], - [{"role": "user", "content": "Write a short story about a robot"}], - [{"role": "user", "content": "What is the capital of France?"}], - [{"role": "user", "content": "Explain machine learning in simple terms"}], - [{"role": "user", "content": "Write a short story about a robot"}]] - - print("Starting continuous batching example with AutoTokenizer...") - print(f"Number of prompts: {len(prompts)}") - print("-" * 50) - - # ------------------------------------------------------------------- - # Run generation - # ------------------------------------------------------------------- - start_time = time.time() - results = generator.generate(prompts, generation_config) - end_time = time.time() - - # ------------------------------------------------------------------- - # Collect and display metrics - # ------------------------------------------------------------------- - generator.collect_metrics(results, prompts, start_time, end_time) - - # Unload the model - generator.unload_model() - -if __name__ == "__main__": - main() + # Add speculative decoding parameters (mutually exclusive per OpenVINO docs) + if config.num_assistant_tokens is not None: + generation_kwargs.num_assistant_tokens = config.num_assistant_tokens + elif config.assistant_confidence_threshold is not None: + generation_kwargs.assistant_confidence_threshold = config.assistant_confidence_threshold + elif getattr(self, 'draft_model_loaded', False): + if self.model_num_assistant_tokens is not None: + generation_kwargs.num_assistant_tokens = self.model_num_assistant_tokens + elif self.model_assistant_confidence_threshold is not None: + generation_kwargs.assistant_confidence_threshold = self.model_assistant_confidence_threshold + else: + default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '4')) + generation_kwargs.num_assistant_tokens = default_tokens + return generation_kwargs diff --git a/src/engine/ov_genai/llm.py b/src/engine/ov_genai/llm.py index 673fdd8f..7529d6a1 100755 --- a/src/engine/ov_genai/llm.py +++ b/src/engine/ov_genai/llm.py @@ -1,6 +1,9 @@ from src.engine.ov_genai.utils import ( apply_temperature, + check_vram_budget, extract_scheduler_config_from_loader, + format_perf_metrics, + load_draft_model, ) import asyncio import gc @@ -9,7 +12,6 @@ import os import openvino as ov -import openvino_genai from openvino_genai import ( GenerationConfig, LLMPipeline, @@ -142,13 +144,6 @@ async def generate_stream(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[ # Chat template tokenization for messages (used by /v1/chat/completions endpoint) prompt_token_ids = self.prepare_inputs(gen_config.messages, gen_config.tools, gen_config.chat_template_kwargs) - # DEBUG: Log what we're about to send - logger.error(f"[DEBUG] draft_model_loaded: {getattr(self, 'draft_model_loaded', False)}") - logger.error(f"[DEBUG] self.model_num_assistant_tokens: {getattr(self, 'model_num_assistant_tokens', 'NOT SET')}") - logger.error(f"[DEBUG] generation_kwargs.num_assistant_tokens: {getattr(generation_kwargs, 'num_assistant_tokens', 'NOT SET')}") - logger.error(f"[DEBUG] generation_kwargs.assistant_confidence_threshold: {getattr(generation_kwargs, 'assistant_confidence_threshold', 'NOT SET')}") - - async def _run_generation(): return await asyncio.to_thread( self.model.generate, @@ -196,7 +191,7 @@ async def cancel(self, request_id: str) -> bool: def collect_metrics(self, gen_config: OVGenAI_GenConfig, perf_metrics) -> Dict[str, Any]: """ Collect and format performance metrics into a dictionary. - + Args: gen_config: OVGenAI_GenConfig perf_metrics: PerfMetrics @@ -204,29 +199,7 @@ def collect_metrics(self, gen_config: OVGenAI_GenConfig, perf_metrics) -> Dict[s Returns: metrics: Dict[str, Any] """ - # Compute prefill throughput = input tokens / ttft (in seconds) - # Inspired by section 2.2 (https://arxiv.org/pdf/2404.14294v3) - ttft_seconds = perf_metrics.get_ttft().mean / 1000 - input_tokens = perf_metrics.get_num_input_tokens() - prefill_throughput = round(input_tokens / ttft_seconds, 2) if ttft_seconds > 0 else 0 - - metrics: Dict[str, Any] = { - 'load_time (s)': round(perf_metrics.get_load_time() / 1000, 2), - 'ttft (s)': round(perf_metrics.get_ttft().mean / 1000, 2), - 'tpot (ms)': round(perf_metrics.get_tpot().mean, 5), - 'prefill_throughput (tokens/s)': prefill_throughput, - 'decode_throughput (tokens/s)': round(perf_metrics.get_throughput().mean, 5), - 'decode_duration (s)': round(perf_metrics.get_generate_duration().mean / 1000, 5), - 'input_token': input_tokens, - 'new_token': perf_metrics.get_num_generated_tokens(), - 'total_token': input_tokens + perf_metrics.get_num_generated_tokens(), - 'stream': gen_config.stream, - } - # Include streaming-specific fields - if gen_config.stream and hasattr(gen_config, "stream_chunk_tokens"): - metrics['stream_chunk_tokens'] = gen_config.stream_chunk_tokens - - return metrics + return format_perf_metrics(gen_config, perf_metrics) def load_model(self, loader: ModelLoadConfig): """Load model using a ModelLoadConfig configuration and cache the tokenizer. @@ -238,45 +211,15 @@ def load_model(self, loader: ModelLoadConfig): logger.info(f"{loader.model_name} loading...") logger.info(f"{loader.model_type} on {loader.device} with {loader.runtime_config}") + check_vram_budget(loader) + # Load draft model for speculative decoding if provided - draft_model = None - if loader.draft_model_path: - try: - # Cache the draft model alongside the main model. OpenVINO keys - # cache blobs by model content, so sharing one CACHE_DIR is safe. - draft_model_properties = {} - if loader.cache_dir: - draft_model_properties['CACHE_DIR'] = loader.cache_dir - draft_model = openvino_genai.draft_model( - loader.draft_model_path, - loader.draft_device, - **draft_model_properties - ) - logger.info(f"Loaded draft model from {loader.draft_model_path} on {loader.draft_device}") - self.draft_model_loaded = True - - # Ensure we always have exactly one parameter set (XOR requirement) - if loader.num_assistant_tokens is not None: - self.model_num_assistant_tokens = loader.num_assistant_tokens - self.model_assistant_confidence_threshold = None - elif loader.assistant_confidence_threshold is not None: - self.model_num_assistant_tokens = None - self.model_assistant_confidence_threshold = loader.assistant_confidence_threshold - else: - default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '3')) - self.model_num_assistant_tokens = default_tokens - self.model_assistant_confidence_threshold = None - logger.info(f"Using default num_assistant_tokens={default_tokens} for speculative decoding") - - except Exception as e: - logger.warning(f"Failed to load draft model: {e}, continuing without speculative decoding") - self.draft_model_loaded = False - self.model_num_assistant_tokens = None - self.model_assistant_confidence_threshold = None - else: - self.draft_model_loaded = False - self.model_num_assistant_tokens = None - self.model_assistant_confidence_threshold = None + ( + draft_model, + self.draft_model_loaded, + self.model_num_assistant_tokens, + self.model_assistant_confidence_threshold, + ) = load_draft_model(loader) pipeline_kwargs = {**(loader.runtime_config or {})} scheduler_config = extract_scheduler_config_from_loader(loader) @@ -295,6 +238,18 @@ def load_model(self, loader: ModelLoadConfig): self.encoder_tokenizer = AutoTokenizer.from_pretrained(loader.model_path) logging.info(f"{loader.model_name} loaded successfully") + # Warm up: one short generation so OpenVINO compiles its GPU kernels + # at load time instead of on the first request. OPENARC_WARMUP=0 opts out. + if os.getenv('OPENARC_WARMUP', '1') != '0': + try: + warm_cfg = self.model.get_generation_config() + warm_cfg.max_new_tokens = 8 + warm_cfg.do_sample = False + self.model.generate("Hello", warm_cfg) + logger.info(f"{loader.model_name} warm-up generation complete") + except Exception as e: + logger.warning(f"{loader.model_name} warm-up generation failed: {e}") + async def unload_model(self, registry: ModelRegistry, model_name: str) -> bool: """Unregister model from registry and free memory resources. @@ -348,6 +303,6 @@ def create_generation_config(self, config: OVGenAI_GenConfig) -> GenerationConfi elif self.model_assistant_confidence_threshold is not None: generation_kwargs.assistant_confidence_threshold = self.model_assistant_confidence_threshold else: - default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '3')) + default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '4')) generation_kwargs.num_assistant_tokens = default_tokens return generation_kwargs diff --git a/src/engine/ov_genai/streamers.py b/src/engine/ov_genai/streamers.py index a2bb1bce..e9eac8ac 100644 --- a/src/engine/ov_genai/streamers.py +++ b/src/engine/ov_genai/streamers.py @@ -9,20 +9,74 @@ from src.engine.ov_genai.tool_parse import qwen35 as qwen35_tool_parse -class ChunkStreamer(StreamerBase): +class IncrementalChunkDecoder: """ - Streams decoded text in chunks of N tokens. - - tokens_len == 1 → token-by-token streaming. + Incremental detokenizer that emits decoded text in chunks of N tokens. + - tokens_len == 1 → token-by-token emission. - tokens_len > 1 → emit after every N tokens. - Uses cumulative decode + delta slicing to avoid subword boundary artifacts. + + Only the tokens since the last emit are decoded (BPE decode is + concatenative, so decoding the pending segment yields exactly the suffix a + full re-decode would produce). A decoded segment containing U+FFFD is held + back until more tokens complete the character, matching the old full-cache + behavior. Cost per emit is constant instead of growing with the square of + the output length. + + Plain helper shared by ChunkStreamer and the continuous-batching engine; + it owns no queues or callbacks. + """ + def __init__(self, decoder_tokenizer, tokens_len: int): + self.decoder_tokenizer = decoder_tokenizer + self.tokens_len = tokens_len # enforce at least 1 + self.tokens_pending: List[int] = [] # tokens not yet emitted as text + self.since_last_emit: int = 0 # tokens collected since last emit + + def feed(self, token_ids: Union[int, List[int]]) -> Optional[str]: + """Buffer token ids; return decoded text at a chunk boundary, else None. + + None is also returned when the decoded segment holds a partial UTF-8 + character (U+FFFD): the pending buffer keeps every unemitted token, so + the next attempt decodes the same suffix the old full-cache decode + would. + """ + # Normalize input to a list of ints + if isinstance(token_ids, list): + self.tokens_pending.extend(token_ids) + self.since_last_emit += len(token_ids) + else: + self.tokens_pending.append(token_ids) + self.since_last_emit += 1 + + # Only emit when we've reached the chunk boundary + if self.since_last_emit >= self.tokens_len: + text = self.decoder_tokenizer.decode(self.tokens_pending) + if chr(65533) in text: + return None + self.tokens_pending = [] + self.since_last_emit = 0 + return text if text else None + return None + + def flush(self) -> Optional[str]: + """Decode and return any tokens still pending at end of generation.""" + if not self.tokens_pending: + return None + text = self.decoder_tokenizer.decode(self.tokens_pending) + self.tokens_pending = [] + self.since_last_emit = 0 + return text if text else None + + +class ChunkStreamer(StreamerBase): + """ + Streams decoded text in chunks of N tokens via IncrementalChunkDecoder. + Enqueues emitted chunks (and the None EOF sentinel) on .text_queue. """ def __init__(self, decoder_tokenizer, gen_config: OVGenAI_GenConfig): super().__init__() - self.decoder_tokenizer = decoder_tokenizer - self.tokens_len = (gen_config.stream_chunk_tokens) # enforce at least 1 - self.tokens_cache: List[int] = [] # cumulative token buffer - self.since_last_emit: int = 0 # tokens collected since last emit - self.last_print_len: int = 0 # length of decoded text we've already emitted + self._decoder = IncrementalChunkDecoder( + decoder_tokenizer, gen_config.stream_chunk_tokens + ) self.text_queue: "asyncio.Queue[Optional[str]]" = asyncio.Queue() self._cancelled = asyncio.Event() try: @@ -43,27 +97,9 @@ def write(self, token: Union[int, List[int]]) -> openvino_genai.StreamingStatus: self._enqueue(None) return openvino_genai.StreamingStatus.CANCEL - # Normalize input to a list of ints - if isinstance(token, list): - self.tokens_cache.extend(token) - self.since_last_emit += len(token) - else: - self.tokens_cache.append(token) - self.since_last_emit += 1 - - # Only emit when we've reached the chunk boundary - if self.since_last_emit >= self.tokens_len: - text = self.decoder_tokenizer.decode(self.tokens_cache) - # Emit only the newly materialized portion - if len(text) > self.last_print_len: - chunk = text[self.last_print_len:] - if chr(65533) in chunk: - self.since_last_emit -= 1 - return openvino_genai.StreamingStatus.RUNNING - if chunk: - self._enqueue(chunk) - self.last_print_len = len(text) - self.since_last_emit = 0 + text = self._decoder.feed(token) + if text: + self._enqueue(text) return openvino_genai.StreamingStatus.RUNNING @@ -77,11 +113,9 @@ def is_cancelled(self) -> bool: def end(self) -> None: # Flush any remaining tokens at the end - text = self.decoder_tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - chunk = text[self.last_print_len:] - if chunk: - self._enqueue(chunk) + text = self._decoder.flush() + if text: + self._enqueue(text) self._enqueue(None) diff --git a/src/engine/ov_genai/tool_parse/gemma4.py b/src/engine/ov_genai/tool_parse/gemma4.py index c6061404..23f700ba 100644 --- a/src/engine/ov_genai/tool_parse/gemma4.py +++ b/src/engine/ov_genai/tool_parse/gemma4.py @@ -536,8 +536,7 @@ def __init__(self, tokenizer, gen_config): ) self._protocol_ids[ids[0]] = tag_id self._protocol_tags[tag_id] = tag - self.tokens_cache: List[int] = [] - self.last_print_len = 0 + self.tokens_pending: List[int] = [] # tokens not yet processed as text self.text_queue: "asyncio.Queue" = asyncio.Queue() self._cancelled = asyncio.Event() try: @@ -572,28 +571,29 @@ def _process_delta(self, text: str, delta_tokens) -> None: self._enqueue({"chat_delta": deltas}) def _decode_available(self) -> None: - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - delta = text[self.last_print_len:] - if chr(65533) in delta: - # partial UTF-8 at the boundary; wait for more tokens - return - self.last_print_len = len(text) - self._process_delta(delta, []) + # Incremental detokenization: decode only the pending segment (BPE + # decode is concatenative, so this equals the suffix a full re-decode + # would produce) at constant cost per token. + text = self.tokenizer.decode(self.tokens_pending) + if chr(65533) in text: + # partial UTF-8 at the boundary; wait for more tokens + return + if text: + self._process_delta(text, []) + self.tokens_pending = [] def _flush_text(self) -> None: - """Decode and process everything left in the cache, then reset it. + """Decode and process everything left pending, then reset it. Called before an intercepted protocol token: pre-boundary text must be processed before the boundary itself. """ - if not self.tokens_cache: + if not self.tokens_pending: return - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - self._process_delta(text[self.last_print_len:], []) - self.tokens_cache = [] - self.last_print_len = 0 + text = self.tokenizer.decode(self.tokens_pending) + if text: + self._process_delta(text, []) + self.tokens_pending = [] def write(self, token) -> StreamingStatus: if self._cancelled.is_set(): @@ -607,7 +607,7 @@ def write(self, token) -> StreamingStatus: self._raw_parts.append(self._protocol_tags[protocol_id]) self._process_delta("", [protocol_id]) else: - self.tokens_cache.append(int(tid)) + self.tokens_pending.append(int(tid)) self._decode_available() return self.tool_parser.status diff --git a/src/engine/ov_genai/tool_parse/museglimmer.py b/src/engine/ov_genai/tool_parse/museglimmer.py index d2faf696..8f0fc8e1 100644 --- a/src/engine/ov_genai/tool_parse/museglimmer.py +++ b/src/engine/ov_genai/tool_parse/museglimmer.py @@ -739,8 +739,7 @@ def __init__(self, tokenizer, gen_config): "it will not be intercepted", tag, ids, ) - self.tokens_cache: List[int] = [] - self.last_print_len = 0 + self.tokens_pending: List[int] = [] # tokens not yet processed as text self.text_queue: "asyncio.Queue" = asyncio.Queue() self._cancelled = asyncio.Event() try: @@ -778,28 +777,29 @@ def _process_delta(self, text: str, delta_tokens) -> None: self._enqueue({"chat_delta": deltas}) def _decode_available(self) -> None: - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - delta = text[self.last_print_len:] - if chr(65533) in delta: - # partial UTF-8 at the boundary; wait for more tokens - return - self.last_print_len = len(text) - self._process_delta(delta, []) + # Incremental detokenization: decode only the pending segment (BPE + # decode is concatenative, so this equals the suffix a full re-decode + # would produce) at constant cost per token. + text = self.tokenizer.decode(self.tokens_pending) + if chr(65533) in text: + # partial UTF-8 at the boundary; wait for more tokens + return + if text: + self._process_delta(text, []) + self.tokens_pending = [] def _flush_text(self) -> None: - """Decode and process everything left in the cache, then reset it. + """Decode and process everything left pending, then reset it. Called before an intercepted control token: pre-boundary text must be processed before the boundary itself. """ - if not self.tokens_cache: + if not self.tokens_pending: return - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - self._process_delta(text[self.last_print_len:], []) - self.tokens_cache = [] - self.last_print_len = 0 + text = self.tokenizer.decode(self.tokens_pending) + if text: + self._process_delta(text, []) + self.tokens_pending = [] def write(self, token) -> StreamingStatus: if self._cancelled.is_set(): @@ -813,7 +813,7 @@ def write(self, token) -> StreamingStatus: self._raw_parts.append(tag) self._process_delta("", [int(tid)]) else: - self.tokens_cache.append(int(tid)) + self.tokens_pending.append(int(tid)) self._decode_available() return self.tool_parser.status diff --git a/src/engine/ov_genai/tool_parse/qwen35.py b/src/engine/ov_genai/tool_parse/qwen35.py index 1df7523e..14feab2c 100644 --- a/src/engine/ov_genai/tool_parse/qwen35.py +++ b/src/engine/ov_genai/tool_parse/qwen35.py @@ -565,8 +565,7 @@ def __init__(self, tokenizer, gen_config): ids = tokenizer.encode(tag).input_ids.data.tolist()[0] if len(ids) == 1: self._wrapper_ids[ids[0]] = tag - self.tokens_cache: List[int] = [] - self.last_print_len = 0 + self.tokens_pending: List[int] = [] # tokens not yet processed as text self.text_queue: "asyncio.Queue" = asyncio.Queue() self._cancelled = asyncio.Event() try: @@ -600,28 +599,29 @@ def _process_delta(self, text: str, delta_tokens) -> None: self._enqueue({"chat_delta": deltas}) def _decode_available(self) -> None: - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - delta = text[self.last_print_len:] - if chr(65533) in delta: - # partial UTF-8 at the boundary; wait for more tokens - return - self.last_print_len = len(text) - self._process_delta(delta, []) + # Incremental detokenization: decode only the pending segment (BPE + # decode is concatenative, so this equals the suffix a full re-decode + # would produce) at constant cost per token. + text = self.tokenizer.decode(self.tokens_pending) + if chr(65533) in text: + # partial UTF-8 at the boundary; wait for more tokens + return + if text: + self._process_delta(text, []) + self.tokens_pending = [] def _flush_text(self) -> None: - """Decode and process everything left in the cache, then reset it. + """Decode and process everything left pending, then reset it. Called before an atomically-detected boundary token: pre-boundary text must be processed before the boundary itself. """ - if not self.tokens_cache: + if not self.tokens_pending: return - text = self.tokenizer.decode(self.tokens_cache) - if len(text) > self.last_print_len: - self._process_delta(text[self.last_print_len:], []) - self.tokens_cache = [] - self.last_print_len = 0 + text = self.tokenizer.decode(self.tokens_pending) + if text: + self._process_delta(text, []) + self.tokens_pending = [] def write(self, token) -> StreamingStatus: if self._cancelled.is_set(): @@ -634,7 +634,7 @@ def write(self, token) -> StreamingStatus: self._flush_text() self._process_delta(tag, [int(tid)]) else: - self.tokens_cache.append(int(tid)) + self.tokens_pending.append(int(tid)) self._decode_available() return self.tool_parser.status diff --git a/src/engine/ov_genai/utils.py b/src/engine/ov_genai/utils.py index 746a6440..052eda35 100644 --- a/src/engine/ov_genai/utils.py +++ b/src/engine/ov_genai/utils.py @@ -1,15 +1,108 @@ """OV GenAI engine utilities.""" import logging -from typing import Literal +import os +from pathlib import Path +from typing import Any, Dict, Literal, Optional, Tuple +import openvino_genai from openvino_genai import GenerationConfig, SchedulerConfig -from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import SchedulerConfigSchema +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import ( + OVGenAI_GenConfig, + SchedulerConfigSchema, +) from src.server.schemas.registration import ModelLoadConfig logger = logging.getLogger(__name__) +def format_perf_metrics(gen_config: OVGenAI_GenConfig, perf_metrics) -> Dict[str, Any]: + """Collect and format performance metrics into a dictionary. + + Shared by the LLM, VLM, and continuous-batching engines. + + Args: + gen_config: OVGenAI_GenConfig + perf_metrics: PerfMetrics + + Returns: + metrics: Dict[str, Any] + """ + # Compute prefill throughput = input tokens / ttft (in seconds) + # Inspired by section 2.2 (https://arxiv.org/pdf/2404.14294v3) + ttft_seconds = perf_metrics.get_ttft().mean / 1000 + input_tokens = perf_metrics.get_num_input_tokens() + prefill_throughput = round(input_tokens / ttft_seconds, 2) if ttft_seconds > 0 else 0 + + metrics: Dict[str, Any] = { + 'load_time (s)': round(perf_metrics.get_load_time() / 1000, 2), + 'ttft (s)': round(perf_metrics.get_ttft().mean / 1000, 2), + 'tpot (ms)': round(perf_metrics.get_tpot().mean, 5), + 'prefill_throughput (tokens/s)': prefill_throughput, + 'decode_throughput (tokens/s)': round(perf_metrics.get_throughput().mean, 5), + 'decode_duration (s)': round(perf_metrics.get_generate_duration().mean / 1000, 5), + 'input_token': input_tokens, + 'new_token': perf_metrics.get_num_generated_tokens(), + 'total_token': input_tokens + perf_metrics.get_num_generated_tokens(), + 'stream': gen_config.stream, + } + # Include streaming-specific fields + if gen_config.stream and hasattr(gen_config, "stream_chunk_tokens"): + metrics['stream_chunk_tokens'] = gen_config.stream_chunk_tokens + + return metrics + + +def load_draft_model(loader: ModelLoadConfig) -> Tuple[Any, bool, Optional[int], Optional[float]]: + """Load the speculative-decoding draft model configured on the loader. + + Returns (draft_model, draft_model_loaded, num_assistant_tokens, + assistant_confidence_threshold). draft_model is None when no draft model + is configured or when loading fails (a warning is logged and generation + continues without speculation). Exactly one of num_assistant_tokens / + assistant_confidence_threshold is set (XOR requirement). + + The draft model is cached alongside the main model: OpenVINO keys cache + blobs by model content, so sharing one CACHE_DIR is safe. + """ + if not loader.draft_model_path: + return None, False, None, None + + draft_model = None + draft_model_loaded = False + num_assistant_tokens = None + assistant_confidence_threshold = None + try: + draft_model_properties = {} + if loader.cache_dir: + draft_model_properties['CACHE_DIR'] = loader.cache_dir + draft_model = openvino_genai.draft_model( + loader.draft_model_path, + loader.draft_device, + **draft_model_properties + ) + logger.info(f"Loaded draft model from {loader.draft_model_path} on {loader.draft_device}") + draft_model_loaded = True + + # Ensure we always have exactly one parameter set (XOR requirement) + if loader.num_assistant_tokens is not None: + num_assistant_tokens = loader.num_assistant_tokens + elif loader.assistant_confidence_threshold is not None: + assistant_confidence_threshold = loader.assistant_confidence_threshold + else: + default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '4')) + num_assistant_tokens = default_tokens + logger.info(f"Using default num_assistant_tokens={default_tokens} for speculative decoding") + except Exception as e: + logger.warning(f"Failed to load draft model: {e}, continuing without speculative decoding") + draft_model = None + draft_model_loaded = False + num_assistant_tokens = None + assistant_confidence_threshold = None + + return draft_model, draft_model_loaded, num_assistant_tokens, assistant_confidence_threshold + + def apply_temperature(generation_config: GenerationConfig, temperature: float) -> None: """Set the sampling temperature, falling back to greedy decoding at zero. @@ -61,9 +154,67 @@ def extract_scheduler_config_from_loader(loader: ModelLoadConfig) -> dict[Litera pipeline_kwargs = loader.runtime_config or {} sched_config = loader.scheduler_config or SchedulerConfigSchema() sched_config_dict = sched_config.model_dump(exclude_unset=True) - if pipeline_kwargs.get("ATTENTION_BACKEND") == "SDPA" and sched_config_dict: - logger.error("Cannot set scheduler_config for model: scheduler config is unsupported for SDPA backends") - return {} - if not sched_config_dict: + if pipeline_kwargs.get("ATTENTION_BACKEND") == "SDPA": + if sched_config_dict: + logger.error("Cannot set scheduler_config for model: scheduler config is unsupported for SDPA backends") return {} + # Arc-tuned default for the PA backend: prefix caching keeps the KV blocks + # of earlier prompts around, which pays off on every client that resends + # long histories (chat UIs, agents). Author enable_prefix_caching: false in + # the model's scheduler_config to opt out. + if "enable_prefix_caching" not in sched_config_dict: + sched_config.enable_prefix_caching = True + # Prefill chunk tuning, swept on an Arc Pro B70 (8K prompt, Qwen3.8-27B + # int4, cold prefill): max_num_batched_tokens 256 -> 8192 with + # dynamic_split_fuse raised prefill from 1464 to 1968 tok/s (TTFT -26%). + # dynamic_split_fuse must stay on: with it off, prompts longer than + # max_num_batched_tokens are rejected outright. + if "max_num_batched_tokens" not in sched_config_dict: + sched_config.max_num_batched_tokens = 8192 + if "dynamic_split_fuse" not in sched_config_dict: + sched_config.dynamic_split_fuse = True return generate_ov_scheduler_config(sched_config) + + +def check_vram_budget(loader: ModelLoadConfig) -> None: + """Refuse a load that would oversubscribe VRAM. + + On the xe driver, allocations beyond VRAM do not fail: buffers spill into + system RAM and the host can livelock until a watchdog reset (observed by + Strata on an Arc Pro B70, 2026-09-29). Estimate weights (.bin bytes) + + the configured KV pool (cache_size) + ~1.5 GB of headroom and compare + against the device's total memory. When cache_size is unset OpenVINO + sizes the KV pool dynamically from what is free, so only the weights + + headroom floor is checked. OPENARC_VRAM_GUARD=0 disables the check. + """ + device = (loader.device or "").upper() + if not device.startswith("GPU"): + return + if os.getenv("OPENARC_VRAM_GUARD", "1") == "0": + return + import openvino as ov + try: + total = ov.Core().get_property(loader.device, "GPU_DEVICE_TOTAL_MEM_SIZE") + except Exception as e: + logger.warning(f"VRAM guard: could not query {loader.device} total memory ({e}); skipping check") + return + + def _bin_bytes(path: str) -> int: + return sum(f.stat().st_size for f in Path(path).glob("*.bin")) + + weights = _bin_bytes(loader.model_path) + if loader.draft_model_path: + weights += _bin_bytes(loader.draft_model_path) + sched = loader.scheduler_config + kv_gb = sched.cache_size if (sched and sched.cache_size) else 0 + headroom = int(1.5e9) + needed = weights + int(kv_gb * 1e9) + headroom + if needed > total: + raise RuntimeError( + f"VRAM guard: '{loader.model_name}' needs at least ~{needed / 1e9:.1f} GB on " + f"{loader.device} ({weights / 1e9:.1f} GB weights + {kv_gb} GB KV pool + " + f"1.5 GB headroom) but the device has {total / 1e9:.1f} GB. On the xe driver, " + f"oversubscribing VRAM spills into system RAM and can hang the host. Use a " + f"smaller model/quantization, lower scheduler_config.cache_size, or set " + f"OPENARC_VRAM_GUARD=0 to bypass this check." + ) diff --git a/src/engine/ov_genai/vlm.py b/src/engine/ov_genai/vlm.py index 392ac199..479bc2eb 100644 --- a/src/engine/ov_genai/vlm.py +++ b/src/engine/ov_genai/vlm.py @@ -1,6 +1,8 @@ from src.engine.ov_genai.utils import ( apply_temperature, + check_vram_budget, extract_scheduler_config_from_loader, + format_perf_metrics, ) import asyncio import base64 @@ -324,25 +326,7 @@ def collect_metrics(self, gen_config: OVGenAI_GenConfig, perf_metrics) -> Dict[s """ Collect and format performance metrics into a dictionary. """ - ttft_seconds = perf_metrics.get_ttft().mean / 1000 - input_tokens = perf_metrics.get_num_input_tokens() - prefill_throughput = round(input_tokens / ttft_seconds, 2) if ttft_seconds > 0 else 0 - - metrics: Dict[str, Any] = { - "load_time (s)": round(perf_metrics.get_load_time() / 1000, 2), - "ttft (s)": round(perf_metrics.get_ttft().mean / 1000, 2), - "tpot (ms)": round(perf_metrics.get_tpot().mean, 5), - "prefill_throughput (tokens/s)": prefill_throughput, - "decode_throughput (tokens/s)": round(perf_metrics.get_throughput().mean, 5), - "decode_duration (s)": round(perf_metrics.get_generate_duration().mean / 1000, 5), - "input_token": input_tokens, - "new_token": perf_metrics.get_num_generated_tokens(), - "total_token": input_tokens + perf_metrics.get_num_generated_tokens(), - "stream": gen_config.stream, - } - if gen_config.stream and hasattr(gen_config, "stream_chunk_tokens"): - metrics["stream_chunk_tokens"] = gen_config.stream_chunk_tokens - return metrics + return format_perf_metrics(gen_config, perf_metrics) def load_model(self, loader: ModelLoadConfig): """ @@ -351,6 +335,8 @@ def load_model(self, loader: ModelLoadConfig): try: logger.info(f"{loader.model_type} on {loader.device} with {loader.runtime_config}") + check_vram_budget(loader) + scheduler_config = extract_scheduler_config_from_loader(loader) pipeline_kwargs = {**(loader.runtime_config or {})} if loader.cache_dir: @@ -372,6 +358,20 @@ def load_model(self, loader: ModelLoadConfig): logger.info(f"{loader.model_name} loaded successfully") + # Warm up: one short generation so OpenVINO compiles its GPU + # kernels at load time instead of on the first request. + # OPENARC_WARMUP=0 opts out. + if os.getenv('OPENARC_WARMUP', '1') != '0': + try: + warm_cfg = self.model_path.get_generation_config() + warm_cfg.max_new_tokens = 8 + warm_cfg.do_sample = False + warm_cfg.apply_chat_template = False + self.model_path.generate(prompt="Hello", generation_config=warm_cfg) + logger.info(f"{loader.model_name} warm-up generation complete") + except Exception as e: + logger.warning(f"{loader.model_name} warm-up generation failed: {e}") + except Exception as e: logger.error(f"[{loader.model_name}] Failed to initialize VLMPipeline: {e}", exc_info=True) raise diff --git a/src/server/model_registry.py b/src/server/model_registry.py index fe8b0ffc..a06e365e 100644 --- a/src/server/model_registry.py +++ b/src/server/model_registry.py @@ -337,6 +337,8 @@ async def readiness(self) -> dict: MODEL_CLASS_REGISTRY = { (EngineType.OV_GENAI, ModelType.LLM): "src.engine.ov_genai.llm.OVGenAI_LLM", (EngineType.OV_GENAI, ModelType.VLM): "src.engine.ov_genai.vlm.OVGenAI_VLM", + (EngineType.OV_GENAI_CB, ModelType.LLM): "src.engine.ov_genai.continuous_batch_llm.OVGenAI_ContinuousBatch", + (EngineType.OV_GENAI_CB, ModelType.VLM): "src.engine.ov_genai.continuous_batch_llm.OVGenAI_ContinuousBatch", (EngineType.OV_GENAI, ModelType.WHISPER): "src.engine.ov_genai.whisper.OVGenAI_Whisper", (EngineType.OPENVINO, ModelType.QWEN3_ASR): "src.engine.openvino.qwen3_asr.qwen3_asr.OVQwen3ASR", (EngineType.OPENVINO, ModelType.KOKORO): "src.engine.openvino.kokoro.OV_Kokoro", diff --git a/src/server/schemas/registration.py b/src/server/schemas/registration.py index 09d97839..d122d0ab 100644 --- a/src/server/schemas/registration.py +++ b/src/server/schemas/registration.py @@ -53,10 +53,13 @@ class EngineType(str, Enum): Options: - optimum: Optimum-Intel engine - - ovgenai: OpenVINO GenAI engine""" + - ovgenai: OpenVINO GenAI engine + - ovgenai_cb: OpenVINO GenAI ContinuousBatchingPipeline engine (llm/vlm only; + requires the nightly-ov dependency group)""" OV_OPTIMUM = "optimum" OV_GENAI = "ovgenai" + OV_GENAI_CB = "ovgenai_cb" OPENVINO = "openvino" diff --git a/src/server/worker_registry.py b/src/server/worker_registry.py index f13638e9..b4318acd 100644 --- a/src/server/worker_registry.py +++ b/src/server/worker_registry.py @@ -11,6 +11,7 @@ from src.engine.ov_genai.llm import OVGenAI_LLM from src.engine.ov_genai.vlm import OVGenAI_VLM +from src.engine.ov_genai.continuous_batch_llm import OVGenAI_ContinuousBatch from src.engine.ov_genai.whisper import OVGenAI_Whisper from src.engine.audio import AudioDecodeError from src.engine.openvino.kokoro import OV_Kokoro @@ -115,6 +116,26 @@ def _commit_completed_packet( packet.result_future.set_result(completed) return False +async def _consume_llm_packet( + model_name: str, + packet: WorkerPacket, + llm_model, + registry: ModelRegistry, +) -> bool: + """Run one packet through an LLM-style engine and commit the result. + + Shared by the strict-FIFO queue_worker_llm and the concurrent + queue_worker_cb. Returns True when the worker should exit (hard inference + error, model unload triggered). + """ + completed_packet = await InferWorker.infer_llm(packet, llm_model) + + if completed_packet.metrics: + logger.info(f"[LLM Worker: {model_name}] Metrics: {completed_packet.metrics}") + + return _commit_completed_packet(packet, completed_packet, model_name, registry) + + class InferWorker: """ Handles generation for individual packets. @@ -420,17 +441,44 @@ async def queue_worker_llm(model_name: str, model_queue: asyncio.Queue, llm_mode logger.info(f"[LLM Worker: {model_name}] Shutdown signal received.") break - completed_packet = await InferWorker.infer_llm(packet, llm_model) + should_exit = await _consume_llm_packet(model_name, packet, llm_model, registry) - if completed_packet.metrics: - logger.info(f"[LLM Worker: {model_name}] Metrics: {completed_packet.metrics}") + model_queue.task_done() + if should_exit: + break - if _commit_completed_packet(packet, completed_packet, model_name, registry): - model_queue.task_done() + @staticmethod + async def queue_worker_cb(model_name: str, model_queue: asyncio.Queue, cb_model: OVGenAI_ContinuousBatch, registry: ModelRegistry): + """Continuous-batching inference worker. + + Unlike queue_worker_llm (strict FIFO, one request at a time), every + packet gets its own consume task as soon as it arrives so requests + overlap in the engine's ContinuousBatchingPipeline. The None sentinel + shuts the worker down after the outstanding consume tasks complete. + """ + logger.info(f"[CB Worker: {model_name}] Started, waiting for packets...") + outstanding: set[asyncio.Task] = set() + + async def _consume(packet: WorkerPacket) -> None: + if await _consume_llm_packet(model_name, packet, cb_model, registry): + # Hard inference error: the model is unloading. Requeue the + # sentinel so the main loop stops accepting packets. + await model_queue.put(None) + + while True: + packet = await model_queue.get() + if packet is None: + logger.info(f"[CB Worker: {model_name}] Shutdown signal received.") break + task = asyncio.create_task(_consume(packet)) + outstanding.add(task) + task.add_done_callback(outstanding.discard) model_queue.task_done() + if outstanding: + await asyncio.gather(*outstanding, return_exceptions=True) + @staticmethod async def queue_worker_vlm(model_name: str, model_queue: asyncio.Queue, vlm_model: OVGenAI_VLM, registry: ModelRegistry): """Image model inference worker that processes packets from queue""" @@ -643,7 +691,16 @@ async def _on_model_loaded(self, record: ModelRecord) -> None: instance = record.model_instance async with self._lock: - if mt == ModelType.LLM and isinstance(instance, OVGenAI_LLM): + if mt in (ModelType.LLM, ModelType.VLM) and isinstance(instance, OVGenAI_ContinuousBatch): + # Continuous-batching engine: same queue structures as LLM, + # but the worker spawns a consume task per packet. + if record.model_name not in self._model_queues_llm: + q: asyncio.Queue = asyncio.Queue() + self._model_queues_llm[record.model_name] = q + task = asyncio.create_task(QueueWorker.queue_worker_cb(record.model_name, q, instance, self._model_registry)) + self._model_tasks_llm[record.model_name] = task + + elif mt == ModelType.LLM and isinstance(instance, OVGenAI_LLM): if record.model_name not in self._model_queues_llm: q: asyncio.Queue = asyncio.Queue() self._model_queues_llm[record.model_name] = q diff --git a/tests/unit/test_ov_genai_continuous_batch_unit.py b/tests/unit/test_ov_genai_continuous_batch_unit.py new file mode 100644 index 00000000..79795cff --- /dev/null +++ b/tests/unit/test_ov_genai_continuous_batch_unit.py @@ -0,0 +1,595 @@ +import asyncio +import threading +from typing import Dict, List, Optional +from unittest.mock import AsyncMock, MagicMock + +import pytest # type: ignore[import] +from openvino_genai import GenerationStatus + +import src.engine.ov_genai.continuous_batch_llm as cb_module +import src.engine.ov_genai.utils as utils_module +from src.engine.ov_genai.continuous_batch_llm import OVGenAI_ContinuousBatch +from src.engine.ov_genai.streamers import ChunkStreamer +from src.server.model_registry import MODEL_CLASS_REGISTRY +from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType +from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig + + +MODEL_PATH = "some_fake_url/Qwen3-0.6B-fp16-ov" + + +class DummyMeanValue: + def __init__(self, mean: float) -> None: + self.mean = mean + + +class DummyPerfMetrics: + def get_load_time(self) -> int: + return 5000 + + def get_ttft(self) -> DummyMeanValue: + return DummyMeanValue(250.0) + + def get_tpot(self) -> DummyMeanValue: + return DummyMeanValue(7.5) + + def get_throughput(self) -> DummyMeanValue: + return DummyMeanValue(12.34567) + + def get_generate_duration(self) -> DummyMeanValue: + return DummyMeanValue(1000.0) + + def get_num_input_tokens(self) -> int: + return 25 + + def get_num_generated_tokens(self) -> int: + return 10 + + +class FakeOVTokenizer: + """Stands in for the pipeline's openvino_genai Tokenizer. + + decode(1D) -> str, decode(2D) -> [str], mirroring the real API surface the + streamers and the engine use. + """ + + def __init__(self, vocab: Optional[Dict[int, str]] = None) -> None: + self.vocab = vocab or {} + + def decode(self, ids, skip_special_tokens: bool = True): + if ids and isinstance(ids[0], list): + return [self._decode_row(row) for row in ids] + return self._decode_row(ids) + + def _decode_row(self, row) -> str: + return "".join(self.vocab.get(i, f"<{i}>") for i in row) + + +class FakeGenerationOutput: + def __init__(self, ids: List[int]) -> None: + self.generated_ids = ids + self.finish_reason = None + self.score = 0.0 + + +class FakeHandle: + """Scripted GenerationHandle. + + script is a list of per-step token id batches; the string "IGNORED" as a + step flips the handle to GenerationStatus.IGNORED (per-request KV OOM). + When the script is exhausted and every token has been read, the handle + reports FINISHED. + """ + + def __init__(self, request_id: int, script: list, perf_raises: bool = False) -> None: + self.request_id = request_id + self._script = list(script) + self._pending: List[int] = [] + self._status = GenerationStatus.RUNNING + self._perf_raises = perf_raises + self.cancelled = False + self.stopped = False + + def advance(self) -> None: + if self._status != GenerationStatus.RUNNING: + return + if self._script: + step = self._script.pop(0) + if step == "IGNORED": + self._status = GenerationStatus.IGNORED + return + self._pending.extend(step) + + def can_read(self) -> bool: + return bool(self._pending) + + def read(self) -> Dict[int, FakeGenerationOutput]: + ids, self._pending = self._pending, [] + return {0: FakeGenerationOutput(ids)} + + def get_status(self) -> GenerationStatus: + if ( + self._status == GenerationStatus.RUNNING + and not self._script + and not self._pending + ): + self._status = GenerationStatus.FINISHED + return self._status + + def cancel(self) -> None: + self.cancelled = True + if self._status == GenerationStatus.RUNNING: + self._status = GenerationStatus.CANCEL + + def stop(self) -> None: + self.stopped = True + if self._status == GenerationStatus.RUNNING: + self._status = GenerationStatus.STOP + + def get_perf_metrics(self) -> DummyPerfMetrics: + if self._perf_raises: + raise RuntimeError("perf metrics unavailable for this handle state") + return DummyPerfMetrics() + + +class FakeCBPipeline: + """Stands in for openvino_genai.ContinuousBatchingPipeline. + + scripts maps the internal (monotonic int) request id to a handle script. + first_step_gate, when set, blocks the first step() call until the test + releases it (the executor thread waits on a threading.Event). + """ + + def __init__( + self, + tokenizer: FakeOVTokenizer, + scripts: Optional[Dict[int, list]] = None, + first_step_gate: Optional[threading.Event] = None, + perf_raises_ids: Optional[set] = None, + ) -> None: + self._tokenizer = tokenizer + self._scripts = scripts or {} + self._perf_raises_ids = perf_raises_ids or set() + self._first_step_gate = first_step_gate + self.handles: Dict[int, FakeHandle] = {} + self.add_calls: list = [] + self.step_count = 0 + + def add_request(self, request_id, *args) -> FakeHandle: + self.add_calls.append((request_id,) + args) + handle = FakeHandle( + request_id, + self._scripts.get(request_id, [[1]]), + perf_raises=request_id in self._perf_raises_ids, + ) + self.handles[request_id] = handle + return handle + + def step(self) -> None: + if self._first_step_gate is not None: + gate, self._first_step_gate = self._first_step_gate, None + gate.wait(timeout=10) + self.step_count += 1 + for handle in list(self.handles.values()): + handle.advance() + + def has_non_finished_requests(self) -> bool: + return any( + h.get_status() == GenerationStatus.RUNNING for h in self.handles.values() + ) + + def get_tokenizer(self) -> FakeOVTokenizer: + return self._tokenizer + + def get_config(self): + from openvino_genai import GenerationConfig + + return GenerationConfig() + + +@pytest.fixture +def load_config() -> ModelLoadConfig: + return ModelLoadConfig( + model_path=str(MODEL_PATH), + model_name="test-cb-model", + model_type=ModelType.LLM, + engine=EngineType.OV_GENAI_CB, + device="CPU", + runtime_config={}, + ) + + +@pytest.fixture +def vlm_load_config() -> ModelLoadConfig: + return ModelLoadConfig( + model_path=str(MODEL_PATH), + model_name="test-cb-vlm", + model_type=ModelType.VLM, + engine=EngineType.OV_GENAI_CB, + device="CPU", + runtime_config={}, + ) + + +def _make_engine(load_config: ModelLoadConfig, pipeline: FakeCBPipeline) -> OVGenAI_ContinuousBatch: + engine = OVGenAI_ContinuousBatch(load_config) + engine.model = pipeline + engine.encoder_tokenizer = MagicMock() + return engine + + +async def _collect(gen_config: OVGenAI_GenConfig, engine: OVGenAI_ContinuousBatch, events: list, tag: str) -> list: + items = [] + async for item in engine.generate_type(gen_config): + items.append(item) + events.append((tag, item)) + return items + + +async def _wait_for_handles(pipeline: FakeCBPipeline, count: int) -> None: + for _ in range(1000): + if len(pipeline.handles) >= count: + return + await asyncio.sleep(0.005) + raise AssertionError(f"timed out waiting for {count} handles, have {len(pipeline.handles)}") + + +@pytest.mark.asyncio +async def test_concurrent_consumers_interleave(load_config: ModelLoadConfig) -> None: + vocab = {10: "a1", 11: "a2", 20: "b1"} + gate = threading.Event() + pipeline = FakeCBPipeline( + FakeOVTokenizer(vocab), + scripts={0: [[10], [], [11]], 1: [[], [20]]}, + first_step_gate=gate, + ) + engine = _make_engine(load_config, pipeline) + events: list = [] + + cfg_a = OVGenAI_GenConfig(stream=True, prompt="prompt-a", request_id="req-a") + cfg_b = OVGenAI_GenConfig(stream=True, prompt="prompt-b", request_id="req-b") + + task_a = asyncio.create_task(_collect(cfg_a, engine, events, "A")) + task_b = asyncio.create_task(_collect(cfg_b, engine, events, "B")) + await _wait_for_handles(pipeline, 2) + # Deterministic admission order: req-a got internal id 0, req-b id 1. + assert pipeline.add_calls[0][1] == "prompt-a" + assert pipeline.add_calls[1][1] == "prompt-b" + gate.set() + + items_a, items_b = await asyncio.wait_for(asyncio.gather(task_a, task_b), timeout=10) + + assert items_a[:2] == ["a1", "a2"] + assert isinstance(items_a[2], dict) # trailing metrics + assert items_b[0] == "b1" + assert isinstance(items_b[1], dict) + + # No serialization: b1 lands between a1 and a2. + chunk_events = [(tag, item) for tag, item in events if isinstance(item, str)] + assert chunk_events[0] == ("A", "a1") + assert chunk_events.index(("B", "b1")) < chunk_events.index(("A", "a2")) + + +@pytest.mark.asyncio +async def test_stream_parity_with_chunk_streamer(load_config: ModelLoadConfig) -> None: + vocab = {i: f"t{i}" for i in range(1, 6)} + tokenizer = FakeOVTokenizer(vocab) + id_batches = [[1, 2], [3, 4], [5]] + pipeline = FakeCBPipeline(tokenizer, scripts={0: id_batches}) + engine = _make_engine(load_config, pipeline) + + gen_config = OVGenAI_GenConfig( + stream=True, stream_chunk_tokens=2, prompt="p", request_id="req-parity" + ) + + # Reference: drive a ChunkStreamer directly with the same token ids. + reference = ChunkStreamer(tokenizer, gen_config) + for batch in id_batches: + reference.write(batch) + reference.end() + await asyncio.sleep(0) # flush call_soon_threadsafe callbacks + reference_text = "" + while True: + chunk = reference.text_queue.get_nowait() + if chunk is None: + break + reference_text += chunk + + streamed = "" + async for item in engine.generate_type(gen_config): + if isinstance(item, str): + streamed += item + + assert streamed == reference_text + assert streamed == "t1t2t3t4t5" + + +@pytest.mark.asyncio +async def test_non_stream_yields_metrics_then_text(load_config: ModelLoadConfig) -> None: + vocab = {5: "x", 6: "y"} + pipeline = FakeCBPipeline(FakeOVTokenizer(vocab), scripts={0: [[5, 6]]}) + engine = _make_engine(load_config, pipeline) + + gen_config = OVGenAI_GenConfig(stream=False, prompt="p") + items = [] + async for item in engine.generate_type(gen_config): + items.append(item) + + assert len(items) == 2 + metrics, text = items + assert isinstance(metrics, dict) + assert metrics["stream"] is False + assert metrics["new_token"] == 10 + assert text == "xy" + + +@pytest.mark.asyncio +async def test_cancel_terminates_consumer(load_config: ModelLoadConfig) -> None: + gate = threading.Event() + pipeline = FakeCBPipeline( + FakeOVTokenizer({1: "a"}), + scripts={0: [[1], [1], [1]]}, + first_step_gate=gate, + ) + engine = _make_engine(load_config, pipeline) + events: list = [] + + cfg = OVGenAI_GenConfig(stream=True, prompt="p", request_id="req-cancel") + task = asyncio.create_task(_collect(cfg, engine, events, "A")) + await _wait_for_handles(pipeline, 1) + + assert await engine.cancel("req-cancel") is True + assert pipeline.handles[0].cancelled is True + gate.set() + + items = await asyncio.wait_for(task, timeout=10) + # Consumer terminated (no hang); a metrics dict closes the stream. + assert isinstance(items[-1], dict) + assert await engine.cancel("req-cancel") is False # already gone + + +@pytest.mark.asyncio +async def test_ignored_request_errors_other_continues(load_config: ModelLoadConfig) -> None: + gate = threading.Event() + vocab = {7: "z1", 8: "z2"} + pipeline = FakeCBPipeline( + FakeOVTokenizer(vocab), + scripts={0: ["IGNORED"], 1: [[7], [8]]}, + first_step_gate=gate, + ) + engine = _make_engine(load_config, pipeline) + events: list = [] + + cfg_a = OVGenAI_GenConfig(stream=True, prompt="prompt-a", request_id="req-ignored") + cfg_b = OVGenAI_GenConfig(stream=True, prompt="prompt-b", request_id="req-ok") + + async def _collect_raising(): + return await _collect(cfg_a, engine, events, "A") + + task_a = asyncio.create_task(_collect_raising()) + task_b = asyncio.create_task(_collect(cfg_b, engine, events, "B")) + await _wait_for_handles(pipeline, 2) + gate.set() + + with pytest.raises(RuntimeError, match="ignored by the continuous batching scheduler"): + await asyncio.wait_for(task_a, timeout=10) + + items_b = await asyncio.wait_for(task_b, timeout=10) + assert items_b[:2] == ["z1", "z2"] + assert isinstance(items_b[2], dict) + + +@pytest.mark.asyncio +async def test_perf_metrics_raise_on_cancel_still_terminates(load_config: ModelLoadConfig) -> None: + gate = threading.Event() + pipeline = FakeCBPipeline( + FakeOVTokenizer({1: "a"}), + scripts={0: [[1], [1]]}, + first_step_gate=gate, + perf_raises_ids={0}, + ) + engine = _make_engine(load_config, pipeline) + events: list = [] + + cfg = OVGenAI_GenConfig(stream=True, prompt="p", request_id="req-cancel-perf") + task = asyncio.create_task(_collect(cfg, engine, events, "A")) + await _wait_for_handles(pipeline, 1) + + await engine.cancel("req-cancel-perf") + gate.set() + + items = await asyncio.wait_for(task, timeout=10) + # Fallback metrics (get_perf_metrics threw on the CANCEL handle). + assert items[-1] == {"stream": True} + + +@pytest.mark.asyncio +async def test_unload_with_in_flight_requests(load_config: ModelLoadConfig) -> None: + gate = threading.Event() + pipeline = FakeCBPipeline( + FakeOVTokenizer({1: "a"}), + scripts={0: [[1], [1], [1]]}, + first_step_gate=gate, + ) + engine = _make_engine(load_config, pipeline) + events: list = [] + + cfg = OVGenAI_GenConfig(stream=True, prompt="p", request_id="req-inflight") + task = asyncio.create_task(_collect(cfg, engine, events, "A")) + await _wait_for_handles(pipeline, 1) + + registry = MagicMock() + registry.register_unload = AsyncMock(return_value=True) + + unload_task = asyncio.create_task(engine.unload_model(registry, "test-cb-model")) + await asyncio.sleep(0.05) # let unload reach the blocked step task + gate.set() + + removed = await asyncio.wait_for(unload_task, timeout=10) + assert removed is True + registry.register_unload.assert_called_once_with("test-cb-model") + + # The in-flight consumer terminates instead of hanging. + await asyncio.wait_for(task, timeout=10) + + assert engine.model is None + assert engine.encoder_tokenizer is None + assert engine._step_task is None + assert engine._executor is None + + +def test_load_model_constructs_pipeline(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENARC_WARMUP", "0") + pipeline_instance = MagicMock() + pipeline_factory = MagicMock(return_value=pipeline_instance) + monkeypatch.setattr(cb_module, "ContinuousBatchingPipeline", pipeline_factory) + tokenizer_instance = MagicMock() + monkeypatch.setattr( + cb_module.AutoTokenizer, + "from_pretrained", + MagicMock(return_value=tokenizer_instance), + ) + + loader = ModelLoadConfig( + model_path=str(MODEL_PATH), + model_name="loader-model", + model_type=ModelType.LLM, + engine=EngineType.OV_GENAI_CB, + device="CPU", + runtime_config={"hint": "value"}, + cache_dir="/tmp/ov_cache", + ) + + engine = OVGenAI_ContinuousBatch(loader) + engine.load_model(loader) + + pipeline_factory.assert_called_once() + args, kwargs = pipeline_factory.call_args + assert args == (loader.model_path,) + assert kwargs["device"] == loader.device + assert kwargs["properties"]["hint"] == "value" + assert kwargs["properties"]["CACHE_DIR"] == "/tmp/ov_cache" + # PA-backend default: prefix caching on. + assert kwargs["scheduler_config"].enable_prefix_caching is True + cb_module.AutoTokenizer.from_pretrained.assert_called_once_with(loader.model_path) + assert engine.model is pipeline_instance + assert engine.encoder_tokenizer is tokenizer_instance + + +def test_load_model_forwards_draft_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENARC_WARMUP", "0") + pipeline_factory = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(cb_module, "ContinuousBatchingPipeline", pipeline_factory) + monkeypatch.setattr( + cb_module.AutoTokenizer, + "from_pretrained", + MagicMock(return_value=MagicMock()), + ) + draft = object() + draft_factory = MagicMock(return_value=draft) + monkeypatch.setattr(utils_module.openvino_genai, "draft_model", draft_factory) + + loader = ModelLoadConfig( + model_path=str(MODEL_PATH), + model_name="loader-model", + model_type=ModelType.LLM, + engine=EngineType.OV_GENAI_CB, + device="CPU", + runtime_config={}, + cache_dir="/tmp/ov_cache", + draft_model_path="/models/draft", + draft_device="CPU", + ) + + OVGenAI_ContinuousBatch(loader).load_model(loader) + + draft_factory.assert_called_once_with("/models/draft", "CPU", CACHE_DIR="/tmp/ov_cache") + _, kwargs = pipeline_factory.call_args + assert kwargs["properties"]["draft_model"] is draft + + +def test_load_model_vlm_resolves_vision_token(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENARC_WARMUP", "0") + monkeypatch.setattr(cb_module, "ContinuousBatchingPipeline", MagicMock(return_value=MagicMock())) + monkeypatch.setattr( + cb_module.AutoTokenizer, + "from_pretrained", + MagicMock(return_value=MagicMock()), + ) + vision_mock = MagicMock(return_value="<|image_pad|>") + monkeypatch.setattr(cb_module, "resolve_vlm_vision_token", vision_mock) + + loader = ModelLoadConfig( + model_path=str(MODEL_PATH), + model_name="loader-vlm", + model_type=ModelType.VLM, + engine=EngineType.OV_GENAI_CB, + device="CPU", + runtime_config={}, + ) + + engine = OVGenAI_ContinuousBatch(loader) + engine.load_model(loader) + + vision_mock.assert_called_once_with(loader.model_path) + assert engine.vision_token == "<|image_pad|>" + + +@pytest.mark.asyncio +async def test_vlm_admit_uses_prompt_overload(vlm_load_config: ModelLoadConfig) -> None: + pipeline = FakeCBPipeline(FakeOVTokenizer({1: "a"}), scripts={0: [[1]]}) + engine = _make_engine(vlm_load_config, pipeline) + engine.vision_token = "<|image_pad|>" + + gen_config = OVGenAI_GenConfig(stream=False, prompt="describe this") + items = [] + async for item in engine.generate_type(gen_config): + items.append(item) + + # (internal_id, prompt, generation_config): no images -> str-prompt overload. + call = pipeline.add_calls[0] + assert call[0] == 0 + assert call[1] == "describe this" + assert len(items) == 2 + + +@pytest.mark.asyncio +async def test_vlm_admit_strips_stray_vision_token(vlm_load_config: ModelLoadConfig) -> None: + pipeline = FakeCBPipeline(FakeOVTokenizer({1: "a"}), scripts={0: [[1]]}) + engine = _make_engine(vlm_load_config, pipeline) + engine.vision_token = "<|image_pad|>" + + gen_config = OVGenAI_GenConfig(stream=False, prompt="what does <|image_pad|> mean?") + async for _ in engine.generate_type(gen_config): + pass + + call = pipeline.add_calls[0] + assert "<|image_pad|>" not in call[1] + + +def test_create_generation_config_zero_temperature_disables_sampling( + monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig +) -> None: + class DummyGenerationConfig: + pass + + monkeypatch.setattr(cb_module, "GenerationConfig", DummyGenerationConfig) + engine = OVGenAI_ContinuousBatch(load_config) + engine.model = None + + config = engine.create_generation_config(OVGenAI_GenConfig(temperature=0.0)) + + assert config.do_sample is False + assert config.temperature == 0.0 + assert config.apply_chat_template is False + + +def test_registry_maps_cb_engine() -> None: + assert ( + MODEL_CLASS_REGISTRY[(EngineType.OV_GENAI_CB, ModelType.LLM)] + == "src.engine.ov_genai.continuous_batch_llm.OVGenAI_ContinuousBatch" + ) + assert ( + MODEL_CLASS_REGISTRY[(EngineType.OV_GENAI_CB, ModelType.VLM)] + == "src.engine.ov_genai.continuous_batch_llm.OVGenAI_ContinuousBatch" + ) diff --git a/tests/unit/test_ov_genai_llm_unit.py b/tests/unit/test_ov_genai_llm_unit.py index b136e5f9..3b009d69 100644 --- a/tests/unit/test_ov_genai_llm_unit.py +++ b/tests/unit/test_ov_genai_llm_unit.py @@ -3,6 +3,7 @@ import pytest # type: ignore[import] import src.engine.ov_genai.llm as llm_module +import src.engine.ov_genai.utils as utils_module from src.engine.ov_genai.llm import OVGenAI_LLM from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig @@ -162,11 +163,12 @@ def test_load_model_sets_pipeline_and_tokenizer(monkeypatch: pytest.MonkeyPatch, llm.load_model(loader) - pipeline_factory.assert_called_once_with( - loader.model_path, - loader.device, - **loader.runtime_config, - ) + pipeline_factory.assert_called_once() + args, kwargs = pipeline_factory.call_args + assert args == (loader.model_path, loader.device) + assert kwargs["hint"] == "value" + # PA-backend default: a scheduler_config with prefix caching on is emitted. + assert kwargs["scheduler_config"].enable_prefix_caching is True llm_module.AutoTokenizer.from_pretrained.assert_called_once_with(loader.model_path) assert llm.model is pipeline_instance assert llm.encoder_tokenizer is tokenizer_instance @@ -193,12 +195,12 @@ def test_load_model_forwards_cache_dir(monkeypatch: pytest.MonkeyPatch) -> None: OVGenAI_LLM(loader).load_model(loader) - pipeline_factory.assert_called_once_with( - loader.model_path, - loader.device, - hint="value", - CACHE_DIR="/tmp/ov_cache", - ) + pipeline_factory.assert_called_once() + args, kwargs = pipeline_factory.call_args + assert args == (loader.model_path, loader.device) + assert kwargs["hint"] == "value" + assert kwargs["CACHE_DIR"] == "/tmp/ov_cache" + assert kwargs["scheduler_config"].enable_prefix_caching is True def _draft_loader(cache_dir): @@ -223,7 +225,7 @@ def _patch_llm_load(monkeypatch): MagicMock(return_value=MagicMock()), ) draft_factory = MagicMock(return_value=object()) - monkeypatch.setattr(llm_module.openvino_genai, "draft_model", draft_factory) + monkeypatch.setattr(utils_module.openvino_genai, "draft_model", draft_factory) return draft_factory diff --git a/tests/unit/test_ov_genai_scheduler_config_unit.py b/tests/unit/test_ov_genai_scheduler_config_unit.py index 9c61c965..0e6cb2cb 100644 --- a/tests/unit/test_ov_genai_scheduler_config_unit.py +++ b/tests/unit/test_ov_genai_scheduler_config_unit.py @@ -2,8 +2,9 @@ ``scheduler_config.max_num_batched_tokens`` (configured in ``config.yaml``). These tests pin that the operator's own ``scheduler_config`` flows through to the -compiled ``openvino.genai.SchedulerConfig`` -- and that nothing is emitted when -the operator has left it unset. +compiled ``openvino.genai.SchedulerConfig`` -- and that a PA-backend loader +always emits one, because prefix caching defaults on for PA (operator knobs +override the defaults; SDPA still emits nothing). Needs ``openvino_genai`` (real pipeline types). """ @@ -59,17 +60,21 @@ def test_operator_value_becomes_compiled_max(self): assert "scheduler_config" in out assert out["scheduler_config"].max_num_batched_tokens == 131072 - def test_no_scheduler_block_emits_nothing(self): - # No scheduler block at all -> emit no scheduler_config, so openvino uses - # its own (uncapped) default rather than a phantom one. + def test_no_scheduler_block_emits_only_defaults(self): + # No scheduler block at all -> the PA defaults are emitted (prefix + # caching on, 8K prefill chunks); nothing operator-set. lo = _loader() - assert "scheduler_config" not in extract_scheduler_config_from_loader(lo) + sched_config = extract_scheduler_config_from_loader(lo)["scheduler_config"] + assert sched_config.enable_prefix_caching is True + assert sched_config.max_num_batched_tokens == 8192 - def test_empty_scheduler_block_emits_nothing(self): - # A present-but-fully-unset scheduler block (all fields left None) is the - # same as none: no operator knob was set, so nothing is emitted. + def test_empty_scheduler_block_emits_only_defaults(self): + # A present-but-fully-unset scheduler block (all fields left None) is + # the same as none: the PA defaults, nothing operator-set. lo = _loader(scheduler_config=SchedulerConfigSchema()) - assert "scheduler_config" not in extract_scheduler_config_from_loader(lo) + sched_config = extract_scheduler_config_from_loader(lo)["scheduler_config"] + assert sched_config.enable_prefix_caching is True + assert sched_config.max_num_batched_tokens == 8192 def test_operator_max_and_kv_pool_coexist(self): # max_num_batched_tokens (per-sequence content cap) and num_kv_blocks @@ -99,10 +104,13 @@ def test_ignores_sdpa_backend(self): def test_context_window_does_not_drive_scheduler(): """The context window is advertisement only: it never enters the compiled scheduler config. An opt-in context_window (a pinned int or "auto") with an - all-unset scheduler block must emit no scheduler_config at all -- proving - the two are decoupled. + all-unset scheduler block must leave every operator knob at openvino's + built-in default -- proving the two are decoupled. (The PA prefix-caching + default is still emitted.) """ lo = _loader(context_window=131072) # advertised value, no scheduler block - assert extract_scheduler_config_from_loader(lo) == {} + sched_config = extract_scheduler_config_from_loader(lo)["scheduler_config"] + assert sched_config.max_num_batched_tokens == 8192 # the tuned default, untouched + assert sched_config.enable_prefix_caching is True # And the advertised value lands on the loader untouched, as-is. assert lo.context_window == 131072 diff --git a/tests/unit/test_ov_genai_vlm_unit.py b/tests/unit/test_ov_genai_vlm_unit.py index c29b28dc..64fd79b5 100644 --- a/tests/unit/test_ov_genai_vlm_unit.py +++ b/tests/unit/test_ov_genai_vlm_unit.py @@ -174,11 +174,15 @@ def test_load_model_sets_pipeline_and_vision_token(monkeypatch: pytest.MonkeyPat vlm.load_model(load_config) - pipeline_factory.assert_called_once_with( - load_config.model_path, - load_config.device, + pipeline_factory.assert_called_once() + args, kwargs = pipeline_factory.call_args + assert args == (load_config.model_path, load_config.device) + assert kwargs == { **load_config.runtime_config, - ) + "scheduler_config": kwargs["scheduler_config"], + } + # PA-backend default: a scheduler_config with prefix caching on is emitted. + assert kwargs["scheduler_config"].enable_prefix_caching is True vlm_module.AutoTokenizer.from_pretrained.assert_called_once_with(load_config.model_path) assert vlm.model_path is pipeline_instance assert vlm.tokenizer is tokenizer_instance diff --git a/uv.lock b/uv.lock index b3acc056..5d95c007 100644 --- a/uv.lock +++ b/uv.lock @@ -1857,6 +1857,11 @@ dependencies = [ ] [package.dev-dependencies] +nightly-ov = [ + { name = "openvino" }, + { name = "openvino-genai" }, + { name = "openvino-tokenizers" }, +] test = [ { name = "httpx" }, { name = "pytest" }, @@ -1894,6 +1899,11 @@ requires-dist = [ ] [package.metadata.requires-dev] +nightly-ov = [ + { name = "openvino", specifier = ">=2026.5.0.dev20261007" }, + { name = "openvino-genai", specifier = ">=2026.5.0.0.dev20261007" }, + { name = "openvino-tokenizers", specifier = ">=2026.5.0.0.dev20261007" }, +] test = [ { name = "httpx", specifier = ">=0.27" }, { name = "pytest", specifier = ">=8.4.2" }, @@ -1902,55 +1912,87 @@ test = [ [[package]] name = "openvino" -version = "2026.3.1" -source = { registry = "https://pypi.org/simple" } +version = "2026.5.0.dev20261007" +source = { registry = "https://storage.openvinotoolkit.org/simple/wheels/nightly" } dependencies = [ { name = "numpy" }, { name = "openvino-telemetry" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/83/f1/21f1503b5d4a6671730d75e4d0006e9483dfd21ed9899b1ea956a1e683b8/openvino-2026.3.1-22476-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3b23fc1669e63b22bdeb3f7cbfcd59b6d6e39ede6e68310ab74a8aaaf3912e70", size = 31612248, upload-time = "2026-08-26T09:17:23.804Z" }, - { url = "https://files.pythonhosted.org/packages/37/91/bb4cee4b1cbc244d6090fdb7625a0b3e1fe4f368bd2075bff67455451fbe/openvino-2026.3.1-22476-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:40bf3724ec70bce0fce4f9547cd1f19a576ef299172ce4ce2552249d10064acc", size = 57510470, upload-time = "2026-08-26T09:17:27.775Z" }, - { url = "https://files.pythonhosted.org/packages/88/c1/4291be34f2677af39933516356d805177effcd7457bf5cf99d289ecbb849/openvino-2026.3.1-22476-cp312-cp312-manylinux_2_35_aarch64.whl", hash = "sha256:e6b78918aef61ee4650bd6bbd6b17e5b8cd90f55f145adff5d7f6def907f33be", size = 28778134, upload-time = "2026-08-26T09:17:30.917Z" }, - { url = "https://files.pythonhosted.org/packages/27/79/77c474c3d8600185792bcb830638198ff3d15fe702028902bd07f4581026/openvino-2026.3.1-22476-cp312-cp312-win_amd64.whl", hash = "sha256:b686302a47abf7c87b48cb265b3a894f787a1b6380f4ab9f5a511de838ba3c64", size = 75818370, upload-time = "2026-08-26T09:17:35.803Z" }, - { url = "https://files.pythonhosted.org/packages/75/cc/66b460cc6266c1b4d49a458dc56ab1d10a2fd7016613010b0971330dd879/openvino-2026.3.1-22476-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5b0acfa02cfb822de0b5a38a314ecb1c67f122b55a491a3c6eb0c4945c550d96", size = 31612322, upload-time = "2026-08-26T09:17:39.719Z" }, - { url = "https://files.pythonhosted.org/packages/73/f9/b40f5c5921cac3906f74911ca038d28e6a2905b632e37a51289137ae1817/openvino-2026.3.1-22476-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bb39ba741cea93277cc6c80cf7f70d1c19dea9a0f2a37f07543e0b4a7e00e0c4", size = 57510484, upload-time = "2026-08-26T09:17:43.482Z" }, - { url = "https://files.pythonhosted.org/packages/27/0c/a430eb38645fe164928f2387d81d7214599138ca18b95a187076856243e7/openvino-2026.3.1-22476-cp313-cp313-manylinux_2_35_aarch64.whl", hash = "sha256:45304d238719b98d51a605752df8ba4696e8bcf6c4560acc00ffd676ef7e70c0", size = 28778137, upload-time = "2026-08-26T09:17:46.985Z" }, - { url = "https://files.pythonhosted.org/packages/fa/79/4b4a115d47e95e48ea76fda5bc6fc00200c441b65bbc38aa5f136eba7712/openvino-2026.3.1-22476-cp313-cp313-win_amd64.whl", hash = "sha256:a17b7e982b5ca5f9b7eb06bc15c1eb3f7ff13115700129123ca20d3d0038c6bb", size = 75818554, upload-time = "2026-08-26T09:17:51.341Z" }, - { url = "https://files.pythonhosted.org/packages/21/9f/2a19ea7e1875e555aea1a70fc07ae6426f9973b842353388ff36dfed6bf1/openvino-2026.3.1-22476-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:488495920a257f2791c734adbf197d173ad1ca1de6423047e4a1911df6e305d0", size = 31588360, upload-time = "2026-08-26T09:17:55.128Z" }, - { url = "https://files.pythonhosted.org/packages/7d/3b/105ac2b28e82e25840c4042656a6f2b678fe45c51e3692666d9f04932e56/openvino-2026.3.1-22476-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:f9b2c14441258ef3ba7b8c5d0bc91d72c73e539f09c983b294abc61bb102b094", size = 57512175, upload-time = "2026-08-26T09:17:59.506Z" }, - { url = "https://files.pythonhosted.org/packages/2d/38/4cf91a579c30851ffcabf1fbeaef7b2f69f2d49ccc9b460be4e55be3e1e9/openvino-2026.3.1-22476-cp314-cp314-manylinux_2_35_aarch64.whl", hash = "sha256:163a96f9ed7b072eb96d074905e2a0fc867ea14bf6d10529a9391015a95b1f27", size = 28785232, upload-time = "2026-08-26T09:18:02.974Z" }, - { url = "https://files.pythonhosted.org/packages/07/0b/15dc2770828feba43c3be4bbe886370b2ce1f6582cbb733691d2af0a569f/openvino-2026.3.1-22476-cp314-cp314-win_amd64.whl", hash = "sha256:7f3f14f4b53f0f07e80ea11ecd1845f2c1e60336aeb932abd7779b0f784ce9bb", size = 75820057, upload-time = "2026-08-26T09:18:07.684Z" }, - { url = "https://files.pythonhosted.org/packages/7c/cd/1b327a37810d919a46af15101910f55afe6247ad0e94b2903a7c1983d1cf/openvino-2026.3.1-22476-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c0688cdf0bb3214882ed3451de4fbeeda69cf2a719a5bcb13c60e50122850a0", size = 31820679, upload-time = "2026-08-26T09:18:11.14Z" }, - { url = "https://files.pythonhosted.org/packages/ae/a0/790da0221392aa3d83a3aaca0e63d1f43f55b62b27b3456b38ddd7e837c7/openvino-2026.3.1-22476-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b72b0a5de3df4511d3fa36d6d94eb037170377a733781a06b4dfb1b529a7fd73", size = 57556712, upload-time = "2026-08-26T09:18:16.523Z" }, - { url = "https://files.pythonhosted.org/packages/ec/0f/1e4b3bbe5e545660e0d8840a791edd16dc3e5487ebda78543aee70b433c6/openvino-2026.3.1-22476-cp314-cp314t-manylinux_2_35_aarch64.whl", hash = "sha256:5180c7268f9247b6793ca96f7f2d6bf7eabd48340b903b10dd1f2368802b33e8", size = 25917348, upload-time = "2026-08-26T09:18:19.499Z" }, - { url = "https://files.pythonhosted.org/packages/96/92/b2a1adda60931e24f2c7c4f22ab32d06a09c5652b92d6212edf27ad92e8b/openvino-2026.3.1-22476-cp314-cp314t-win_amd64.whl", hash = "sha256:4d26c91008a7f71c6b287f3d67c00f46f9651ba21f8f907a0f4d51ee6e127c33", size = 75984170, upload-time = "2026-08-26T09:18:24.159Z" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp312-cp312-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp312-cp312-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp312-cp312-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp312-cp312-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp313-cp313-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp313-cp313-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp313-cp313-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp313-cp313-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314t-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314t-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314t-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23368-cp314-cp314t-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp312-cp312-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp312-cp312-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp312-cp312-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp312-cp312-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp313-cp313-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp313-cp313-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp313-cp313-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp313-cp313-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314t-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314t-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314t-manylinux_2_35_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino/openvino-2026.5.0.dev20261007-23376-cp314-cp314t-win_amd64.whl" }, ] [[package]] name = "openvino-genai" -version = "2026.3.1.0" -source = { registry = "https://pypi.org/simple" } +version = "2026.5.0.0.dev20261007" +source = { registry = "https://storage.openvinotoolkit.org/simple/wheels/nightly" } dependencies = [ { name = "openvino-tokenizers" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/5b/8bf09b86867b3de3428c7ff15cd513d63c4e129bd22307f28bc09e5a1217/openvino_genai-2026.3.1.0-2499-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f859fc864a3f6b7e19fa6814c7c437804b8d50c70d1e85deeadcd2076dea997", size = 4242920, upload-time = "2026-08-26T09:26:29.289Z" }, - { url = "https://files.pythonhosted.org/packages/3f/e7/36418a3bdbce5a93ee7aafba69b7f5bbcafdda5300b4e2bcda39bace7419/openvino_genai-2026.3.1.0-2499-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a85b8545a035506b6910a0ef1fa921375d44281969ff827f4e2b95f9c5ecb18c", size = 6246025, upload-time = "2026-08-26T09:26:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/30/c1/6d3eeabb1184b76083d4ea8e48618de0a05edbe7b901a2940d585ff9ec8f/openvino_genai-2026.3.1.0-2499-cp312-cp312-manylinux_2_31_aarch64.whl", hash = "sha256:b87d2ffc51997c5c84f7e8b9bc7e60ac64690a1c99a5663ee317f214671e18d9", size = 5464592, upload-time = "2026-08-26T09:26:32.674Z" }, - { url = "https://files.pythonhosted.org/packages/bd/26/f89400b7403cbd9d85c613ca8e7d868f1858002bc31bc219a3d90987291d/openvino_genai-2026.3.1.0-2499-cp312-cp312-win_amd64.whl", hash = "sha256:e1eddd432240368e89fd2578c6b347b3324899f028087c20b3cf046736ce5a79", size = 3684545, upload-time = "2026-08-26T09:26:34.457Z" }, - { url = "https://files.pythonhosted.org/packages/77/93/174dacd88831827ac440faca0300ac52794ff253795e12cd6ca90d3579ca/openvino_genai-2026.3.1.0-2499-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b3f83298347b8de1657c73beb8f5f0ba4a14be77cdbdad09e052aa750270f4ce", size = 4243057, upload-time = "2026-08-26T09:26:36.119Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4e/b925d8fc7a5c532ccf61737ec5ec82aca24160b0f96df9a98ab1fbd06629/openvino_genai-2026.3.1.0-2499-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ff10cf990077992aec38db1895ef8eb7078efe78d0c00f51518df1abf99637a3", size = 6246018, upload-time = "2026-08-26T09:26:37.837Z" }, - { url = "https://files.pythonhosted.org/packages/aa/67/f375d13d5f2f2bc11f91c13843d19dd739902b493d53b4173789c5cec37c/openvino_genai-2026.3.1.0-2499-cp313-cp313-manylinux_2_31_aarch64.whl", hash = "sha256:f1b4d6bd5ad7ae08b236ffac54ccb17f95d14e482bbba457381ec79b0282e3cb", size = 5463609, upload-time = "2026-08-26T09:26:39.354Z" }, - { url = "https://files.pythonhosted.org/packages/ad/62/4394c5e38d358ec751252ba0d7d67b9bf6360fb081e72e4da9643daf1306/openvino_genai-2026.3.1.0-2499-cp313-cp313-win_amd64.whl", hash = "sha256:d061984f20e1ccdabbe9bd62829de3afcefd5d8f09ae2969a0735b631e3b1407", size = 3684665, upload-time = "2026-08-26T09:26:40.788Z" }, - { url = "https://files.pythonhosted.org/packages/f7/d7/f7c289399e7f8adca050d2af447a91b4f5212a906965584c60436b08c9ae/openvino_genai-2026.3.1.0-2499-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ce4b9a7a509b65a4f519a55fa42874e7df71957a65c909ade788c4883e8d1cc4", size = 4245346, upload-time = "2026-08-26T09:26:42.415Z" }, - { url = "https://files.pythonhosted.org/packages/5a/28/5202ed174592038c444368d58165a951c24325f05a40ceba9bf80864843f/openvino_genai-2026.3.1.0-2499-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:18192a5e6aa6382d834d8ca79f4aa9c1c1a0245a7eaba625b54f1824348cd12a", size = 6247562, upload-time = "2026-08-26T09:26:44.14Z" }, - { url = "https://files.pythonhosted.org/packages/3c/34/678229fa26dac2b7dae3b1a13ced8a325372b976236e5882aa941fa67f44/openvino_genai-2026.3.1.0-2499-cp314-cp314-manylinux_2_31_aarch64.whl", hash = "sha256:bd7b9f42bf52b1276c9f42ecdba6cdbfa2f3f74fbbdbfa6a40cf940b3914ee4b", size = 5469002, upload-time = "2026-08-26T09:26:45.906Z" }, - { url = "https://files.pythonhosted.org/packages/f6/57/9a7be92520024ac566f633535f2b17b7abfe2a3b7ae6598e4ce07fa14696/openvino_genai-2026.3.1.0-2499-cp314-cp314-win_amd64.whl", hash = "sha256:72420dd15042c6932e546e5328cb22dd1dd2ecee70c3464c4925d640b7d6c714", size = 3686035, upload-time = "2026-08-26T09:26:47.302Z" }, - { url = "https://files.pythonhosted.org/packages/da/f1/1059e94f8f1bc4c22d0c8742140288a68777da5abf4f71fa805fc066f48c/openvino_genai-2026.3.1.0-2499-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:48d87da2038416ad59f16101a4891171e54133d220eef3078031738b05f87123", size = 4339823, upload-time = "2026-08-26T09:26:48.928Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a7/4a362cbc0605d0d0dc0a5270d9d98eca3064ec090e42304721c3383e1ad8/openvino_genai-2026.3.1.0-2499-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c230a072177769146cb3ee30b07e8d64447cde215c0f5f5d9005750d800f0587", size = 6256251, upload-time = "2026-08-26T09:26:50.469Z" }, - { url = "https://files.pythonhosted.org/packages/61/a0/350eaaa0bfa47cace253147262a492e071c66d7e73d3ca54ca84dcbf3b76/openvino_genai-2026.3.1.0-2499-cp314-cp314t-manylinux_2_31_aarch64.whl", hash = "sha256:92a0d7c844ad0715a6aa9910f63169feea01ea94d7d1e699950b397891b0f24b", size = 5481961, upload-time = "2026-08-26T09:26:52.118Z" }, - { url = "https://files.pythonhosted.org/packages/a3/64/1aa95a4c0b6ebe223253a85956370f3310da241ff065e8803d29cfe28234/openvino_genai-2026.3.1.0-2499-cp314-cp314t-win_amd64.whl", hash = "sha256:1f6f4ebf24e0190bb52f3f3359b9b49559cbe4d8b9cfc0896822300164c89fec", size = 3686275, upload-time = "2026-08-26T09:26:53.583Z" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp312-cp312-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp312-cp312-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp312-cp312-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp312-cp312-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp313-cp313-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp313-cp313-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp313-cp313-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp313-cp313-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314t-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314t-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314t-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2740-cp314-cp314t-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp312-cp312-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp312-cp312-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp312-cp312-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp312-cp312-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp313-cp313-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp313-cp313-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp313-cp313-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp313-cp313-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314-win_amd64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314t-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314t-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314t-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-genai/openvino_genai-2026.5.0.0.dev20261007-2742-cp314-cp314t-win_amd64.whl" }, ] [[package]] @@ -1964,16 +2006,16 @@ wheels = [ [[package]] name = "openvino-tokenizers" -version = "2026.3.1.0" -source = { registry = "https://pypi.org/simple" } +version = "2026.5.0.0.dev20261007" +source = { registry = "https://storage.openvinotoolkit.org/simple/wheels/nightly" } dependencies = [ { name = "openvino" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/ee/43aae7a36b1ce8b4e525cddee0e3d01115ecb8a4b742eeece35d03c1d372/openvino_tokenizers-2026.3.1.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:881950692961f5fdc5628437210777fee9b97952737ba361830e5bb0a5a17539", size = 1750191, upload-time = "2026-08-26T09:20:55.183Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2a/e50406892a96b98bb1adeaf5197adf7a7ef6ead6638837e67ea0f2d8259e/openvino_tokenizers-2026.3.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:4dc45f6b62e31b1a9d7ae0a84c1897d6f385688e89a6c87398c43e23944e6edd", size = 1833713, upload-time = "2026-08-26T09:20:56.732Z" }, - { url = "https://files.pythonhosted.org/packages/76/70/4192b1ecfd06d5af9da63df513c02143940a14dc8931fa28291e40baf89a/openvino_tokenizers-2026.3.1.0-py3-none-manylinux_2_31_aarch64.whl", hash = "sha256:596c686cf92b942b48e94e071fa13ef3f3e53c3be097419a08240f57656e0d32", size = 1745649, upload-time = "2026-08-26T09:20:58.393Z" }, - { url = "https://files.pythonhosted.org/packages/13/d0/fdc569ef7dea6b679154a13ff30412733b04c4b9fc0018b4f23818e486ca/openvino_tokenizers-2026.3.1.0-py3-none-win_amd64.whl", hash = "sha256:9a18ac537abc02b8a8cc0d7165cffe79fe16e5087e3fe6625d40d0b84bccb1b9", size = 1541935, upload-time = "2026-08-26T09:20:59.758Z" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-tokenizers/openvino_tokenizers-2026.5.0.0.dev20261007-py3-none-macosx_11_0_arm64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-tokenizers/openvino_tokenizers-2026.5.0.0.dev20261007-py3-none-manylinux_2_28_x86_64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-tokenizers/openvino_tokenizers-2026.5.0.0.dev20261007-py3-none-manylinux_2_31_aarch64.whl" }, + { url = "https://storage.openvinotoolkit.org/wheels/nightly/openvino-tokenizers/openvino_tokenizers-2026.5.0.0.dev20261007-py3-none-win_amd64.whl" }, ] [[package]] @@ -2529,10 +2571,10 @@ name = "pyobjc-framework-applicationservices" version = "12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-coretext", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-coretext" }, + { name = "pyobjc-framework-quartz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/ab/1776ac62687cb3d81e59517ca55970f26efa5a96bbf3ebcea57afcdd6b06/pyobjc_framework_applicationservices-12.2.tar.gz", hash = "sha256:4f6c4027405f709872e5b098f3cd86961bdf262fb80679a548725a02171bb0cf", size = 109325, upload-time = "2026-05-30T12:30:18.7Z" } wheels = [ @@ -2550,7 +2592,7 @@ name = "pyobjc-framework-cocoa" version = "12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/cc/927169225e72bab9c9b44285656768fb75052a0bc85fdbca62740e1ca43c/pyobjc_framework_cocoa-12.2.tar.gz", hash = "sha256:20b392e2b7241caad0538dfde12143343e5dfe48f72e7df660a7548e635903dc", size = 3125555, upload-time = "2026-05-30T12:35:09.273Z" } wheels = [ @@ -2568,9 +2610,9 @@ name = "pyobjc-framework-coretext" version = "12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-quartz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/b0/e7ef99240f853d4dddde82c9c0114cc525de7355661b2bf2d5e04cfb1582/pyobjc_framework_coretext-12.2.tar.gz", hash = "sha256:82def2c281347e0677866315675124c84c36e9bc21651d62870cfdcecb7da34e", size = 97343, upload-time = "2026-05-30T12:37:00.996Z" } wheels = [ @@ -2588,8 +2630,8 @@ name = "pyobjc-framework-quartz" version = "12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, ] sdist = { url = "https://files.pythonhosted.org/packages/91/a3/5ae4c90c13999b46315f549694f25c374c48a9f7ab18f98ace6e74f4a5c1/pyobjc_framework_quartz-12.2.tar.gz", hash = "sha256:b343395d4790323b0376fe20c83ac468510ba19f65429323ca211708c939d107", size = 3215525, upload-time = "2026-05-30T12:44:27.759Z" } wheels = [ @@ -3483,8 +3525,8 @@ name = "standard-aifc" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, - { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, + { name = "audioop-lts" }, + { name = "standard-chunk" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } wheels = [ @@ -3505,7 +3547,7 @@ name = "standard-sunau" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "audioop-lts" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/e3/ce8d38cb2d70e05ffeddc28bb09bad77cfef979eb0a299c9117f7ed4e6a9/standard_sunau-3.13.0.tar.gz", hash = "sha256:b319a1ac95a09a2378a8442f403c66f4fd4b36616d6df6ae82b8e536ee790908", size = 9368, upload-time = "2024-10-30T16:01:41.626Z" } wheels = [ @@ -3690,13 +3732,13 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform == 'darwin'", ] dependencies = [ - { name = "filelock", marker = "sys_platform == 'darwin'" }, - { name = "fsspec", marker = "sys_platform == 'darwin'" }, - { name = "jinja2", marker = "sys_platform == 'darwin'" }, - { name = "networkx", marker = "sys_platform == 'darwin'" }, - { name = "setuptools", marker = "sys_platform == 'darwin'" }, - { name = "sympy", marker = "sys_platform == 'darwin'" }, - { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b41339df93d491435e790ff8bcbae1c0ce777175889bfd1281d119862793e6a2", upload-time = "2026-05-12T16:20:12Z" }, @@ -3715,13 +3757,13 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform != 'darwin'", ] dependencies = [ - { name = "filelock", marker = "sys_platform != 'darwin'" }, - { name = "fsspec", marker = "sys_platform != 'darwin'" }, - { name = "jinja2", marker = "sys_platform != 'darwin'" }, - { name = "networkx", marker = "sys_platform != 'darwin'" }, - { name = "setuptools", marker = "sys_platform != 'darwin'" }, - { name = "sympy", marker = "sys_platform != 'darwin'" }, - { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:b9d0e8eed0af9321ffb12b75f4aca371b071254f12cf75875d5a8e7cc8f52b51", upload-time = "2026-05-12T23:16:33Z" }, @@ -3757,9 +3799,9 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", marker = "sys_platform == 'darwin'" }, - { name = "pillow", marker = "sys_platform == 'darwin'" }, - { name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" } }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.27.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:1a6dd742a150645126df9e0b2e449874c1d635897c773b322c2e067e98382dfe", upload-time = "2026-05-12T16:20:37Z" }, @@ -3778,9 +3820,9 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", marker = "sys_platform != 'darwin'" }, - { name = "pillow", marker = "sys_platform != 'darwin'" }, - { name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" } }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.27.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1b06f42d48b62098114923d8a3fe9fa864182715db06584a515155db0aa8eb30", upload-time = "2026-05-12T16:20:36Z" },