perf: continuous batching engine (ovgenai_cb) + Arc-tuned serving improvements

docs/openvino-nightly.md

@@ -0,0 +1,50 @@
+# OpenVINO nightly builds
+
+OpenArc pins stable `openvino-genai` (see `dependencies` in `pyproject.toml`).
+Some models require a newer nightly build of OpenVINO / GenAI. Example:
+`OpenVINO/Qwen3.8-27B-int4-ov` requires OpenVINO 2026.4+ and **segfaults while
+loading** on the pinned 2026.3.1 stable.
+
+## Enabling the nightly (managed by uv)
+
+The `nightly-ov` dependency group in `pyproject.toml` carries minimum nightly
+versions for `openvino`, `openvino-genai`, and `openvino-tokenizers`:
+
+```bash
+uv sync --group nightly-ov \
+  --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \
+  --index-strategy unsafe-best-match
+```
+
+Note: a plain `uv sync` afterwards resolves back to the stable pins and will
+revert the environment. Either always sync with `--group nightly-ov`, or use
+`uv run --no-sync ...` after installing.
+
+## Manual override (no lockfile changes)
+
+```bash
+uv pip install --pre -U openvino openvino-genai openvino-tokenizers \
+  --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \
+  --index-strategy unsafe-best-match
+uv run --no-sync ...   # plain `uv run` / `uv sync` reverts to the stable pins
+```
+
+## Known-good nightly
+
+| Date | openvino | openvino-genai / openvino-tokenizers | Used for |
+|---|---|---|---|
+| 2026-10-07 | 2026.5.0.dev20261007 | 2026.5.0.0.dev20261007 | Qwen3.8-27B-int4-ov (VLM engine) |
+
+Record the exact nightly in every benchmark result produced while it is
+installed.
+
+## Companion upgrades (2026-10-08)
+
+- `transformers` was upgraded 5.0.0 → 5.19.0 (5.0.0 does not recognize the
+  `qwen3_5` architecture; needed for any Qwen3.5/3.8 export work) and
+  `optimum[openvino]` to the latest release. Note: even with these,
+  optimum-intel cannot export qwen3_5 for `text-generation-with-past` yet
+  (see PLAN.md 3.2), so Qwen3.5/3.8 IRs still come from pre-exported hubs.
+- `chat_template_kwargs` does **not** reach the Qwen3.5 chat template through
+  `apply_chat_template` on transformers 5.x — pass `enable_thinking` as a
+  direct keyword instead (OpenArc's VLM path already splats it correctly).

pyproject.toml

@@ -44,6 +44,17 @@ test = [
     "pytest-asyncio>=1.4.0",
     "httpx>=0.27",
 ]
+# OpenVINO nightly wheels (opt-in). Some models require a GenAI nightly newer
+# than the pinned stable release. Sync with:
+#   uv sync --group nightly-ov \
+#     --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \
+#     --index-strategy unsafe-best-match
+# Pin the exact nightly here once validated (see docs/openvino-nightly.md).
+nightly-ov = [
+    "openvino>=2026.5.0.dev20261007",
+    "openvino-genai>=2026.5.0.0.dev20261007",
+    "openvino-tokenizers>=2026.5.0.0.dev20261007",
+]
 
 [build-system]
 requires = ["setuptools>=68", "wheel"]

src/cli/groups/add.py

@@ -2,6 +2,7 @@
 Add command - Add a model configuration to the config file.
 """
 import json
+import shutil
 from pathlib import Path
 from typing import NoReturn
 
@@ -66,9 +67,9 @@ def _fail(extra: str = "") -> NoReturn:
     required=True,
     help='Path to OpenVINO IR converted model.')
 @click.option('--engine', '--en',
-    type=click.Choice(['ovgenai', 'openvino', 'optimum']),
+    type=click.Choice(['ovgenai', 'ovgenai_cb', 'openvino', 'optimum']),
     required=True,
-    help='Engine used to load the model (ovgenai, openvino, optimum)')
+    help='Engine used to load the model (ovgenai, ovgenai_cb, openvino, optimum)')
 @click.option('--model-type', '--mt',
     type=click.Choice([
         'llm', 'vlm', 'whisper', 'qwen3_asr', 'kokoro',
@@ -153,14 +154,25 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config,
             console.print(f"[red]Error parsing runtime_config JSON:[/red] {e}")
             console.print('[yellow]Example format: \'{"MODEL_DISTRIBUTION_POLICY": "PIPELINE_PARALLEL"}\'[/yellow]')
             ctx.exit(1)
-    # llm/vlm always get a model cache: compiled blobs live alongside the IR.
-    # An explicit CACHE_DIR in --runtime-config wins. Resolved to an absolute
-    # path here because runtime_config values are passed to OpenVINO verbatim,
-    # unlike model_path they are not resolved against the config file.
+    # llm/vlm always get a compiled-model cache. OpenVINO keys cache blobs by
+    # model content, so one shared directory is safe for any number of models
+    # (and for draft models). An explicit CACHE_DIR in --runtime-config wins.
+    # Resolved to an absolute path here because runtime_config values are
+    # passed to OpenVINO verbatim, unlike model_path they are not resolved
+    # against the config file. Blobs can reach a sizeable fraction of the IR
+    # size, so warn when the disk looks tight.
     if model_type in ("llm", "vlm"):
-        parsed_runtime_config.setdefault(
-            "CACHE_DIR", str((Path(model_path) / "model_cache").resolve())
-        )
+        default_cache_dir = Path.home() / ".cache" / "openarc" / "ov_cache"
+        parsed_runtime_config.setdefault("CACHE_DIR", str(default_cache_dir))
+        if parsed_runtime_config["CACHE_DIR"] == str(default_cache_dir):
+            free_gb = shutil.disk_usage(Path.home()).free / 1e9
+            if free_gb < 20:
+                console.print(
+                    f"[yellow]Warning: only {free_gb:.1f} GB free on the home filesystem; "
+                    f"the OpenVINO compiled-model cache at {default_cache_dir} can take "
+                    f"several GB per model. Free space or set CACHE_DIR via "
+                    f"--runtime-config if loads fail.[/yellow]"
+                )
 
     # Route the contract-flagged values into their blocks. Validation happens
     # against the contracts themselves, so a bad value is caught here rather

src/cli/groups/status.py

@@ -9,9 +9,21 @@
 
 
 @cli.command()
+@click.option("--doctor", is_flag=True, help="Run local Arc/GPU diagnostics (PCIe link, driver, compute-runtime, render access) instead of querying the server.")
 @click.pass_context
-def status(ctx):
+def status(ctx, doctor):
     """- GET Status of loaded models."""
+    if doctor:
+        from ..modules.doctor import collect_diagnostics, evaluate
+        console.print("[blue]Running local GPU diagnostics...[/blue]")
+        findings = evaluate(collect_diagnostics())
+        for level, message in findings:
+            style = "green" if level == "ok" else "yellow"
+            marker = "OK  " if level == "ok" else "WARN"
+            console.print(f"[{style}]{marker}[/{style}] {message}")
+        if any(level == "warn" for level, _ in findings):
+            ctx.exit(1)
+        return
     cli_instance = OpenArcCLI(server_config=ctx.obj.server_config)
 
     url = f"{cli_instance.base_url}/openarc/status"

src/cli/modules/doctor.py

@@ -0,0 +1,151 @@
+"""Local Intel Arc diagnostics for `openarc status --doctor`.
+
+Reports the things that silently cost performance or break inference on Arc:
+the PCIe link the card actually trained at (vs. what card and slot allow),
+the kernel driver in use (xe vs i915), the Intel compute-runtime version, and
+render-node group membership. Everything is read from sysfs / the local
+package manager; no server or root needed. PCIe link-walking follows
+Strata's sycl/serve/xe_telemetry.py.
+"""
+import glob
+import grp
+import os
+import subprocess
+
+
+def _read(path: str):
+    try:
+        with open(path) as f:
+            return f.read().strip()
+    except OSError:
+        return None
+
+
+def _find_intel_gpus():
+    """All Intel DRM devices; discrete Arc cards have a tile0 entry (iGPUs don't)."""
+    gpus = []
+    for dev in sorted(glob.glob("/sys/class/drm/card[0-9]*/device")):
+        if _read(os.path.join(dev, "vendor")) == "0x8086":
+            real = os.path.realpath(dev)
+            gpus.append({
+                "card": dev.split("/")[-2],
+                "path": real,
+                "discrete": os.path.isdir(os.path.join(real, "tile0")),
+            })
+    return gpus
+
+
+def _driver(gpu_path: str):
+    link = os.path.join(gpu_path, "driver")
+    if os.path.islink(link):
+        return os.path.basename(os.path.realpath(link))
+    return None
+
+
+def _pcie_link(gpu_path: str):
+    """(current_gen, max_gen, current_width, max_width) for the card's external link.
+
+    The card's own functions sit behind an internal x1 switch, so the reading
+    comes from the first upstream port wider than x1; the max is what card AND
+    slot allow (the root port caps it).
+    """
+    gen = {"2.5": 1, "5.0": 2, "8.0": 3, "16.0": 4, "32.0": 5, "64.0": 6}
+    d, chain = gpu_path, []
+    while d and d.startswith("/sys/devices/pci") and os.path.exists(os.path.join(d, "current_link_speed")):
+        chain.append(d)
+        d = os.path.dirname(d)
+
+    def g(path, kind):
+        return gen.get((_read(os.path.join(path, f"{kind}_link_speed")) or "").split(" ")[0])
+
+    for port in chain:
+        width = int(_read(os.path.join(port, "current_link_width")) or 0)
+        if width > 1:
+            card_max = g(port, "max")
+            slot_max = g(chain[-1], "max")
+            max_gen = min(x for x in (card_max, slot_max) if x) if (card_max or slot_max) else None
+            max_width_raw = _read(os.path.join(port, "max_link_width"))
+            max_width = int(max_width_raw) if max_width_raw else None
+            return g(port, "current"), max_gen, width, max_width
+    return None, None, None, None
+
+
+def _compute_runtime_version():
+    """Intel compute-runtime (Level Zero) version from the local package manager."""
+    for cmd in (["pacman", "-Q", "intel-compute-runtime"],
+                ["dpkg-query", "-W", "-f=${Version}", "intel-level-zero-gpu"]):
+        try:
+            out = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+            if out.returncode == 0 and out.stdout.strip():
+                return out.stdout.strip().split()[-1]
+        except (OSError, subprocess.SubprocessError):
+            continue
+    return None
+
+
+def _render_access():
+    """(node, group, accessible) for each render node."""
+    gids = set(os.getgroups()) | {os.getgid()}
+    rows = []
+    for node in sorted(glob.glob("/dev/dri/renderD*")):
+        try:
+            st = os.stat(node)
+        except OSError:
+            continue
+        try:
+            group = grp.getgrgid(st.st_gid).gr_name
+        except KeyError:
+            group = str(st.st_gid)
+        rows.append((node, group, os.access(node, os.R_OK | os.W_OK)))
+    return rows
+
+
+def collect_diagnostics():
+    """Gather all doctor checks. Returns a dict; never raises on missing data."""
+    gpus = _find_intel_gpus()
+    for gpu in gpus:
+        gpu["driver"] = _driver(gpu["path"])
+        cur_gen, max_gen, width, max_width = _pcie_link(gpu["path"])
+        gpu["pcie"] = {"gen": cur_gen, "gen_max": max_gen, "width": width, "width_max": max_width}
+    return {
+        "gpus": gpus,
+        "compute_runtime": _compute_runtime_version(),
+        "render_nodes": _render_access(),
+    }
+
+
+def evaluate(diag):
+    """Turn diagnostics into (level, message) findings. level: ok / warn."""
+    findings = []
+    if not diag["gpus"]:
+        findings.append(("warn", "no Intel GPU found under /sys/class/drm"))
+        return findings
+    for gpu in diag["gpus"]:
+        label = "discrete Arc" if gpu["discrete"] else "integrated GPU"
+        driver = gpu["driver"] or "unknown"
+        if gpu["discrete"] and driver == "i915":
+            findings.append(("warn", f"{gpu['card']} ({label}) is bound to i915; Arc discrete cards want the xe driver"))
+        else:
+            findings.append(("ok", f"{gpu['card']} ({label}) driver: {driver}"))
+        pcie = gpu["pcie"]
+        if gpu["discrete"] and pcie["gen"]:
+            cur = f"Gen{pcie['gen']} x{pcie['width']}"
+            best = f"Gen{pcie['gen_max']} x{pcie['width_max']}" if pcie["gen_max"] else "unknown"
+            if (pcie["gen_max"] and pcie["gen"] < pcie["gen_max"]) or \
+               (pcie["width_max"] and pcie["width"] < pcie["width_max"]):
+                findings.append(("warn", f"{gpu['card']} PCIe link trained at {cur}, but card and slot allow {best}; "
+                                         "prefill and model loads are link-bound (reseat, riser, or BIOS link settings)"))
+            else:
+                findings.append(("ok", f"{gpu['card']} PCIe link: {cur} (max {best})"))
+    rt = diag["compute_runtime"]
+    if rt:
+        findings.append(("ok", f"Intel compute-runtime: {rt}"))
+    else:
+        findings.append(("warn", "Intel compute-runtime (Level Zero) not found via pacman/dpkg"))
+    for node, group, accessible in diag["render_nodes"]:
+        if accessible:
+            findings.append(("ok", f"{node}: group '{group}', accessible"))
+        else:
+            findings.append(("warn", f"{node}: group '{group}', NOT accessible by this user "
+                                     "(add the user to the render group)"))
+    return findings