From 9a1c3254e1b46c8ddfc0f6e4d40d99a8cf91a75e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:29:50 -0400
Subject: [PATCH 001/128] Add provider API inference smoke-test harness

---
 tools/provider_api/smoke_test.py | 237 +++++++++++++++++++++++++++++++
 1 file changed, 237 insertions(+)
 create mode 100644 tools/provider_api/smoke_test.py

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
new file mode 100644
index 0000000..993a1e4
--- /dev/null
+++ b/tools/provider_api/smoke_test.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Minimal, non-streaming provider inference smoke tests.
+
+Uses only Python's standard library. Credentials are read from *_API_KEY_FLOSSWARE
+environment variables, never accepted as command-line arguments, and never emitted.
+No automatic retries are performed.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+PROMPT = "Reply with exactly this token and nothing else: FLOSSWARE_PROVIDER_SMOKE_OK"
+EXPECTED = "FLOSSWARE_PROVIDER_SMOKE_OK"
+
+# The default batch is restricted to catalogue-marked OpenRouter :free models and
+# Gemini models visible in the authenticated catalogue. Paid/promotional-credit
+# comparisons require --include-paid.
+CASES: list[dict[str, str]] = [
+    {"provider": "GOOGLE", "model": "models/gemini-2.5-flash-lite", "access": "free-tier-candidate"},
+    {"provider": "GOOGLE", "model": "models/gemini-2.5-flash", "access": "free-tier-candidate"},
+    {"provider": "OPENROUTER", "model": "nvidia/nemotron-3.5-lightning:free", "access": "catalogue-zero-price"},
+    {"provider": "OPENROUTER", "model": "google/gemma-4-31b-it:free", "access": "catalogue-zero-price"},
+    {"provider": "OPENROUTER", "model": "thinkingmachines/inkling:free", "access": "catalogue-zero-price"},
+    {"provider": "OPENROUTER", "model": "thinkingmachines/inkling-small:free", "access": "catalogue-zero-price"},
+    {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
+    {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
+    {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
+    {"provider": "DEEPSEEK", "model": "deepseek-flash", "access": "paid-unless-account-credit"},
+    {"provider": "GROQ", "model": "openai/gpt-oss-120b", "access": "paid"},
+]
+
+OPENAI_COMPATIBLE = {
+    "CEREBRAS": "https://api.cerebras.ai/v1/chat/completions",
+    "DEEPSEEK": "https://api.deepseek.com/chat/completions",
+    "GROQ": "https://api.groq.com/openai/v1/chat/completions",
+    "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
+}
+ENV_VARS = {
+    "CEREBRAS": "CEREBRAS_API_KEY_FLOSSWARE",
+    "DEEPSEEK": "DEEPSEEK_API_KEY_FLOSSWARE",
+    "GOOGLE": "GOOGLE_API_KEY_FLOSSWARE",
+    "GROQ": "GROQ_API_KEY_FLOSSWARE",
+    "OPENROUTER": "OPENROUTER_API_KEY_FLOSSWARE",
+}
+
+
+def safe_text(value: Any, secret: str) -> str:
+    text = str(value)
+    if secret:
+        text = text.replace(secret, "[REDACTED]")
+    return text[:2000]
+
+
+def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
+    provider, model = case["provider"], case["model"]
+    env_name = ENV_VARS[provider]
+    key = os.environ.get(env_name, "")
+    result: dict[str, Any] = {
+        "observed_at": datetime.now(timezone.utc).isoformat(),
+        "provider": provider,
+        "model_requested": model,
+        "access_classification": case["access"],
+        "credential_env": env_name,
+        "credential_configured": bool(key),
+        "http_status": None,
+        "latency_seconds": None,
+        "provider_request_id": None,
+        "model_returned": None,
+        "finish_reason": None,
+        "usage": None,
+        "response_text": None,
+        "semantic_match": None,
+        "rate_limit_headers": {},
+        "error": None,
+        "retry_performed": False,
+    }
+    if not key:
+        result["status"] = "skipped-no-credential"
+        return result
+
+    if provider == "GOOGLE":
+        url = f"https://generativelanguage.googleapis.com/v1beta/{model}:generateContent"
+        payload = {
+            "contents": [{"role": "user", "parts": [{"text": PROMPT}]}],
+            "generationConfig": {"maxOutputTokens": max_tokens, "temperature": 0},
+        }
+        headers = {"x-goog-api-key": key, "Content-Type": "application/json"}
+    else:
+        url = OPENAI_COMPATIBLE[provider]
+        payload = {
+            "model": model,
+            "messages": [{"role": "user", "content": PROMPT}],
+            "max_tokens": max_tokens,
+            "temperature": 0,
+            "stream": False,
+        }
+        headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
+        if provider == "OPENROUTER":
+            headers["HTTP-Referer"] = "https://flossware.org"
+            headers["X-Title"] = "FlossWare provider API smoke test"
+
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=headers,
+        method="POST",
+    )
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            body_bytes = response.read()
+            result["http_status"] = response.status
+            result["rate_limit_headers"] = {
+                name.lower(): value
+                for name, value in response.headers.items()
+                if "ratelimit" in name.lower() or name.lower() in {
+                    "retry-after", "x-request-id", "request-id", "cf-ray"
+                }
+            }
+            result["provider_request_id"] = (
+                response.headers.get("x-request-id")
+                or response.headers.get("request-id")
+                or response.headers.get("cf-ray")
+            )
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        data = json.loads(body_bytes.decode("utf-8"))
+
+        if provider == "GOOGLE":
+            candidates = data.get("candidates") or []
+            candidate = candidates[0] if candidates else {}
+            parts = (candidate.get("content") or {}).get("parts") or []
+            text = "".join(part.get("text", "") for part in parts if isinstance(part, dict))
+            result["model_returned"] = data.get("modelVersion") or model
+            result["finish_reason"] = candidate.get("finishReason")
+            result["usage"] = data.get("usageMetadata")
+        else:
+            choices = data.get("choices") or []
+            choice = choices[0] if choices else {}
+            message = choice.get("message") or {}
+            text = message.get("content") or ""
+            result["model_returned"] = data.get("model") or model
+            result["finish_reason"] = choice.get("finish_reason")
+            result["usage"] = data.get("usage")
+            result["provider_request_id"] = result["provider_request_id"] or data.get("id")
+
+        result["response_text"] = safe_text(text, key)
+        result["semantic_match"] = text.strip() == EXPECTED
+        result["status"] = "passed" if result["semantic_match"] else "response-mismatch"
+    except urllib.error.HTTPError as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["http_status"] = exc.code
+        result["provider_request_id"] = (
+            exc.headers.get("x-request-id")
+            or exc.headers.get("request-id")
+            or exc.headers.get("cf-ray")
+        )
+        result["rate_limit_headers"] = {
+            name.lower(): value
+            for name, value in exc.headers.items()
+            if "ratelimit" in name.lower() or name.lower() in {"retry-after", "x-request-id", "request-id", "cf-ray"}
+        }
+        try:
+            detail = exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            detail = str(exc)
+        result["error"] = safe_text(detail, key)
+        result["status"] = "http-error"
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["error"] = safe_text(exc, key)
+        result["status"] = "transport-error"
+    except (json.JSONDecodeError, UnicodeDecodeError, KeyError, TypeError, ValueError) as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["error"] = safe_text(f"Response parsing failed: {type(exc).__name__}: {exc}", key)
+        result["status"] = "response-parse-error"
+    finally:
+        # Avoid retaining the key in local references longer than necessary.
+        key = ""
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--provider", action="append", choices=sorted(ENV_VARS),
+                        help="Run only this provider; repeat to select several.")
+    parser.add_argument("--include-paid", action="store_true",
+                        help="Also run paid/promotional-credit comparison candidates.")
+    parser.add_argument("--timeout", type=float, default=45.0)
+    parser.add_argument("--max-output-tokens", type=int, default=32)
+    parser.add_argument("--output", type=Path, default=Path("provider-smoke-results.jsonl"))
+    args = parser.parse_args()
+
+    if args.timeout <= 0 or args.max_output_tokens < 1:
+        parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
+
+    selected = [
+        case for case in CASES
+        if (not args.provider or case["provider"] in args.provider)
+        and (args.include_paid or case["access"] not in {
+            "promotional-or-paid", "paid-unless-account-credit", "paid"
+        })
+    ]
+    if not selected:
+        parser.error("No test cases selected.")
+
+    results = [request_case(case, args.timeout, args.max_output_tokens) for case in selected]
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("w", encoding="utf-8") as stream:
+        for item in results:
+            stream.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+    for item in results:
+        status = item["status"]
+        http = item["http_status"] if item["http_status"] is not None else "-"
+        latency = item["latency_seconds"] if item["latency_seconds"] is not None else "-"
+        print(f'{item["provider"]:10} {item["model_requested"]:46} {status:24} HTTP={http} latency={latency}s')
+
+    passed = sum(item["status"] == "passed" for item in results)
+    skipped = sum(item["status"] == "skipped-no-credential" for item in results)
+    failed = len(results) - passed - skipped
+    print(f"\nSummary: {passed} passed, {failed} failed, {skipped} skipped; {len(results)} selected.")
+    print(f"Evidence: {args.output}")
+    print("No retries performed. Review provider billing/usage independently; catalogue price is not a bill.")
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From 2561a5e68f943ff84224fa2a89c49c0eece281b3 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:29:57 -0400
Subject: [PATCH 002/128] Add provider catalogue inventory utility

---
 tools/provider_api/catalogue_inventory.sh | 226 ++++++++++++++++++++++
 1 file changed, 226 insertions(+)
 create mode 100644 tools/provider_api/catalogue_inventory.sh

diff --git a/tools/provider_api/catalogue_inventory.sh b/tools/provider_api/catalogue_inventory.sh
new file mode 100644
index 0000000..677ea03
--- /dev/null
+++ b/tools/provider_api/catalogue_inventory.sh
@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# Read authenticated model catalogues and normalize public model metadata.
+# This script does NOT invoke models. It never prints or writes credential values.
+
+OUT="${1:-provider-model-catalogue.json}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+command -v curl >/dev/null || { echo "curl required" >&2; exit 1; }
+command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; }
+
+declare -A KEYS=(
+  [CEREBRAS]=CEREBRAS_API_KEY_FLOSSWARE
+  [COHERE]=COHERE_API_KEY_FLOSSWARE
+  [DEEPINFRA]=DEEPINFRA_API_KEY_FLOSSWARE
+  [DEEPSEEK]=DEEPSEEK_API_KEY_FLOSSWARE
+  [GOOGLE]=GOOGLE_API_KEY_FLOSSWARE
+  [GROQ]=GROQ_API_KEY_FLOSSWARE
+  [HUGGINGFACE]=HUGGINGFACE_API_KEY_FLOSSWARE
+  [MISTRAL]=MISTRAL_API_KEY_FLOSSWARE
+  [NVIDIA]=NVIDIA_API_KEY_FLOSSWARE
+  [OPENAI]=OPENAI_API_KEY_FLOSSWARE
+  [OPENROUTER]=OPENROUTER_API_KEY_FLOSSWARE
+)
+
+declare -A URLS=(
+  [CEREBRAS]="https://api.cerebras.ai/v1/models"
+  [COHERE]="https://api.cohere.com/v1/models"
+  [DEEPINFRA]="https://api.deepinfra.com/v1/openai/models"
+  [DEEPSEEK]="https://api.deepseek.com/models"
+  [GOOGLE]="https://generativelanguage.googleapis.com/v1beta/models"
+  [GROQ]="https://api.groq.com/openai/v1/models"
+  [HUGGINGFACE]="https://router.huggingface.co/v1/models"
+  [MISTRAL]="https://api.mistral.ai/v1/models"
+  [NVIDIA]="https://integrate.api.nvidia.com/v1/models"
+  [OPENAI]="https://api.openai.com/v1/models"
+  [OPENROUTER]="https://openrouter.ai/api/v1/models"
+)
+
+RESULTS="$TMP/results.jsonl"
+: > "$RESULTS"
+
+for provider in CEREBRAS COHERE DEEPINFRA DEEPSEEK GOOGLE GROQ                 HUGGINGFACE MISTRAL NVIDIA OPENAI OPENROUTER; do
+  var="${KEYS[$provider]}"
+  key="${!var:-}"
+  body="$TMP/$provider.json"
+  status="not_configured"
+  elapsed=""
+  reason=""
+
+  if [[ -n "$key" ]]; then
+    if [[ "$provider" == GOOGLE ]]; then
+      config="$(printf 'header = "x-goog-api-key: %s"\n' "$key")"
+    else
+      config="$(printf 'header = "Authorization: Bearer %s"\n' "$key")"
+    fi
+
+    result="$(
+      printf '%s' "$config" |
+      curl --config - -sS         --connect-timeout 8 --max-time 30         -o "$body"         -w '%{http_code} %{time_total}'         "${URLS[$provider]}" 2>/dev/null || true
+    )"
+
+    read -r status elapsed <<< "$result"
+    status="${status:-000}"
+    unset config
+  else
+    reason="Credential not configured"
+  fi
+
+  python3 - "$provider" "$var" "$status" "$elapsed" "$body" "$reason" >> "$RESULTS" <<'PY'
+import json
+import re
+import sys
+
+provider, envvar, status, elapsed, path, reason = sys.argv[1:]
+
+def first(obj, *keys):
+    for key in keys:
+        value = obj.get(key) if isinstance(obj, dict) else None
+        if value is not None:
+            return value
+    return None
+
+def as_number(value):
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+def is_zero(value):
+    try:
+        return float(value) == 0.0
+    except (TypeError, ValueError):
+        return False
+
+models = []
+parse_error = None
+
+if status == "200":
+    try:
+        with open(path, encoding="utf-8") as f:
+            payload = json.load(f)
+
+        if isinstance(payload, list):
+            raw_models = payload
+        else:
+            raw_models = first(payload, "data", "models", "modelData") or []
+
+        if not isinstance(raw_models, list):
+            raw_models = []
+
+        for item in raw_models:
+            if not isinstance(item, dict):
+                continue
+
+            model_id = first(item, "id", "name", "model", "model_id")
+            if not model_id:
+                continue
+
+            pricing = first(item, "pricing", "prices")
+            if not isinstance(pricing, dict):
+                pricing = None
+
+            prompt_price = first(pricing, "prompt", "input") if pricing else None
+            completion_price = first(pricing, "completion", "output") if pricing else None
+
+            free = None
+            if prompt_price is not None and completion_price is not None:
+                free = is_zero(prompt_price) and is_zero(completion_price)
+            elif str(model_id).endswith(":free"):
+                free = True
+
+            methods = first(item, "supportedGenerationMethods", "supported_generation_methods")
+            endpoints = first(item, "endpoints", "supported_endpoints", "supportedEndpoints")
+            capabilities = first(item, "capabilities", "tasks")
+
+            model = {
+                "id": str(model_id),
+                "display_name": first(item, "displayName", "display_name", "name"),
+                "owned_by": first(item, "owned_by", "publisher", "organization"),
+                "context_length": as_number(first(item, "context_length", "context_window", "inputTokenLimit", "max_input_tokens")),
+                "max_output_tokens": as_number(first(item, "max_output_tokens", "max_completion_tokens", "outputTokenLimit")),
+                "input_modalities": first(item, "input_modalities", "inputModalities"),
+                "output_modalities": first(item, "output_modalities", "outputModalities"),
+                "generation_methods": methods,
+                "endpoints": endpoints,
+                "capabilities": capabilities,
+                "pricing": pricing,
+                "free_by_catalogue_metadata": free
+            }
+            models.append(model)
+    except Exception as exc:
+        parse_error = type(exc).__name__
+
+# Remove obvious non-chat-only entries from the initial general-inference shortlist.
+non_chat = re.compile(r"(embed|rerank|whisper|transcri|text-to-speech|tts|moderation|prompt.guard)", re.I)
+
+def chat_candidate(model):
+    if provider == "GOOGLE":
+        methods = model.get("generation_methods") or []
+        return "generateContent" in methods
+    text = " ".join(str(model.get(k) or "") for k in ("id", "endpoints", "capabilities"))
+    return not non_chat.search(text)
+
+candidates = [m for m in models if chat_candidate(m)]
+
+def rank(model):
+    free = model.get("free_by_catalogue_metadata")
+    ctx = model.get("context_length") or 0
+    return (0 if free is True else 1 if free is None else 2, -ctx, model["id"])
+
+candidates.sort(key=rank)
+
+record = {
+    "provider": provider,
+    "environment_variable": envvar,
+    "http_status": status,
+    "response_seconds": float(elapsed) if elapsed else None,
+    "catalogue_count": len(models),
+    "models": models,
+    "chat_candidates": candidates[:20],
+    "parse_error": parse_error,
+    "reason": reason or None
+}
+print(json.dumps(record))
+PY
+
+  printf '%-14s HTTP %-3s catalogue=%s\n' "$provider" "$status"     "$(python3 -c 'import json,sys; x=json.load(sys.stdin); print(x["catalogue_count"])' < <(tail -n 1 "$RESULTS"))"
+done
+
+python3 - "$RESULTS" "$OUT" <<'PY'
+import json
+import sys
+from datetime import datetime, timezone
+
+source, destination = sys.argv[1:]
+with open(source, encoding="utf-8") as f:
+    providers = [json.loads(line) for line in f if line.strip()]
+
+providers.sort(key=lambda p: p["provider"])
+report = {
+    "generated_at": datetime.now(timezone.utc).isoformat(),
+    "purpose": "Model catalogue inventory only; no inference requests made.",
+    "providers": providers
+}
+
+with open(destination, "w", encoding="utf-8") as f:
+    json.dump(report, f, indent=2, ensure_ascii=False)
+    f.write("\n")
+
+print("\nCandidate shortlist (up to 20 per provider):")
+for provider in providers:
+    if provider["http_status"] != "200":
+        print(f'{provider["provider"]}: HTTP {provider["http_status"]}')
+        continue
+    print(f'\n{provider["provider"]} ({provider["catalogue_count"]} models)')
+    for model in provider["chat_candidates"]:
+        free = model["free_by_catalogue_metadata"]
+        marker = "catalogue-$0" if free is True else "price-unknown" if free is None else "priced"
+        print(f'  {model["id"]} [{marker}]')
+
+print(f"\nFull normalized catalogue written to: {destination}")
+print("No credential values or inference outputs were written.")
+PY

From 02da0ee58793dac86c975816bbc34848df3c3d5d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:30:21 -0400
Subject: [PATCH 003/128] Document provider API research utilities

---
 README.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/README.md b/README.md
index f32e2cd..19b15a5 100644
--- a/README.md
+++ b/README.md
@@ -30,6 +30,38 @@ Loom defines execution and evidence semantics. `evaluation` supplies reusable ev
 
 Evaluation results should distinguish measured facts, evaluator judgments, confidence, and provenance. A successful transport call is not proof that the intended outcome was achieved.
 
+## Provider API research utilities
+
+Experimental provider-discovery and smoke-test scripts live under `tools/provider_api/`. They support the cross-project investigation documented in [FlossWare/research/provider-api-inference](https://github.com/FlossWare/research/tree/research/provider-api-inference-2026-11/provider-api-inference).
+
+These scripts are **research utilities, not part of the published `evaluation-ai` package**:
+
+- `catalogue_inventory.sh` reads authenticated provider model catalogues and writes a normalized local JSON report. It does not invoke models.
+- `smoke_test.py` sends a tiny synthetic prompt to selected catalogue-listed models and records transport/response evidence as JSONL.
+
+The harness does not own provider selection, routing, credential storage, retries, or production invocation. Those responsibilities remain with `model-gateway`. The harness reads only FlossWare-scoped environment variables and never accepts or prints credential values.
+
+### Run
+
+From the repository root, with the relevant `*_API_KEY_FLOSSWARE` variables exported:
+
+```bash
+bash tools/provider_api/catalogue_inventory.sh
+python3 tools/provider_api/smoke_test.py
+```
+
+The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
+
+The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. Paid or promotional-credit comparison models require explicit opt-in:
+
+```bash
+python3 tools/provider_api/smoke_test.py --include-paid
+```
+
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, uses a small output limit, and performs no automatic retries. It captures total latency, HTTP status, request IDs, rate-limit headers, usage, finish reason, and whether the exact synthetic response was returned. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+
+A zero price in a model catalogue is not a guarantee of availability or account eligibility. Check provider billing and account terms independently. Never send real user data, proprietary code, or secrets in these tests.
+
 ## Packaging note
 
 The repository is named `evaluation`. The current distribution/import identifiers remain `evaluation-ai` and `evaluation_ai`; repository naming does not automatically rename a published package or Python imports. Change those identifiers only through an explicit compatibility and release decision.

From edda04452c8ab0f8e007304ed4bc0479b3338f17 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:30:26 -0400
Subject: [PATCH 004/128] Ignore local provider API research outputs

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 386f7d7..90c5a94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,10 @@ env/
 .coverage
 htmlcov/
 
+# Provider API research outputs (local evidence, never commit by default)
+provider-model-catalogue.json
+provider-smoke-results*.jsonl
+
 # OS
 .DS_Store
 Thumbs.db

From c853eb989d25487be251aeb74662cde32fe48403 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:30:51 -0400
Subject: [PATCH 005/128] Require positive metadata for chat candidate
 classification

---
 tools/provider_api/catalogue_inventory.sh | 25 +++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/catalogue_inventory.sh b/tools/provider_api/catalogue_inventory.sh
index 677ea03..1b2abf2 100644
--- a/tools/provider_api/catalogue_inventory.sh
+++ b/tools/provider_api/catalogue_inventory.sh
@@ -161,8 +161,29 @@ def chat_candidate(model):
     if provider == "GOOGLE":
         methods = model.get("generation_methods") or []
         return "generateContent" in methods
-    text = " ".join(str(model.get(k) or "") for k in ("id", "endpoints", "capabilities"))
-    return not non_chat.search(text)
+
+    inputs = model.get("input_modalities")
+    outputs = model.get("output_modalities")
+    if inputs is not None or outputs is not None:
+        input_text = isinstance(inputs, list) and "text" in [str(x).lower() for x in inputs]
+        output_text = isinstance(outputs, list) and "text" in [str(x).lower() for x in outputs]
+        return input_text and output_text
+
+    endpoints = model.get("endpoints")
+    if isinstance(endpoints, list):
+        return any(
+            marker in str(endpoint).lower()
+            for endpoint in endpoints
+            for marker in ("chat/completions", "/messages", "/responses")
+        )
+
+    capabilities = model.get("capabilities")
+    if isinstance(capabilities, list):
+        known_chat_tasks = {"text-generation", "conversational", "chat-completion"}
+        return bool(known_chat_tasks.intersection(str(x).lower() for x in capabilities))
+
+    # Missing metadata is unknown, not evidence of chat capability.
+    return False
 
 candidates = [m for m in models if chat_candidate(m)]
 

From 026275a2445ac3d5adacfeef3dd1424b38aa71cc Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:39:17 -0400
Subject: [PATCH 006/128] Classify incomplete reasoning responses in provider
 smoke tests

---
 tools/provider_api/smoke_test.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 993a1e4..27c138e 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -154,7 +154,31 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
 
         result["response_text"] = safe_text(text, key)
         result["semantic_match"] = text.strip() == EXPECTED
-        result["status"] = "passed" if result["semantic_match"] else "response-mismatch"
+
+        # Preserve provider-native usage while exposing a normalized reasoning count.
+        usage = result["usage"] or {}
+        if provider == "GOOGLE":
+            reasoning_tokens = usage.get("thoughtsTokenCount", 0)
+        else:
+            completion_details = usage.get("completion_tokens_details") or {}
+            reasoning_tokens = completion_details.get("reasoning_tokens", usage.get("reasoning_tokens", 0))
+        result["reasoning_tokens"] = reasoning_tokens or 0
+        result["visible_output_chars"] = len(text.strip())
+
+        if result["semantic_match"]:
+            result["response_completeness"] = "complete"
+            result["status"] = "passed"
+        elif not text.strip() and result["finish_reason"] in {"length", "MAX_TOKENS"}:
+            result["response_completeness"] = "incomplete"
+            result["status"] = (
+                "incomplete-reasoning" if result["reasoning_tokens"] else "incomplete-output"
+            )
+        elif not text.strip():
+            result["response_completeness"] = "empty"
+            result["status"] = "empty-response"
+        else:
+            result["response_completeness"] = "complete"
+            result["status"] = "response-mismatch"
     except urllib.error.HTTPError as exc:
         result["latency_seconds"] = round(time.monotonic() - started, 3)
         result["http_status"] = exc.code
@@ -195,7 +219,8 @@ def main() -> int:
     parser.add_argument("--include-paid", action="store_true",
                         help="Also run paid/promotional-credit comparison candidates.")
     parser.add_argument("--timeout", type=float, default=45.0)
-    parser.add_argument("--max-output-tokens", type=int, default=32)
+    parser.add_argument("--max-output-tokens", type=int, default=128,
+                        help="Maximum generated tokens per request (default: 128).")
     parser.add_argument("--output", type=Path, default=Path("provider-smoke-results.jsonl"))
     args = parser.parse_args()
 

From f8bfb275f38de4a41cfc247c3894b6c5c33ef473 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:39:19 -0400
Subject: [PATCH 007/128] Filter task-specific models from chat candidate
 shortlist

---
 tools/provider_api/catalogue_inventory.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/catalogue_inventory.sh b/tools/provider_api/catalogue_inventory.sh
index 1b2abf2..1ca2ba0 100644
--- a/tools/provider_api/catalogue_inventory.sh
+++ b/tools/provider_api/catalogue_inventory.sh
@@ -155,9 +155,20 @@ if status == "200":
         parse_error = type(exc).__name__
 
 # Remove obvious non-chat-only entries from the initial general-inference shortlist.
-non_chat = re.compile(r"(embed|rerank|whisper|transcri|text-to-speech|tts|moderation|prompt.guard)", re.I)
+non_chat = re.compile(
+    r"(embed|rerank|whisper|transcri|text-to-speech|tts|moderation|prompt.guard|"
+    r"lyria|music-generation|image-generation|text-to-image|video-generation|"
+    r"speech-generation|voice-generation)",
+    re.I,
+)
 
 def chat_candidate(model):
+    # Task-specific models can expose generic generation endpoints without being
+    # conversational models. Exclude known non-chat families before endpoint checks.
+    identity = " ".join(str(model.get(key) or "") for key in ("id", "display_name"))
+    if non_chat.search(identity):
+        return False
+
     if provider == "GOOGLE":
         methods = model.get("generation_methods") or []
         return "generateContent" in methods

From 4984968c2efc0f58eb35d97dc899c707ae13da7d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:39:21 -0400
Subject: [PATCH 008/128] Document normalized reasoning evidence and smoke
 budget

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 19b15a5..5aeaada 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ The default smoke batch is limited to catalogue-marked OpenRouter `:free` models
 python3 tools/provider_api/smoke_test.py --include-paid
 ```
 
-Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, uses a small output limit, and performs no automatic retries. It captures total latency, HTTP status, request IDs, rate-limit headers, usage, finish reason, and whether the exact synthetic response was returned. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. An empty response ending at the output limit is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; it is not mislabeled as a semantic mismatch. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
 A zero price in a model catalogue is not a guarantee of availability or account eligibility. Check provider billing and account terms independently. Never send real user data, proprietary code, or secrets in these tests.
 

From 70db0508479f44b57762457625a1665cc85b0610 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:46:44 -0400
Subject: [PATCH 009/128] Add exact model selection to provider smoke tests

---
 tools/provider_api/smoke_test.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 27c138e..e99eb29 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -216,6 +216,8 @@ def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--provider", action="append", choices=sorted(ENV_VARS),
                         help="Run only this provider; repeat to select several.")
+    parser.add_argument("--model",
+                        help="Run only this exact configured model ID; combine with --provider to validate its provider.")
     parser.add_argument("--include-paid", action="store_true",
                         help="Also run paid/promotional-credit comparison candidates.")
     parser.add_argument("--timeout", type=float, default=45.0)
@@ -227,13 +229,26 @@ def main() -> int:
     if args.timeout <= 0 or args.max_output_tokens < 1:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
+    # Validate exact model selection against the configured case catalogue first.
+    # This keeps --model narrow and prevents arbitrary model IDs from reaching providers.
+    if args.model:
+        matching_models = [case for case in CASES if case["model"] == args.model]
+        if not matching_models:
+            parser.error(f"Model is not configured in this smoke-test catalogue: {args.model}")
+        if args.provider and not any(case["provider"] in args.provider for case in matching_models):
+            configured = ", ".join(sorted({case["provider"] for case in matching_models}))
+            parser.error(f"Model {args.model!r} is configured for {configured}, not the selected provider(s).")
+
     selected = [
         case for case in CASES
         if (not args.provider or case["provider"] in args.provider)
+        and (not args.model or case["model"] == args.model)
         and (args.include_paid or case["access"] not in {
             "promotional-or-paid", "paid-unless-account-credit", "paid"
         })
     ]
+    if not selected and args.model:
+        parser.error(f"Model {args.model!r} is excluded from the default batch; use --include-paid if it is a paid/promotional candidate.")
     if not selected:
         parser.error("No test cases selected.")
 

From 8bf881c2763f86f37e5e3abdeed9a535969bb6a3 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:59:29 -0400
Subject: [PATCH 010/128] Correct truncated response classification and retain
 smoke evidence

---
 tools/provider_api/smoke_test.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index e99eb29..a4e6dfb 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -68,6 +68,7 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
         "observed_at": datetime.now(timezone.utc).isoformat(),
         "provider": provider,
         "model_requested": model,
+        "requested_max_output_tokens": max_tokens,
         "access_classification": case["access"],
         "credential_env": env_name,
         "credential_configured": bool(key),
@@ -165,14 +166,19 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
         result["reasoning_tokens"] = reasoning_tokens or 0
         result["visible_output_chars"] = len(text.strip())
 
-        if result["semantic_match"]:
-            result["response_completeness"] = "complete"
-            result["status"] = "passed"
-        elif not text.strip() and result["finish_reason"] in {"length", "MAX_TOKENS"}:
+        finish_reason = str(result["finish_reason"] or "").upper()
+        hit_output_limit = finish_reason in {"LENGTH", "MAX_TOKENS"}
+
+        # A length stop takes precedence over text matching: partial output must
+        # never be reported as complete, even when it contains some visible text.
+        if hit_output_limit:
             result["response_completeness"] = "incomplete"
             result["status"] = (
                 "incomplete-reasoning" if result["reasoning_tokens"] else "incomplete-output"
             )
+        elif result["semantic_match"]:
+            result["response_completeness"] = "complete"
+            result["status"] = "passed"
         elif not text.strip():
             result["response_completeness"] = "empty"
             result["status"] = "empty-response"
@@ -254,7 +260,7 @@ def main() -> int:
 
     results = [request_case(case, args.timeout, args.max_output_tokens) for case in selected]
     args.output.parent.mkdir(parents=True, exist_ok=True)
-    with args.output.open("w", encoding="utf-8") as stream:
+    with args.output.open("a", encoding="utf-8") as stream:
         for item in results:
             stream.write(json.dumps(item, ensure_ascii=False) + "\n")
 

From 6540bf13777de7ae6bc194d7f00e209bb0a32007 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:59:44 -0400
Subject: [PATCH 011/128] Extract response classification for deterministic
 tests

---
 tools/provider_api/smoke_test.py | 57 +++++++++++++++-----------------
 1 file changed, 26 insertions(+), 31 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index a4e6dfb..3cfb29f 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -60,6 +60,23 @@ def safe_text(value: Any, secret: str) -> str:
     return text[:2000]
 
 
+def classify_response(text: str, finish_reason: str | None, reasoning_tokens: int) -> tuple[str, str, bool]:
+    """Return (completeness, status, semantic_match) for one model response."""
+    semantic_match = text.strip() == EXPECTED
+    hit_output_limit = str(finish_reason or "").upper() in {"LENGTH", "MAX_TOKENS"}
+
+    # A length stop takes precedence over text matching: partial output must
+    # never be reported as complete, even when it contains some visible text.
+    if hit_output_limit:
+        status = "incomplete-reasoning" if reasoning_tokens else "incomplete-output"
+        return "incomplete", status, semantic_match
+    if semantic_match:
+        return "complete", "passed", True
+    if not text.strip():
+        return "empty", "empty-response", False
+    return "complete", "response-mismatch", False
+
+
 def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
     provider, model = case["provider"], case["model"]
     env_name = ENV_VARS[provider]
@@ -154,37 +171,15 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             result["provider_request_id"] = result["provider_request_id"] or data.get("id")
 
         result["response_text"] = safe_text(text, key)
-        result["semantic_match"] = text.strip() == EXPECTED
-
-        # Preserve provider-native usage while exposing a normalized reasoning count.
-        usage = result["usage"] or {}
-        if provider == "GOOGLE":
-            reasoning_tokens = usage.get("thoughtsTokenCount", 0)
-        else:
-            completion_details = usage.get("completion_tokens_details") or {}
-            reasoning_tokens = completion_details.get("reasoning_tokens", usage.get("reasoning_tokens", 0))
-        result["reasoning_tokens"] = reasoning_tokens or 0
-        result["visible_output_chars"] = len(text.strip())
-
-        finish_reason = str(result["finish_reason"] or "").upper()
-        hit_output_limit = finish_reason in {"LENGTH", "MAX_TOKENS"}
-
-        # A length stop takes precedence over text matching: partial output must
-        # never be reported as complete, even when it contains some visible text.
-        if hit_output_limit:
-            result["response_completeness"] = "incomplete"
-            result["status"] = (
-                "incomplete-reasoning" if result["reasoning_tokens"] else "incomplete-output"
-            )
-        elif result["semantic_match"]:
-            result["response_completeness"] = "complete"
-            result["status"] = "passed"
-        elif not text.strip():
-            result["response_completeness"] = "empty"
-            result["status"] = "empty-response"
-        else:
-            result["response_completeness"] = "complete"
-            result["status"] = "response-mismatch"
+        (
+            result["response_completeness"],
+            result["status"],
+            result["semantic_match"],
+        ) = classify_response(
+            text,
+            result["finish_reason"],
+            result["reasoning_tokens"],
+        )
     except urllib.error.HTTPError as exc:
         result["latency_seconds"] = round(time.monotonic() - started, 3)
         result["http_status"] = exc.code

From a9526a81e5b3b67314d31e1b4c1c2371d9aa4daf Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:59:54 -0400
Subject: [PATCH 012/128] Restore normalized usage fields before classification

---
 tools/provider_api/smoke_test.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 3cfb29f..9c0c3f1 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -171,6 +171,17 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             result["provider_request_id"] = result["provider_request_id"] or data.get("id")
 
         result["response_text"] = safe_text(text, key)
+
+        # Preserve provider-native usage while exposing normalized reasoning usage.
+        usage = result["usage"] or {}
+        if provider == "GOOGLE":
+            reasoning_tokens = usage.get("thoughtsTokenCount", 0)
+        else:
+            completion_details = usage.get("completion_tokens_details") or {}
+            reasoning_tokens = completion_details.get("reasoning_tokens", usage.get("reasoning_tokens", 0))
+        result["reasoning_tokens"] = reasoning_tokens or 0
+        result["visible_output_chars"] = len(text.strip())
+
         (
             result["response_completeness"],
             result["status"],

From e4b89a40bc2bf1950d24e93401df74ac0cfe3c02 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:00:04 -0500
Subject: [PATCH 013/128] Add deterministic smoke response classification tests

---
 tests/test_provider_api_smoke_test.py | 52 +++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 tests/test_provider_api_smoke_test.py

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
new file mode 100644
index 0000000..c2a99c5
--- /dev/null
+++ b/tests/test_provider_api_smoke_test.py
@@ -0,0 +1,52 @@
+"""Deterministic tests for provider smoke-test response classification."""
+import unittest
+
+from tools.provider_api.smoke_test import EXPECTED, classify_response
+
+
+class ClassifyResponseTests(unittest.TestCase):
+    def test_exact_response_with_stop_passes(self):
+        self.assertEqual(
+            classify_response(EXPECTED, "stop", 0),
+            ("complete", "passed", True),
+        )
+
+    def test_partial_text_with_length_is_incomplete_reasoning(self):
+        self.assertEqual(
+            classify_response("FLOSS", "length", 316),
+            ("incomplete", "incomplete-reasoning", False),
+        )
+
+    def test_partial_text_with_length_without_reasoning_is_incomplete_output(self):
+        self.assertEqual(
+            classify_response("FLOSS", "length", 0),
+            ("incomplete", "incomplete-output", False),
+        )
+
+    def test_exact_text_with_length_is_still_incomplete(self):
+        self.assertEqual(
+            classify_response(EXPECTED, "length", 4),
+            ("incomplete", "incomplete-reasoning", True),
+        )
+
+    def test_empty_response_without_length_is_empty(self):
+        self.assertEqual(
+            classify_response("", "stop", 0),
+            ("empty", "empty-response", False),
+        )
+
+    def test_nonmatching_complete_text_is_semantic_mismatch(self):
+        self.assertEqual(
+            classify_response("some other answer", "stop", 0),
+            ("complete", "response-mismatch", False),
+        )
+
+    def test_google_max_tokens_is_incomplete(self):
+        self.assertEqual(
+            classify_response("partial", "MAX_TOKENS", 12),
+            ("incomplete", "incomplete-reasoning", False),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

From 6f0efdda7a79e6816ff2102aad065a32d4fa2fdd Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:00:18 -0500
Subject: [PATCH 014/128] Document corrected classification and append-only
 evidence

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5aeaada..324ad68 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,9 @@ The default smoke batch is limited to catalogue-marked OpenRouter `:free` models
 python3 tools/provider_api/smoke_test.py --include-paid
 ```
 
-Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. An empty response ending at the output limit is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; it is not mislabeled as a semantic mismatch. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+
+Run the deterministic classification tests with `python3 -m unittest discover -s tests -v`.
 
 A zero price in a model catalogue is not a guarantee of availability or account eligibility. Check provider billing and account terms independently. Never send real user data, proprietary code, or secrets in these tests.
 

From 631d12d76016ac7dd2d32f6d08004b14e7b211ee Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:13:46 -0500
Subject: [PATCH 015/128] Identify provider smoke-test HTTP client explicitly

---
 tools/provider_api/smoke_test.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 9c0c3f1..decc737 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -60,6 +60,25 @@ def safe_text(value: Any, secret: str) -> str:
     return text[:2000]
 
 
+def build_headers(provider: str, key: str) -> dict[str, str]:
+    """Build explicit application-identifying headers for provider API requests."""
+    headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        # Cloudflare Error 1010 can reject urllib's default Python-urllib user agent.
+        # Identify the client honestly; do not impersonate a browser.
+        "User-Agent": "FlossWare-ProviderSmoke/0.1",
+    }
+    if provider == "GOOGLE":
+        headers["x-goog-api-key"] = key
+    else:
+        headers["Authorization"] = f"Bearer {key}"
+    if provider == "OPENROUTER":
+        headers["HTTP-Referer"] = "https://flossware.org"
+        headers["X-Title"] = "FlossWare provider API smoke test"
+    return headers
+
+
 def classify_response(text: str, finish_reason: str | None, reasoning_tokens: int) -> tuple[str, str, bool]:
     """Return (completeness, status, semantic_match) for one model response."""
     semantic_match = text.strip() == EXPECTED
@@ -111,7 +130,6 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             "contents": [{"role": "user", "parts": [{"text": PROMPT}]}],
             "generationConfig": {"maxOutputTokens": max_tokens, "temperature": 0},
         }
-        headers = {"x-goog-api-key": key, "Content-Type": "application/json"}
     else:
         url = OPENAI_COMPATIBLE[provider]
         payload = {
@@ -121,10 +139,7 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             "temperature": 0,
             "stream": False,
         }
-        headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
-        if provider == "OPENROUTER":
-            headers["HTTP-Referer"] = "https://flossware.org"
-            headers["X-Title"] = "FlossWare provider API smoke test"
+    headers = build_headers(provider, key)
 
     request = urllib.request.Request(
         url,

From ea66724bae986033de6d52448ca6609b66a7f8f9 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:13:50 -0500
Subject: [PATCH 016/128] Test explicit provider HTTP client headers

---
 tests/test_provider_api_smoke_test.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index c2a99c5..c625569 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,7 +1,20 @@
 """Deterministic tests for provider smoke-test response classification."""
 import unittest
 
-from tools.provider_api.smoke_test import EXPECTED, classify_response
+from tools.provider_api.smoke_test import EXPECTED, build_headers, classify_response
+
+
+class RequestHeaderTests(unittest.TestCase):
+    def test_provider_requests_identify_the_application(self):
+        headers = build_headers("CEREBRAS", "test-key")
+        self.assertEqual(headers["User-Agent"], "FlossWare-ProviderSmoke/0.1")
+        self.assertEqual(headers["Accept"], "application/json")
+        self.assertEqual(headers["Authorization"], "Bearer test-key")
+
+    def test_google_uses_its_api_key_header(self):
+        headers = build_headers("GOOGLE", "test-key")
+        self.assertEqual(headers["x-goog-api-key"], "test-key")
+        self.assertNotIn("Authorization", headers)
 
 
 class ClassifyResponseTests(unittest.TestCase):

From 86e45f47cdfda7778d2ea76c1bf4d9581b2ad557 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:16:27 -0500
Subject: [PATCH 017/128] Add DeepInfra representative inference candidate

---
 tools/provider_api/smoke_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index decc737..c3b31b8 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -32,6 +32,7 @@
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling-small:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
+    {"provider": "DEEPINFRA", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "access": "paid"},
     {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
     {"provider": "DEEPSEEK", "model": "deepseek-flash", "access": "paid-unless-account-credit"},
@@ -40,12 +41,14 @@
 
 OPENAI_COMPATIBLE = {
     "CEREBRAS": "https://api.cerebras.ai/v1/chat/completions",
+    "DEEPINFRA": "https://api.deepinfra.com/v1/openai/chat/completions",
     "DEEPSEEK": "https://api.deepseek.com/chat/completions",
     "GROQ": "https://api.groq.com/openai/v1/chat/completions",
     "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
 }
 ENV_VARS = {
     "CEREBRAS": "CEREBRAS_API_KEY_FLOSSWARE",
+    "DEEPINFRA": "DEEPINFRA_API_KEY_FLOSSWARE",
     "DEEPSEEK": "DEEPSEEK_API_KEY_FLOSSWARE",
     "GOOGLE": "GOOGLE_API_KEY_FLOSSWARE",
     "GROQ": "GROQ_API_KEY_FLOSSWARE",

From 81b630364a6cd50c16a61cc12d085182eb648424 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:16:31 -0500
Subject: [PATCH 018/128] Test DeepInfra endpoint configuration

---
 tests/test_provider_api_smoke_test.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index c625569..4c46c07 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,7 +1,7 @@
 """Deterministic tests for provider smoke-test response classification."""
 import unittest
 
-from tools.provider_api.smoke_test import EXPECTED, build_headers, classify_response
+from tools.provider_api.smoke_test import EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response
 
 
 class RequestHeaderTests(unittest.TestCase):
@@ -16,6 +16,12 @@ def test_google_uses_its_api_key_header(self):
         self.assertEqual(headers["x-goog-api-key"], "test-key")
         self.assertNotIn("Authorization", headers)
 
+    def test_deepinfra_uses_openai_compatible_chat_endpoint(self):
+        self.assertEqual(
+            OPENAI_COMPATIBLE["DEEPINFRA"],
+            "https://api.deepinfra.com/v1/openai/chat/completions",
+        )
+
 
 class ClassifyResponseTests(unittest.TestCase):
     def test_exact_response_with_stop_passes(self):

From 30bff47e0e3d4c5f0a5671d5cb0e54937edccd7a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:16:34 -0500
Subject: [PATCH 019/128] Document DeepInfra smoke-test candidate

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 324ad68..97f3a73 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ python3 tools/provider_api/smoke_test.py
 
 The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
 
-The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. Paid or promotional-credit comparison models require explicit opt-in:
+The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Paid or promotional-credit comparison models require explicit opt-in:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --include-paid

From 9616530eadaf5de0432a01b9477e5954e7f27540 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:17:48 -0500
Subject: [PATCH 020/128] Add NVIDIA NIM representative inference candidate

---
 tools/provider_api/smoke_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index c3b31b8..b7dad43 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -33,6 +33,7 @@
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling-small:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
     {"provider": "DEEPINFRA", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "access": "paid"},
+    {"provider": "NVIDIA", "model": "openai/gpt-oss-120b", "access": "paid"},
     {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
     {"provider": "DEEPSEEK", "model": "deepseek-flash", "access": "paid-unless-account-credit"},
@@ -42,6 +43,7 @@
 OPENAI_COMPATIBLE = {
     "CEREBRAS": "https://api.cerebras.ai/v1/chat/completions",
     "DEEPINFRA": "https://api.deepinfra.com/v1/openai/chat/completions",
+    "NVIDIA": "https://integrate.api.nvidia.com/v1/chat/completions",
     "DEEPSEEK": "https://api.deepseek.com/chat/completions",
     "GROQ": "https://api.groq.com/openai/v1/chat/completions",
     "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
@@ -49,6 +51,7 @@
 ENV_VARS = {
     "CEREBRAS": "CEREBRAS_API_KEY_FLOSSWARE",
     "DEEPINFRA": "DEEPINFRA_API_KEY_FLOSSWARE",
+    "NVIDIA": "NVIDIA_API_KEY_FLOSSWARE",
     "DEEPSEEK": "DEEPSEEK_API_KEY_FLOSSWARE",
     "GOOGLE": "GOOGLE_API_KEY_FLOSSWARE",
     "GROQ": "GROQ_API_KEY_FLOSSWARE",

From 123aab9dade350c59fe86580cf2246bec4002d71 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:17:52 -0500
Subject: [PATCH 021/128] Test NVIDIA NIM endpoint configuration

---
 tests/test_provider_api_smoke_test.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 4c46c07..6f4637a 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -22,6 +22,12 @@ def test_deepinfra_uses_openai_compatible_chat_endpoint(self):
             "https://api.deepinfra.com/v1/openai/chat/completions",
         )
 
+    def test_nvidia_uses_nim_chat_completions_endpoint(self):
+        self.assertEqual(
+            OPENAI_COMPATIBLE["NVIDIA"],
+            "https://integrate.api.nvidia.com/v1/chat/completions",
+        )
+
 
 class ClassifyResponseTests(unittest.TestCase):
     def test_exact_response_with_stop_passes(self):

From 2492ee18e1ba76d193d2f5a62af3d5767fb326b8 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:17:56 -0500
Subject: [PATCH 022/128] Document NVIDIA NIM smoke-test candidate

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 97f3a73..5939f72 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ python3 tools/provider_api/smoke_test.py
 
 The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
 
-The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Paid or promotional-credit comparison models require explicit opt-in:
+The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. It also includes NVIDIA NIM's `openai/gpt-oss-120b` candidate via NVIDIA's OpenAI-compatible `/v1/chat/completions` API. Paid or promotional-credit comparison models require explicit opt-in:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --include-paid

From 77e37d2b8f3fbf782f320f2d8620a4f14324f701 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:19:40 -0500
Subject: [PATCH 023/128] Add Hugging Face Inference Providers smoke candidate

---
 tools/provider_api/smoke_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index b7dad43..1393f4f 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -34,6 +34,7 @@
     {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
     {"provider": "DEEPINFRA", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "access": "paid"},
     {"provider": "NVIDIA", "model": "openai/gpt-oss-120b", "access": "paid"},
+    {"provider": "HUGGINGFACE", "model": "openai/gpt-oss-120b:fastest", "access": "paid-or-account-credit"},
     {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
     {"provider": "DEEPSEEK", "model": "deepseek-flash", "access": "paid-unless-account-credit"},
@@ -44,6 +45,7 @@
     "CEREBRAS": "https://api.cerebras.ai/v1/chat/completions",
     "DEEPINFRA": "https://api.deepinfra.com/v1/openai/chat/completions",
     "NVIDIA": "https://integrate.api.nvidia.com/v1/chat/completions",
+    "HUGGINGFACE": "https://router.huggingface.co/v1/chat/completions",
     "DEEPSEEK": "https://api.deepseek.com/chat/completions",
     "GROQ": "https://api.groq.com/openai/v1/chat/completions",
     "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
@@ -52,6 +54,7 @@
     "CEREBRAS": "CEREBRAS_API_KEY_FLOSSWARE",
     "DEEPINFRA": "DEEPINFRA_API_KEY_FLOSSWARE",
     "NVIDIA": "NVIDIA_API_KEY_FLOSSWARE",
+    "HUGGINGFACE": "HUGGINGFACE_API_KEY_FLOSSWARE",
     "DEEPSEEK": "DEEPSEEK_API_KEY_FLOSSWARE",
     "GOOGLE": "GOOGLE_API_KEY_FLOSSWARE",
     "GROQ": "GROQ_API_KEY_FLOSSWARE",

From 9f8d6b7a290f8fff502d0901a17df4cd40fbe08e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:21:12 -0500
Subject: [PATCH 024/128] Add safe all-configured provider batch mode

---
 tools/provider_api/smoke_test.py | 69 +++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 15 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 1393f4f..74a3119 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -252,6 +252,8 @@ def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--provider", action="append", choices=sorted(ENV_VARS),
                         help="Run only this provider; repeat to select several.")
+    parser.add_argument("--all-configured", action="store_true",
+                        help="Run one representative candidate per provider; paid-only candidates require --include-paid.")
     parser.add_argument("--model",
                         help="Run only this exact configured model ID; combine with --provider to validate its provider.")
     parser.add_argument("--include-paid", action="store_true",
@@ -265,6 +267,9 @@ def main() -> int:
     if args.timeout <= 0 or args.max_output_tokens < 1:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
+    if args.all_configured and (args.provider or args.model):
+        parser.error("--all-configured cannot be combined with --provider or --model")
+
     # Validate exact model selection against the configured case catalogue first.
     # This keeps --model narrow and prevents arbitrary model IDs from reaching providers.
     if args.model:
@@ -275,20 +280,54 @@ def main() -> int:
             configured = ", ".join(sorted({case["provider"] for case in matching_models}))
             parser.error(f"Model {args.model!r} is configured for {configured}, not the selected provider(s).")
 
-    selected = [
-        case for case in CASES
-        if (not args.provider or case["provider"] in args.provider)
-        and (not args.model or case["model"] == args.model)
-        and (args.include_paid or case["access"] not in {
-            "promotional-or-paid", "paid-unless-account-credit", "paid"
-        })
-    ]
-    if not selected and args.model:
-        parser.error(f"Model {args.model!r} is excluded from the default batch; use --include-paid if it is a paid/promotional candidate.")
-    if not selected:
-        parser.error("No test cases selected.")
-
-    results = [request_case(case, args.timeout, args.max_output_tokens) for case in selected]
+    paid_access = {
+        "paid", "paid-or-account-credit", "paid-unless-account-credit",
+        "promotional-or-paid",
+    }
+
+    if args.all_configured:
+        # One representative model per provider. Never fan out across every
+        # catalogue candidate, and never opt into paid candidates implicitly.
+        selected_by_provider: dict[str, dict[str, str]] = {}
+        for case in CASES:
+            if args.include_paid or case["access"] not in paid_access:
+                selected_by_provider.setdefault(case["provider"], case)
+
+        results = []
+        for provider in sorted(ENV_VARS):
+            key_configured = bool(os.environ.get(ENV_VARS[provider], ""))
+            case = selected_by_provider.get(provider)
+            if not key_configured:
+                results.append({
+                    "provider": provider, "model_requested": case["model"] if case else None,
+                    "credential_configured": False, "http_status": None,
+                    "latency_seconds": None, "status": "skipped-no-credential",
+                    "retry_performed": False,
+                })
+            elif case is None:
+                paid_case = next((c for c in CASES if c["provider"] == provider), None)
+                results.append({
+                    "provider": provider,
+                    "model_requested": paid_case["model"] if paid_case else None,
+                    "credential_configured": True, "http_status": None,
+                    "latency_seconds": None, "status": "skipped-paid-opt-in-required",
+                    "access_classification": paid_case["access"] if paid_case else None,
+                    "retry_performed": False,
+                })
+            else:
+                results.append(request_case(case, args.timeout, args.max_output_tokens))
+    else:
+        selected = [
+            case for case in CASES
+            if (not args.provider or case["provider"] in args.provider)
+            and (not args.model or case["model"] == args.model)
+            and (args.include_paid or case["access"] not in paid_access)
+        ]
+        if not selected and args.model:
+            parser.error(f"Model {args.model!r} is excluded from the default batch; use --include-paid if it is a paid/promotional candidate.")
+        if not selected:
+            parser.error("No test cases selected.")
+        results = [request_case(case, args.timeout, args.max_output_tokens) for case in selected]
     args.output.parent.mkdir(parents=True, exist_ok=True)
     with args.output.open("a", encoding="utf-8") as stream:
         for item in results:
@@ -301,7 +340,7 @@ def main() -> int:
         print(f'{item["provider"]:10} {item["model_requested"]:46} {status:24} HTTP={http} latency={latency}s')
 
     passed = sum(item["status"] == "passed" for item in results)
-    skipped = sum(item["status"] == "skipped-no-credential" for item in results)
+    skipped = sum(item["status"].startswith("skipped-") for item in results)
     failed = len(results) - passed - skipped
     print(f"\nSummary: {passed} passed, {failed} failed, {skipped} skipped; {len(results)} selected.")
     print(f"Evidence: {args.output}")

From c394bf6ff3024e3d7047fd9353a8b60eb69c2b58 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:21:24 -0500
Subject: [PATCH 025/128] Extract representative provider selection

---
 tools/provider_api/smoke_test.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 74a3119..025d0f6 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -105,6 +105,21 @@ def classify_response(text: str, finish_reason: str | None, reasoning_tokens: in
     return "complete", "response-mismatch", False
 
 
+PAID_ACCESS = {
+    "paid", "paid-or-account-credit", "paid-unless-account-credit",
+    "promotional-or-paid",
+}
+
+
+def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
+    """Choose at most one catalogue candidate per provider, preserving catalogue order."""
+    selected: dict[str, dict[str, str]] = {}
+    for case in CASES:
+        if include_paid or case["access"] not in PAID_ACCESS:
+            selected.setdefault(case["provider"], case)
+    return selected
+
+
 def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
     provider, model = case["provider"], case["model"]
     env_name = ENV_VARS[provider]
@@ -280,18 +295,10 @@ def main() -> int:
             configured = ", ".join(sorted({case["provider"] for case in matching_models}))
             parser.error(f"Model {args.model!r} is configured for {configured}, not the selected provider(s).")
 
-    paid_access = {
-        "paid", "paid-or-account-credit", "paid-unless-account-credit",
-        "promotional-or-paid",
-    }
-
     if args.all_configured:
         # One representative model per provider. Never fan out across every
         # catalogue candidate, and never opt into paid candidates implicitly.
-        selected_by_provider: dict[str, dict[str, str]] = {}
-        for case in CASES:
-            if args.include_paid or case["access"] not in paid_access:
-                selected_by_provider.setdefault(case["provider"], case)
+        selected_by_provider = representative_cases(args.include_paid)
 
         results = []
         for provider in sorted(ENV_VARS):
@@ -321,7 +328,7 @@ def main() -> int:
             case for case in CASES
             if (not args.provider or case["provider"] in args.provider)
             and (not args.model or case["model"] == args.model)
-            and (args.include_paid or case["access"] not in paid_access)
+            and (args.include_paid or case["access"] not in PAID_ACCESS)
         ]
         if not selected and args.model:
             parser.error(f"Model {args.model!r} is excluded from the default batch; use --include-paid if it is a paid/promotional candidate.")

From f3453cc3c3a26b90afd7decbb73bba0bc5fe0a37 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:21:31 -0500
Subject: [PATCH 026/128] Test all-configured batch selection

---
 tests/test_provider_api_smoke_test.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 6f4637a..db5ecee 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,7 +1,7 @@
 """Deterministic tests for provider smoke-test response classification."""
 import unittest
 
-from tools.provider_api.smoke_test import EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response
+from tools.provider_api.smoke_test import CASES, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases
 
 
 class RequestHeaderTests(unittest.TestCase):
@@ -29,6 +29,23 @@ def test_nvidia_uses_nim_chat_completions_endpoint(self):
         )
 
 
+class BatchSelectionTests(unittest.TestCase):
+    def test_batch_selects_at_most_one_candidate_per_provider(self):
+        selected = representative_cases(include_paid=False)
+        self.assertEqual(len(selected), len({case["provider"] for case in selected.values()}))
+        self.assertEqual(selected["GOOGLE"]["model"], "models/gemini-2.5-flash-lite")
+        self.assertEqual(selected["OPENROUTER"]["model"], "nvidia/nemotron-3.5-lightning:free")
+        self.assertNotIn("CEREBRAS", selected)
+        self.assertNotIn("NVIDIA", selected)
+
+    def test_paid_opt_in_adds_paid_only_providers(self):
+        selected = representative_cases(include_paid=True)
+        self.assertIn("CEREBRAS", selected)
+        self.assertIn("DEEPINFRA", selected)
+        self.assertIn("NVIDIA", selected)
+        self.assertEqual(selected["NVIDIA"]["model"], "openai/gpt-oss-120b")
+
+
 class ClassifyResponseTests(unittest.TestCase):
     def test_exact_response_with_stop_passes(self):
         self.assertEqual(

From 3da4fb84a59c0e54a5d32dcb74ac7faf08885af7 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:21:39 -0500
Subject: [PATCH 027/128] Document all-configured provider batch mode

---
 README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/README.md b/README.md
index 5939f72..73b11a7 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,20 @@ The default smoke batch is limited to catalogue-marked OpenRouter `:free` models
 python3 tools/provider_api/smoke_test.py --include-paid
 ```
 
+Run one representative candidate for every provider in a single batch:
+
+```bash
+python3 tools/provider_api/smoke_test.py --all-configured
+```
+
+This checks one non-paid candidate per provider where available. Providers without credentials are recorded as skipped; providers whose only configured candidate is paid or promotional-credit based are also skipped unless `--include-paid` is supplied. To explicitly include those candidates:
+
+```bash
+python3 tools/provider_api/smoke_test.py --all-configured --include-paid
+```
+
+The batch never fans out across every model in a provider catalogue. It selects at most one representative model per provider, continues after individual failures, and appends all outcomes to the same JSONL evidence file. `--all-configured` cannot be combined with `--provider` or `--model`.
+
 Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
 Run the deterministic classification tests with `python3 -m unittest discover -s tests -v`.

From eec66447c17b9eacd935a4381b43ca36d6daf7ea Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:27:45 -0500
Subject: [PATCH 028/128] Add Mistral and Cohere smoke tests; refresh Gemini
 candidate

---
 tools/provider_api/smoke_test.py | 49 ++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 5 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 025d0f6..2e8ea98 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -25,7 +25,9 @@
 # Gemini models visible in the authenticated catalogue. Paid/promotional-credit
 # comparisons require --include-paid.
 CASES: list[dict[str, str]] = [
-    {"provider": "GOOGLE", "model": "models/gemini-2.5-flash-lite", "access": "free-tier-candidate"},
+    {"provider": "GOOGLE", "model": "models/gemini-3.5-flash-lite", "access": "free-tier-candidate"},
+    {"provider": "MISTRAL", "model": "mistral-small-latest", "access": "free-tier-candidate"},
+    {"provider": "COHERE", "model": "command-r7b-12-2024", "access": "free-trial-candidate"},
     {"provider": "GOOGLE", "model": "models/gemini-2.5-flash", "access": "free-tier-candidate"},
     {"provider": "OPENROUTER", "model": "nvidia/nemotron-3.5-lightning:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "google/gemma-4-31b-it:free", "access": "catalogue-zero-price"},
@@ -33,7 +35,7 @@
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling-small:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
     {"provider": "DEEPINFRA", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "access": "paid"},
-    {"provider": "NVIDIA", "model": "openai/gpt-oss-120b", "access": "paid"},
+    {"provider": "NVIDIA", "model": "openai/gpt-oss-120b", "access": "deprecated-endpoint"},
     {"provider": "HUGGINGFACE", "model": "openai/gpt-oss-120b:fastest", "access": "paid-or-account-credit"},
     {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
@@ -47,6 +49,7 @@
     "NVIDIA": "https://integrate.api.nvidia.com/v1/chat/completions",
     "HUGGINGFACE": "https://router.huggingface.co/v1/chat/completions",
     "DEEPSEEK": "https://api.deepseek.com/chat/completions",
+    "MISTRAL": "https://api.mistral.ai/v1/chat/completions",
     "GROQ": "https://api.groq.com/openai/v1/chat/completions",
     "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
 }
@@ -59,6 +62,8 @@
     "GOOGLE": "GOOGLE_API_KEY_FLOSSWARE",
     "GROQ": "GROQ_API_KEY_FLOSSWARE",
     "OPENROUTER": "OPENROUTER_API_KEY_FLOSSWARE",
+    "MISTRAL": "MISTRAL_API_KEY_FLOSSWARE",
+    "COHERE": "COHERE_API_KEY_FLOSSWARE",
 }
 
 
@@ -82,6 +87,8 @@ def build_headers(provider: str, key: str) -> dict[str, str]:
         headers["x-goog-api-key"] = key
     else:
         headers["Authorization"] = f"Bearer {key}"
+    if provider == "COHERE":
+        headers["X-Client-Name"] = "FlossWare provider API smoke test"
     if provider == "OPENROUTER":
         headers["HTTP-Referer"] = "https://flossware.org"
         headers["X-Title"] = "FlossWare provider API smoke test"
@@ -105,6 +112,9 @@ def classify_response(text: str, finish_reason: str | None, reasoning_tokens: in
     return "complete", "response-mismatch", False
 
 
+DEPRECATED_ACCESS = {"deprecated-endpoint"}
+
+
 PAID_ACCESS = {
     "paid", "paid-or-account-credit", "paid-unless-account-credit",
     "promotional-or-paid",
@@ -115,7 +125,7 @@ def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
     """Choose at most one catalogue candidate per provider, preserving catalogue order."""
     selected: dict[str, dict[str, str]] = {}
     for case in CASES:
-        if include_paid or case["access"] not in PAID_ACCESS:
+        if case["access"] not in DEPRECATED_ACCESS and (include_paid or case["access"] not in PAID_ACCESS):
             selected.setdefault(case["provider"], case)
     return selected
 
@@ -154,6 +164,15 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             "contents": [{"role": "user", "parts": [{"text": PROMPT}]}],
             "generationConfig": {"maxOutputTokens": max_tokens, "temperature": 0},
         }
+    elif provider == "COHERE":
+        url = "https://api.cohere.com/v2/chat"
+        payload = {
+            "model": model,
+            "messages": [{"role": "user", "content": PROMPT}],
+            "max_tokens": max_tokens,
+            "temperature": 0,
+            "stream": False,
+        }
     else:
         url = OPENAI_COMPATIBLE[provider]
         payload = {
@@ -199,6 +218,16 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
             result["model_returned"] = data.get("modelVersion") or model
             result["finish_reason"] = candidate.get("finishReason")
             result["usage"] = data.get("usageMetadata")
+        elif provider == "COHERE":
+            message = data.get("message") or {}
+            content = message.get("content") or []
+            text = "".join(
+                part.get("text", "") for part in content
+                if isinstance(part, dict) and part.get("type") == "text"
+            )
+            result["model_returned"] = data.get("model") or model
+            result["finish_reason"] = data.get("finish_reason")
+            result["usage"] = data.get("usage")
         else:
             choices = data.get("choices") or []
             choice = choices[0] if choices else {}
@@ -215,6 +244,8 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
         usage = result["usage"] or {}
         if provider == "GOOGLE":
             reasoning_tokens = usage.get("thoughtsTokenCount", 0)
+        elif provider == "COHERE":
+            reasoning_tokens = 0
         else:
             completion_details = usage.get("completion_tokens_details") or {}
             reasoning_tokens = completion_details.get("reasoning_tokens", usage.get("reasoning_tokens", 0))
@@ -313,11 +344,17 @@ def main() -> int:
                 })
             elif case is None:
                 paid_case = next((c for c in CASES if c["provider"] == provider), None)
+                skip_status = (
+                    "skipped-deprecated-endpoint"
+                    if paid_case and paid_case["access"] in DEPRECATED_ACCESS
+                    else "skipped-paid-opt-in-required"
+                    if paid_case else "skipped-no-chat-candidate"
+                )
                 results.append({
                     "provider": provider,
                     "model_requested": paid_case["model"] if paid_case else None,
                     "credential_configured": True, "http_status": None,
-                    "latency_seconds": None, "status": "skipped-paid-opt-in-required",
+                    "latency_seconds": None, "status": skip_status,
                     "access_classification": paid_case["access"] if paid_case else None,
                     "retry_performed": False,
                 })
@@ -328,6 +365,7 @@ def main() -> int:
             case for case in CASES
             if (not args.provider or case["provider"] in args.provider)
             and (not args.model or case["model"] == args.model)
+            and case["access"] not in DEPRECATED_ACCESS
             and (args.include_paid or case["access"] not in PAID_ACCESS)
         ]
         if not selected and args.model:
@@ -344,7 +382,8 @@ def main() -> int:
         status = item["status"]
         http = item["http_status"] if item["http_status"] is not None else "-"
         latency = item["latency_seconds"] if item["latency_seconds"] is not None else "-"
-        print(f'{item["provider"]:10} {item["model_requested"]:46} {status:24} HTTP={http} latency={latency}s')
+        model_label = item["model_requested"] or "-"
+        print(f'{item["provider"]:10} {model_label:46} {status:30} HTTP={http} latency={latency}s')
 
     passed = sum(item["status"] == "passed" for item in results)
     skipped = sum(item["status"].startswith("skipped-") for item in results)

From bdede8718234d283a928f6d85da887a29420bb62 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:27:54 -0500
Subject: [PATCH 029/128] Test refreshed free-tier candidates and deprecated
 NVIDIA endpoint

---
 tests/test_provider_api_smoke_test.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index db5ecee..1bc4e1f 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,7 +1,7 @@
 """Deterministic tests for provider smoke-test response classification."""
 import unittest
 
-from tools.provider_api.smoke_test import CASES, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases
+from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases
 
 
 class RequestHeaderTests(unittest.TestCase):
@@ -22,19 +22,24 @@ def test_deepinfra_uses_openai_compatible_chat_endpoint(self):
             "https://api.deepinfra.com/v1/openai/chat/completions",
         )
 
-    def test_nvidia_uses_nim_chat_completions_endpoint(self):
+    def test_mistral_uses_openai_compatible_chat_endpoint(self):
         self.assertEqual(
-            OPENAI_COMPATIBLE["NVIDIA"],
-            "https://integrate.api.nvidia.com/v1/chat/completions",
+            OPENAI_COMPATIBLE["MISTRAL"],
+            "https://api.mistral.ai/v1/chat/completions",
         )
 
+    def test_nvidia_candidate_is_marked_deprecated(self):
+        self.assertIn("deprecated-endpoint", DEPRECATED_ACCESS)
+
 
 class BatchSelectionTests(unittest.TestCase):
     def test_batch_selects_at_most_one_candidate_per_provider(self):
         selected = representative_cases(include_paid=False)
         self.assertEqual(len(selected), len({case["provider"] for case in selected.values()}))
-        self.assertEqual(selected["GOOGLE"]["model"], "models/gemini-2.5-flash-lite")
+        self.assertEqual(selected["GOOGLE"]["model"], "models/gemini-3.5-flash-lite")
         self.assertEqual(selected["OPENROUTER"]["model"], "nvidia/nemotron-3.5-lightning:free")
+        self.assertEqual(selected["MISTRAL"]["model"], "mistral-small-latest")
+        self.assertEqual(selected["COHERE"]["model"], "command-r7b-12-2024")
         self.assertNotIn("CEREBRAS", selected)
         self.assertNotIn("NVIDIA", selected)
 
@@ -42,8 +47,7 @@ def test_paid_opt_in_adds_paid_only_providers(self):
         selected = representative_cases(include_paid=True)
         self.assertIn("CEREBRAS", selected)
         self.assertIn("DEEPINFRA", selected)
-        self.assertIn("NVIDIA", selected)
-        self.assertEqual(selected["NVIDIA"]["model"], "openai/gpt-oss-120b")
+        self.assertNotIn("NVIDIA", selected)
 
 
 class ClassifyResponseTests(unittest.TestCase):

From 4d9b0f921f7b106beb9013fe4ce698d36e0b1548 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:28:06 -0500
Subject: [PATCH 030/128] Remove obsolete Gemini 2.5 smoke candidate

---
 tools/provider_api/smoke_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 2e8ea98..532a05b 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -28,7 +28,6 @@
     {"provider": "GOOGLE", "model": "models/gemini-3.5-flash-lite", "access": "free-tier-candidate"},
     {"provider": "MISTRAL", "model": "mistral-small-latest", "access": "free-tier-candidate"},
     {"provider": "COHERE", "model": "command-r7b-12-2024", "access": "free-trial-candidate"},
-    {"provider": "GOOGLE", "model": "models/gemini-2.5-flash", "access": "free-tier-candidate"},
     {"provider": "OPENROUTER", "model": "nvidia/nemotron-3.5-lightning:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "google/gemma-4-31b-it:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling:free", "access": "catalogue-zero-price"},

From f6357e89f432828eb00516cf8d2ba242a970450a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:28:15 -0500
Subject: [PATCH 031/128] Document new free-tier providers and deprecated
 NVIDIA endpoint

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 73b11a7..2679e34 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ python3 tools/provider_api/smoke_test.py
 
 The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
 
-The default smoke batch is limited to catalogue-marked OpenRouter `:free` models and the Gemini models observed in the initial catalogue. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. It also includes NVIDIA NIM's `openai/gpt-oss-120b` candidate via NVIDIA's OpenAI-compatible `/v1/chat/completions` API. Paid or promotional-credit comparison models require explicit opt-in:
+The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), and catalogue-marked OpenRouter `:free` models. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --include-paid
@@ -64,7 +64,7 @@ Run one representative candidate for every provider in a single batch:
 python3 tools/provider_api/smoke_test.py --all-configured
 ```
 
-This checks one non-paid candidate per provider where available. Providers without credentials are recorded as skipped; providers whose only configured candidate is paid or promotional-credit based are also skipped unless `--include-paid` is supplied. To explicitly include those candidates:
+This checks one non-paid candidate per provider where available, including Gemini 3.5 Flash-Lite, Mistral Small, Cohere Command R7B, and OpenRouter free candidates. Providers without credentials are recorded as skipped; providers whose only configured candidate is paid or promotional-credit based are also skipped unless `--include-paid` is supplied. Deprecated endpoints are always skipped. To explicitly include those candidates:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --all-configured --include-paid

From 4c632d26809529be98f23f416a905849232727bb Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:33:42 -0500
Subject: [PATCH 032/128] Improve provider 402 diagnostics

---
 tools/provider_api/smoke_test.py | 39 ++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 532a05b..a1b7648 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -73,6 +73,40 @@ def safe_text(value: Any, secret: str) -> str:
     return text[:2000]
 
 
+def classify_http_error(status: int | None, detail: str | None) -> str | None:
+    """Classify provider HTTP errors without confusing billing with inference failure."""
+    if status != 402:
+        return None
+    message = (detail or "").lower()
+    if "team" in message and ("subscription" in message or "credit" in message):
+        return "organization-subscription-or-credit-required"
+    if "insufficient balance" in message or "no balance" in message:
+        return "insufficient-account-balance"
+    if any(term in message for term in ("credit", "billing", "payment required", "payment_required")):
+        return "account-credit-or-billing-required"
+    return "payment-required-unclassified"
+
+
+def error_summary(detail: str | None) -> str | None:
+    """Extract a compact provider message for console output; never print credentials."""
+    if not detail:
+        return None
+    message = detail
+    try:
+        parsed = json.loads(detail)
+    except (json.JSONDecodeError, TypeError):
+        parsed = None
+    if isinstance(parsed, dict):
+        error = parsed.get("error")
+        if isinstance(error, dict):
+            message = str(error.get("message") or error.get("detail") or error)
+        elif isinstance(error, str):
+            message = error
+        else:
+            message = str(parsed.get("message") or parsed.get("detail") or detail)
+    return " ".join(message.split())[:240]
+
+
 def build_headers(provider: str, key: str) -> dict[str, str]:
     """Build explicit application-identifying headers for provider API requests."""
     headers = {
@@ -278,6 +312,7 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
         except Exception:
             detail = str(exc)
         result["error"] = safe_text(detail, key)
+        result["http_error_category"] = classify_http_error(exc.code, result["error"])
         result["status"] = "http-error"
     except (urllib.error.URLError, TimeoutError, OSError) as exc:
         result["latency_seconds"] = round(time.monotonic() - started, 3)
@@ -383,6 +418,10 @@ def main() -> int:
         latency = item["latency_seconds"] if item["latency_seconds"] is not None else "-"
         model_label = item["model_requested"] or "-"
         print(f'{item["provider"]:10} {model_label:46} {status:30} HTTP={http} latency={latency}s')
+        detail = error_summary(item.get("error"))
+        if detail:
+            category = item.get("http_error_category") or "provider-error"
+            print(f"  detail [{category}]: {detail}")
 
     passed = sum(item["status"] == "passed" for item in results)
     skipped = sum(item["status"].startswith("skipped-") for item in results)

From a82fe7ef2d3ada250d7a5c2c684c61cd92b6db8d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:33:45 -0500
Subject: [PATCH 033/128] Test provider billing error classification

---
 tests/test_provider_api_smoke_test.py | 34 +++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 1bc4e1f..f1b0c34 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -32,6 +32,40 @@ def test_nvidia_candidate_is_marked_deprecated(self):
         self.assertIn("deprecated-endpoint", DEPRECATED_ACCESS)
 
 
+class HttpErrorClassificationTests(unittest.TestCase):
+    def test_deepseek_insufficient_balance_is_not_called_model_failure(self):
+        from tools.provider_api.smoke_test import classify_http_error
+        self.assertEqual(
+            classify_http_error(402, '{"error":{"message":"Insufficient Balance"}}'),
+            "insufficient-account-balance",
+        )
+
+    def test_cerebras_team_subscription_error_is_identified(self):
+        from tools.provider_api.smoke_test import classify_http_error
+        self.assertEqual(
+            classify_http_error(402, "Team org requires active subscription or available credit"),
+            "organization-subscription-or-credit-required",
+        )
+
+    def test_huggingface_credit_error_is_identified(self):
+        from tools.provider_api.smoke_test import classify_http_error
+        self.assertEqual(
+            classify_http_error(402, "Inference Providers credits exhausted"),
+            "account-credit-or-billing-required",
+        )
+
+    def test_unknown_402_is_not_overclassified(self):
+        from tools.provider_api.smoke_test import classify_http_error
+        self.assertEqual(
+            classify_http_error(402, "Payment required"),
+            "account-credit-or-billing-required",
+        )
+
+    def test_non_402_is_not_a_billing_failure(self):
+        from tools.provider_api.smoke_test import classify_http_error
+        self.assertIsNone(classify_http_error(403, "Forbidden"))
+
+
 class BatchSelectionTests(unittest.TestCase):
     def test_batch_selects_at_most_one_candidate_per_provider(self):
         selected = representative_cases(include_paid=False)

From a9cf7e483357486d663c86c564d192bd47317e40 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:33:47 -0500
Subject: [PATCH 034/128] Document sanitized provider error diagnostics

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2679e34..5a0f5aa 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ python3 tools/provider_api/smoke_test.py --all-configured --include-paid
 
 The batch never fans out across every model in a provider catalogue. It selects at most one representative model per provider, continues after individual failures, and appends all outcomes to the same JSONL evidence file. `--all-configured` cannot be combined with `--provider` or `--model`.
 
-Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are additionally classified as account-balance, account-credit/billing, or organization-subscription issues when the provider response supports that distinction; a compact sanitized error detail is printed so account restrictions are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
 Run the deterministic classification tests with `python3 -m unittest discover -s tests -v`.
 

From e1127b18d118b7776b5228d16caccd470576d254 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:34:41 -0500
Subject: [PATCH 035/128] Add read-only provider error inspection and free
 DeepSeek routes

---
 tools/provider_api/smoke_test.py | 37 ++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index a1b7648..ea4696d 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -29,12 +29,14 @@
     {"provider": "MISTRAL", "model": "mistral-small-latest", "access": "free-tier-candidate"},
     {"provider": "COHERE", "model": "command-r7b-12-2024", "access": "free-trial-candidate"},
     {"provider": "OPENROUTER", "model": "nvidia/nemotron-3.5-lightning:free", "access": "catalogue-zero-price"},
+    {"provider": "OPENROUTER", "model": "deepseek/deepseek-v4-flash-0731:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "google/gemma-4-31b-it:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "thinkingmachines/inkling-small:free", "access": "catalogue-zero-price"},
     {"provider": "OPENROUTER", "model": "openrouter/free", "access": "catalogue-zero-price-router"},
     {"provider": "DEEPINFRA", "model": "deepseek-ai/DeepSeek-V4-Flash-0731", "access": "paid"},
     {"provider": "NVIDIA", "model": "openai/gpt-oss-120b", "access": "deprecated-endpoint"},
+    {"provider": "HUGGINGFACE", "model": "deepseek-ai/DeepSeek-V4-Flash-0731:deepinfra", "access": "free-monthly-credit"},
     {"provider": "HUGGINGFACE", "model": "openai/gpt-oss-120b:fastest", "access": "paid-or-account-credit"},
     {"provider": "CEREBRAS", "model": "gpt-oss-120b", "access": "promotional-or-paid"},
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
@@ -338,6 +340,8 @@ def main() -> int:
                         help="Run only this exact configured model ID; combine with --provider to validate its provider.")
     parser.add_argument("--include-paid", action="store_true",
                         help="Also run paid/promotional-credit comparison candidates.")
+    parser.add_argument("--show-latest-errors", action="store_true",
+                        help="Print the latest recorded errors for Cerebras, DeepInfra, DeepSeek, and Hugging Face without making API calls.")
     parser.add_argument("--timeout", type=float, default=45.0)
     parser.add_argument("--max-output-tokens", type=int, default=128,
                         help="Maximum generated tokens per request (default: 128).")
@@ -347,6 +351,39 @@ def main() -> int:
     if args.timeout <= 0 or args.max_output_tokens < 1:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
+    if args.show_latest_errors:
+        if args.all_configured or args.provider or args.model or args.include_paid:
+            parser.error("--show-latest-errors cannot be combined with inference-selection options")
+        if not args.output.exists():
+            parser.error(f"No evidence file found: {args.output}")
+        wanted = {"CEREBRAS", "DEEPINFRA", "DEEPSEEK", "HUGGINGFACE"}
+        latest: dict[str, dict[str, Any]] = {}
+        with args.output.open(encoding="utf-8") as stream:
+            for line in stream:
+                try:
+                    item = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                provider = item.get("provider")
+                if provider in wanted and (item.get("error") or item.get("http_status") == 402):
+                    latest[provider] = item
+        for provider in sorted(wanted):
+            item = latest.get(provider)
+            if not item:
+                print(f"{provider:10} no recorded error")
+                continue
+            detail = error_summary(item.get("error")) or "No response body recorded"
+            category = item.get("http_error_category") or classify_http_error(
+                item.get("http_status"), item.get("error")
+            ) or "provider-error"
+            print(
+                f'{provider:10} {item.get("model_requested") or "-"} '
+                f'HTTP={item.get("http_status") or "-"} [{category}]'
+            )
+            print(f"  {detail}")
+        print("Read-only: no provider requests were made.")
+        return 0
+
     if args.all_configured and (args.provider or args.model):
         parser.error("--all-configured cannot be combined with --provider or --model")
 

From 01b854b35f5f78d74be1f1368d1e0b2e92740252 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:34:45 -0500
Subject: [PATCH 036/128] Cover free DeepSeek provider routes

---
 tests/test_provider_api_smoke_test.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index f1b0c34..3a5790e 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -72,8 +72,16 @@ def test_batch_selects_at_most_one_candidate_per_provider(self):
         self.assertEqual(len(selected), len({case["provider"] for case in selected.values()}))
         self.assertEqual(selected["GOOGLE"]["model"], "models/gemini-3.5-flash-lite")
         self.assertEqual(selected["OPENROUTER"]["model"], "nvidia/nemotron-3.5-lightning:free")
+        self.assertIn(
+            {"provider": "OPENROUTER", "model": "deepseek/deepseek-v4-flash-0731:free", "access": "catalogue-zero-price"},
+            CASES,
+        )
         self.assertEqual(selected["MISTRAL"]["model"], "mistral-small-latest")
         self.assertEqual(selected["COHERE"]["model"], "command-r7b-12-2024")
+        self.assertEqual(
+            selected["HUGGINGFACE"]["model"],
+            "deepseek-ai/DeepSeek-V4-Flash-0731:deepinfra",
+        )
         self.assertNotIn("CEREBRAS", selected)
         self.assertNotIn("NVIDIA", selected)
 

From 45069204d73ffa1ececd31647e190834cfeae85f Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 01:34:47 -0500
Subject: [PATCH 037/128] Document no-cost DeepSeek routes and error inspection

---
 README.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5a0f5aa..ca7a326 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ python3 tools/provider_api/smoke_test.py
 
 The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
 
-The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), and catalogue-marked OpenRouter `:free` models. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
+The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --include-paid
@@ -74,6 +74,12 @@ The batch never fans out across every model in a provider catalogue. It selects
 
 Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are additionally classified as account-balance, account-credit/billing, or organization-subscription issues when the provider response supports that distinction; a compact sanitized error detail is printed so account restrictions are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
+Inspect the latest recorded HTTP errors without making any network requests:
+
+```bash
+python3 tools/provider_api/smoke_test.py --show-latest-errors
+```
+
 Run the deterministic classification tests with `python3 -m unittest discover -s tests -v`.
 
 A zero price in a model catalogue is not a guarantee of availability or account eligibility. Check provider billing and account terms independently. Never send real user data, proprietary code, or secrets in these tests.

From aaee23d24d385d25efacc92f808826540ae08053 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:05:37 -0500
Subject: [PATCH 038/128] Support conventional provider credential environment
 variables

---
 tools/provider_api/smoke_test.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index ea4696d..73d5025 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -67,6 +67,31 @@
     "COHERE": "COHERE_API_KEY_FLOSSWARE",
 }
 
+# Accept standard provider names and common SDK conventions.
+# The FlossWare-specific variable takes precedence when both are present.
+ENV_VAR_ALIASES = {
+    "CEREBRAS": ("CEREBRAS_API_KEY",),
+    "DEEPINFRA": ("DEEPINFRA_API_KEY",),
+    "NVIDIA": ("NVIDIA_API_KEY",),
+    "HUGGINGFACE": ("HUGGINGFACE_API_KEY", "HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"),
+    "DEEPSEEK": ("DEEPSEEK_API_KEY",),
+    "GOOGLE": ("GOOGLE_API_KEY", "GEMINI_API_KEY"),
+    "GROQ": ("GROQ_API_KEY",),
+    "OPENROUTER": ("OPENROUTER_API_KEY",),
+    "MISTRAL": ("MISTRAL_API_KEY",),
+    "COHERE": ("COHERE_API_KEY",),
+}
+
+
+def resolve_credential(provider: str) -> tuple[str, str | None]:
+    """Return credential and source variable name without exposing its value."""
+    candidates = (ENV_VARS[provider], *ENV_VAR_ALIASES.get(provider, ()))
+    for env_name in candidates:
+        value = os.environ.get(env_name, "")
+        if value.strip():
+            return value.strip(), env_name
+    return "", None
+
 
 def safe_text(value: Any, secret: str) -> str:
     text = str(value)
@@ -167,8 +192,7 @@ def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
 
 def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
     provider, model = case["provider"], case["model"]
-    env_name = ENV_VARS[provider]
-    key = os.environ.get(env_name, "")
+    key, env_name = resolve_credential(provider)
     result: dict[str, Any] = {
         "observed_at": datetime.now(timezone.utc).isoformat(),
         "provider": provider,
@@ -404,7 +428,7 @@ def main() -> int:
 
         results = []
         for provider in sorted(ENV_VARS):
-            key_configured = bool(os.environ.get(ENV_VARS[provider], ""))
+            key_configured = bool(resolve_credential(provider)[0])
             case = selected_by_provider.get(provider)
             if not key_configured:
                 results.append({

From 5f4125cdfdafbaf7ff7caf3f8e7c717986481aea Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:05:51 -0500
Subject: [PATCH 039/128] Test provider credential aliases and precedence

---
 tests/test_provider_api_smoke_test.py | 57 ++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 3a5790e..1c6455f 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,8 +1,61 @@
 """Deterministic tests for provider smoke-test response classification."""
 import unittest
 
-from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases
-
+from unittest.mock import patch
+
+from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases, resolve_credential
+
+
+
+class CredentialResolutionTests(unittest.TestCase):
+    def test_standard_provider_environment_names_are_supported(self):
+        cases = {
+            "CEREBRAS": "CEREBRAS_API_KEY",
+            "DEEPINFRA": "DEEPINFRA_API_KEY",
+            "DEEPSEEK": "DEEPSEEK_API_KEY",
+            "HUGGINGFACE": "HF_TOKEN",
+            "GOOGLE": "GEMINI_API_KEY",
+            "GROQ": "GROQ_API_KEY",
+            "OPENROUTER": "OPENROUTER_API_KEY",
+            "MISTRAL": "MISTRAL_API_KEY",
+            "COHERE": "COHERE_API_KEY",
+            "NVIDIA": "NVIDIA_API_KEY",
+        }
+        for provider, env_name in cases.items():
+            with self.subTest(provider=provider):
+                with patch.dict("os.environ", {env_name: "test-secret"}, clear=True):
+                    self.assertEqual(resolve_credential(provider), ("test-secret", env_name))
+
+    def test_flossware_variable_has_precedence(self):
+        with patch.dict("os.environ", {
+            "CEREBRAS_API_KEY_FLOSSWARE": "preferred-secret",
+            "CEREBRAS_API_KEY": "fallback-secret",
+        }, clear=True):
+            self.assertEqual(
+                resolve_credential("CEREBRAS"),
+                ("preferred-secret", "CEREBRAS_API_KEY_FLOSSWARE"),
+            )
+
+    def test_huggingface_sdk_alias_is_supported(self):
+        with patch.dict("os.environ", {"HUGGING_FACE_HUB_TOKEN": "hf-secret"}, clear=True):
+            self.assertEqual(
+                resolve_credential("HUGGINGFACE"),
+                ("hf-secret", "HUGGING_FACE_HUB_TOKEN"),
+            )
+
+    def test_blank_variables_are_ignored(self):
+        with patch.dict("os.environ", {
+            "DEEPSEEK_API_KEY_FLOSSWARE": "  ",
+            "DEEPSEEK_API_KEY": "deepseek-secret",
+        }, clear=True):
+            self.assertEqual(
+                resolve_credential("DEEPSEEK"),
+                ("deepseek-secret", "DEEPSEEK_API_KEY"),
+            )
+
+    def test_missing_credentials_return_no_value_or_source(self):
+        with patch.dict("os.environ", {}, clear=True):
+            self.assertEqual(resolve_credential("GROQ"), ("", None))
 
 class RequestHeaderTests(unittest.TestCase):
     def test_provider_requests_identify_the_application(self):

From cb07e3d3e27532776ea2e8a4a4bb23e916103335 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:06:05 -0500
Subject: [PATCH 040/128] Document provider credential environment aliases

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ca7a326..ffce42a 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ The harness does not own provider selection, routing, credential storage, retrie
 
 ### Run
 
-From the repository root, with the relevant `*_API_KEY_FLOSSWARE` variables exported:
+From the repository root, with the relevant provider credentials exported. The harness first checks each `*_API_KEY_FLOSSWARE` name, then falls back to the provider's conventional name (for example, `CEREBRAS_API_KEY`, `DEEPSEEK_API_KEY`, `GOOGLE_API_KEY`, `GROQ_API_KEY`, `OPENROUTER_API_KEY`, `MISTRAL_API_KEY`, or `COHERE_API_KEY`). Hugging Face accepts `HUGGINGFACE_API_KEY`, `HF_TOKEN`, and `HUGGING_FACE_HUB_TOKEN`; Google also accepts `GEMINI_API_KEY`. When more than one is set, the FlossWare-specific variable takes precedence. Only the selected variable's name is recorded, never its value:
 
 ```bash
 bash tools/provider_api/catalogue_inventory.sh

From de573cbbd17b0e93e4d136b276d2862e52d14d14 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:06:40 -0500
Subject: [PATCH 041/128] Support conventional credentials in catalogue
 inventory

---
 tools/provider_api/catalogue_inventory.sh | 25 +++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tools/provider_api/catalogue_inventory.sh b/tools/provider_api/catalogue_inventory.sh
index 1ca2ba0..061f985 100644
--- a/tools/provider_api/catalogue_inventory.sh
+++ b/tools/provider_api/catalogue_inventory.sh
@@ -25,6 +25,21 @@ declare -A KEYS=(
   [OPENROUTER]=OPENROUTER_API_KEY_FLOSSWARE
 )
 
+# Space-separated aliases; FlossWare-scoped names retain precedence.
+declare -A ALIASES=(
+  [CEREBRAS]="CEREBRAS_API_KEY"
+  [COHERE]="COHERE_API_KEY"
+  [DEEPINFRA]="DEEPINFRA_API_KEY"
+  [DEEPSEEK]="DEEPSEEK_API_KEY"
+  [GOOGLE]="GOOGLE_API_KEY GEMINI_API_KEY"
+  [GROQ]="GROQ_API_KEY"
+  [HUGGINGFACE]="HUGGINGFACE_API_KEY HF_TOKEN HUGGING_FACE_HUB_TOKEN"
+  [MISTRAL]="MISTRAL_API_KEY"
+  [NVIDIA]="NVIDIA_API_KEY"
+  [OPENAI]="OPENAI_API_KEY"
+  [OPENROUTER]="OPENROUTER_API_KEY"
+)
+
 declare -A URLS=(
   [CEREBRAS]="https://api.cerebras.ai/v1/models"
   [COHERE]="https://api.cohere.com/v1/models"
@@ -45,6 +60,16 @@ RESULTS="$TMP/results.jsonl"
 for provider in CEREBRAS COHERE DEEPINFRA DEEPSEEK GOOGLE GROQ                 HUGGINGFACE MISTRAL NVIDIA OPENAI OPENROUTER; do
   var="${KEYS[$provider]}"
   key="${!var:-}"
+  if [[ -z "$key" ]]; then
+    for alias in ${ALIASES[$provider]:-}; do
+      candidate="${!alias:-}"
+      if [[ -n "$candidate" ]]; then
+        var="$alias"
+        key="$candidate"
+        break
+      fi
+    done
+  fi
   body="$TMP/$provider.json"
   status="not_configured"
   elapsed=""

From 7f6dd9262113b149458613642825cb1228c62018 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:06:50 -0500
Subject: [PATCH 042/128] Clarify credential fallback behavior in provider
 research docs

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ffce42a..c3bc418 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ These scripts are **research utilities, not part of the published `evaluation-ai
 - `catalogue_inventory.sh` reads authenticated provider model catalogues and writes a normalized local JSON report. It does not invoke models.
 - `smoke_test.py` sends a tiny synthetic prompt to selected catalogue-listed models and records transport/response evidence as JSONL.
 
-The harness does not own provider selection, routing, credential storage, retries, or production invocation. Those responsibilities remain with `model-gateway`. The harness reads only FlossWare-scoped environment variables and never accepts or prints credential values.
+The harness does not own provider selection, routing, credential storage, retries, or production invocation. Those responsibilities remain with `model-gateway`. Both utilities prefer FlossWare-scoped environment variables and fall back to conventional provider/SDK names. They never accept credentials as command-line arguments or print/write credential values.
 
 ### Run
 

From 11d323166eaab86688cdf23704b7fef270aea1e2 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:07:26 -0500
Subject: [PATCH 043/128] Add read-only credential source diagnostics

---
 tools/provider_api/smoke_test.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 73d5025..96696a1 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -366,6 +366,8 @@ def main() -> int:
                         help="Also run paid/promotional-credit comparison candidates.")
     parser.add_argument("--show-latest-errors", action="store_true",
                         help="Print the latest recorded errors for Cerebras, DeepInfra, DeepSeek, and Hugging Face without making API calls.")
+    parser.add_argument("--show-credential-sources", action="store_true",
+                        help="Show which credential variable name is selected for each provider, without printing values or making API calls.")
     parser.add_argument("--timeout", type=float, default=45.0)
     parser.add_argument("--max-output-tokens", type=int, default=128,
                         help="Maximum generated tokens per request (default: 128).")
@@ -375,8 +377,17 @@ def main() -> int:
     if args.timeout <= 0 or args.max_output_tokens < 1:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
+    if args.show_credential_sources:
+        if args.show_latest_errors or args.all_configured or args.provider or args.model or args.include_paid:
+            parser.error("--show-credential-sources cannot be combined with other selection options")
+        for provider in sorted(ENV_VARS):
+            _, env_name = resolve_credential(provider)
+            print(f"{provider:10} {env_name or 'not configured'}")
+        print("Read-only: no credential values were displayed and no provider requests were made.")
+        return 0
+
     if args.show_latest_errors:
-        if args.all_configured or args.provider or args.model or args.include_paid:
+        if args.all_configured or args.provider or args.model or args.include_paid or args.show_credential_sources:
             parser.error("--show-latest-errors cannot be combined with inference-selection options")
         if not args.output.exists():
             parser.error(f"No evidence file found: {args.output}")

From 155064e1d0010b829fa35949f870580e6d1b94dd Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:07:33 -0500
Subject: [PATCH 044/128] Document read-only credential source diagnostic

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index c3bc418..be1e118 100644
--- a/README.md
+++ b/README.md
@@ -74,6 +74,12 @@ The batch never fans out across every model in a provider catalogue. It selects
 
 Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are additionally classified as account-balance, account-credit/billing, or organization-subscription issues when the provider response supports that distinction; a compact sanitized error detail is printed so account restrictions are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
+Inspect which credential variable name is selected for each provider without displaying values or making network requests:
+
+```bash
+python3 tools/provider_api/smoke_test.py --show-credential-sources
+```
+
 Inspect the latest recorded HTTP errors without making any network requests:
 
 ```bash

From d24932299d9a4cfcb906e3b1711ad1756a20fe9d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:09:25 -0500
Subject: [PATCH 045/128] Plan and report every configured provider in batch
 runs

---
 tools/provider_api/smoke_test.py | 96 +++++++++++++++++++++-----------
 1 file changed, 65 insertions(+), 31 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 96696a1..4e09b35 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -182,7 +182,7 @@ def classify_response(text: str, finish_reason: str | None, reasoning_tokens: in
 
 
 def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
-    """Choose at most one catalogue candidate per provider, preserving catalogue order."""
+    """Choose at most one eligible catalogue candidate per provider."""
     selected: dict[str, dict[str, str]] = {}
     for case in CASES:
         if case["access"] not in DEPRECATED_ACCESS and (include_paid or case["access"] not in PAID_ACCESS):
@@ -190,6 +190,40 @@ def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
     return selected
 
 
+def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
+    """Build a complete provider plan without making network requests."""
+    selected = representative_cases(include_paid)
+    plan: list[dict[str, Any]] = []
+    for provider in sorted(ENV_VARS):
+        _, env_name = resolve_credential(provider)
+        candidates = [case for case in CASES if case["provider"] == provider]
+        case = selected.get(provider)
+        item: dict[str, Any] = {
+            "provider": provider,
+            "credential_env": env_name,
+            "credential_configured": bool(env_name),
+            "model_requested": case["model"] if case else None,
+            "access_classification": case["access"] if case else None,
+        }
+        if not env_name:
+            item["status"] = "skipped-no-credential"
+        elif case:
+            item["status"] = "ready"
+        elif candidates and all(c["access"] in DEPRECATED_ACCESS for c in candidates):
+            item["status"] = "skipped-deprecated-endpoint"
+            item["model_requested"] = candidates[0]["model"]
+            item["access_classification"] = candidates[0]["access"]
+        elif candidates and not include_paid and all(c["access"] in PAID_ACCESS or c["access"] in DEPRECATED_ACCESS for c in candidates):
+            item["status"] = "skipped-paid-opt-in-required"
+            eligible = next((c for c in candidates if c["access"] in PAID_ACCESS), candidates[0])
+            item["model_requested"] = eligible["model"]
+            item["access_classification"] = eligible["access"]
+        else:
+            item["status"] = "skipped-no-chat-candidate"
+        plan.append(item)
+    return plan
+
+
 def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
     provider, model = case["provider"], case["model"]
     key, env_name = resolve_credential(provider)
@@ -359,7 +393,7 @@ def main() -> int:
     parser.add_argument("--provider", action="append", choices=sorted(ENV_VARS),
                         help="Run only this provider; repeat to select several.")
     parser.add_argument("--all-configured", action="store_true",
-                        help="Run one representative candidate per provider; paid-only candidates require --include-paid.")
+                        help="Iterate every known provider; run one free/trial candidate where available and report all others as skipped.")
     parser.add_argument("--model",
                         help="Run only this exact configured model ID; combine with --provider to validate its provider.")
     parser.add_argument("--include-paid", action="store_true",
@@ -433,39 +467,24 @@ def main() -> int:
             parser.error(f"Model {args.model!r} is configured for {configured}, not the selected provider(s).")
 
     if args.all_configured:
-        # One representative model per provider. Never fan out across every
-        # catalogue candidate, and never opt into paid candidates implicitly.
-        selected_by_provider = representative_cases(args.include_paid)
-
+        # Plan every provider first. Only free/trial/zero-price candidates run
+        # by default; paid-only and deprecated providers are explicitly recorded.
+        # At most one candidate per provider is ever invoked in this batch.
         results = []
-        for provider in sorted(ENV_VARS):
-            key_configured = bool(resolve_credential(provider)[0])
-            case = selected_by_provider.get(provider)
-            if not key_configured:
-                results.append({
-                    "provider": provider, "model_requested": case["model"] if case else None,
-                    "credential_configured": False, "http_status": None,
-                    "latency_seconds": None, "status": "skipped-no-credential",
-                    "retry_performed": False,
-                })
-            elif case is None:
-                paid_case = next((c for c in CASES if c["provider"] == provider), None)
-                skip_status = (
-                    "skipped-deprecated-endpoint"
-                    if paid_case and paid_case["access"] in DEPRECATED_ACCESS
-                    else "skipped-paid-opt-in-required"
-                    if paid_case else "skipped-no-chat-candidate"
+        for item in plan_all_configured(args.include_paid):
+            if item["status"] == "ready":
+                case = next(
+                    c for c in CASES
+                    if c["provider"] == item["provider"] and c["model"] == item["model_requested"]
                 )
-                results.append({
-                    "provider": provider,
-                    "model_requested": paid_case["model"] if paid_case else None,
-                    "credential_configured": True, "http_status": None,
-                    "latency_seconds": None, "status": skip_status,
-                    "access_classification": paid_case["access"] if paid_case else None,
+                results.append(request_case(case, args.timeout, args.max_output_tokens))
+            else:
+                item.update({
+                    "http_status": None,
+                    "latency_seconds": None,
                     "retry_performed": False,
                 })
-            else:
-                results.append(request_case(case, args.timeout, args.max_output_tokens))
+                results.append(item)
     else:
         selected = [
             case for case in CASES
@@ -498,6 +517,21 @@ def main() -> int:
     passed = sum(item["status"] == "passed" for item in results)
     skipped = sum(item["status"].startswith("skipped-") for item in results)
     failed = len(results) - passed - skipped
+    if args.all_configured:
+        by_reason = {
+            reason: sum(item["status"] == reason for item in results)
+            for reason in (
+                "skipped-no-credential",
+                "skipped-paid-opt-in-required",
+                "skipped-deprecated-endpoint",
+                "skipped-no-chat-candidate",
+            )
+        }
+        print(f"\nProvider inventory: {len(results)} providers examined.")
+        print(f"Eligible requests: {len(results) - skipped}; passed: {passed}; failed: {failed}.")
+        for reason, count in by_reason.items():
+            if count:
+                print(f"  {reason}: {count}")
     print(f"\nSummary: {passed} passed, {failed} failed, {skipped} skipped; {len(results)} selected.")
     print(f"Evidence: {args.output}")
     print("No retries performed. Review provider billing/usage independently; catalogue price is not a bill.")

From 6034cb8aa98037dca48507908288899a7a349a13 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:09:41 -0500
Subject: [PATCH 046/128] Test complete all-provider batch planning

---
 tests/test_provider_api_smoke_test.py | 51 ++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 1c6455f..f721ab0 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -3,7 +3,7 @@
 
 from unittest.mock import patch
 
-from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, representative_cases, resolve_credential
+from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, ENV_VARS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, plan_all_configured, representative_cases, resolve_credential
 
 
 
@@ -57,6 +57,55 @@ def test_missing_credentials_return_no_value_or_source(self):
         with patch.dict("os.environ", {}, clear=True):
             self.assertEqual(resolve_credential("GROQ"), ("", None))
 
+
+class AllConfiguredPlanningTests(unittest.TestCase):
+    def test_plan_includes_every_provider_and_never_opts_into_paid_candidates(self):
+        aliases = {
+            "CEREBRAS_API_KEY": "CEREBRAS_API_KEY",
+            "DEEPINFRA_API_KEY": "DEEPINFRA_API_KEY",
+            "NVIDIA_API_KEY": "NVIDIA_API_KEY",
+            "HF_TOKEN": "HF_TOKEN",
+            "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY",
+            "GEMINI_API_KEY": "GEMINI_API_KEY",
+            "GROQ_API_KEY": "GROQ_API_KEY",
+            "OPENROUTER_API_KEY": "OPENROUTER_API_KEY",
+            "MISTRAL_API_KEY": "MISTRAL_API_KEY",
+            "COHERE_API_KEY": "COHERE_API_KEY",
+        }
+        with patch.dict("os.environ", {name: "test-secret" for name in aliases}, clear=True):
+            plan = plan_all_configured()
+        self.assertEqual([item["provider"] for item in plan], sorted(ENV_VARS))
+        by_provider = {item["provider"]: item for item in plan}
+        self.assertEqual(by_provider["GOOGLE"]["status"], "ready")
+        self.assertEqual(by_provider["HUGGINGFACE"]["status"], "ready")
+        self.assertEqual(by_provider["OPENROUTER"]["status"], "ready")
+        self.assertEqual(by_provider["CEREBRAS"]["status"], "skipped-paid-opt-in-required")
+        self.assertEqual(by_provider["DEEPINFRA"]["status"], "skipped-paid-opt-in-required")
+        self.assertEqual(by_provider["DEEPSEEK"]["status"], "skipped-paid-opt-in-required")
+        self.assertEqual(by_provider["GROQ"]["status"], "skipped-paid-opt-in-required")
+        self.assertEqual(by_provider["NVIDIA"]["status"], "skipped-deprecated-endpoint")
+        self.assertTrue(all(item["credential_configured"] for item in plan))
+
+    def test_plan_reports_every_missing_credential(self):
+        with patch.dict("os.environ", {}, clear=True):
+            plan = plan_all_configured()
+        self.assertEqual(len(plan), len(ENV_VARS))
+        self.assertTrue(all(item["status"] == "skipped-no-credential" for item in plan))
+        self.assertTrue(all(item["credential_env"] is None for item in plan))
+
+    def test_paid_opt_in_only_changes_the_execution_plan(self):
+        with patch.dict("os.environ", {
+            "CEREBRAS_API_KEY": "test-secret",
+            "DEEPINFRA_API_KEY": "test-secret",
+            "DEEPSEEK_API_KEY": "test-secret",
+            "GROQ_API_KEY": "test-secret",
+        }, clear=True):
+            plan = {item["provider"]: item for item in plan_all_configured(include_paid=True)}
+        self.assertEqual(plan["CEREBRAS"]["status"], "ready")
+        self.assertEqual(plan["DEEPINFRA"]["status"], "ready")
+        self.assertEqual(plan["DEEPSEEK"]["status"], "ready")
+        self.assertEqual(plan["GROQ"]["status"], "ready")
+
 class RequestHeaderTests(unittest.TestCase):
     def test_provider_requests_identify_the_application(self):
         headers = build_headers("CEREBRAS", "test-key")

From 303b5e35eba0d2765af29ddafb9a562e4ab677d2 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:09:58 -0500
Subject: [PATCH 047/128] Document complete all-provider evaluation behavior

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index be1e118..20ccdd5 100644
--- a/README.md
+++ b/README.md
@@ -58,13 +58,13 @@ The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small
 python3 tools/provider_api/smoke_test.py --include-paid
 ```
 
-Run one representative candidate for every provider in a single batch:
+Run the complete provider inventory in one batch:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --all-configured
 ```
 
-This checks one non-paid candidate per provider where available, including Gemini 3.5 Flash-Lite, Mistral Small, Cohere Command R7B, and OpenRouter free candidates. Providers without credentials are recorded as skipped; providers whose only configured candidate is paid or promotional-credit based are also skipped unless `--include-paid` is supplied. Deprecated endpoints are always skipped. To explicitly include those candidates:
+This iterates every known provider, not merely the providers with a ready model. It runs at most one representative candidate for each provider with a configured credential and a free/trial/zero-price candidate, including Gemini 3.5 Flash-Lite, Mistral Small, Cohere Command R7B, Hugging Face's monthly included-credit route, and OpenRouter free candidates. Every other provider is still written to the evidence file with an explicit skip reason: no credential, paid candidate requiring opt-in, deprecated endpoint, or no supported chat candidate. The final summary reports the complete provider count and skip categories. Paid/promotional-credit candidates are never invoked by default; `--include-paid` is a separate explicit opt-in and may incur charges. Deprecated endpoints are always skipped. To explicitly include those candidates:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --all-configured --include-paid

From ef17a65d4e8d3eb62d5580e946a5026b73fb9cee Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:10:28 -0500
Subject: [PATCH 048/128] Clarify provider credential lookup documentation

---
 tools/provider_api/smoke_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 4e09b35..7ba750b 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 """Minimal, non-streaming provider inference smoke tests.
 
-Uses only Python's standard library. Credentials are read from *_API_KEY_FLOSSWARE
-environment variables, never accepted as command-line arguments, and never emitted.
-No automatic retries are performed.
+Uses only Python's standard library. Credentials prefer *_API_KEY_FLOSSWARE and
+fall back to conventional provider/SDK environment names. They are never accepted
+as command-line arguments or emitted. No automatic retries are performed.
 """
 from __future__ import annotations
 

From 454199dbd90ba1af7582ac62a87b470d5c2ac808 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:15:45 -0500
Subject: [PATCH 049/128] Discover additional provider credentials from the
 environment

---
 tools/provider_api/smoke_test.py | 56 +++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 5 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 7ba750b..8c896f6 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -10,6 +10,7 @@
 import argparse
 import json
 import os
+import re
 import sys
 import time
 import urllib.error
@@ -42,6 +43,7 @@
     {"provider": "CEREBRAS", "model": "qwen-3.8-27b", "access": "promotional-or-paid"},
     {"provider": "DEEPSEEK", "model": "deepseek-flash", "access": "paid-unless-account-credit"},
     {"provider": "GROQ", "model": "openai/gpt-oss-120b", "access": "paid"},
+    {"provider": "OPENAI", "model": "gpt-4.1-mini", "access": "paid"},
 ]
 
 OPENAI_COMPATIBLE = {
@@ -53,6 +55,7 @@
     "MISTRAL": "https://api.mistral.ai/v1/chat/completions",
     "GROQ": "https://api.groq.com/openai/v1/chat/completions",
     "OPENROUTER": "https://openrouter.ai/api/v1/chat/completions",
+    "OPENAI": "https://api.openai.com/v1/chat/completions",
 }
 ENV_VARS = {
     "CEREBRAS": "CEREBRAS_API_KEY_FLOSSWARE",
@@ -65,6 +68,8 @@
     "OPENROUTER": "OPENROUTER_API_KEY_FLOSSWARE",
     "MISTRAL": "MISTRAL_API_KEY_FLOSSWARE",
     "COHERE": "COHERE_API_KEY_FLOSSWARE",
+    "OPENAI": "OPENAI_API_KEY_FLOSSWARE",
+    "ANTHROPIC": "ANTHROPIC_API_KEY_FLOSSWARE",
 }
 
 # Accept standard provider names and common SDK conventions.
@@ -80,13 +85,52 @@
     "OPENROUTER": ("OPENROUTER_API_KEY",),
     "MISTRAL": ("MISTRAL_API_KEY",),
     "COHERE": ("COHERE_API_KEY",),
+    "OPENAI": ("OPENAI_API_KEY",),
+    "ANTHROPIC": ("ANTHROPIC_API_KEY",),
 }
 
 
+def provider_from_env_name(env_name: str) -> str | None:
+    """Map a provider credential variable name to its canonical provider."""
+    explicit = {
+        "GEMINI_API_KEY": "GOOGLE",
+        "GEMINI_API_KEY_FLOSSWARE": "GOOGLE",
+        "HF_TOKEN": "HUGGINGFACE",
+        "HUGGING_FACE_HUB_TOKEN": "HUGGINGFACE",
+    }
+    if env_name in explicit:
+        return explicit[env_name]
+    match = re.fullmatch(r"([A-Z][A-Z0-9_]*)_(?:API_KEY(?:_FLOSSWARE)?|API_TOKEN)", env_name)
+    if not match:
+        return None
+    provider = match.group(1)
+    return {"HF": "HUGGINGFACE", "HUGGING_FACE": "HUGGINGFACE", "GEMINI": "GOOGLE"}.get(provider, provider)
+
+
+def discovered_provider_names() -> list[str]:
+    """Return known providers plus providers inferred from non-empty credential variables."""
+    discovered = {
+        provider
+        for name, value in os.environ.items()
+        if value.strip() and (provider := provider_from_env_name(name))
+    }
+    return sorted(set(ENV_VARS) | discovered)
+
+
+def credential_variable_names(provider: str) -> tuple[str, ...]:
+    """Return ordered credential variable names for a provider, without reading values."""
+    primary = ENV_VARS.get(provider, f"{provider}_API_KEY_FLOSSWARE")
+    names = [primary, *ENV_VAR_ALIASES.get(provider, ())]
+    names.extend(
+        name for name in os.environ
+        if provider_from_env_name(name) == provider
+    )
+    return tuple(dict.fromkeys(names))
+
+
 def resolve_credential(provider: str) -> tuple[str, str | None]:
     """Return credential and source variable name without exposing its value."""
-    candidates = (ENV_VARS[provider], *ENV_VAR_ALIASES.get(provider, ()))
-    for env_name in candidates:
+    for env_name in credential_variable_names(provider):
         value = os.environ.get(env_name, "")
         if value.strip():
             return value.strip(), env_name
@@ -194,7 +238,7 @@ def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
     """Build a complete provider plan without making network requests."""
     selected = representative_cases(include_paid)
     plan: list[dict[str, Any]] = []
-    for provider in sorted(ENV_VARS):
+    for provider in discovered_provider_names():
         _, env_name = resolve_credential(provider)
         candidates = [case for case in CASES if case["provider"] == provider]
         case = selected.get(provider)
@@ -213,6 +257,8 @@ def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
             item["status"] = "skipped-deprecated-endpoint"
             item["model_requested"] = candidates[0]["model"]
             item["access_classification"] = candidates[0]["access"]
+        elif provider not in {"GOOGLE", "COHERE", *OPENAI_COMPATIBLE}:
+            item["status"] = "skipped-no-provider-adapter"
         elif candidates and not include_paid and all(c["access"] in PAID_ACCESS or c["access"] in DEPRECATED_ACCESS for c in candidates):
             item["status"] = "skipped-paid-opt-in-required"
             eligible = next((c for c in candidates if c["access"] in PAID_ACCESS), candidates[0])
@@ -390,7 +436,7 @@ def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--provider", action="append", choices=sorted(ENV_VARS),
+    parser.add_argument("--provider", action="append", choices=discovered_provider_names(),
                         help="Run only this provider; repeat to select several.")
     parser.add_argument("--all-configured", action="store_true",
                         help="Iterate every known provider; run one free/trial candidate where available and report all others as skipped.")
@@ -414,7 +460,7 @@ def main() -> int:
     if args.show_credential_sources:
         if args.show_latest_errors or args.all_configured or args.provider or args.model or args.include_paid:
             parser.error("--show-credential-sources cannot be combined with other selection options")
-        for provider in sorted(ENV_VARS):
+        for provider in discovered_provider_names():
             _, env_name = resolve_credential(provider)
             print(f"{provider:10} {env_name or 'not configured'}")
         print("Read-only: no credential values were displayed and no provider requests were made.")

From fb1ad2ca8db6e4418f122acca8140163826cc774 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:15:58 -0500
Subject: [PATCH 050/128] Test discovery of unlisted provider API credentials

---
 tests/test_provider_api_smoke_test.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index f721ab0..5592a31 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -3,7 +3,7 @@
 
 from unittest.mock import patch
 
-from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, ENV_VARS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, plan_all_configured, representative_cases, resolve_credential
+from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, ENV_VARS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, discovered_provider_names, plan_all_configured, provider_from_env_name, representative_cases, resolve_credential
 
 
 
@@ -20,6 +20,8 @@ def test_standard_provider_environment_names_are_supported(self):
             "MISTRAL": "MISTRAL_API_KEY",
             "COHERE": "COHERE_API_KEY",
             "NVIDIA": "NVIDIA_API_KEY",
+            "OPENAI": "OPENAI_API_KEY",
+            "ANTHROPIC": "ANTHROPIC_API_KEY",
         }
         for provider, env_name in cases.items():
             with self.subTest(provider=provider):
@@ -71,6 +73,8 @@ def test_plan_includes_every_provider_and_never_opts_into_paid_candidates(self):
             "OPENROUTER_API_KEY": "OPENROUTER_API_KEY",
             "MISTRAL_API_KEY": "MISTRAL_API_KEY",
             "COHERE_API_KEY": "COHERE_API_KEY",
+            "OPENAI_API_KEY": "OPENAI_API_KEY",
+            "ANTHROPIC_API_KEY": "ANTHROPIC_API_KEY",
         }
         with patch.dict("os.environ", {name: "test-secret" for name in aliases}, clear=True):
             plan = plan_all_configured()
@@ -93,6 +97,26 @@ def test_plan_reports_every_missing_credential(self):
         self.assertTrue(all(item["status"] == "skipped-no-credential" for item in plan))
         self.assertTrue(all(item["credential_env"] is None for item in plan))
 
+
+    def test_unlisted_provider_credentials_are_discovered_without_secret_values(self):
+        with patch.dict("os.environ", {
+            "TOGETHER_API_KEY": "together-secret",
+            "FIREWORKS_API_KEY": "fireworks-secret",
+        }, clear=True):
+            self.assertEqual(provider_from_env_name("TOGETHER_API_KEY"), "TOGETHER")
+            self.assertEqual(
+                resolve_credential("TOGETHER"),
+                ("together-secret", "TOGETHER_API_KEY"),
+            )
+            self.assertIn("FIREWORKS", discovered_provider_names())
+            plan = {item["provider"]: item for item in plan_all_configured()}
+            self.assertEqual(plan["TOGETHER"]["status"], "skipped-no-provider-adapter")
+            self.assertEqual(plan["FIREWORKS"]["status"], "skipped-no-provider-adapter")
+
+    def test_non_provider_tokens_are_not_misclassified_as_api_credentials(self):
+        self.assertIsNone(provider_from_env_name("GITHUB_TOKEN"))
+        self.assertIsNone(provider_from_env_name("AWS_SESSION_TOKEN"))
+
     def test_paid_opt_in_only_changes_the_execution_plan(self):
         with patch.dict("os.environ", {
             "CEREBRAS_API_KEY": "test-secret",

From b30b364b717f84a2902daaccc3fdf2c7ae8b516b Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:16:09 -0500
Subject: [PATCH 051/128] Add Anthropic to credentialed catalogue inventory

---
 tools/provider_api/catalogue_inventory.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/catalogue_inventory.sh b/tools/provider_api/catalogue_inventory.sh
index 061f985..489ab3a 100644
--- a/tools/provider_api/catalogue_inventory.sh
+++ b/tools/provider_api/catalogue_inventory.sh
@@ -22,6 +22,7 @@ declare -A KEYS=(
   [MISTRAL]=MISTRAL_API_KEY_FLOSSWARE
   [NVIDIA]=NVIDIA_API_KEY_FLOSSWARE
   [OPENAI]=OPENAI_API_KEY_FLOSSWARE
+  [ANTHROPIC]=ANTHROPIC_API_KEY_FLOSSWARE
   [OPENROUTER]=OPENROUTER_API_KEY_FLOSSWARE
 )
 
@@ -37,6 +38,7 @@ declare -A ALIASES=(
   [MISTRAL]="MISTRAL_API_KEY"
   [NVIDIA]="NVIDIA_API_KEY"
   [OPENAI]="OPENAI_API_KEY"
+  [ANTHROPIC]="ANTHROPIC_API_KEY"
   [OPENROUTER]="OPENROUTER_API_KEY"
 )
 
@@ -51,13 +53,14 @@ declare -A URLS=(
   [MISTRAL]="https://api.mistral.ai/v1/models"
   [NVIDIA]="https://integrate.api.nvidia.com/v1/models"
   [OPENAI]="https://api.openai.com/v1/models"
+  [ANTHROPIC]="https://api.anthropic.com/v1/models"
   [OPENROUTER]="https://openrouter.ai/api/v1/models"
 )
 
 RESULTS="$TMP/results.jsonl"
 : > "$RESULTS"
 
-for provider in CEREBRAS COHERE DEEPINFRA DEEPSEEK GOOGLE GROQ                 HUGGINGFACE MISTRAL NVIDIA OPENAI OPENROUTER; do
+for provider in ANTHROPIC CEREBRAS COHERE DEEPINFRA DEEPSEEK GOOGLE GROQ HUGGINGFACE MISTRAL NVIDIA OPENAI OPENROUTER; do
   var="${KEYS[$provider]}"
   key="${!var:-}"
   if [[ -z "$key" ]]; then
@@ -78,6 +81,8 @@ for provider in CEREBRAS COHERE DEEPINFRA DEEPSEEK GOOGLE GROQ                 H
   if [[ -n "$key" ]]; then
     if [[ "$provider" == GOOGLE ]]; then
       config="$(printf 'header = "x-goog-api-key: %s"\n' "$key")"
+    elif [[ "$provider" == ANTHROPIC ]]; then
+      config="$(printf 'header = "x-api-key: %s"\nheader = "anthropic-version: 2023-06-01"\n' "$key")"
     else
       config="$(printf 'header = "Authorization: Bearer %s"\n' "$key")"
     fi

From e71e986ece25be6fd2b2b6688ab687409e20a513 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:16:25 -0500
Subject: [PATCH 052/128] Report unsupported discovered providers in batch
 summary

---
 tools/provider_api/smoke_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 8c896f6..6749283 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -100,7 +100,7 @@ def provider_from_env_name(env_name: str) -> str | None:
     }
     if env_name in explicit:
         return explicit[env_name]
-    match = re.fullmatch(r"([A-Z][A-Z0-9_]*)_(?:API_KEY(?:_FLOSSWARE)?|API_TOKEN)", env_name)
+    match = re.fullmatch(r"([A-Z][A-Z0-9_]*)_(?:API_KEY(?:_FLOSSWARE)?|API_TOKEN|AUTH_TOKEN)", env_name)
     if not match:
         return None
     provider = match.group(1)
@@ -571,6 +571,7 @@ def main() -> int:
                 "skipped-paid-opt-in-required",
                 "skipped-deprecated-endpoint",
                 "skipped-no-chat-candidate",
+                "skipped-no-provider-adapter",
             )
         }
         print(f"\nProvider inventory: {len(results)} providers examined.")

From b44e3a043c105f579bd22a7d790be239b22ac8fb Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:16:37 -0500
Subject: [PATCH 053/128] Document dynamic provider credential discovery

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 20ccdd5..07c540b 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ The harness does not own provider selection, routing, credential storage, retrie
 
 ### Run
 
-From the repository root, with the relevant provider credentials exported. The harness first checks each `*_API_KEY_FLOSSWARE` name, then falls back to the provider's conventional name (for example, `CEREBRAS_API_KEY`, `DEEPSEEK_API_KEY`, `GOOGLE_API_KEY`, `GROQ_API_KEY`, `OPENROUTER_API_KEY`, `MISTRAL_API_KEY`, or `COHERE_API_KEY`). Hugging Face accepts `HUGGINGFACE_API_KEY`, `HF_TOKEN`, and `HUGGING_FACE_HUB_TOKEN`; Google also accepts `GEMINI_API_KEY`. When more than one is set, the FlossWare-specific variable takes precedence. Only the selected variable's name is recorded, never its value:
+From the repository root, with the relevant provider credentials exported. The harness first checks each `*_API_KEY_FLOSSWARE` name, then falls back to conventional provider/SDK names. The built-in catalogue includes Cerebras, Cohere, DeepInfra, DeepSeek, Google/Gemini, Groq, Hugging Face, Mistral, NVIDIA, OpenAI, OpenRouter, and Anthropic. Hugging Face accepts `HUGGINGFACE_API_KEY`, `HF_TOKEN`, and `HUGGING_FACE_HUB_TOKEN`; Google also accepts `GEMINI_API_KEY`. In addition, the Python harness discovers non-empty environment variables ending in `_API_KEY`, `_API_KEY_FLOSSWARE`, `_API_TOKEN`, or `_AUTH_TOKEN`, and reports providers without an adapter as unsupported rather than silently omitting them. This is a discovery inventory, not a claim that every discovered service has an inference integration. When more than one credential is set, the FlossWare-specific variable takes precedence. Only variable names are recorded, never values:
 
 ```bash
 bash tools/provider_api/catalogue_inventory.sh
@@ -64,7 +64,7 @@ Run the complete provider inventory in one batch:
 python3 tools/provider_api/smoke_test.py --all-configured
 ```
 
-This iterates every known provider, not merely the providers with a ready model. It runs at most one representative candidate for each provider with a configured credential and a free/trial/zero-price candidate, including Gemini 3.5 Flash-Lite, Mistral Small, Cohere Command R7B, Hugging Face's monthly included-credit route, and OpenRouter free candidates. Every other provider is still written to the evidence file with an explicit skip reason: no credential, paid candidate requiring opt-in, deprecated endpoint, or no supported chat candidate. The final summary reports the complete provider count and skip categories. Paid/promotional-credit candidates are never invoked by default; `--include-paid` is a separate explicit opt-in and may incur charges. Deprecated endpoints are always skipped. To explicitly include those candidates:
+This iterates every known provider, not merely the providers with a ready model. It runs at most one representative candidate for each provider with a configured credential and a free/trial/zero-price candidate, including Gemini 3.5 Flash-Lite, Mistral Small, Cohere Command R7B, Hugging Face's monthly included-credit route, and OpenRouter free candidates. Every provider discovered from the environment is written to the evidence file with an explicit status: no credential, paid candidate requiring opt-in, deprecated endpoint, no supported chat candidate, or no provider adapter. OpenAI is included as a paid-only representative and is not called by default. Anthropic is included in catalogue credential discovery; inference remains skipped until an appropriate message API candidate/adapter is explicitly added. The final summary reports the complete discovered provider count and skip categories. Paid/promotional-credit candidates are never invoked by default; `--include-paid` is a separate explicit opt-in and may incur charges. Deprecated endpoints are always skipped. To explicitly include those candidates:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --all-configured --include-paid

From 21046d815836abdb285867246008d125531854a2 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:24:49 -0500
Subject: [PATCH 054/128] Discover and test account-scoped provider credentials

---
 tools/provider_api/smoke_test.py | 150 +++++++++++++++++++++----------
 1 file changed, 103 insertions(+), 47 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 6749283..d6c42fa 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -100,13 +100,22 @@ def provider_from_env_name(env_name: str) -> str | None:
     }
     if env_name in explicit:
         return explicit[env_name]
-    match = re.fullmatch(r"([A-Z][A-Z0-9_]*)_(?:API_KEY(?:_FLOSSWARE)?|API_TOKEN|AUTH_TOKEN)", env_name)
+    match = re.fullmatch(
+        r"([A-Z][A-Z0-9_]*)_(?:API_KEY(?:_(?:FLOSSWARE|HOTMAIL|NCRR))?|API_TOKEN|AUTH_TOKEN)",
+        env_name,
+    )
     if not match:
         return None
     provider = match.group(1)
     return {"HF": "HUGGINGFACE", "HUGGING_FACE": "HUGGINGFACE", "GEMINI": "GOOGLE"}.get(provider, provider)
 
 
+def credential_account_from_env_name(env_name: str) -> str:
+    """Return the account namespace encoded in a credential variable name."""
+    match = re.fullmatch(r"[A-Z][A-Z0-9_]*_API_KEY_(FLOSSWARE|HOTMAIL|NCRR)", env_name)
+    return match.group(1) if match else "DEFAULT"
+
+
 def discovered_provider_names() -> list[str]:
     """Return known providers plus providers inferred from non-empty credential variables."""
     discovered = {
@@ -118,22 +127,40 @@ def discovered_provider_names() -> list[str]:
 
 
 def credential_variable_names(provider: str) -> tuple[str, ...]:
-    """Return ordered credential variable names for a provider, without reading values."""
-    primary = ENV_VARS.get(provider, f"{provider}_API_KEY_FLOSSWARE")
-    names = [primary, *ENV_VAR_ALIASES.get(provider, ())]
-    names.extend(
-        name for name in os.environ
-        if provider_from_env_name(name) == provider
-    )
-    return tuple(dict.fromkeys(names))
+    """Return all configured credential variable names, with FlossWare first."""
+    names = [name for name in os.environ if provider_from_env_name(name) == provider]
+    aliases = ENV_VAR_ALIASES.get(provider, ())
+    names.extend(name for name in aliases if name in os.environ)
+    primary = ENV_VARS.get(provider)
+    if primary and primary in os.environ:
+        names.append(primary)
+    rank = {"FLOSSWARE": 0, "HOTMAIL": 1, "NCRR": 2, "DEFAULT": 3}
+    unique = dict.fromkeys(names)
+    return tuple(sorted(
+        unique,
+        key=lambda name: (rank.get(credential_account_from_env_name(name), 3), name),
+    ))
+
+
+def credential_accounts(provider: str) -> list[dict[str, str]]:
+    """Return each configured account credential separately; never merge accounts."""
+    accounts: list[dict[str, str]] = []
+    for env_name in credential_variable_names(provider):
+        value = os.environ.get(env_name, "").strip()
+        if value:
+            accounts.append({
+                "account": credential_account_from_env_name(env_name),
+                "credential_env": env_name,
+                "credential": value,
+            })
+    return accounts
 
 
 def resolve_credential(provider: str) -> tuple[str, str | None]:
-    """Return credential and source variable name without exposing its value."""
-    for env_name in credential_variable_names(provider):
-        value = os.environ.get(env_name, "")
-        if value.strip():
-            return value.strip(), env_name
+    """Return the preferred credential and source variable name."""
+    accounts = credential_accounts(provider)
+    if accounts:
+        return accounts[0]["credential"], accounts[0]["credential_env"]
     return "", None
 
 
@@ -235,47 +262,66 @@ def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
 
 
 def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
-    """Build a complete provider plan without making network requests."""
+    """Build a provider/account plan without making network requests."""
     selected = representative_cases(include_paid)
     plan: list[dict[str, Any]] = []
     for provider in discovered_provider_names():
-        _, env_name = resolve_credential(provider)
+        accounts = credential_accounts(provider)
+        account_rows = accounts or [{
+            "account": "DEFAULT",
+            "credential_env": None,
+            "credential": "",
+        }]
         candidates = [case for case in CASES if case["provider"] == provider]
         case = selected.get(provider)
-        item: dict[str, Any] = {
-            "provider": provider,
-            "credential_env": env_name,
-            "credential_configured": bool(env_name),
-            "model_requested": case["model"] if case else None,
-            "access_classification": case["access"] if case else None,
-        }
-        if not env_name:
-            item["status"] = "skipped-no-credential"
-        elif case:
-            item["status"] = "ready"
-        elif candidates and all(c["access"] in DEPRECATED_ACCESS for c in candidates):
-            item["status"] = "skipped-deprecated-endpoint"
-            item["model_requested"] = candidates[0]["model"]
-            item["access_classification"] = candidates[0]["access"]
-        elif provider not in {"GOOGLE", "COHERE", *OPENAI_COMPATIBLE}:
-            item["status"] = "skipped-no-provider-adapter"
-        elif candidates and not include_paid and all(c["access"] in PAID_ACCESS or c["access"] in DEPRECATED_ACCESS for c in candidates):
-            item["status"] = "skipped-paid-opt-in-required"
-            eligible = next((c for c in candidates if c["access"] in PAID_ACCESS), candidates[0])
-            item["model_requested"] = eligible["model"]
-            item["access_classification"] = eligible["access"]
-        else:
-            item["status"] = "skipped-no-chat-candidate"
-        plan.append(item)
+        for account in account_rows:
+            env_name = account["credential_env"]
+            item: dict[str, Any] = {
+                "provider": provider,
+                "account": account["account"],
+                "credential_env": env_name,
+                "credential_configured": bool(env_name),
+                "model_requested": case["model"] if case else None,
+                "access_classification": case["access"] if case else None,
+            }
+            if not env_name:
+                item["status"] = "skipped-no-credential"
+            elif case:
+                item["status"] = "ready"
+            elif candidates and all(c["access"] in DEPRECATED_ACCESS for c in candidates):
+                item["status"] = "skipped-deprecated-endpoint"
+                item["model_requested"] = candidates[0]["model"]
+                item["access_classification"] = candidates[0]["access"]
+            elif provider not in {"GOOGLE", "COHERE", *OPENAI_COMPATIBLE}:
+                item["status"] = "skipped-no-provider-adapter"
+            elif candidates and not include_paid and all(c["access"] in PAID_ACCESS or c["access"] in DEPRECATED_ACCESS for c in candidates):
+                item["status"] = "skipped-paid-opt-in-required"
+                eligible = next((c for c in candidates if c["access"] in PAID_ACCESS), candidates[0])
+                item["model_requested"] = eligible["model"]
+                item["access_classification"] = eligible["access"]
+            else:
+                item["status"] = "skipped-no-chat-candidate"
+            plan.append(item)
     return plan
 
 
-def request_case(case: dict[str, str], timeout: float, max_tokens: int) -> dict[str, Any]:
+def request_case(
+    case: dict[str, str],
+    timeout: float,
+    max_tokens: int,
+    credential_env: str | None = None,
+    account: str = "DEFAULT",
+) -> dict[str, Any]:
     provider, model = case["provider"], case["model"]
-    key, env_name = resolve_credential(provider)
+    if credential_env:
+        key = os.environ.get(credential_env, "").strip()
+        env_name = credential_env
+    else:
+        key, env_name = resolve_credential(provider)
     result: dict[str, Any] = {
         "observed_at": datetime.now(timezone.utc).isoformat(),
         "provider": provider,
+        "account": account,
         "model_requested": model,
         "requested_max_output_tokens": max_tokens,
         "access_classification": case["access"],
@@ -461,8 +507,11 @@ def main() -> int:
         if args.show_latest_errors or args.all_configured or args.provider or args.model or args.include_paid:
             parser.error("--show-credential-sources cannot be combined with other selection options")
         for provider in discovered_provider_names():
-            _, env_name = resolve_credential(provider)
-            print(f"{provider:10} {env_name or 'not configured'}")
+            names = [
+                name for name in credential_variable_names(provider)
+                if os.environ.get(name, "").strip()
+            ]
+            print(f"{provider:16} {', '.join(names) if names else 'not configured'}")
         print("Read-only: no credential values were displayed and no provider requests were made.")
         return 0
 
@@ -523,7 +572,13 @@ def main() -> int:
                     c for c in CASES
                     if c["provider"] == item["provider"] and c["model"] == item["model_requested"]
                 )
-                results.append(request_case(case, args.timeout, args.max_output_tokens))
+                results.append(request_case(
+                    case,
+                    args.timeout,
+                    args.max_output_tokens,
+                    credential_env=item.get("credential_env"),
+                    account=item.get("account", "DEFAULT"),
+                ))
             else:
                 item.update({
                     "http_status": None,
@@ -554,7 +609,8 @@ def main() -> int:
         http = item["http_status"] if item["http_status"] is not None else "-"
         latency = item["latency_seconds"] if item["latency_seconds"] is not None else "-"
         model_label = item["model_requested"] or "-"
-        print(f'{item["provider"]:10} {model_label:46} {status:30} HTTP={http} latency={latency}s')
+        account_label = item.get("account", "DEFAULT")
+        print(f'{item["provider"]:10} {account_label:10} {model_label:46} {status:30} HTTP={http} latency={latency}s')
         detail = error_summary(item.get("error"))
         if detail:
             category = item.get("http_error_category") or "provider-error"

From f826362386803bdf3c9f0778ee6867391b746867 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:24:51 -0500
Subject: [PATCH 055/128] Test separate provider account credential discovery

---
 tests/test_provider_api_smoke_test.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 5592a31..fd2ae02 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -113,9 +113,31 @@ def test_unlisted_provider_credentials_are_discovered_without_secret_values(self
             self.assertEqual(plan["TOGETHER"]["status"], "skipped-no-provider-adapter")
             self.assertEqual(plan["FIREWORKS"]["status"], "skipped-no-provider-adapter")
 
-    def test_non_provider_tokens_are_not_misclassified_as_api_credentials(self):
+    def test_account_scoped_api_keys_are_discovered_separately(self):
+        from tools.provider_api.smoke_test import credential_accounts
+        with patch.dict("os.environ", {
+            "CEREBRAS_API_KEY_FLOSSWARE": "fw-secret",
+            "CEREBRAS_API_KEY_HOTMAIL": "hotmail-secret",
+            "CEREBRAS_API_KEY_NCRR": "ncrr-secret",
+            "GROQ_ORG_ID_FLOSSWARE": "org-id",
+        }, clear=True):
+            self.assertEqual(provider_from_env_name("CEREBRAS_API_KEY_HOTMAIL"), "CEREBRAS")
+            self.assertEqual(provider_from_env_name("CEREBRAS_API_KEY_NCRR"), "CEREBRAS")
+            accounts = credential_accounts("CEREBRAS")
+            self.assertEqual(
+                [(a["account"], a["credential_env"]) for a in accounts],
+                [
+                    ("FLOSSWARE", "CEREBRAS_API_KEY_FLOSSWARE"),
+                    ("HOTMAIL", "CEREBRAS_API_KEY_HOTMAIL"),
+                    ("NCRR", "CEREBRAS_API_KEY_NCRR"),
+                ],
+            )
+            plan = [p for p in plan_all_configured() if p["provider"] == "CEREBRAS"]
+            self.assertEqual([p["account"] for p in plan], ["FLOSSWARE", "HOTMAIL", "NCRR"])
+            self.assertTrue(all(p["status"] == "skipped-paid-opt-in-required" for p in plan))
         self.assertIsNone(provider_from_env_name("GITHUB_TOKEN"))
         self.assertIsNone(provider_from_env_name("AWS_SESSION_TOKEN"))
+        self.assertIsNone(provider_from_env_name("GROQ_ORG_ID_FLOSSWARE"))
 
     def test_paid_opt_in_only_changes_the_execution_plan(self):
         with patch.dict("os.environ", {

From 0643fa80c9c2adb64daed8b4db2c9c49e620498c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:24:53 -0500
Subject: [PATCH 056/128] Document provider account-scoped credential handling

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 07c540b..2b35de1 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ The harness does not own provider selection, routing, credential storage, retrie
 
 ### Run
 
-From the repository root, with the relevant provider credentials exported. The harness first checks each `*_API_KEY_FLOSSWARE` name, then falls back to conventional provider/SDK names. The built-in catalogue includes Cerebras, Cohere, DeepInfra, DeepSeek, Google/Gemini, Groq, Hugging Face, Mistral, NVIDIA, OpenAI, OpenRouter, and Anthropic. Hugging Face accepts `HUGGINGFACE_API_KEY`, `HF_TOKEN`, and `HUGGING_FACE_HUB_TOKEN`; Google also accepts `GEMINI_API_KEY`. In addition, the Python harness discovers non-empty environment variables ending in `_API_KEY`, `_API_KEY_FLOSSWARE`, `_API_TOKEN`, or `_AUTH_TOKEN`, and reports providers without an adapter as unsupported rather than silently omitting them. This is a discovery inventory, not a claim that every discovered service has an inference integration. When more than one credential is set, the FlossWare-specific variable takes precedence. Only variable names are recorded, never values:
+From the repository root, with the relevant provider credentials exported. The harness first checks each `*_API_KEY_FLOSSWARE` name, then falls back to conventional provider/SDK names. The built-in catalogue includes Cerebras, Cohere, DeepInfra, DeepSeek, Google/Gemini, Groq, Hugging Face, Mistral, NVIDIA, OpenAI, OpenRouter, and Anthropic. Hugging Face accepts `HUGGINGFACE_API_KEY`, `HF_TOKEN`, and `HUGGING_FACE_HUB_TOKEN`; Google also accepts `GEMINI_API_KEY`. In addition, the Python harness discovers non-empty environment variables ending in `_API_KEY`, `_API_KEY_FLOSSWARE`, `_API_KEY_HOTMAIL`, `_API_KEY_NCRR`, `_API_TOKEN`, or `_AUTH_TOKEN`, and reports providers without an adapter as unsupported rather than silently omitting them. Account suffixes `FLOSSWARE`, `HOTMAIL`, and `NCRR` are treated as separate credential identities, never merged. The all-configured batch creates a separate inventory row for each account and tests each account independently where a supported, eligible candidate exists. The FlossWare account is preferred only by single-credential convenience lookups; it does not suppress other account-scoped keys in the batch. Non-credential settings such as `GROQ_ORG_ID_FLOSSWARE` are not treated as API keys. This is a discovery inventory, not a claim that every discovered service has an inference integration. Only variable names are recorded, never values:
 
 ```bash
 bash tools/provider_api/catalogue_inventory.sh

From a81b0d3e676a3d38421765e13dae969202a3ee1f Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:31:35 -0500
Subject: [PATCH 057/128] Normalize Think Machines account credential aliases

---
 tools/provider_api/smoke_test.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index d6c42fa..6495fb6 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -107,7 +107,12 @@ def provider_from_env_name(env_name: str) -> str | None:
     if not match:
         return None
     provider = match.group(1)
-    return {"HF": "HUGGINGFACE", "HUGGING_FACE": "HUGGINGFACE", "GEMINI": "GOOGLE"}.get(provider, provider)
+    return {
+        "HF": "HUGGINGFACE",
+        "HUGGING_FACE": "HUGGINGFACE",
+        "GEMINI": "GOOGLE",
+        "THINKINGMACHINES": "THINKMACHINES",
+    }.get(provider, provider)
 
 
 def credential_account_from_env_name(env_name: str) -> str:
@@ -630,7 +635,7 @@ def main() -> int:
                 "skipped-no-provider-adapter",
             )
         }
-        print(f"\nProvider inventory: {len(results)} providers examined.")
+        print(f"\nProvider/account inventory: {len(results)} rows examined.")
         print(f"Eligible requests: {len(results) - skipped}; passed: {passed}; failed: {failed}.")
         for reason, count in by_reason.items():
             if count:

From 24b14e13b85fe25402667755b0d08abc0a0c6b3e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:31:37 -0500
Subject: [PATCH 058/128] Test Think Machines credential namespace alias

---
 tests/test_provider_api_smoke_test.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index fd2ae02..1a449ad 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -135,6 +135,26 @@ def test_account_scoped_api_keys_are_discovered_separately(self):
             plan = [p for p in plan_all_configured() if p["provider"] == "CEREBRAS"]
             self.assertEqual([p["account"] for p in plan], ["FLOSSWARE", "HOTMAIL", "NCRR"])
             self.assertTrue(all(p["status"] == "skipped-paid-opt-in-required" for p in plan))
+
+        with patch.dict("os.environ", {
+            "THINKMACHINES_API_KEY_FLOSSWARE": "fw-secret",
+            "THINKMACHINES_API_KEY_HOTMAIL": "hotmail-secret",
+            "THINKINGMACHINES_API_KEY_NCRR": "ncrr-secret",
+        }, clear=True):
+            self.assertEqual(
+                provider_from_env_name("THINKINGMACHINES_API_KEY_NCRR"),
+                "THINKMACHINES",
+            )
+            aliases = credential_accounts("THINKMACHINES")
+            self.assertEqual(
+                [(a["account"], a["credential_env"]) for a in aliases],
+                [
+                    ("FLOSSWARE", "THINKMACHINES_API_KEY_FLOSSWARE"),
+                    ("HOTMAIL", "THINKMACHINES_API_KEY_HOTMAIL"),
+                    ("NCRR", "THINKINGMACHINES_API_KEY_NCRR"),
+                ],
+            )
+
         self.assertIsNone(provider_from_env_name("GITHUB_TOKEN"))
         self.assertIsNone(provider_from_env_name("AWS_SESSION_TOKEN"))
         self.assertIsNone(provider_from_env_name("GROQ_ORG_ID_FLOSSWARE"))

From 4f8cb13e0ecc70f35ba1e597c6376eeeb86ef952 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:37:44 -0500
Subject: [PATCH 059/128] Raise default smoke-test output budget to 512 tokens

---
 tools/provider_api/smoke_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 6495fb6..1840f9c 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -500,8 +500,8 @@ def main() -> int:
     parser.add_argument("--show-credential-sources", action="store_true",
                         help="Show which credential variable name is selected for each provider, without printing values or making API calls.")
     parser.add_argument("--timeout", type=float, default=45.0)
-    parser.add_argument("--max-output-tokens", type=int, default=128,
-                        help="Maximum generated tokens per request (default: 128).")
+    parser.add_argument("--max-output-tokens", type=int, default=512,
+                        help="Maximum generated tokens per request (default: 512).")
     parser.add_argument("--output", type=Path, default=Path("provider-smoke-results.jsonl"))
     args = parser.parse_args()
 

From f8232be4be232ef5c8c48b6fa5d2057f47eb512c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:37:47 -0500
Subject: [PATCH 060/128] Test the default smoke-test output budget

---
 tests/test_provider_api_smoke_test.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 1a449ad..92deac2 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -260,6 +260,16 @@ def test_paid_opt_in_adds_paid_only_providers(self):
         self.assertNotIn("NVIDIA", selected)
 
 
+class DefaultOutputBudgetTests(unittest.TestCase):
+    def test_default_output_budget_is_512_tokens(self):
+        import inspect
+        from tools.provider_api.smoke_test import main
+
+        source = inspect.getsource(main)
+        self.assertIn('type=int, default=512', source)
+        self.assertIn('default: 512', source)
+
+
 class ClassifyResponseTests(unittest.TestCase):
     def test_exact_response_with_stop_passes(self):
         self.assertEqual(

From e8305de09bf9b4dd723e59da50722b31b2e230fb Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:38:01 -0500
Subject: [PATCH 061/128] Centralize the default output token budget

---
 tools/provider_api/smoke_test.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 1840f9c..8685e53 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -21,6 +21,7 @@
 
 PROMPT = "Reply with exactly this token and nothing else: FLOSSWARE_PROVIDER_SMOKE_OK"
 EXPECTED = "FLOSSWARE_PROVIDER_SMOKE_OK"
+DEFAULT_MAX_OUTPUT_TOKENS = 512
 
 # The default batch is restricted to catalogue-marked OpenRouter :free models and
 # Gemini models visible in the authenticated catalogue. Paid/promotional-credit
@@ -500,8 +501,8 @@ def main() -> int:
     parser.add_argument("--show-credential-sources", action="store_true",
                         help="Show which credential variable name is selected for each provider, without printing values or making API calls.")
     parser.add_argument("--timeout", type=float, default=45.0)
-    parser.add_argument("--max-output-tokens", type=int, default=512,
-                        help="Maximum generated tokens per request (default: 512).")
+    parser.add_argument("--max-output-tokens", type=int, default=DEFAULT_MAX_OUTPUT_TOKENS,
+                        help=f"Maximum generated tokens per request (default: {DEFAULT_MAX_OUTPUT_TOKENS}).")
     parser.add_argument("--output", type=Path, default=Path("provider-smoke-results.jsonl"))
     args = parser.parse_args()
 

From 540534281468c302aa965b345f1845de0ad53ec3 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:41:10 -0500
Subject: [PATCH 062/128] Fix output-budget regression test to use constant

---
 tests/test_provider_api_smoke_test.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 92deac2..a9ae435 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -3,7 +3,7 @@
 
 from unittest.mock import patch
 
-from tools.provider_api.smoke_test import CASES, DEPRECATED_ACCESS, ENV_VARS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, discovered_provider_names, plan_all_configured, provider_from_env_name, representative_cases, resolve_credential
+from tools.provider_api.smoke_test import CASES, DEFAULT_MAX_OUTPUT_TOKENS, DEPRECATED_ACCESS, ENV_VARS, EXPECTED, OPENAI_COMPATIBLE, build_headers, classify_response, discovered_provider_names, plan_all_configured, provider_from_env_name, representative_cases, resolve_credential
 
 
 
@@ -265,9 +265,10 @@ def test_default_output_budget_is_512_tokens(self):
         import inspect
         from tools.provider_api.smoke_test import main
 
+        self.assertEqual(DEFAULT_MAX_OUTPUT_TOKENS, 512)
         source = inspect.getsource(main)
-        self.assertIn('type=int, default=512', source)
-        self.assertIn('default: 512', source)
+        self.assertIn('default=DEFAULT_MAX_OUTPUT_TOKENS', source)
+        self.assertIn('default: {DEFAULT_MAX_OUTPUT_TOKENS}', source)
 
 
 class ClassifyResponseTests(unittest.TestCase):

From 6d772b6a82f39e2900538fe47aec9c5808cd0a83 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:45:35 -0500
Subject: [PATCH 063/128] Fix account identity in direct provider runs

---
 tools/provider_api/smoke_test.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 8685e53..5b581eb 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -324,6 +324,10 @@ def request_case(
         env_name = credential_env
     else:
         key, env_name = resolve_credential(provider)
+        if env_name:
+            # Direct single-provider runs must record the account namespace
+            # encoded in the selected credential, not the default parameter.
+            account = credential_account_from_env_name(env_name)
     result: dict[str, Any] = {
         "observed_at": datetime.now(timezone.utc).isoformat(),
         "provider": provider,

From 47a13e702da12fea871f89e7b3c045c969785e2b Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:45:38 -0500
Subject: [PATCH 064/128] Test account identity in direct provider runs

---
 tests/test_provider_api_smoke_test.py | 44 +++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index a9ae435..458808a 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -260,6 +260,50 @@ def test_paid_opt_in_adds_paid_only_providers(self):
         self.assertNotIn("NVIDIA", selected)
 
 
+class DirectRequestAccountTests(unittest.TestCase):
+    def test_direct_request_records_the_selected_credential_account(self):
+        import json
+        from tools.provider_api.smoke_test import request_case
+
+        class FakeResponse:
+            status = 200
+            headers = {}
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *args):
+                return False
+
+            def read(self):
+                return json.dumps({
+                    "id": "test-request",
+                    "model": "nvidia/nemotron-3.5-lightning:free",
+                    "choices": [{
+                        "message": {"content": EXPECTED},
+                        "finish_reason": "stop",
+                    }],
+                    "usage": {},
+                }).encode("utf-8")
+
+        case = {
+            "provider": "OPENROUTER",
+            "model": "nvidia/nemotron-3.5-lightning:free",
+            "access": "catalogue-zero-price",
+        }
+        with patch.dict("os.environ", {
+            "OPENROUTER_API_KEY_FLOSSWARE": "test-secret",
+        }, clear=True), patch(
+            "tools.provider_api.smoke_test.urllib.request.urlopen",
+            return_value=FakeResponse(),
+        ):
+            result = request_case(case, timeout=1, max_tokens=512)
+
+        self.assertEqual(result["status"], "passed")
+        self.assertEqual(result["credential_env"], "OPENROUTER_API_KEY_FLOSSWARE")
+        self.assertEqual(result["account"], "FLOSSWARE")
+
+
 class DefaultOutputBudgetTests(unittest.TestCase):
     def test_default_output_budget_is_512_tokens(self):
         import inspect

From b57867b6381d45a20c64fb0f31925d35b79a0d17 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:49:44 -0500
Subject: [PATCH 065/128] Add explicit OpenRouter free model account matrix

---
 tools/provider_api/smoke_test.py | 37 +++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 5b581eb..d78d1a0 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -267,6 +267,15 @@ def representative_cases(include_paid: bool) -> dict[str, dict[str, str]]:
     return selected
 
 
+def openrouter_free_cases() -> list[dict[str, str]]:
+    """Return explicit, catalogue-zero-price OpenRouter models (not the dynamic router alias)."""
+    return [
+        case for case in CASES
+        if case["provider"] == "OPENROUTER"
+        and case["access"] == "catalogue-zero-price"
+    ]
+
+
 def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
     """Build a provider/account plan without making network requests."""
     selected = representative_cases(include_paid)
@@ -496,6 +505,8 @@ def main() -> int:
                         help="Run only this provider; repeat to select several.")
     parser.add_argument("--all-configured", action="store_true",
                         help="Iterate every known provider; run one free/trial candidate where available and report all others as skipped.")
+    parser.add_argument("--openrouter-free-matrix", action="store_true",
+                        help="Run every explicit catalogue-zero-price OpenRouter model once for each configured OpenRouter credential account; excludes the dynamic router alias and all paid candidates.")
     parser.add_argument("--model",
                         help="Run only this exact configured model ID; combine with --provider to validate its provider.")
     parser.add_argument("--include-paid", action="store_true",
@@ -558,6 +569,12 @@ def main() -> int:
         print("Read-only: no provider requests were made.")
         return 0
 
+    if args.openrouter_free_matrix and (
+        args.all_configured or args.provider or args.model or args.include_paid
+        or args.show_latest_errors or args.show_credential_sources
+    ):
+        parser.error("--openrouter-free-matrix cannot be combined with other selection options")
+
     if args.all_configured and (args.provider or args.model):
         parser.error("--all-configured cannot be combined with --provider or --model")
 
@@ -571,7 +588,25 @@ def main() -> int:
             configured = ", ".join(sorted({case["provider"] for case in matching_models}))
             parser.error(f"Model {args.model!r} is configured for {configured}, not the selected provider(s).")
 
-    if args.all_configured:
+    if args.openrouter_free_matrix:
+        accounts = credential_accounts("OPENROUTER")
+        cases = openrouter_free_cases()
+        if not accounts:
+            parser.error("No configured OpenRouter credential accounts found.")
+        if not cases:
+            parser.error("No explicit catalogue-zero-price OpenRouter models are configured.")
+        results = [
+            request_case(
+                case,
+                args.timeout,
+                args.max_output_tokens,
+                credential_env=account["credential_env"],
+                account=account["account"],
+            )
+            for account in accounts
+            for case in cases
+        ]
+    elif args.all_configured:
         # Plan every provider first. Only free/trial/zero-price candidates run
         # by default; paid-only and deprecated providers are explicitly recorded.
         # At most one candidate per provider is ever invoked in this batch.

From f008e49a83e304c7fb1699fe0e63f2a900f86f8c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 02:49:48 -0500
Subject: [PATCH 066/128] Test OpenRouter free model matrix selection

---
 tests/test_provider_api_smoke_test.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 458808a..cde35e2 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -260,6 +260,26 @@ def test_paid_opt_in_adds_paid_only_providers(self):
         self.assertNotIn("NVIDIA", selected)
 
 
+class OpenRouterFreeMatrixTests(unittest.TestCase):
+    def test_matrix_contains_only_explicit_zero_price_models(self):
+        from tools.provider_api.smoke_test import openrouter_free_cases
+
+        cases = openrouter_free_cases()
+        self.assertEqual(
+            [case["model"] for case in cases],
+            [
+                "nvidia/nemotron-3.5-lightning:free",
+                "deepseek/deepseek-v4-flash-0731:free",
+                "google/gemma-4-31b-it:free",
+                "thinkingmachines/inkling:free",
+                "thinkingmachines/inkling-small:free",
+            ],
+        )
+        self.assertTrue(all(case["provider"] == "OPENROUTER" for case in cases))
+        self.assertTrue(all(case["access"] == "catalogue-zero-price" for case in cases))
+        self.assertNotIn("openrouter/free", [case["model"] for case in cases])
+
+
 class DirectRequestAccountTests(unittest.TestCase):
     def test_direct_request_records_the_selected_credential_account(self):
         import json

From 952dd9330b9f5c6bfdfcace80c8aec6b3f4af6e4 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:02:44 -0500
Subject: [PATCH 067/128] Classify provider rate limits and sanitize errors

---
 tools/provider_api/smoke_test.py | 68 +++++++++++++++++++++++++++++---
 1 file changed, 62 insertions(+), 6 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index d78d1a0..a343561 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -177,8 +177,56 @@ def safe_text(value: Any, secret: str) -> str:
     return text[:2000]
 
 
+def sanitize_error_detail(detail: str | None, secret: str = "") -> str | None:
+    """Remove provider user identifiers and redact secrets before evidence is persisted."""
+    if detail is None:
+        return None
+    try:
+        parsed = json.loads(detail)
+    except (json.JSONDecodeError, TypeError):
+        return safe_text(detail, secret)
+
+    def scrub(value: Any) -> Any:
+        if isinstance(value, dict):
+            cleaned = {}
+            for key, nested in value.items():
+                normalized_key = re.sub(r"[^a-z]", "", str(key).lower())
+                if normalized_key in {"userid", "useridentifier"}:
+                    continue
+                cleaned[key] = scrub(nested)
+            return cleaned
+        if isinstance(value, list):
+            return [scrub(item) for item in value]
+        if isinstance(value, str) and secret:
+            return value.replace(secret, "[REDACTED]")
+        return value
+
+    sanitized = json.dumps(scrub(parsed), ensure_ascii=False, separators=(",", ":"))
+    return safe_text(sanitized, secret)
+
+
 def classify_http_error(status: int | None, detail: str | None) -> str | None:
-    """Classify provider HTTP errors without confusing billing with inference failure."""
+    """Classify billing and rate-limit responses using provider metadata where available."""
+    try:
+        parsed = json.loads(detail or "")
+    except (json.JSONDecodeError, TypeError):
+        parsed = {}
+
+    error = parsed.get("error", {}) if isinstance(parsed, dict) else {}
+    metadata = error.get("metadata", {}) if isinstance(error, dict) else {}
+
+    if status == 429:
+        limit_source = str(metadata.get("limit_source") or "").lower()
+        if limit_source == "upstream_provider_shared_pool":
+            return "upstream-shared-pool-rate-limit"
+        if metadata.get("is_byok") is True:
+            return "byok-provider-rate-limit"
+        if "upstream" in limit_source:
+            return "upstream-provider-rate-limit"
+        if limit_source:
+            return "provider-rate-limit"
+        return "rate-limit-unclassified"
+
     if status != 402:
         return None
     message = (detail or "").lower()
@@ -203,7 +251,13 @@ def error_summary(detail: str | None) -> str | None:
     if isinstance(parsed, dict):
         error = parsed.get("error")
         if isinstance(error, dict):
-            message = str(error.get("message") or error.get("detail") or error)
+            metadata = error.get("metadata") or {}
+            message = str(
+                metadata.get("raw")
+                or error.get("message")
+                or error.get("detail")
+                or error
+            )
         elif isinstance(error, str):
             message = error
         else:
@@ -482,7 +536,7 @@ def request_case(
             detail = exc.read().decode("utf-8", errors="replace")
         except Exception:
             detail = str(exc)
-        result["error"] = safe_text(detail, key)
+        result["error"] = sanitize_error_detail(detail, key)
         result["http_error_category"] = classify_http_error(exc.code, result["error"])
         result["status"] = "http-error"
     except (urllib.error.URLError, TimeoutError, OSError) as exc:
@@ -512,7 +566,7 @@ def main() -> int:
     parser.add_argument("--include-paid", action="store_true",
                         help="Also run paid/promotional-credit comparison candidates.")
     parser.add_argument("--show-latest-errors", action="store_true",
-                        help="Print the latest recorded errors for Cerebras, DeepInfra, DeepSeek, and Hugging Face without making API calls.")
+                        help="Print the latest recorded errors for Cerebras, DeepInfra, DeepSeek, Hugging Face, and OpenRouter without making API calls.")
     parser.add_argument("--show-credential-sources", action="store_true",
                         help="Show which credential variable name is selected for each provider, without printing values or making API calls.")
     parser.add_argument("--timeout", type=float, default=45.0)
@@ -541,7 +595,7 @@ def main() -> int:
             parser.error("--show-latest-errors cannot be combined with inference-selection options")
         if not args.output.exists():
             parser.error(f"No evidence file found: {args.output}")
-        wanted = {"CEREBRAS", "DEEPINFRA", "DEEPSEEK", "HUGGINGFACE"}
+        wanted = {"CEREBRAS", "DEEPINFRA", "DEEPSEEK", "HUGGINGFACE", "OPENROUTER"}
         latest: dict[str, dict[str, Any]] = {}
         with args.output.open(encoding="utf-8") as stream:
             for line in stream:
@@ -550,7 +604,9 @@ def main() -> int:
                 except json.JSONDecodeError:
                     continue
                 provider = item.get("provider")
-                if provider in wanted and (item.get("error") or item.get("http_status") == 402):
+                if provider in wanted and (
+                    item.get("error") or item.get("http_status") in {402, 429}
+                ):
                     latest[provider] = item
         for provider in sorted(wanted):
             item = latest.get(provider)

From da620f52e80568a24bf47b053c078586e3b53de4 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:03:10 -0500
Subject: [PATCH 068/128] Test rate-limit classification and error sanitization

---
 tests/test_provider_api_smoke_test.py | 71 +++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index cde35e2..7a44262 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -233,6 +233,77 @@ def test_non_402_is_not_a_billing_failure(self):
         from tools.provider_api.smoke_test import classify_http_error
         self.assertIsNone(classify_http_error(403, "Forbidden"))
 
+    def test_openrouter_shared_pool_429_is_classified_as_upstream_rate_limit(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        detail = json.dumps({
+            "error": {
+                "metadata": {
+                    "provider_name": "Google AI Studio",
+                    "is_byok": False,
+                    "limit_source": "upstream_provider_shared_pool",
+                }
+            }
+        })
+        self.assertEqual(
+            classify_http_error(429, detail),
+            "upstream-shared-pool-rate-limit",
+        )
+
+    def test_byok_429_is_distinguished_from_shared_pool_limit(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        detail = json.dumps({"error": {"metadata": {"is_byok": True}}})
+        self.assertEqual(
+            classify_http_error(429, detail),
+            "byok-provider-rate-limit",
+        )
+
+    def test_unclassified_429_is_still_identified_as_rate_limit(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        self.assertEqual(
+            classify_http_error(429, "Too many requests"),
+            "rate-limit-unclassified",
+        )
+
+    def test_provider_error_sanitizer_removes_user_ids_and_redacts_secrets(self):
+        from tools.provider_api.smoke_test import sanitize_error_detail
+
+        detail = json.dumps({
+            "error": {
+                "message": "Provider returned error",
+                "metadata": {
+                    "raw": "request rejected using secret-value",
+                    "user_id": "provider-user-123",
+                },
+            },
+            "userId": "top-level-user-456",
+        })
+        sanitized = sanitize_error_detail(detail, "secret-value")
+
+        self.assertNotIn("provider-user-123", sanitized)
+        self.assertNotIn("top-level-user-456", sanitized)
+        self.assertNotIn("user_id", sanitized)
+        self.assertNotIn("userId", sanitized)
+        self.assertNotIn("secret-value", sanitized)
+        self.assertIn("[REDACTED]", sanitized)
+        self.assertIn("Google", sanitized) if "Google" in sanitized else None
+
+    def test_error_summary_prefers_provider_diagnostic_detail(self):
+        from tools.provider_api.smoke_test import error_summary
+
+        detail = json.dumps({
+            "error": {
+                "message": "Provider returned error",
+                "metadata": {
+                    "raw": "temporarily rate-limited upstream",
+                    "provider_name": "Google AI Studio",
+                },
+            }
+        })
+        self.assertEqual(error_summary(detail), "temporarily rate-limited upstream")
+
 
 class BatchSelectionTests(unittest.TestCase):
     def test_batch_selects_at_most_one_candidate_per_provider(self):

From ad5324e21f4e07a81619ecb1a3ad8fca5f1df0b4 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:03:13 -0500
Subject: [PATCH 069/128] Document rate-limit diagnostics and error
 sanitization

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2b35de1..6d14031 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ python3 tools/provider_api/smoke_test.py --all-configured --include-paid
 
 The batch never fans out across every model in a provider catalogue. It selects at most one representative model per provider, continues after individual failures, and appends all outcomes to the same JSONL evidence file. `--all-configured` cannot be combined with `--provider` or `--model`.
 
-Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 128-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are additionally classified as account-balance, account-credit/billing, or organization-subscription issues when the provider response supports that distinction; a compact sanitized error detail is printed so account restrictions are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 512-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are classified as account-balance, account-credit/billing, or organization-subscription issues when provider details support that distinction. HTTP 429 responses are classified separately, including upstream shared-pool limits and BYOK provider limits when metadata is available. Error evidence removes provider-supplied user identifiers and redacts the active credential before persistence; compact diagnostics prefer the provider's underlying error detail so account restrictions and shared-pool throttling are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
 Inspect which credential variable name is selected for each provider without displaying values or making network requests:
 
@@ -80,7 +80,7 @@ Inspect which credential variable name is selected for each provider without dis
 python3 tools/provider_api/smoke_test.py --show-credential-sources
 ```
 
-Inspect the latest recorded HTTP errors without making any network requests:
+Inspect the latest recorded HTTP errors for the supported diagnostic providers, including OpenRouter, without making any network requests:
 
 ```bash
 python3 tools/provider_api/smoke_test.py --show-latest-errors

From 763bf638ccff6313b1b9b3bb1011ebe245b74e5f Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:03:27 -0500
Subject: [PATCH 070/128] Fix sanitizer regression test imports and assertions

---
 tests/test_provider_api_smoke_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 7a44262..955d6ab 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -1,4 +1,5 @@
 """Deterministic tests for provider smoke-test response classification."""
+import json
 import unittest
 
 from unittest.mock import patch
@@ -276,6 +277,7 @@ def test_provider_error_sanitizer_removes_user_ids_and_redacts_secrets(self):
                 "metadata": {
                     "raw": "request rejected using secret-value",
                     "user_id": "provider-user-123",
+                    "provider_name": "Google AI Studio",
                 },
             },
             "userId": "top-level-user-456",
@@ -288,7 +290,7 @@ def test_provider_error_sanitizer_removes_user_ids_and_redacts_secrets(self):
         self.assertNotIn("userId", sanitized)
         self.assertNotIn("secret-value", sanitized)
         self.assertIn("[REDACTED]", sanitized)
-        self.assertIn("Google", sanitized) if "Google" in sanitized else None
+        self.assertIn("Google AI Studio", sanitized)
 
     def test_error_summary_prefers_provider_diagnostic_detail(self):
         from tools.provider_api.smoke_test import error_summary

From 7af45df0cd1cc48449fccec5000e5bb30ea178e6 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:03:36 -0500
Subject: [PATCH 071/128] Harden provider error metadata parsing

---
 tools/provider_api/smoke_test.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index a343561..55e20f6 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -214,6 +214,8 @@ def classify_http_error(status: int | None, detail: str | None) -> str | None:
 
     error = parsed.get("error", {}) if isinstance(parsed, dict) else {}
     metadata = error.get("metadata", {}) if isinstance(error, dict) else {}
+    if not isinstance(metadata, dict):
+        metadata = {}
 
     if status == 429:
         limit_source = str(metadata.get("limit_source") or "").lower()
@@ -252,6 +254,8 @@ def error_summary(detail: str | None) -> str | None:
         error = parsed.get("error")
         if isinstance(error, dict):
             metadata = error.get("metadata") or {}
+            if not isinstance(metadata, dict):
+                metadata = {}
             message = str(
                 metadata.get("raw")
                 or error.get("message")

From bf815d43288ea71e5eb2e16e6415aab2b80f555c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:06:28 -0500
Subject: [PATCH 072/128] Classify agentic-only OpenRouter model restrictions

---
 tools/provider_api/smoke_test.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 55e20f6..c1486db 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -217,6 +217,15 @@ def classify_http_error(status: int | None, detail: str | None) -> str | None:
     if not isinstance(metadata, dict):
         metadata = {}
 
+    if status == 403:
+        message = str(metadata.get("raw") or error.get("message") or detail or "").lower()
+        if (
+            "only available on agentic harnesses" in message
+            or "agentic harness" in message and "coding agent" in message
+        ):
+            return "agentic-harness-required"
+        return None
+
     if status == 429:
         limit_source = str(metadata.get("limit_source") or "").lower()
         if limit_source == "upstream_provider_shared_pool":

From a49be29e23c39d371a4adda53d23c50fb47e83c3 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:06:39 -0500
Subject: [PATCH 073/128] Test agentic harness restriction classification

---
 tests/test_provider_api_smoke_test.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 955d6ab..279eae0 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -234,6 +234,32 @@ def test_non_402_is_not_a_billing_failure(self):
         from tools.provider_api.smoke_test import classify_http_error
         self.assertIsNone(classify_http_error(403, "Forbidden"))
 
+    def test_openrouter_agentic_only_403_is_classified(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        detail = json.dumps({
+            "error": {
+                "message": "Provider returned error",
+                "metadata": {
+                    "raw": (
+                        "thinkingmachines/inkling-small:free is only available "
+                        "on agentic harnesses. Try plugging it into a coding agent."
+                    ),
+                    "provider_name": "Thinking Machines",
+                },
+            }
+        })
+        self.assertEqual(
+            classify_http_error(403, detail),
+            "agentic-harness-required",
+        )
+
+    def test_generic_403_remains_unclassified(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        self.assertIsNone(classify_http_error(403, "Forbidden"))
+
+
     def test_openrouter_shared_pool_429_is_classified_as_upstream_rate_limit(self):
         from tools.provider_api.smoke_test import classify_http_error
 

From 126de584742b524f96faabdf45cc5558b8b026ba Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:06:49 -0500
Subject: [PATCH 074/128] Document agentic-harness-only error classification

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6d14031..07e6a3b 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ python3 tools/provider_api/smoke_test.py --all-configured --include-paid
 
 The batch never fans out across every model in a provider catalogue. It selects at most one representative model per provider, continues after individual failures, and appends all outcomes to the same JSONL evidence file. `--all-configured` cannot be combined with `--provider` or `--model`.
 
-Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 512-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are classified as account-balance, account-credit/billing, or organization-subscription issues when provider details support that distinction. HTTP 429 responses are classified separately, including upstream shared-pool limits and BYOK provider limits when metadata is available. Error evidence removes provider-supplied user identifiers and redacts the active credential before persistence; compact diagnostics prefer the provider's underlying error detail so account restrictions and shared-pool throttling are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
+Select providers with repeatable `--provider` arguments. The script makes one non-streaming request per selected model, defaults to a 512-token output limit (configurable with `--max-output-tokens`), and performs no automatic retries. It captures the requested output-token budget, total latency, HTTP status, request IDs, rate-limit headers, provider-native usage, normalized reasoning-token usage, finish reason, visible output length, and whether the exact synthetic response was returned. HTTP 402 responses are classified as account-balance, account-credit/billing, or organization-subscription issues when provider details support that distinction. HTTP 429 responses are classified separately, including upstream shared-pool limits and BYOK provider limits when metadata is available. HTTP 403 responses explicitly identifying agentic-harness-only access are classified as `agentic-harness-required`; other 403 responses remain generic provider errors. Error evidence removes provider-supplied user identifiers and redacts the active credential before persistence; compact diagnostics prefer the provider's underlying error detail so account restrictions and shared-pool throttling are distinguishable from model/transport failures. Any response ending with `length` (OpenAI-compatible APIs) or `MAX_TOKENS` (Google) is classified as incomplete, even when partial or apparently matching text is present. It is classified as `incomplete-reasoning` when reasoning tokens were consumed, or `incomplete-output` otherwise; partial output is never mislabeled as a complete semantic mismatch. Other non-empty incorrect responses are `response-mismatch`, while empty responses without a length stop are `empty-response`. JSONL evidence is appended across invocations so successive experiments are retained. Router-selected model IDs are recorded separately from the requested router alias. It does not measure time-to-first-token; streaming belongs in the later capability suite.
 
 Inspect which credential variable name is selected for each provider without displaying values or making network requests:
 

From c30a4127fbe3c42fc7798d37f52438e5c876f96f Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:17:48 -0500
Subject: [PATCH 075/128] Add provider capability and adapter catalogue

---
 tools/provider_api/provider-catalogue.json | 158 +++++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100644 tools/provider_api/provider-catalogue.json

diff --git a/tools/provider_api/provider-catalogue.json b/tools/provider_api/provider-catalogue.json
new file mode 100644
index 0000000..0167c1c
--- /dev/null
+++ b/tools/provider_api/provider-catalogue.json
@@ -0,0 +1,158 @@
+{
+  "schema_version": "0.1",
+  "description": "Provider API capability and adapter catalogue for the evaluation runner. Only listed adapters are implemented; catalogue metadata never implies free usage.",
+  "capability_families": [
+    "chat",
+    "embeddings",
+    "reranking",
+    "image-generation",
+    "document-processing"
+  ],
+  "providers": {
+    "CEREBRAS": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.cerebras.ai/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "COHERE": {
+      "adapter": "cohere-v2",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.cohere.com/v2/chat",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "DEEPINFRA": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.deepinfra.com/v1/openai/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "DEEPSEEK": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.deepseek.com/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "GOOGLE": {
+      "adapter": "google-generative-language",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://generativelanguage.googleapis.com/v1beta/{model}:generateContent",
+      "authentication": "x-goog-api-key",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "GROQ": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.groq.com/openai/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "HUGGINGFACE": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://router.huggingface.co/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "MISTRAL": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.mistral.ai/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "NVIDIA": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://integrate.api.nvidia.com/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case",
+      "status": "deprecated-endpoint"
+    },
+    "OPENAI": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.openai.com/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "OPENROUTER": {
+      "adapter": "openai-compatible",
+      "implemented_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://openrouter.ai/api/v1/chat/completions",
+      "authentication": "bearer",
+      "operations": [
+        "non-streaming-smoke"
+      ],
+      "pricing_source": "model-case"
+    },
+    "ANTHROPIC": {
+      "adapter": null,
+      "implemented_capabilities": [],
+      "known_capabilities": [
+        "chat"
+      ],
+      "endpoint": "https://api.anthropic.com/v1/messages",
+      "authentication": "x-api-key",
+      "operations": [],
+      "status": "adapter-not-implemented",
+      "pricing_source": "provider-model-catalogue"
+    }
+  }
+}

From d3bfe37d20c4e62007d91bdc511e2329b2a9a2dd Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:18:30 -0500
Subject: [PATCH 076/128] Add unified inventory capability planning and
 evidence reporting

---
 tools/provider_api/smoke_test.py | 170 +++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index c1486db..aec006e 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -90,6 +90,142 @@
     "ANTHROPIC": ("ANTHROPIC_API_KEY",),
 }
 
+PROVIDER_CATALOGUE_PATH = Path(__file__).with_name("provider-catalogue.json")
+with PROVIDER_CATALOGUE_PATH.open(encoding="utf-8") as catalogue_stream:
+    PROVIDER_CATALOGUE = json.load(catalogue_stream)
+
+
+def provider_catalogue_entry(provider: str) -> dict[str, Any]:
+    """Return metadata for a provider, or an explicit unknown-adapter record."""
+    return PROVIDER_CATALOGUE.get("providers", {}).get(provider, {
+        "adapter": None,
+        "implemented_capabilities": [],
+        "known_capabilities": [],
+        "endpoint": None,
+        "authentication": None,
+        "operations": [],
+        "status": "provider-not-catalogued",
+    })
+
+
+def inventory_rows() -> list[dict[str, Any]]:
+    """Describe every known/discovered provider and each configured account without secrets."""
+    rows: list[dict[str, Any]] = []
+    for provider in discovered_provider_names():
+        entry = provider_catalogue_entry(provider)
+        accounts = credential_accounts(provider) or [{
+            "account": "DEFAULT",
+            "credential_env": None,
+            "credential": "",
+        }]
+        for account in accounts:
+            rows.append({
+                "provider": provider,
+                "account": account["account"],
+                "credential_env": account["credential_env"],
+                "credential_configured": bool(account["credential_env"]),
+                "adapter": entry.get("adapter"),
+                "implemented_capabilities": entry.get("implemented_capabilities", []),
+                "known_capabilities": entry.get("known_capabilities", []),
+                "endpoint": entry.get("endpoint"),
+                "authentication": entry.get("authentication"),
+                "status": (
+                    "ready" if account["credential_env"] and entry.get("adapter")
+                    else "skipped-no-credential" if not account["credential_env"]
+                    else entry.get("status", "skipped-no-provider-adapter")
+                ),
+            })
+    return rows
+
+
+def capability_plan(capability: str, providers: list[str] | None = None) -> list[dict[str, Any]]:
+    """Build a no-network plan for a capability family not yet backed by adapters."""
+    selected = set(providers or discovered_provider_names())
+    rows: list[dict[str, Any]] = []
+    for provider in sorted(selected):
+        entry = provider_catalogue_entry(provider)
+        accounts = credential_accounts(provider) or [{
+            "account": "DEFAULT", "credential_env": None, "credential": ""
+        }]
+        for account in accounts:
+            configured = bool(account["credential_env"])
+            supported = capability in entry.get("implemented_capabilities", [])
+            rows.append({
+                "provider": provider,
+                "account": account["account"],
+                "credential_env": account["credential_env"],
+                "credential_configured": configured,
+                "capability": capability,
+                "adapter": entry.get("adapter"),
+                "status": (
+                    "skipped-no-credential" if not configured
+                    else "ready" if supported
+                    else "skipped-no-capability-adapter"
+                ),
+                "http_status": None,
+                "latency_seconds": None,
+                "retry_performed": False,
+            })
+    return rows
+
+
+def print_inventory() -> None:
+    """Print a safe provider/account inventory; never print credential values."""
+    rows = inventory_rows()
+    print(f"{'PROVIDER':16} {'ACCOUNT':10} {'CREDENTIAL':9} {'ADAPTER':24} CAPABILITIES")
+    for item in rows:
+        credential = "yes" if item["credential_configured"] else "no"
+        adapter = item["adapter"] or "-"
+        capabilities = ",".join(item["implemented_capabilities"]) or "-"
+        print(
+            f'{item["provider"]:16} {item["account"]:10} {credential:9} '
+            f'{adapter:24} {capabilities}'
+        )
+        if item["credential_env"]:
+            print(f'  source: {item["credential_env"]}')
+        if item["endpoint"]:
+            print(f'  endpoint: {item["endpoint"]}')
+        if item["status"] not in {"ready"}:
+            print(f'  status: {item["status"]}')
+    print(f"\n{len(rows)} provider/account rows. Inventory only; no network requests or credential values.")
+
+
+def print_report(path: Path) -> int:
+    """Summarize accumulated JSONL evidence without making provider requests."""
+    if not path.exists():
+        print(f"No evidence file found: {path}", file=sys.stderr)
+        return 2
+    totals: dict[tuple[str, str, str], int] = {}
+    errors: dict[tuple[str, str, str], int] = {}
+    count = 0
+    with path.open(encoding="utf-8") as stream:
+        for line in stream:
+            try:
+                item = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            provider = str(item.get("provider") or "UNKNOWN")
+            account = str(item.get("account") or "DEFAULT")
+            capability = str(item.get("capability") or "chat")
+            status = str(item.get("status") or "unknown")
+            totals[(provider, account, status)] = totals.get((provider, account, status), 0) + 1
+            if status not in {"passed"} and not status.startswith("skipped-"):
+                category = str(item.get("http_error_category") or status)
+                errors[(provider, account, category)] = errors.get((provider, account, category), 0) + 1
+            count += 1
+    print(f"Evidence report: {path}")
+    print(f"Records: {count}")
+    print(f"{'PROVIDER':16} {'ACCOUNT':10} {'STATUS':34} COUNT")
+    for (provider, account, status), total in sorted(totals.items()):
+        print(f"{provider:16} {account:10} {status:34} {total}")
+    if errors:
+        print("\nFailure categories:")
+        for (provider, account, category), total in sorted(errors.items()):
+            print(f"  {provider}/{account}: {category} ({total})")
+    print("Read-only: no provider requests were made.")
+    return 0
+
+
 
 def provider_from_env_name(env_name: str) -> str | None:
     """Map a provider credential variable name to its canonical provider."""
@@ -407,6 +543,7 @@ def request_case(
     result: dict[str, Any] = {
         "observed_at": datetime.now(timezone.utc).isoformat(),
         "provider": provider,
+        "capability": "chat",
         "account": account,
         "model_requested": model,
         "requested_max_output_tokens": max_tokens,
@@ -568,6 +705,12 @@ def request_case(
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--inventory", action="store_true",
+                        help="Show all known/discovered provider accounts, credential source names, endpoints, and implemented capabilities without network requests.")
+    parser.add_argument("--report", action="store_true",
+                        help="Summarize accumulated JSONL evidence without network requests.")
+    parser.add_argument("--capability", choices=PROVIDER_CATALOGUE["capability_families"], default="chat",
+                        help="Capability family to evaluate (only implemented adapters execute requests).")
     parser.add_argument("--provider", action="append", choices=discovered_provider_names(),
                         help="Run only this provider; repeat to select several.")
     parser.add_argument("--all-configured", action="store_true",
@@ -588,6 +731,17 @@ def main() -> int:
     parser.add_argument("--output", type=Path, default=Path("provider-smoke-results.jsonl"))
     args = parser.parse_args()
 
+    if args.inventory:
+        if args.report or args.provider or args.model or args.all_configured or args.include_paid or args.openrouter_free_matrix or args.show_latest_errors or args.show_credential_sources or args.capability != "chat":
+            parser.error("--inventory cannot be combined with inference, report, or capability-selection options")
+        print_inventory()
+        return 0
+
+    if args.report:
+        if args.inventory or args.provider or args.model or args.all_configured or args.include_paid or args.openrouter_free_matrix or args.show_latest_errors or args.show_credential_sources or args.capability != "chat":
+            parser.error("--report cannot be combined with inference, inventory, or capability-selection options")
+        return print_report(args.output)
+
     if args.timeout <= 0 or args.max_output_tokens < 1:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
@@ -638,6 +792,22 @@ def main() -> int:
         print("Read-only: no provider requests were made.")
         return 0
 
+    if args.capability != "chat":
+        if args.model or args.include_paid or args.openrouter_free_matrix:
+            parser.error("--model, --include-paid, and --openrouter-free-matrix currently apply only to chat")
+        results = capability_plan(args.capability, args.provider)
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        with args.output.open("a", encoding="utf-8") as stream:
+            for item in results:
+                stream.write(json.dumps(item, ensure_ascii=False) + "\\n")
+        unsupported = sum(item["status"] == "skipped-no-capability-adapter" for item in results)
+        missing = sum(item["status"] == "skipped-no-credential" for item in results)
+        print(f"Capability: {args.capability}")
+        print(f"Provider/account rows: {len(results)}; adapters not implemented: {unsupported}; no credentials: {missing}.")
+        print("No provider requests were made. These rows are explicit adapter backlog, not inference failures.")
+        print(f"Evidence: {args.output}")
+        return 0
+
     if args.openrouter_free_matrix and (
         args.all_configured or args.provider or args.model or args.include_paid
         or args.show_latest_errors or args.show_credential_sources

From 0624d4690f5d0142be115c06d54e1baaec9178ee Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:18:56 -0500
Subject: [PATCH 077/128] Fix evidence report grouping and JSONL line endings

---
 tools/provider_api/smoke_test.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index aec006e..644e0b9 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -195,7 +195,7 @@ def print_report(path: Path) -> int:
     if not path.exists():
         print(f"No evidence file found: {path}", file=sys.stderr)
         return 2
-    totals: dict[tuple[str, str, str], int] = {}
+    totals: dict[tuple[str, str, str, str], int] = {}
     errors: dict[tuple[str, str, str], int] = {}
     count = 0
     with path.open(encoding="utf-8") as stream:
@@ -208,16 +208,16 @@ def print_report(path: Path) -> int:
             account = str(item.get("account") or "DEFAULT")
             capability = str(item.get("capability") or "chat")
             status = str(item.get("status") or "unknown")
-            totals[(provider, account, status)] = totals.get((provider, account, status), 0) + 1
+            totals[(provider, account, capability, status)] = totals.get((provider, account, capability, status), 0) + 1
             if status not in {"passed"} and not status.startswith("skipped-"):
                 category = str(item.get("http_error_category") or status)
                 errors[(provider, account, category)] = errors.get((provider, account, category), 0) + 1
             count += 1
     print(f"Evidence report: {path}")
     print(f"Records: {count}")
-    print(f"{'PROVIDER':16} {'ACCOUNT':10} {'STATUS':34} COUNT")
-    for (provider, account, status), total in sorted(totals.items()):
-        print(f"{provider:16} {account:10} {status:34} {total}")
+    print(f"{'PROVIDER':16} {'ACCOUNT':10} {'CAPABILITY':20} {'STATUS':34} COUNT")
+    for (provider, account, capability, status), total in sorted(totals.items()):
+        print(f"{provider:16} {account:10} {capability:20} {status:34} {total}")
     if errors:
         print("\nFailure categories:")
         for (provider, account, category), total in sorted(errors.items()):
@@ -799,7 +799,7 @@ def main() -> int:
         args.output.parent.mkdir(parents=True, exist_ok=True)
         with args.output.open("a", encoding="utf-8") as stream:
             for item in results:
-                stream.write(json.dumps(item, ensure_ascii=False) + "\\n")
+                stream.write(json.dumps(item, ensure_ascii=False) + "\n")
         unsupported = sum(item["status"] == "skipped-no-capability-adapter" for item in results)
         missing = sum(item["status"] == "skipped-no-credential" for item in results)
         print(f"Capability: {args.capability}")

From 5b2eadecd6375f001467f8f3374c1f1828f21ce0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:19:17 -0500
Subject: [PATCH 078/128] Test unified provider inventory capability planning
 and reports

---
 tests/test_provider_api_smoke_test.py | 80 +++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 279eae0..52841e2 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -478,5 +478,85 @@ def test_google_max_tokens_is_incomplete(self):
         )
 
 
+class UnifiedRunnerInventoryTests(unittest.TestCase):
+    def test_inventory_lists_each_account_without_exposing_credentials(self):
+        import io
+        from contextlib import redirect_stdout
+        from tools.provider_api.smoke_test import inventory_rows, print_inventory
+
+        secrets = {
+            "OPENROUTER_API_KEY_FLOSSWARE": "flossware-secret",
+            "OPENROUTER_API_KEY_HOTMAIL": "hotmail-secret",
+            "VOYAGEAI_API_KEY_NCRR": "voyage-secret",
+        }
+        with patch.dict("os.environ", secrets, clear=True):
+            rows = inventory_rows()
+            openrouter = [row for row in rows if row["provider"] == "OPENROUTER"]
+            self.assertEqual(
+                [row["account"] for row in openrouter],
+                ["FLOSSWARE", "HOTMAIL"],
+            )
+            self.assertTrue(all(row["credential_configured"] for row in openrouter))
+            output = io.StringIO()
+            with redirect_stdout(output):
+                print_inventory()
+
+        rendered = output.getvalue()
+        for secret in secrets.values():
+            self.assertNotIn(secret, rendered)
+        self.assertIn("OPENROUTER_API_KEY_FLOSSWARE", rendered)
+        self.assertIn("OPENROUTER_API_KEY_HOTMAIL", rendered)
+        self.assertIn("VOYAGEAI", rendered)
+
+    def test_unimplemented_capability_is_reported_without_requests(self):
+        from tools.provider_api.smoke_test import capability_plan
+
+        with patch.dict("os.environ", {
+            "OPENROUTER_API_KEY_FLOSSWARE": "test-secret",
+        }, clear=True):
+            rows = capability_plan("embeddings", ["OPENROUTER"])
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["capability"], "embeddings")
+        self.assertEqual(rows[0]["status"], "skipped-no-capability-adapter")
+        self.assertIsNone(rows[0]["http_status"])
+        self.assertFalse(rows[0]["retry_performed"])
+
+    def test_evidence_report_groups_by_provider_account_capability_and_status(self):
+        import io
+        import tempfile
+        from contextlib import redirect_stdout
+        from pathlib import Path
+        from tools.provider_api.smoke_test import print_report
+
+        with tempfile.TemporaryDirectory() as directory:
+            evidence = Path(directory) / "evidence.jsonl"
+            evidence.write_text(
+                json.dumps({
+                    "provider": "OPENROUTER",
+                    "account": "FLOSSWARE",
+                    "capability": "chat",
+                    "status": "passed",
+                }) + "\\n" +
+                json.dumps({
+                    "provider": "OPENROUTER",
+                    "account": "FLOSSWARE",
+                    "capability": "embeddings",
+                    "status": "skipped-no-capability-adapter",
+                }) + "\\n",
+                encoding="utf-8",
+            )
+            output = io.StringIO()
+            with redirect_stdout(output):
+                status = print_report(evidence)
+
+        self.assertEqual(status, 0)
+        self.assertIn("Records: 2", output.getvalue())
+        self.assertIn("chat", output.getvalue())
+        self.assertIn("embeddings", output.getvalue())
+        self.assertIn("Read-only", output.getvalue())
+
+
+
 if __name__ == "__main__":
     unittest.main()

From c5d2265e32da0b6841ebd66946f8d3af8a95019b Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:19:43 -0500
Subject: [PATCH 079/128] Correct evidence fixture JSONL separators

---
 tests/test_provider_api_smoke_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 52841e2..e05021e 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -537,13 +537,13 @@ def test_evidence_report_groups_by_provider_account_capability_and_status(self):
                     "account": "FLOSSWARE",
                     "capability": "chat",
                     "status": "passed",
-                }) + "\\n" +
+                }) + "\n" +
                 json.dumps({
                     "provider": "OPENROUTER",
                     "account": "FLOSSWARE",
                     "capability": "embeddings",
                     "status": "skipped-no-capability-adapter",
-                }) + "\\n",
+                }) + "\n",
                 encoding="utf-8",
             )
             output = io.StringIO()

From 98cfc30156426bdfd690196393dbe129bb34ea69 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:19:51 -0500
Subject: [PATCH 080/128] Harden unified runner option validation

---
 tools/provider_api/smoke_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 644e0b9..0732ae2 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -746,7 +746,7 @@ def main() -> int:
         parser.error("--timeout must be positive and --max-output-tokens must be at least 1")
 
     if args.show_credential_sources:
-        if args.show_latest_errors or args.all_configured or args.provider or args.model or args.include_paid:
+        if args.show_latest_errors or args.all_configured or args.provider or args.model or args.include_paid or args.report or args.capability != "chat":
             parser.error("--show-credential-sources cannot be combined with other selection options")
         for provider in discovered_provider_names():
             names = [
@@ -758,7 +758,7 @@ def main() -> int:
         return 0
 
     if args.show_latest_errors:
-        if args.all_configured or args.provider or args.model or args.include_paid or args.show_credential_sources:
+        if args.all_configured or args.provider or args.model or args.include_paid or args.show_credential_sources or args.inventory or args.report or args.capability != "chat":
             parser.error("--show-latest-errors cannot be combined with inference-selection options")
         if not args.output.exists():
             parser.error(f"No evidence file found: {args.output}")

From 2614505e44e5db9a667072b8f2272ca6c0748ef0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:20:20 -0500
Subject: [PATCH 081/128] Document unified provider runner inventory and
 capabilities

---
 README.md | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 07e6a3b..f6178bc 100644
--- a/README.md
+++ b/README.md
@@ -50,7 +50,27 @@ bash tools/provider_api/catalogue_inventory.sh
 python3 tools/provider_api/smoke_test.py
 ```
 
-The catalogue command writes `provider-model-catalogue.json`. The smoke test writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
+The catalogue helper writes `provider-model-catalogue.json`. The unified runner writes `provider-smoke-results.jsonl`. Both are local evidence artifacts and should be reviewed before sharing.
+
+### Unified runner inventory and reporting
+
+The Python runner is the single command-line entry point for provider/account discovery, capability selection, inference execution, and evidence reporting. Its static adapter metadata lives in `tools/provider_api/provider-catalogue.json`; that catalogue records endpoints, authentication schemes, implemented capabilities, and operations. Model-level price/eligibility remains attached to individual model cases, because provider-wide pricing labels are misleading.
+
+```bash
+# Inventory known and environment-discovered providers/accounts, without requests.
+python3 tools/provider_api/smoke_test.py --inventory
+
+# Run all currently implemented chat adapters, preserving separate account identities.
+python3 tools/provider_api/smoke_test.py --all-configured
+
+# Record the current embeddings adapter backlog; this makes no provider requests.
+python3 tools/provider_api/smoke_test.py --capability embeddings
+
+# Summarize accumulated JSONL evidence, without requests.
+python3 tools/provider_api/smoke_test.py --report
+```
+
+Capability families are declared in the provider catalogue: `chat`, `embeddings`, `reranking`, `image-generation`, and `document-processing`. Only adapters listed under `implemented_capabilities` execute network requests. Other capability selections append explicit `skipped-no-capability-adapter` planning records, not false inference failures. Chat is the only implemented capability family in this initial registry. Adding an adapter means declaring its endpoint/authentication/operations in the catalogue, implementing its request/response contract, and adding deterministic tests. One runner does not mean forcing incompatible APIs through a chat-completions payload.
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
 

From c8c370658f473b2bd9e4ecb661c839243ada6d56 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:21:12 -0500
Subject: [PATCH 082/128] Normalize capability planning evidence metadata

---
 tools/provider_api/smoke_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 0732ae2..77fa7ba 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -130,7 +130,7 @@ def inventory_rows() -> list[dict[str, Any]]:
                 "endpoint": entry.get("endpoint"),
                 "authentication": entry.get("authentication"),
                 "status": (
-                    "ready" if account["credential_env"] and entry.get("adapter")
+                    "adapter-available" if account["credential_env"] and entry.get("adapter")
                     else "skipped-no-credential" if not account["credential_env"]
                     else entry.get("status", "skipped-no-provider-adapter")
                 ),
@@ -151,6 +151,7 @@ def capability_plan(capability: str, providers: list[str] | None = None) -> list
             configured = bool(account["credential_env"])
             supported = capability in entry.get("implemented_capabilities", [])
             rows.append({
+                "observed_at": datetime.now(timezone.utc).isoformat(),
                 "provider": provider,
                 "account": account["account"],
                 "credential_env": account["credential_env"],

From 43c8cba88c626f94bff50a2e37ca208e147424c0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:26:19 -0500
Subject: [PATCH 083/128] Implement provider-native embedding adapters and
 execution

---
 tools/provider_api/smoke_test.py | 291 +++++++++++++++++++++++++++++++
 1 file changed, 291 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 77fa7ba..0961deb 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -9,6 +9,7 @@
 
 import argparse
 import json
+import math
 import os
 import re
 import sys
@@ -47,6 +48,19 @@
     {"provider": "OPENAI", "model": "gpt-4.1-mini", "access": "paid"},
 ]
 
+
+EMBEDDING_TEXT = (
+    "FlossWare evaluates embedding APIs for semantic retrieval and vector search."
+)
+EMBEDDING_CASES: list[dict[str, str]] = [
+    {"provider": "VOYAGEAI", "model": "voyage-4-lite", "access": "paid-or-account-credit"},
+    {"provider": "JINA", "model": "jina-embeddings-v3", "access": "free-tier-candidate"},
+    {"provider": "COHERE", "model": "embed-v4.0", "access": "free-trial-candidate"},
+    {"provider": "GOOGLE", "model": "models/gemini-embedding-001", "access": "free-tier-candidate"},
+    {"provider": "OPENAI", "model": "text-embedding-3-small", "access": "paid"},
+    {"provider": "MISTRAL", "model": "mistral-embed", "access": "paid"},
+]
+
 OPENAI_COMPATIBLE = {
     "CEREBRAS": "https://api.cerebras.ai/v1/chat/completions",
     "DEEPINFRA": "https://api.deepinfra.com/v1/openai/chat/completions",
@@ -524,6 +538,283 @@ def plan_all_configured(include_paid: bool = False) -> list[dict[str, Any]]:
     return plan
 
 
+
+def embedding_request_spec(case: dict[str, str]) -> tuple[str, dict[str, Any]]:
+    """Build a provider-native embedding request without performing network I/O."""
+    provider = case["provider"]
+    model = case["model"]
+    if provider == "GOOGLE":
+        return (
+            f"https://generativelanguage.googleapis.com/v1beta/{model}:embedContent",
+            {
+                "content": {"parts": [{"text": EMBEDDING_TEXT}]},
+                "taskType": "RETRIEVAL_DOCUMENT",
+            },
+        )
+    if provider == "COHERE":
+        return (
+            "https://api.cohere.com/v2/embed",
+            {
+                "model": model,
+                "texts": [EMBEDDING_TEXT],
+                "input_type": "search_document",
+                "embedding_types": ["float"],
+            },
+        )
+    if provider == "VOYAGEAI":
+        return (
+            "https://api.voyageai.com/v1/embeddings",
+            {
+                "model": model,
+                "input": [EMBEDDING_TEXT],
+                "input_type": "document",
+                "output_dtype": "float",
+            },
+        )
+    if provider == "JINA":
+        return (
+            "https://api.jina.ai/v1/embeddings",
+            {
+                "model": model,
+                "input": [EMBEDDING_TEXT],
+                "task": "retrieval.passage",
+                "embedding_type": "float",
+                "normalized": True,
+            },
+        )
+    if provider in {"OPENAI", "MISTRAL"}:
+        endpoint = (
+            "https://api.openai.com/v1/embeddings"
+            if provider == "OPENAI"
+            else "https://api.mistral.ai/v1/embeddings"
+        )
+        return (
+            endpoint,
+            {"model": model, "input": [EMBEDDING_TEXT], "encoding_format": "float"},
+        )
+    raise ValueError(f"No embedding request adapter for provider: {provider}")
+
+
+def extract_embedding_vectors(provider: str, data: dict[str, Any]) -> list[list[float]]:
+    """Normalize supported provider embedding response envelopes to float vectors."""
+    if provider == "GOOGLE":
+        values = (data.get("embedding") or {}).get("values")
+        raw_vectors = [values] if values is not None else []
+    elif provider == "COHERE":
+        raw_vectors = (data.get("embeddings") or {}).get("float") or []
+    else:
+        raw_vectors = [
+            item.get("embedding")
+            for item in data.get("data", [])
+            if isinstance(item, dict) and item.get("embedding") is not None
+        ]
+
+    vectors: list[list[float]] = []
+    for vector in raw_vectors:
+        if not isinstance(vector, list) or not vector:
+            raise ValueError("Embedding response contained an empty or invalid vector")
+        normalized: list[float] = []
+        for value in vector:
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise ValueError("Embedding vector contained a non-numeric value")
+            number = float(value)
+            if not math.isfinite(number):
+                raise ValueError("Embedding vector contained a non-finite value")
+            normalized.append(number)
+        vectors.append(normalized)
+    if not vectors:
+        raise ValueError("Embedding response did not contain any vectors")
+    dimensions = {len(vector) for vector in vectors}
+    if len(dimensions) != 1:
+        raise ValueError("Embedding response contained inconsistent vector dimensions")
+    if not any(value != 0.0 for value in vectors[0]):
+        raise ValueError("Embedding response contained an all-zero vector")
+    return vectors
+
+
+def request_embedding_case(
+    case: dict[str, str],
+    timeout: float,
+    credential_env: str | None = None,
+    account: str = "DEFAULT",
+) -> dict[str, Any]:
+    """Run one bounded embedding smoke request and retain only vector metadata."""
+    provider, model = case["provider"], case["model"]
+    if credential_env:
+        key = os.environ.get(credential_env, "").strip()
+        env_name = credential_env
+    else:
+        key, env_name = resolve_credential(provider)
+        if env_name:
+            account = credential_account_from_env_name(env_name)
+
+    result: dict[str, Any] = {
+        "observed_at": datetime.now(timezone.utc).isoformat(),
+        "provider": provider,
+        "capability": "embeddings",
+        "account": account,
+        "model_requested": model,
+        "access_classification": case["access"],
+        "credential_env": env_name,
+        "credential_configured": bool(key),
+        "http_status": None,
+        "latency_seconds": None,
+        "provider_request_id": None,
+        "model_returned": None,
+        "embedding_count": 0,
+        "embedding_dimensions": None,
+        "embedding_norm": None,
+        "usage": None,
+        "error": None,
+        "retry_performed": False,
+    }
+    if not key:
+        result["status"] = "skipped-no-credential"
+        return result
+
+    url, payload = embedding_request_spec(case)
+    headers = build_headers(provider, key)
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=headers,
+        method="POST",
+    )
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            body_bytes = response.read()
+            result["http_status"] = response.status
+            result["rate_limit_headers"] = {
+                name.lower(): value
+                for name, value in response.headers.items()
+                if "ratelimit" in name.lower() or name.lower() in {
+                    "retry-after", "x-request-id", "request-id", "cf-ray"
+                }
+            }
+            result["provider_request_id"] = (
+                response.headers.get("x-request-id")
+                or response.headers.get("request-id")
+                or response.headers.get("cf-ray")
+            )
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        data = json.loads(body_bytes.decode("utf-8"))
+        vectors = extract_embedding_vectors(provider, data)
+        vector = vectors[0]
+        result["model_returned"] = data.get("model") or data.get("modelVersion") or model
+        result["embedding_count"] = len(vectors)
+        result["embedding_dimensions"] = len(vector)
+        result["embedding_norm"] = round(math.sqrt(sum(value * value for value in vector)), 8)
+        result["usage"] = data.get("usage") or data.get("usageMetadata")
+        result["status"] = "passed"
+    except urllib.error.HTTPError as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["http_status"] = exc.code
+        result["provider_request_id"] = (
+            exc.headers.get("x-request-id")
+            or exc.headers.get("request-id")
+            or exc.headers.get("cf-ray")
+        )
+        result["rate_limit_headers"] = {
+            name.lower(): value
+            for name, value in exc.headers.items()
+            if "ratelimit" in name.lower() or name.lower() in {
+                "retry-after", "x-request-id", "request-id", "cf-ray"
+            }
+        }
+        try:
+            detail = exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            detail = str(exc)
+        result["error"] = sanitize_error_detail(detail, key)
+        result["http_error_category"] = classify_http_error(exc.code, result["error"])
+        result["status"] = "http-error"
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["error"] = safe_text(exc, key)
+        result["status"] = "transport-error"
+    except (json.JSONDecodeError, UnicodeDecodeError, KeyError, TypeError, ValueError) as exc:
+        result["latency_seconds"] = round(time.monotonic() - started, 3)
+        result["error"] = safe_text(
+            f"Embedding response validation failed: {type(exc).__name__}: {exc}", key
+        )
+        result["status"] = "response-parse-error"
+    finally:
+        key = ""
+    return result
+
+
+def plan_capability_all_configured(
+    capability: str,
+    include_paid: bool = False,
+    providers: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Plan each account for a capability, selecting at most one candidate per provider."""
+    candidates = EMBEDDING_CASES if capability == "embeddings" else []
+    selected_providers = sorted(set(providers or discovered_provider_names()))
+    results: list[dict[str, Any]] = []
+    for provider in selected_providers:
+        entry = provider_catalogue_entry(provider)
+        case = next((item for item in candidates if item["provider"] == provider), None)
+        accounts = credential_accounts(provider) or [{
+            "account": "DEFAULT", "credential_env": None, "credential": ""
+        }]
+        for account in accounts:
+            env_name = account["credential_env"]
+            row: dict[str, Any] = {
+                "observed_at": datetime.now(timezone.utc).isoformat(),
+                "provider": provider,
+                "account": account["account"],
+                "credential_env": env_name,
+                "credential_configured": bool(env_name),
+                "capability": capability,
+                "model_requested": case["model"] if case else None,
+                "access_classification": case["access"] if case else None,
+                "adapter": entry.get("capability_adapters", {}).get(capability),
+                "http_status": None,
+                "latency_seconds": None,
+                "retry_performed": False,
+            }
+            if not env_name:
+                row["status"] = "skipped-no-credential"
+            elif not case:
+                row["status"] = (
+                    "skipped-no-provider-adapter"
+                    if capability not in entry.get("implemented_capabilities", [])
+                    else "skipped-no-model-candidate"
+                )
+            elif case["access"] in PAID_ACCESS and not include_paid:
+                row["status"] = "skipped-paid-opt-in-required"
+            else:
+                row["status"] = "ready"
+            results.append(row)
+    return results
+
+
+def execute_embedding_selection(
+    capability: str,
+    timeout: float,
+    include_paid: bool,
+    providers: list[str] | None,
+) -> list[dict[str, Any]]:
+    """Execute eligible embedding cases while preserving account boundaries."""
+    results: list[dict[str, Any]] = []
+    for item in plan_capability_all_configured(capability, include_paid, providers):
+        if item["status"] == "ready":
+            case = next(
+                candidate for candidate in EMBEDDING_CASES
+                if candidate["provider"] == item["provider"]
+            )
+            results.append(request_embedding_case(
+                case,
+                timeout,
+                credential_env=item.get("credential_env"),
+                account=item.get("account", "DEFAULT"),
+            ))
+        else:
+            results.append(item)
+    return results
+
 def request_case(
     case: dict[str, str],
     timeout: float,

From 7eb3b3807258d2d939318d3fa506dbce6e09c839 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:26:28 -0500
Subject: [PATCH 084/128] Register Voyage Jina and native embedding adapters

---
 tools/provider_api/provider-catalogue.json | 99 ++++++++++++++++++++--
 1 file changed, 91 insertions(+), 8 deletions(-)

diff --git a/tools/provider_api/provider-catalogue.json b/tools/provider_api/provider-catalogue.json
index 0167c1c..fc3a8c7 100644
--- a/tools/provider_api/provider-catalogue.json
+++ b/tools/provider_api/provider-catalogue.json
@@ -24,14 +24,24 @@
     "COHERE": {
       "adapter": "cohere-v2",
       "implemented_capabilities": [
-        "chat"
+        "chat",
+        "embeddings"
       ],
       "endpoint": "https://api.cohere.com/v2/chat",
       "authentication": "bearer",
       "operations": [
         "non-streaming-smoke"
       ],
-      "pricing_source": "model-case"
+      "pricing_source": "model-case",
+      "capability_adapters": {
+        "embeddings": "cohere-v2-embed"
+      },
+      "capability_endpoints": {
+        "embeddings": "https://api.cohere.com/v2/embed"
+      },
+      "known_capabilities": [
+        "embeddings"
+      ]
     },
     "DEEPINFRA": {
       "adapter": "openai-compatible",
@@ -60,14 +70,24 @@
     "GOOGLE": {
       "adapter": "google-generative-language",
       "implemented_capabilities": [
-        "chat"
+        "chat",
+        "embeddings"
       ],
       "endpoint": "https://generativelanguage.googleapis.com/v1beta/{model}:generateContent",
       "authentication": "x-goog-api-key",
       "operations": [
         "non-streaming-smoke"
       ],
-      "pricing_source": "model-case"
+      "pricing_source": "model-case",
+      "capability_adapters": {
+        "embeddings": "google-embed-content"
+      },
+      "capability_endpoints": {
+        "embeddings": "https://generativelanguage.googleapis.com/v1beta/{model}:embedContent"
+      },
+      "known_capabilities": [
+        "embeddings"
+      ]
     },
     "GROQ": {
       "adapter": "openai-compatible",
@@ -96,14 +116,24 @@
     "MISTRAL": {
       "adapter": "openai-compatible",
       "implemented_capabilities": [
-        "chat"
+        "chat",
+        "embeddings"
       ],
       "endpoint": "https://api.mistral.ai/v1/chat/completions",
       "authentication": "bearer",
       "operations": [
         "non-streaming-smoke"
       ],
-      "pricing_source": "model-case"
+      "pricing_source": "model-case",
+      "capability_adapters": {
+        "embeddings": "openai-embeddings"
+      },
+      "capability_endpoints": {
+        "embeddings": "https://api.mistral.ai/v1/embeddings"
+      },
+      "known_capabilities": [
+        "embeddings"
+      ]
     },
     "NVIDIA": {
       "adapter": "openai-compatible",
@@ -121,14 +151,24 @@
     "OPENAI": {
       "adapter": "openai-compatible",
       "implemented_capabilities": [
-        "chat"
+        "chat",
+        "embeddings"
       ],
       "endpoint": "https://api.openai.com/v1/chat/completions",
       "authentication": "bearer",
       "operations": [
         "non-streaming-smoke"
       ],
-      "pricing_source": "model-case"
+      "pricing_source": "model-case",
+      "capability_adapters": {
+        "embeddings": "openai-embeddings"
+      },
+      "capability_endpoints": {
+        "embeddings": "https://api.openai.com/v1/embeddings"
+      },
+      "known_capabilities": [
+        "embeddings"
+      ]
     },
     "OPENROUTER": {
       "adapter": "openai-compatible",
@@ -153,6 +193,49 @@
       "operations": [],
       "status": "adapter-not-implemented",
       "pricing_source": "provider-model-catalogue"
+    },
+    "VOYAGEAI": {
+      "adapter": null,
+      "implemented_capabilities": [
+        "embeddings"
+      ],
+      "known_capabilities": [
+        "embeddings"
+      ],
+      "capability_adapters": {
+        "embeddings": "voyage-embeddings"
+      },
+      "endpoint": "https://api.voyageai.com/v1/embeddings",
+      "capability_endpoints": {
+        "embeddings": "https://api.voyageai.com/v1/embeddings"
+      },
+      "authentication": "bearer",
+      "operations": [
+        "single-text-embedding-smoke"
+      ],
+      "pricing_source": "provider-model-catalogue"
+    },
+    "JINA": {
+      "adapter": null,
+      "implemented_capabilities": [
+        "embeddings"
+      ],
+      "known_capabilities": [
+        "embeddings",
+        "reranking"
+      ],
+      "capability_adapters": {
+        "embeddings": "jina-embeddings"
+      },
+      "endpoint": "https://api.jina.ai/v1/embeddings",
+      "capability_endpoints": {
+        "embeddings": "https://api.jina.ai/v1/embeddings"
+      },
+      "authentication": "bearer",
+      "operations": [
+        "single-text-embedding-smoke"
+      ],
+      "pricing_source": "provider-model-catalogue"
     }
   }
 }

From 3abb29f0418afa78004301cdd99e48af7c90e5dd Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:26:46 -0500
Subject: [PATCH 085/128] Enable explicit embedding execution and safe planning
 mode

---
 tools/provider_api/smoke_test.py | 43 +++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 0961deb..f48da6a 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -1085,8 +1085,42 @@ def main() -> int:
         return 0
 
     if args.capability != "chat":
-        if args.model or args.include_paid or args.openrouter_free_matrix:
-            parser.error("--model, --include-paid, and --openrouter-free-matrix currently apply only to chat")
+        if args.model or args.openrouter_free_matrix:
+            parser.error("--model and --openrouter-free-matrix currently apply only to chat")
+        if args.include_paid and not (args.all_configured or args.provider):
+            parser.error("--include-paid requires --all-configured or an explicit --provider selection")
+        if args.all_configured or args.provider:
+            results = execute_embedding_selection(
+                args.capability,
+                args.timeout,
+                args.include_paid,
+                args.provider,
+            )
+            args.output.parent.mkdir(parents=True, exist_ok=True)
+            with args.output.open("a", encoding="utf-8") as stream:
+                for item in results:
+                    stream.write(json.dumps(item, ensure_ascii=False) + "\n")
+            passed = sum(item["status"] == "passed" for item in results)
+            skipped = sum(item["status"].startswith("skipped-") for item in results)
+            failed = len(results) - passed - skipped
+            print(f"Capability: {args.capability}")
+            print(f"Provider/account rows: {len(results)}; passed: {passed}; failed: {failed}; skipped: {skipped}.")
+            for item in results:
+                http = item.get("http_status") if item.get("http_status") is not None else "-"
+                latency = item.get("latency_seconds") if item.get("latency_seconds") is not None else "-"
+                print(
+                    f'{item["provider"]:10} {item.get("account", "DEFAULT"):10} '
+                    f'{item.get("model_requested") or "-":32} {item["status"]:30} '
+                    f'HTTP={http} latency={latency}s dimensions={item.get("embedding_dimensions") or "-"}'
+                )
+                detail = error_summary(item.get("error"))
+                if detail:
+                    category = item.get("http_error_category") or "provider-error"
+                    print(f"  detail [{category}]: {detail}")
+            print(f"Evidence: {args.output}")
+            print("No retries performed. Only free/trial candidates run unless --include-paid is explicitly supplied.")
+            return 0 if failed == 0 else 1
+
         results = capability_plan(args.capability, args.provider)
         args.output.parent.mkdir(parents=True, exist_ok=True)
         with args.output.open("a", encoding="utf-8") as stream:
@@ -1094,9 +1128,10 @@ def main() -> int:
                 stream.write(json.dumps(item, ensure_ascii=False) + "\n")
         unsupported = sum(item["status"] == "skipped-no-capability-adapter" for item in results)
         missing = sum(item["status"] == "skipped-no-credential" for item in results)
+        ready = sum(item["status"] == "ready" for item in results)
         print(f"Capability: {args.capability}")
-        print(f"Provider/account rows: {len(results)}; adapters not implemented: {unsupported}; no credentials: {missing}.")
-        print("No provider requests were made. These rows are explicit adapter backlog, not inference failures.")
+        print(f"Provider/account rows: {len(results)}; adapters available: {ready}; adapters not implemented: {unsupported}; no credentials: {missing}.")
+        print("Planning only: no provider requests were made. Use --all-configured or --provider to execute eligible adapters.")
         print(f"Evidence: {args.output}")
         return 0
 

From 165580e21a04534b704971b93aae5c34a7da869b Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:26:59 -0500
Subject: [PATCH 086/128] Test embedding adapters response validation and paid
 opt-in

---
 tests/test_provider_api_smoke_test.py | 157 ++++++++++++++++++++++++++
 1 file changed, 157 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index e05021e..45f3302 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -558,5 +558,162 @@ def test_evidence_report_groups_by_provider_account_capability_and_status(self):
 
 
 
+class EmbeddingAdapterTests(unittest.TestCase):
+    def test_provider_native_embedding_request_specs(self):
+        from tools.provider_api.smoke_test import embedding_request_spec
+
+        cases = {
+            "GOOGLE": "models/gemini-embedding-001",
+            "COHERE": "embed-v4.0",
+            "VOYAGEAI": "voyage-4-lite",
+            "JINA": "jina-embeddings-v3",
+            "OPENAI": "text-embedding-3-small",
+            "MISTRAL": "mistral-embed",
+        }
+        for provider, model in cases.items():
+            with self.subTest(provider=provider):
+                url, payload = embedding_request_spec({
+                    "provider": provider,
+                    "model": model,
+                    "access": "test",
+                })
+                self.assertTrue(url.startswith("https://"))
+                self.assertEqual(payload["model"], model)
+                self.assertTrue(
+                    payload.get("content") or payload.get("texts") or payload.get("input")
+                )
+        self.assertEqual(
+            embedding_request_spec({
+                "provider": "JINA", "model": "jina-embeddings-v3", "access": "test"
+            })[1]["task"],
+            "retrieval.passage",
+        )
+        self.assertEqual(
+            embedding_request_spec({
+                "provider": "COHERE", "model": "embed-v4.0", "access": "test"
+            })[1]["input_type"],
+            "search_document",
+        )
+
+    def test_embedding_response_envelopes_normalize(self):
+        from tools.provider_api.smoke_test import extract_embedding_vectors
+
+        fixtures = {
+            "OPENAI": {"data": [{"embedding": [0.1, 0.2, 0.3]}]},
+            "MISTRAL": {"data": [{"embedding": [0.1, 0.2, 0.3]}]},
+            "VOYAGEAI": {"data": [{"embedding": [0.1, 0.2, 0.3]}]},
+            "JINA": {"data": [{"embedding": [0.1, 0.2, 0.3]}]},
+            "COHERE": {"embeddings": {"float": [[0.1, 0.2, 0.3]]}},
+            "GOOGLE": {"embedding": {"values": [0.1, 0.2, 0.3]}},
+        }
+        for provider, fixture in fixtures.items():
+            with self.subTest(provider=provider):
+                self.assertEqual(
+                    extract_embedding_vectors(provider, fixture),
+                    [[0.1, 0.2, 0.3]],
+                )
+
+    def test_embedding_response_rejects_invalid_vectors(self):
+        from tools.provider_api.smoke_test import extract_embedding_vectors
+
+        invalid = [
+            {"data": [{"embedding": []}]},
+            {"data": [{"embedding": [0.0, 0.0]}]},
+            {"data": [{"embedding": [0.1, float("nan")]}]},
+            {"data": [{"embedding": [0.1, "bad"]}]},
+            {"data": [{"embedding": [0.1, 0.2]}, {"embedding": [0.1]}]},
+        ]
+        for fixture in invalid:
+            with self.subTest(fixture=fixture):
+                with self.assertRaises(ValueError):
+                    extract_embedding_vectors("OPENAI", fixture)
+
+    def test_embedding_request_records_dimensions_without_persisting_vectors(self):
+        import json
+        from tools.provider_api.smoke_test import request_embedding_case
+
+        class FakeResponse:
+            status = 200
+            headers = {"x-request-id": "embedding-test-id"}
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *args):
+                return False
+
+            def read(self):
+                return json.dumps({
+                    "model": "text-embedding-3-small",
+                    "data": [{"embedding": [0.25, 0.5, 0.75]}],
+                    "usage": {"total_tokens": 8},
+                }).encode("utf-8")
+
+        case = {
+            "provider": "OPENAI",
+            "model": "text-embedding-3-small",
+            "access": "paid",
+        }
+        with patch.dict("os.environ", {
+            "OPENAI_API_KEY_FLOSSWARE": "test-secret",
+        }, clear=True), patch(
+            "tools.provider_api.smoke_test.urllib.request.urlopen",
+            return_value=FakeResponse(),
+        ) as urlopen:
+            result = request_embedding_case(
+                case,
+                timeout=1,
+                credential_env="OPENAI_API_KEY_FLOSSWARE",
+                account="FLOSSWARE",
+            )
+
+        self.assertEqual(result["status"], "passed")
+        self.assertEqual(result["account"], "FLOSSWARE")
+        self.assertEqual(result["capability"], "embeddings")
+        self.assertEqual(result["embedding_count"], 1)
+        self.assertEqual(result["embedding_dimensions"], 3)
+        self.assertGreater(result["embedding_norm"], 0)
+        self.assertNotIn("embedding", result)
+        request = urlopen.call_args.args[0]
+        self.assertEqual(request.full_url, "https://api.openai.com/v1/embeddings")
+        self.assertEqual(request.get_header("Authorization"), "Bearer test-secret")
+
+    def test_embedding_plan_keeps_paid_candidates_opt_in_only(self):
+        from tools.provider_api.smoke_test import plan_capability_all_configured
+
+        env = {
+            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+            "JINA_API_KEY_FLOSSWARE": "jina-secret",
+            "OPENAI_API_KEY_FLOSSWARE": "openai-secret",
+        }
+        with patch.dict("os.environ", env, clear=True):
+            plan = plan_capability_all_configured(
+                "embeddings",
+                include_paid=False,
+                providers=["VOYAGEAI", "JINA", "OPENAI"],
+            )
+
+        statuses = {item["provider"]: item["status"] for item in plan}
+        self.assertEqual(statuses["JINA"], "ready")
+        self.assertEqual(statuses["VOYAGEAI"], "skipped-paid-opt-in-required")
+        self.assertEqual(statuses["OPENAI"], "skipped-paid-opt-in-required")
+
+    def test_embedding_plan_includes_paid_candidates_only_with_opt_in(self):
+        from tools.provider_api.smoke_test import plan_capability_all_configured
+
+        with patch.dict("os.environ", {
+            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+        }, clear=True):
+            plan = plan_capability_all_configured(
+                "embeddings",
+                include_paid=True,
+                providers=["VOYAGEAI"],
+            )
+
+        self.assertEqual(len(plan), 1)
+        self.assertEqual(plan[0]["status"], "ready")
+
+
+
 if __name__ == "__main__":
     unittest.main()

From 2c64215137416f0501a964902a63e40367c940b7 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:27:07 -0500
Subject: [PATCH 087/128] Show capability-only providers as adapter available


From 1e65e7d39737f4305bc1f3ea98f6ea7cea02d239 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:27:13 -0500
Subject: [PATCH 088/128] Account for Google model in embedding URL

---
 tests/test_provider_api_smoke_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 45f3302..b99b948 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -578,7 +578,10 @@ def test_provider_native_embedding_request_specs(self):
                     "access": "test",
                 })
                 self.assertTrue(url.startswith("https://"))
-                self.assertEqual(payload["model"], model)
+                if provider != "GOOGLE":
+                    self.assertEqual(payload["model"], model)
+                else:
+                    self.assertIn(model, url)
                 self.assertTrue(
                     payload.get("content") or payload.get("texts") or payload.get("input")
                 )

From 4792ea7f23a6e34bc1405a44c1ba3ac32bdd2ff0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:27:26 -0500
Subject: [PATCH 089/128] Report embedding-only adapters in inventory and plans

---
 tools/provider_api/smoke_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index f48da6a..de4684a 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -138,13 +138,13 @@ def inventory_rows() -> list[dict[str, Any]]:
                 "account": account["account"],
                 "credential_env": account["credential_env"],
                 "credential_configured": bool(account["credential_env"]),
-                "adapter": entry.get("adapter"),
+                "adapter": entry.get("adapter") or entry.get("capability_adapters", {}).get("embeddings"),
                 "implemented_capabilities": entry.get("implemented_capabilities", []),
                 "known_capabilities": entry.get("known_capabilities", []),
                 "endpoint": entry.get("endpoint"),
                 "authentication": entry.get("authentication"),
                 "status": (
-                    "adapter-available" if account["credential_env"] and entry.get("adapter")
+                    "adapter-available" if account["credential_env"] and (entry.get("adapter") or entry.get("capability_adapters"))
                     else "skipped-no-credential" if not account["credential_env"]
                     else entry.get("status", "skipped-no-provider-adapter")
                 ),
@@ -171,7 +171,7 @@ def capability_plan(capability: str, providers: list[str] | None = None) -> list
                 "credential_env": account["credential_env"],
                 "credential_configured": configured,
                 "capability": capability,
-                "adapter": entry.get("adapter"),
+                "adapter": entry.get("capability_adapters", {}).get(capability) or entry.get("adapter"),
                 "status": (
                     "skipped-no-credential" if not configured
                     else "ready" if supported

From 708837b145400e18c14291245ca3fa64a14e4d14 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:27:40 -0500
Subject: [PATCH 090/128] Document native embedding adapters and explicit
 execution

---
 README.md | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f6178bc..e4125c5 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,32 @@ python3 tools/provider_api/smoke_test.py --capability embeddings
 python3 tools/provider_api/smoke_test.py --report
 ```
 
-Capability families are declared in the provider catalogue: `chat`, `embeddings`, `reranking`, `image-generation`, and `document-processing`. Only adapters listed under `implemented_capabilities` execute network requests. Other capability selections append explicit `skipped-no-capability-adapter` planning records, not false inference failures. Chat is the only implemented capability family in this initial registry. Adding an adapter means declaring its endpoint/authentication/operations in the catalogue, implementing its request/response contract, and adding deterministic tests. One runner does not mean forcing incompatible APIs through a chat-completions payload.
+Capability families are declared in the provider catalogue: `chat`, `embeddings`, `reranking`, `image-generation`, and `document-processing`. Only adapters listed under `implemented_capabilities` execute network requests. Unsupported capability selections append explicit `skipped-no-capability-adapter` planning records, not false inference failures. Chat and text embeddings have initial adapters; reranking, image generation, and document processing remain planned. Adding an adapter means declaring its endpoint/authentication/operations in the catalogue, implementing its request/response contract, and adding deterministic tests. One runner does not mean forcing incompatible APIs through a chat-completions payload.
+
+#### Embedding adapter evaluation
+
+The initial embedding adapters use native provider request/response contracts for Voyage AI, Jina, Cohere, Google Gemini, OpenAI, and Mistral. The bounded smoke request embeds one synthetic FlossWare sentence and validates that the response contains a finite, non-zero vector. Evidence stores model, dimensions, norm, latency, usage, status, and request ID; it deliberately does not persist vector contents.
+
+```bash
+# Plan capability coverage only. No provider requests are made.
+python3 tools/provider_api/smoke_test.py --capability embeddings
+
+# Execute only candidates marked free-tier/trial, across configured accounts.
+python3 tools/provider_api/smoke_test.py --all-configured --capability embeddings
+
+# Execute one provider's free/trial candidate across its configured accounts.
+python3 tools/provider_api/smoke_test.py --capability embeddings --provider JINA
+
+# Explicitly opt into paid/account-credit candidates for one provider.
+python3 tools/provider_api/smoke_test.py --capability embeddings --provider VOYAGEAI --include-paid
+
+# Explicitly opt into all configured embedding candidates, including paid candidates.
+python3 tools/provider_api/smoke_test.py --all-configured --capability embeddings --include-paid
+```
+
+Access labels are conservative candidate classifications, not a promise that a particular account is free or has quota. The default execution excludes `paid` and `paid-or-account-credit` candidates. The explicit `--include-paid` flag may incur charges. There are no retries. The adapter validates vector shape and numeric values without retaining raw embeddings.
+
+Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/#tag/embeddings).
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
 

From dfd26f9ccc00e6ac3ed8519192f6c72daa709aa8 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:27:57 -0500
Subject: [PATCH 091/128] Expose per-capability endpoints and guard unsupported
 capability execution

---
 tools/provider_api/smoke_test.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index de4684a..297c4ec 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -142,6 +142,7 @@ def inventory_rows() -> list[dict[str, Any]]:
                 "implemented_capabilities": entry.get("implemented_capabilities", []),
                 "known_capabilities": entry.get("known_capabilities", []),
                 "endpoint": entry.get("endpoint"),
+                "capability_endpoints": entry.get("capability_endpoints", {}),
                 "authentication": entry.get("authentication"),
                 "status": (
                     "adapter-available" if account["credential_env"] and (entry.get("adapter") or entry.get("capability_adapters"))
@@ -200,6 +201,8 @@ def print_inventory() -> None:
             print(f'  source: {item["credential_env"]}')
         if item["endpoint"]:
             print(f'  endpoint: {item["endpoint"]}')
+        for capability, endpoint in sorted(item["capability_endpoints"].items()):
+            print(f"  {capability} endpoint: {endpoint}")
         if item["status"] not in {"ready"}:
             print(f'  status: {item["status"]}')
     print(f"\n{len(rows)} provider/account rows. Inventory only; no network requests or credential values.")
@@ -1087,6 +1090,10 @@ def main() -> int:
     if args.capability != "chat":
         if args.model or args.openrouter_free_matrix:
             parser.error("--model and --openrouter-free-matrix currently apply only to chat")
+        if args.capability != "embeddings" and (args.all_configured or args.provider or args.include_paid):
+            parser.error("Only embeddings currently has executable non-chat adapters; other capabilities are planning-only")
+        if args.all_configured and args.provider:
+            parser.error("--all-configured cannot be combined with --provider")
         if args.include_paid and not (args.all_configured or args.provider):
             parser.error("--include-paid requires --all-configured or an explicit --provider selection")
         if args.all_configured or args.provider:

From 23307706a2c02404866b8b9dd2d1cfa4562c357e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:28:05 -0500
Subject: [PATCH 092/128] Test embedding-only provider inventory and capability
 plans

---
 tests/test_provider_api_smoke_test.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index b99b948..cb5253e 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -701,6 +701,21 @@ def test_embedding_plan_keeps_paid_candidates_opt_in_only(self):
         self.assertEqual(statuses["VOYAGEAI"], "skipped-paid-opt-in-required")
         self.assertEqual(statuses["OPENAI"], "skipped-paid-opt-in-required")
 
+    def test_embedding_capability_plan_recognizes_embedding_only_providers(self):
+        from tools.provider_api.smoke_test import capability_plan
+
+        with patch.dict("os.environ", {
+            "JINA_API_KEY_FLOSSWARE": "jina-secret",
+            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+        }, clear=True):
+            rows = capability_plan("embeddings", ["JINA", "VOYAGEAI"])
+
+        by_provider = {item["provider"]: item for item in rows}
+        self.assertEqual(by_provider["JINA"]["status"], "ready")
+        self.assertEqual(by_provider["JINA"]["adapter"], "jina-embeddings")
+        self.assertEqual(by_provider["VOYAGEAI"]["status"], "ready")
+        self.assertEqual(by_provider["VOYAGEAI"]["adapter"], "voyage-embeddings")
+
     def test_embedding_plan_includes_paid_candidates_only_with_opt_in(self):
         from tools.provider_api.smoke_test import plan_capability_all_configured
 

From 64e33dc85fcc3ae048455035cf5c8d658b22c7fd Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:28:24 -0500
Subject: [PATCH 093/128] Use current Gemini embedContent config field

---
 tools/provider_api/smoke_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 297c4ec..9f1e3dd 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -551,7 +551,7 @@ def embedding_request_spec(case: dict[str, str]) -> tuple[str, dict[str, Any]]:
             f"https://generativelanguage.googleapis.com/v1beta/{model}:embedContent",
             {
                 "content": {"parts": [{"text": EMBEDDING_TEXT}]},
-                "taskType": "RETRIEVAL_DOCUMENT",
+                "embedContentConfig": {"taskType": "RETRIEVAL_DOCUMENT"},
             },
         )
     if provider == "COHERE":

From 59127b8e21255a8d864881c7938e8cb2fe85cd94 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:28:31 -0500
Subject: [PATCH 094/128] Assert Gemini retrieval task config uses current API
 field

---
 tests/test_provider_api_smoke_test.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index cb5253e..62d7885 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -591,6 +591,12 @@ def test_provider_native_embedding_request_specs(self):
             })[1]["task"],
             "retrieval.passage",
         )
+        self.assertEqual(
+            embedding_request_spec({
+                "provider": "GOOGLE", "model": "models/gemini-embedding-001", "access": "test"
+            })[1]["embedContentConfig"]["taskType"],
+            "RETRIEVAL_DOCUMENT",
+        )
         self.assertEqual(
             embedding_request_spec({
                 "provider": "COHERE", "model": "embed-v4.0", "access": "test"

From c50f945e89baae40ea6857f48de2e7e7d9d79cbf Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:28:45 -0500
Subject: [PATCH 095/128] Link to current Mistral embeddings API reference

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e4125c5..457592c 100644
--- a/README.md
+++ b/README.md
@@ -95,7 +95,7 @@ python3 tools/provider_api/smoke_test.py --all-configured --capability embedding
 
 Access labels are conservative candidate classifications, not a promise that a particular account is free or has quota. The default execution excludes `paid` and `paid-or-account-credit` candidates. The explicit `--include-paid` flag may incur charges. There are no retries. The adapter validates vector shape and numeric values without retaining raw embeddings.
 
-Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/#tag/embeddings).
+Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/endpoint/embeddings).
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:
 

From 52778e88e75d0bb9a260ce64fd48d33d682505a7 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:29:33 -0500
Subject: [PATCH 096/128] Recognize Voyage AI credential names and SDK alias

---
 tools/provider_api/smoke_test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 9f1e3dd..87577d7 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -85,6 +85,7 @@
     "COHERE": "COHERE_API_KEY_FLOSSWARE",
     "OPENAI": "OPENAI_API_KEY_FLOSSWARE",
     "ANTHROPIC": "ANTHROPIC_API_KEY_FLOSSWARE",
+    "VOYAGEAI": "VOYAGEAI_API_KEY_FLOSSWARE",
 }
 
 # Accept standard provider names and common SDK conventions.
@@ -102,6 +103,7 @@
     "COHERE": ("COHERE_API_KEY",),
     "OPENAI": ("OPENAI_API_KEY",),
     "ANTHROPIC": ("ANTHROPIC_API_KEY",),
+    "VOYAGEAI": ("VOYAGE_API_KEY", "VOYAGEAI_API_KEY"),
 }
 
 PROVIDER_CATALOGUE_PATH = Path(__file__).with_name("provider-catalogue.json")

From 968e4aaf42860ff9b4f80aa0d90be66c36595acb Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:29:40 -0500
Subject: [PATCH 097/128] Test canonical Voyage AI credential discovery

---
 tests/test_provider_api_smoke_test.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 62d7885..7930b4d 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -691,7 +691,7 @@ def test_embedding_plan_keeps_paid_candidates_opt_in_only(self):
         from tools.provider_api.smoke_test import plan_capability_all_configured
 
         env = {
-            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+            "VOYAGEAI_API_KEY_FLOSSWARE": "voyage-secret",
             "JINA_API_KEY_FLOSSWARE": "jina-secret",
             "OPENAI_API_KEY_FLOSSWARE": "openai-secret",
         }
@@ -712,7 +712,7 @@ def test_embedding_capability_plan_recognizes_embedding_only_providers(self):
 
         with patch.dict("os.environ", {
             "JINA_API_KEY_FLOSSWARE": "jina-secret",
-            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+            "VOYAGEAI_API_KEY_FLOSSWARE": "voyage-secret",
         }, clear=True):
             rows = capability_plan("embeddings", ["JINA", "VOYAGEAI"])
 
@@ -722,11 +722,24 @@ def test_embedding_capability_plan_recognizes_embedding_only_providers(self):
         self.assertEqual(by_provider["VOYAGEAI"]["status"], "ready")
         self.assertEqual(by_provider["VOYAGEAI"]["adapter"], "voyage-embeddings")
 
+    def test_voyage_sdk_credential_alias_maps_to_voyageai(self):
+        from tools.provider_api.smoke_test import credential_accounts, discovered_provider_names
+
+        with patch.dict("os.environ", {
+            "VOYAGE_API_KEY": "voyage-sdk-secret",
+        }, clear=True):
+            self.assertIn("VOYAGEAI", discovered_provider_names())
+            accounts = credential_accounts("VOYAGEAI")
+
+        self.assertEqual(len(accounts), 1)
+        self.assertEqual(accounts[0]["credential_env"], "VOYAGE_API_KEY")
+        self.assertEqual(accounts[0]["account"], "DEFAULT")
+
     def test_embedding_plan_includes_paid_candidates_only_with_opt_in(self):
         from tools.provider_api.smoke_test import plan_capability_all_configured
 
         with patch.dict("os.environ", {
-            "VOYAGE_API_KEY_FLOSSWARE": "voyage-secret",
+            "VOYAGEAI_API_KEY_FLOSSWARE": "voyage-secret",
         }, clear=True):
             plan = plan_capability_all_configured(
                 "embeddings",

From 1ba6eec1fca7969b1bed0f9edc2a90160d6d833a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:30:36 -0500
Subject: [PATCH 098/128] Include Voyage AI in all-provider credential planning
 fixture

---
 tests/test_provider_api_smoke_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 7930b4d..ec78a85 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -76,6 +76,7 @@ def test_plan_includes_every_provider_and_never_opts_into_paid_candidates(self):
             "COHERE_API_KEY": "COHERE_API_KEY",
             "OPENAI_API_KEY": "OPENAI_API_KEY",
             "ANTHROPIC_API_KEY": "ANTHROPIC_API_KEY",
+            "VOYAGEAI_API_KEY_FLOSSWARE": "VOYAGEAI_API_KEY_FLOSSWARE",
         }
         with patch.dict("os.environ", {name: "test-secret" for name in aliases}, clear=True):
             plan = plan_all_configured()

From a0df67b88c6a775499ddc4fb6cc25c2a2224ea44 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:35:48 -0500
Subject: [PATCH 099/128] Require explicit opt-in for Jina embedding account
 credits

---
 tools/provider_api/smoke_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 87577d7..454d58d 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -54,7 +54,7 @@
 )
 EMBEDDING_CASES: list[dict[str, str]] = [
     {"provider": "VOYAGEAI", "model": "voyage-4-lite", "access": "paid-or-account-credit"},
-    {"provider": "JINA", "model": "jina-embeddings-v3", "access": "free-tier-candidate"},
+    {"provider": "JINA", "model": "jina-embeddings-v3", "access": "paid-or-account-credit"},
     {"provider": "COHERE", "model": "embed-v4.0", "access": "free-trial-candidate"},
     {"provider": "GOOGLE", "model": "models/gemini-embedding-001", "access": "free-tier-candidate"},
     {"provider": "OPENAI", "model": "text-embedding-3-small", "access": "paid"},

From 89317dee28f4a2395026614452a0230ff101ca91 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:36:00 -0500
Subject: [PATCH 100/128] Test Jina account-credit candidate requires explicit
 opt-in

---
 tests/test_provider_api_smoke_test.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index ec78a85..836b2ba 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -704,10 +704,16 @@ def test_embedding_plan_keeps_paid_candidates_opt_in_only(self):
             )
 
         statuses = {item["provider"]: item["status"] for item in plan}
-        self.assertEqual(statuses["JINA"], "ready")
+        self.assertEqual(statuses["JINA"], "skipped-paid-opt-in-required")
         self.assertEqual(statuses["VOYAGEAI"], "skipped-paid-opt-in-required")
         self.assertEqual(statuses["OPENAI"], "skipped-paid-opt-in-required")
 
+    def test_jina_embedding_candidate_requires_paid_opt_in(self):
+        from tools.provider_api.smoke_test import EMBEDDING_CASES
+
+        jina = next(item for item in EMBEDDING_CASES if item["provider"] == "JINA")
+        self.assertEqual(jina["access"], "paid-or-account-credit")
+
     def test_embedding_capability_plan_recognizes_embedding_only_providers(self):
         from tools.provider_api.smoke_test import capability_plan
 

From 602dac743b764961fb8ffd6cb346f35bcad1ed04 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:36:18 -0500
Subject: [PATCH 101/128] Document Jina balance findings and require opt-in for
 retest

---
 README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 457592c..1f1e3fb 100644
--- a/README.md
+++ b/README.md
@@ -74,17 +74,18 @@ Capability families are declared in the provider catalogue: `chat`, `embeddings`
 
 #### Embedding adapter evaluation
 
-The initial embedding adapters use native provider request/response contracts for Voyage AI, Jina, Cohere, Google Gemini, OpenAI, and Mistral. The bounded smoke request embeds one synthetic FlossWare sentence and validates that the response contains a finite, non-zero vector. Evidence stores model, dimensions, norm, latency, usage, status, and request ID; it deliberately does not persist vector contents.
+The initial embedding adapters use native provider request/response contracts for Voyage AI, Jina, Cohere, Google Gemini, OpenAI, and Mistral. Initial account testing found Jina credentials had insufficient balance on all four configured accounts, so Jina is conservatively classified as paid/account-credit and excluded from default execution. The bounded smoke request embeds one synthetic FlossWare sentence and validates that the response contains a finite, non-zero vector. Evidence stores model, dimensions, norm, latency, usage, status, and request ID; it deliberately does not persist vector contents.
 
 ```bash
 # Plan capability coverage only. No provider requests are made.
 python3 tools/provider_api/smoke_test.py --capability embeddings
 
-# Execute only candidates marked free-tier/trial, across configured accounts.
+# Execute candidates classified as free-tier/trial, across configured accounts. Jina is excluded after all four configured accounts reported no balance.
 python3 tools/provider_api/smoke_test.py --all-configured --capability embeddings
 
-# Execute one provider's free/trial candidate across its configured accounts.
-python3 tools/provider_api/smoke_test.py --capability embeddings --provider JINA
+# Jina is classified as paid/account-credit after the initial four-account test.
+# It requires explicit opt-in if you later add credits and want to retest.
+python3 tools/provider_api/smoke_test.py --capability embeddings --provider JINA --include-paid
 
 # Explicitly opt into paid/account-credit candidates for one provider.
 python3 tools/provider_api/smoke_test.py --capability embeddings --provider VOYAGEAI --include-paid

From be4b654a36ee6db03d163c9814a23c4b84166c2a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:36:43 -0500
Subject: [PATCH 102/128] Classify Jina 403 insufficient balance as billing
 limitation

---
 tools/provider_api/smoke_test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/provider_api/smoke_test.py b/tools/provider_api/smoke_test.py
index 454d58d..29e9415 100644
--- a/tools/provider_api/smoke_test.py
+++ b/tools/provider_api/smoke_test.py
@@ -375,6 +375,8 @@ def classify_http_error(status: int | None, detail: str | None) -> str | None:
 
     if status == 403:
         message = str(metadata.get("raw") or error.get("message") or detail or "").lower()
+        if "insufficient account balance" in message or "insufficient balance" in message:
+            return "insufficient-account-balance"
         if (
             "only available on agentic harnesses" in message
             or "agentic harness" in message and "coding agent" in message

From 6c68c16054dde41de295327346f75ddf1def3d03 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:36:59 -0500
Subject: [PATCH 103/128] Test Jina 403 insufficient-balance classification

---
 tests/test_provider_api_smoke_test.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_provider_api_smoke_test.py b/tests/test_provider_api_smoke_test.py
index 836b2ba..7c9a2d1 100644
--- a/tests/test_provider_api_smoke_test.py
+++ b/tests/test_provider_api_smoke_test.py
@@ -231,6 +231,14 @@ def test_unknown_402_is_not_overclassified(self):
             "account-credit-or-billing-required",
         )
 
+    def test_jina_403_insufficient_balance_is_classified_as_billing_limit(self):
+        from tools.provider_api.smoke_test import classify_http_error
+
+        self.assertEqual(
+            classify_http_error(403, "Insufficient account balance. Top up your account."),
+            "insufficient-account-balance",
+        )
+
     def test_non_402_is_not_a_billing_failure(self):
         from tools.provider_api.smoke_test import classify_http_error
         self.assertIsNone(classify_http_error(403, "Forbidden"))

From b2a6049316b86ddafce2f27a80018b96d312d04d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:40:54 -0500
Subject: [PATCH 104/128] Add fixed semantic retrieval benchmark dataset

---
 tools/provider_api/embedding-benchmark.json | 137 ++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 tools/provider_api/embedding-benchmark.json

diff --git a/tools/provider_api/embedding-benchmark.json b/tools/provider_api/embedding-benchmark.json
new file mode 100644
index 0000000..cd3e322
--- /dev/null
+++ b/tools/provider_api/embedding-benchmark.json
@@ -0,0 +1,137 @@
+{
+  "version": 1,
+  "description": "Small, deterministic retrieval benchmark for comparing document/query embedding models. Gold relevance is binary and curated by topic.",
+  "documents": [
+    {
+      "id": "vec-01",
+      "text": "A vector index stores dense numeric embeddings for passages. At query time, the query is embedded in the same model space and nearest neighbors are ranked by cosine similarity or inner product."
+    },
+    {
+      "id": "vec-02",
+      "text": "Document and query embeddings must use compatible model versions, dimensions, and task modes. Mixing vectors from different embedding models in one index makes similarity scores meaningless."
+    },
+    {
+      "id": "vec-03",
+      "text": "Recall at K measures the fraction of known relevant documents retrieved in the first K results. It is useful when missing a relevant passage is costly."
+    },
+    {
+      "id": "vec-04",
+      "text": "Mean reciprocal rank rewards systems that place the first relevant result near the top. For each query, take the reciprocal of the rank of its first relevant result, then average."
+    },
+    {
+      "id": "dist-01",
+      "text": "A distributed service should expose a stable API contract and keep provider-specific implementation details behind adapters. Clients should not depend on a vendor's native request format."
+    },
+    {
+      "id": "dist-02",
+      "text": "Retries can amplify outages and rate limits. Use bounded retry policies with backoff and jitter only for transient failures, and never blindly retry non-idempotent operations."
+    },
+    {
+      "id": "dist-03",
+      "text": "A circuit breaker stops sending requests to a dependency after repeated failures. It allows the dependency time to recover and prevents resource exhaustion in callers."
+    },
+    {
+      "id": "dist-04",
+      "text": "Idempotency keys let a client safely repeat an operation without creating duplicate effects. The server records the key and returns the original result for a repeated request."
+    },
+    {
+      "id": "api-01",
+      "text": "API keys should be supplied through environment variables or a secret manager, never command-line arguments, source files, logs, or committed configuration."
+    },
+    {
+      "id": "api-02",
+      "text": "A useful HTTP error taxonomy distinguishes authentication failures, authorization failures, billing limits, missing resources, validation errors, and rate limiting."
+    },
+    {
+      "id": "api-03",
+      "text": "Observability should record request identifiers, status codes, latency, model identifiers, and token usage while excluding credentials and sensitive payload content."
+    },
+    {
+      "id": "api-04",
+      "text": "Rate limits may be enforced by an upstream shared pool rather than by an individual API key. Multiple credentials therefore do not necessarily provide independent quota."
+    },
+    {
+      "id": "eval-01",
+      "text": "A benchmark needs a fixed dataset, explicit relevance judgments, reproducible configuration, and metrics. A successful HTTP response only proves that an endpoint answered."
+    },
+    {
+      "id": "eval-02",
+      "text": "Normalized discounted cumulative gain evaluates ranking quality while accounting for graded relevance and the position of each relevant result."
+    },
+    {
+      "id": "eval-03",
+      "text": "A semantic retrieval evaluation should use the same passages and queries for every candidate model. Otherwise differences in results may come from the dataset rather than the model."
+    },
+    {
+      "id": "eval-04",
+      "text": "Latency, vector dimensions, storage footprint, and usage cost should be considered alongside retrieval quality when selecting an embedding model."
+    }
+  ],
+  "queries": [
+    {
+      "id": "q-01",
+      "text": "How do we ensure passage vectors and search-query vectors can be compared in one index?",
+      "relevant": [
+        "vec-01",
+        "vec-02"
+      ]
+    },
+    {
+      "id": "q-02",
+      "text": "Which metric tells us what share of relevant passages appeared within the first five search hits?",
+      "relevant": [
+        "vec-03",
+        "eval-01"
+      ]
+    },
+    {
+      "id": "q-03",
+      "text": "How is reciprocal rank used to reward a search engine for putting its first useful result near the top?",
+      "relevant": [
+        "vec-04",
+        "eval-02"
+      ]
+    },
+    {
+      "id": "q-04",
+      "text": "How should a service hide vendor-specific API formats behind a consistent interface?",
+      "relevant": [
+        "dist-01",
+        "api-02"
+      ]
+    },
+    {
+      "id": "q-05",
+      "text": "What prevents repeated transient failures from overwhelming a downstream service?",
+      "relevant": [
+        "dist-02",
+        "dist-03"
+      ]
+    },
+    {
+      "id": "q-06",
+      "text": "Where should API credentials live, and what should never appear in logs or process arguments?",
+      "relevant": [
+        "api-01",
+        "api-03"
+      ]
+    },
+    {
+      "id": "q-07",
+      "text": "Why might using several keys from one provider fail to increase the available request quota?",
+      "relevant": [
+        "api-04",
+        "dist-02"
+      ]
+    },
+    {
+      "id": "q-08",
+      "text": "What should be held constant when comparing embedding models, and what should be measured beyond retrieval relevance?",
+      "relevant": [
+        "eval-03",
+        "eval-04",
+        "eval-01"
+      ]
+    }
+  ]
+}

From 048a664d738d3df9a408a22fbe98a30be0ca7e1c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:41:00 -0500
Subject: [PATCH 105/128] Add semantic retrieval benchmark runner

---
 tools/provider_api/embedding_benchmark.py | 379 ++++++++++++++++++++++
 1 file changed, 379 insertions(+)
 create mode 100644 tools/provider_api/embedding_benchmark.py

diff --git a/tools/provider_api/embedding_benchmark.py b/tools/provider_api/embedding_benchmark.py
new file mode 100644
index 0000000..975dffd
--- /dev/null
+++ b/tools/provider_api/embedding_benchmark.py
@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""Run a small, reproducible semantic retrieval benchmark across embedding APIs.
+
+No vectors are written to disk. A run requires explicit --run consent.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from smoke_test import (
+    EMBEDDING_CASES,
+    PAID_ACCESS,
+    build_headers,
+    classify_http_error,
+    credential_accounts,
+    credential_account_from_env_name,
+    error_summary,
+    sanitize_error_detail,
+)
+
+HERE = Path(__file__).resolve().parent
+DATASET_PATH = HERE / "embedding-benchmark.json"
+DEFAULT_OUTPUT = Path("embedding-benchmark-results.jsonl")
+DEFAULT_PROVIDERS = ("COHERE", "GOOGLE")
+TOP_K = 5
+
+
+def load_dataset(path: Path = DATASET_PATH) -> dict[str, Any]:
+    """Load and validate the fixed benchmark dataset."""
+    data = json.loads(path.read_text(encoding="utf-8"))
+    documents = data.get("documents", [])
+    queries = data.get("queries", [])
+    doc_ids = [item["id"] for item in documents]
+    if not documents or not queries or len(doc_ids) != len(set(doc_ids)):
+        raise ValueError("Dataset must contain unique document IDs and at least one query")
+    known = set(doc_ids)
+    for query in queries:
+        relevant = query.get("relevant", [])
+        if not relevant or not set(relevant).issubset(known):
+            raise ValueError(f"Query {query.get('id')} has missing or unknown relevance IDs")
+    return data
+
+
+def embedding_batch_spec(provider: str, model: str, texts: list[str], mode: str) -> tuple[str, dict[str, Any]]:
+    """Build a provider-native batch request for document or query embeddings."""
+    if mode not in {"document", "query"}:
+        raise ValueError(f"Unsupported embedding mode: {mode}")
+    if provider == "GOOGLE":
+        task = "RETRIEVAL_DOCUMENT" if mode == "document" else "RETRIEVAL_QUERY"
+        return (
+            f"https://generativelanguage.googleapis.com/v1beta/{model}:batchEmbedContents",
+            {"requests": [
+                {"model": model, "content": {"parts": [{"text": text}]}, "taskType": task}
+                for text in texts
+            ]},
+        )
+    if provider == "COHERE":
+        return (
+            "https://api.cohere.com/v2/embed",
+            {
+                "model": model,
+                "texts": texts,
+                "input_type": "search_document" if mode == "document" else "search_query",
+                "embedding_types": ["float"],
+            },
+        )
+    if provider == "VOYAGEAI":
+        return (
+            "https://api.voyageai.com/v1/embeddings",
+            {
+                "model": model,
+                "input": texts,
+                "input_type": mode,
+                "output_dtype": "float",
+            },
+        )
+    if provider == "JINA":
+        return (
+            "https://api.jina.ai/v1/embeddings",
+            {
+                "model": model,
+                "input": texts,
+                "task": "retrieval.passage" if mode == "document" else "retrieval.query",
+                "embedding_type": "float",
+                "normalized": True,
+            },
+        )
+    if provider in {"OPENAI", "MISTRAL"}:
+        endpoint = (
+            "https://api.openai.com/v1/embeddings"
+            if provider == "OPENAI"
+            else "https://api.mistral.ai/v1/embeddings"
+        )
+        return endpoint, {"model": model, "input": texts, "encoding_format": "float"}
+    raise ValueError(f"No embedding batch adapter for provider: {provider}")
+
+
+def extract_batch_vectors(provider: str, data: dict[str, Any]) -> list[list[float]]:
+    """Normalize provider batch responses and validate every vector."""
+    if provider == "GOOGLE":
+        raw = [item.get("values") for item in data.get("embeddings", []) if isinstance(item, dict)]
+    elif provider == "COHERE":
+        raw = (data.get("embeddings") or {}).get("float") or []
+    else:
+        items = data.get("data", [])
+        raw = [
+            item.get("embedding") for item in items
+            if isinstance(item, dict) and item.get("embedding") is not None
+        ]
+    vectors: list[list[float]] = []
+    for vector in raw:
+        if not isinstance(vector, list) or not vector:
+            raise ValueError("Provider returned an empty or invalid embedding")
+        if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in vector):
+            raise ValueError("Provider returned a non-numeric embedding value")
+        normalized = [float(value) for value in vector]
+        if not all(math.isfinite(value) for value in normalized):
+            raise ValueError("Provider returned a non-finite embedding value")
+        if not any(value != 0.0 for value in normalized):
+            raise ValueError("Provider returned an all-zero embedding")
+        vectors.append(normalized)
+    if not vectors:
+        raise ValueError("Provider response contained no embeddings")
+    dimensions = {len(vector) for vector in vectors}
+    if len(dimensions) != 1:
+        raise ValueError("Provider returned embeddings with inconsistent dimensions")
+    return vectors
+
+
+def cosine(left: list[float], right: list[float]) -> float:
+    if len(left) != len(right):
+        raise ValueError("Cannot compare vectors with different dimensions")
+    dot = sum(a * b for a, b in zip(left, right))
+    left_norm = math.sqrt(sum(value * value for value in left))
+    right_norm = math.sqrt(sum(value * value for value in right))
+    if left_norm == 0.0 or right_norm == 0.0:
+        raise ValueError("Cosine similarity is undefined for a zero vector")
+    return dot / (left_norm * right_norm)
+
+
+def ranking_metrics(
+    query_vectors: list[list[float]],
+    document_vectors: list[list[float]],
+    document_ids: list[str],
+    queries: list[dict[str, Any]],
+    k: int = TOP_K,
+) -> dict[str, Any]:
+    """Calculate macro Recall@K, MRR@K, and binary nDCG@K."""
+    if len(query_vectors) != len(queries) or len(document_vectors) != len(document_ids):
+        raise ValueError("Vector counts do not match benchmark records")
+    dimensions = {len(vector) for vector in query_vectors + document_vectors}
+    if len(dimensions) != 1:
+        raise ValueError("Query and document vector dimensions do not match")
+    recalls: list[float] = []
+    reciprocal_ranks: list[float] = []
+    ndcgs: list[float] = []
+    details: list[dict[str, Any]] = []
+    for query, vector in zip(queries, query_vectors):
+        ranked = sorted(
+            zip(document_ids, (cosine(vector, doc) for doc in document_vectors)),
+            key=lambda item: (-item[1], item[0]),
+        )
+        relevant = set(query["relevant"])
+        top = ranked[:k]
+        hits = [rank for rank, (doc_id, _) in enumerate(ranked[:k], start=1) if doc_id in relevant]
+        recalls.append(len({doc_id for doc_id, _ in top} & relevant) / len(relevant))
+        reciprocal_ranks.append(1.0 / hits[0] if hits else 0.0)
+        dcg = sum(1.0 / math.log2(rank + 1) for rank in hits)
+        ideal_hits = min(len(relevant), k)
+        idcg = sum(1.0 / math.log2(rank + 1) for rank in range(1, ideal_hits + 1))
+        ndcgs.append(dcg / idcg if idcg else 0.0)
+        details.append({
+            "query_id": query["id"],
+            "relevant_count": len(relevant),
+            "retrieved_relevant_at_k": len(hits),
+            "reciprocal_rank_at_k": round(reciprocal_ranks[-1], 6),
+            "ndcg_at_k": round(ndcgs[-1], 6),
+            "top_document_ids": [doc_id for doc_id, _ in top],
+        })
+    return {
+        f"recall_at_{k}": round(sum(recalls) / len(recalls), 6),
+        f"mrr_at_{k}": round(sum(reciprocal_ranks) / len(reciprocal_ranks), 6),
+        f"ndcg_at_{k}": round(sum(ndcgs) / len(ndcgs), 6),
+        "query_count": len(queries),
+        "document_count": len(document_ids),
+        "per_query": details,
+    }
+
+
+def post_embeddings(
+    provider: str,
+    model: str,
+    key: str,
+    texts: list[str],
+    mode: str,
+    timeout: float,
+) -> tuple[list[list[float]], dict[str, Any]]:
+    url, payload = embedding_batch_spec(provider, model, texts, mode)
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=build_headers(provider, key),
+        method="POST",
+    )
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            body = json.loads(response.read().decode("utf-8"))
+            metadata = {
+                "http_status": response.status,
+                "latency_seconds": round(time.monotonic() - started, 3),
+                "request_id": response.headers.get("x-request-id") or response.headers.get("request-id"),
+                "usage": body.get("usage") or body.get("usageMetadata"),
+            }
+        vectors = extract_batch_vectors(provider, body)
+        if len(vectors) != len(texts):
+            raise ValueError(
+                f"Expected {len(texts)} vectors for {mode}, received {len(vectors)}"
+            )
+        return vectors, metadata
+    except urllib.error.HTTPError as exc:
+        detail = sanitize_error_detail(exc.read().decode("utf-8", errors="replace"), key)
+        raise ProviderRequestError(exc.code, detail, time.monotonic() - started) from exc
+    except urllib.error.URLError as exc:
+        raise ProviderRequestError(None, str(exc.reason), time.monotonic() - started) from exc
+
+
+class ProviderRequestError(RuntimeError):
+    def __init__(self, status: int | None, detail: str, elapsed: float):
+        super().__init__(detail)
+        self.status = status
+        self.detail = detail
+        self.elapsed = elapsed
+
+
+def select_account(provider: str, all_accounts: bool = False) -> list[dict[str, str]]:
+    accounts = credential_accounts(provider)
+    if all_accounts:
+        return accounts
+    return accounts[:1]
+
+
+def run_provider(
+    provider: str,
+    case: dict[str, str],
+    account: dict[str, str],
+    dataset: dict[str, Any],
+    timeout: float,
+) -> dict[str, Any]:
+    key = account["credential"]
+    docs = dataset["documents"]
+    queries = dataset["queries"]
+    evidence: dict[str, Any] = {
+        "observed_at": datetime.now(timezone.utc).isoformat(),
+        "benchmark": "embedding-semantic-retrieval-v1",
+        "provider": provider,
+        "account": account["account"],
+        "credential_env": account["credential_env"],
+        "model": case["model"],
+        "access_classification": case["access"],
+        "dataset_version": dataset["version"],
+        "top_k": TOP_K,
+        "document_count": len(docs),
+        "query_count": len(queries),
+        "status": "failed",
+        "retry_performed": False,
+        "vectors_persisted": False,
+    }
+    try:
+        document_vectors, document_meta = post_embeddings(
+            provider, case["model"], key, [item["text"] for item in docs], "document", timeout
+        )
+        query_vectors, query_meta = post_embeddings(
+            provider, case["model"], key, [item["text"] for item in queries], "query", timeout
+        )
+        metrics = ranking_metrics(
+            query_vectors, document_vectors, [item["id"] for item in docs], queries
+        )
+        dimensions = len(document_vectors[0])
+        evidence.update({
+            "status": "passed",
+            "dimensions": dimensions,
+            "document_request": document_meta,
+            "query_request": query_meta,
+            "total_latency_seconds": round(
+                document_meta["latency_seconds"] + query_meta["latency_seconds"], 3
+            ),
+            "metrics": metrics,
+        })
+    except ProviderRequestError as exc:
+        evidence.update({
+            "status": "http-error" if exc.status is not None else "transport-error",
+            "http_status": exc.status,
+            "latency_seconds": round(exc.elapsed, 3),
+            "http_error_category": classify_http_error(exc.status, exc.detail),
+            "error": exc.detail,
+        })
+    except (ValueError, KeyError, TypeError) as exc:
+        evidence.update({"status": "invalid-response", "error": str(exc)})
+    return evidence
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--provider", action="append", choices=sorted(
+        {item["provider"] for item in EMBEDDING_CASES}
+    ), help="Provider to benchmark; repeat to select several. Defaults to Cohere and Google.")
+    parser.add_argument("--run", action="store_true", help="Execute provider requests. Without this flag, only print the plan.")
+    parser.add_argument("--include-paid", action="store_true", help="Explicitly allow paid/account-credit candidates.")
+    parser.add_argument("--all-accounts", action="store_true", help="Run once per configured account instead of only the preferred account.")
+    parser.add_argument("--timeout", type=float, default=45.0)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    args = parser.parse_args()
+    if args.timeout <= 0:
+        parser.error("--timeout must be positive")
+    dataset = load_dataset()
+    selected_providers = args.provider or list(DEFAULT_PROVIDERS)
+    cases = {item["provider"]: item for item in EMBEDDING_CASES}
+    plans: list[tuple[str, dict[str, str], dict[str, str]]] = []
+    for provider in selected_providers:
+        case = cases[provider]
+        if case["access"] in PAID_ACCESS and not args.include_paid:
+            print(f"{provider:10} {case['model']:32} skipped-paid-opt-in-required")
+            continue
+        accounts = select_account(provider, args.all_accounts)
+        if not accounts:
+            print(f"{provider:10} {'-':32} skipped-no-credential")
+            continue
+        for account in accounts:
+            plans.append((provider, case, account))
+            print(
+                f"{provider:10} {account['account']:10} {case['model']:32} "
+                f"planned ({len(dataset['documents'])} docs + {len(dataset['queries'])} queries; "
+                "2 batch requests)"
+            )
+    if not args.run:
+        print("Planning only: no provider requests were made. Add --run to execute.")
+        return 0
+    if not plans:
+        print("No eligible configured provider/account combinations.", file=sys.stderr)
+        return 2
+    results = [
+        run_provider(provider, case, account, dataset, args.timeout)
+        for provider, case, account in plans
+    ]
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("a", encoding="utf-8") as stream:
+        for item in results:
+            stream.write(json.dumps(item, ensure_ascii=False) + "\n")
+    for item in results:
+        print(
+            f"{item['provider']:10} {item['account']:10} {item['model']:32} "
+            f"{item['status']:18} dimensions={item.get('dimensions', '-')}"
+        )
+        if item.get("metrics"):
+            metrics = item["metrics"]
+            print(
+                f"  Recall@{TOP_K}={metrics[f'recall_at_{TOP_K}']:.3f} "
+                f"MRR@{TOP_K}={metrics[f'mrr_at_{TOP_K}']:.3f} "
+                f"nDCG@{TOP_K}={metrics[f'ndcg_at_{TOP_K}']:.3f} "
+                f"latency={item['total_latency_seconds']}s"
+            )
+        elif item.get("error"):
+            print(f"  detail [{item.get('http_error_category') or 'provider-error'}]: {error_summary(item['error'])}")
+    return 0 if all(item["status"] == "passed" for item in results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From 72285d2f269e2cc83003e3a3cf292a08c138286d Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:41:06 -0500
Subject: [PATCH 106/128] Test semantic embedding retrieval benchmark

---
 tests/test_embedding_benchmark.py | 115 ++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 tests/test_embedding_benchmark.py

diff --git a/tests/test_embedding_benchmark.py b/tests/test_embedding_benchmark.py
new file mode 100644
index 0000000..7cdbe00
--- /dev/null
+++ b/tests/test_embedding_benchmark.py
@@ -0,0 +1,115 @@
+"""Tests for the deterministic embedding retrieval benchmark."""
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from tools.provider_api.embedding_benchmark import (
+    embedding_batch_spec,
+    extract_batch_vectors,
+    load_dataset,
+    ranking_metrics,
+    select_account,
+)
+
+
+class EmbeddingBenchmarkTests(unittest.TestCase):
+    def test_dataset_has_valid_gold_relevance(self):
+        dataset = load_dataset()
+        doc_ids = {item["id"] for item in dataset["documents"]}
+        self.assertEqual(len(doc_ids), len(dataset["documents"]))
+        self.assertEqual(len(dataset["queries"]), 8)
+        self.assertTrue(all(set(item["relevant"]) <= doc_ids for item in dataset["queries"]))
+
+    def test_dataset_rejects_unknown_relevance_id(self):
+        bad = {
+            "version": 1,
+            "documents": [{"id": "d1", "text": "doc"}],
+            "queries": [{"id": "q1", "text": "query", "relevant": ["missing"]}],
+        }
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "bad.json"
+            path.write_text(json.dumps(bad), encoding="utf-8")
+            with self.assertRaises(ValueError):
+                load_dataset(path)
+
+    def test_provider_native_document_and_query_modes(self):
+        url, payload = embedding_batch_spec(
+            "COHERE", "embed-v4.0", ["text"], "document"
+        )
+        self.assertEqual(url, "https://api.cohere.com/v2/embed")
+        self.assertEqual(payload["input_type"], "search_document")
+        _, payload = embedding_batch_spec("COHERE", "embed-v4.0", ["text"], "query")
+        self.assertEqual(payload["input_type"], "search_query")
+
+        url, payload = embedding_batch_spec(
+            "GOOGLE", "models/gemini-embedding-001", ["text"], "query"
+        )
+        self.assertTrue(url.endswith(":batchEmbedContents"))
+        self.assertEqual(payload["requests"][0]["taskType"], "RETRIEVAL_QUERY")
+
+        _, payload = embedding_batch_spec(
+            "JINA", "jina-embeddings-v3", ["text"], "query"
+        )
+        self.assertEqual(payload["task"], "retrieval.query")
+
+    def test_batch_response_envelopes(self):
+        self.assertEqual(
+            extract_batch_vectors("GOOGLE", {"embeddings": [{"values": [1, 0]}]}),
+            [[1.0, 0.0]],
+        )
+        self.assertEqual(
+            extract_batch_vectors("COHERE", {"embeddings": {"float": [[1, 0]]}}),
+            [[1.0, 0.0]],
+        )
+        self.assertEqual(
+            extract_batch_vectors("OPENAI", {"data": [{"embedding": [1, 0]}]}),
+            [[1.0, 0.0]],
+        )
+
+    def test_batch_response_rejects_non_finite_or_inconsistent_vectors(self):
+        with self.assertRaises(ValueError):
+            extract_batch_vectors("OPENAI", {"data": [{"embedding": [1, float("nan")]}]})
+        with self.assertRaises(ValueError):
+            extract_batch_vectors("OPENAI", {
+                "data": [{"embedding": [1, 0]}, {"embedding": [1]}]
+            })
+
+    def test_metrics_for_perfect_retrieval(self):
+        docs = [[1, 0], [0, 1], [-1, 0]]
+        queries = [[1, 0]]
+        relevance = [{"id": "q1", "relevant": ["d1"]}]
+        metrics = ranking_metrics(queries, docs, ["d1", "d2", "d3"], relevance, k=2)
+        self.assertEqual(metrics["recall_at_2"], 1.0)
+        self.assertEqual(metrics["mrr_at_2"], 1.0)
+        self.assertEqual(metrics["ndcg_at_2"], 1.0)
+        self.assertEqual(metrics["per_query"][0]["top_document_ids"][0], "d1")
+
+    def test_metrics_penalize_lower_rank_and_missing_relevant_docs(self):
+        docs = [[0, 1], [1, 0], [-1, 0]]
+        queries = [[1, 0]]
+        relevance = [{"id": "q1", "relevant": ["d1", "d2"]}]
+        metrics = ranking_metrics(queries, docs, ["d1", "d2", "d3"], relevance, k=2)
+        self.assertLess(metrics["mrr_at_2"], 1.0)
+        self.assertEqual(metrics["recall_at_2"], 1.0)
+
+    def test_metrics_reject_dimension_mismatch(self):
+        with self.assertRaises(ValueError):
+            ranking_metrics([[1, 0]], [[1, 0, 0]], ["d1"], [{"id": "q", "relevant": ["d1"]}])
+
+    def test_default_account_selection_is_single_preferred_account(self):
+        with patch.dict("os.environ", {
+            "COHERE_API_KEY_HOTMAIL": "hotmail-secret",
+            "COHERE_API_KEY_FLOSSWARE": "flossware-secret",
+        }, clear=True):
+            accounts = select_account("COHERE")
+            self.assertEqual(len(accounts), 1)
+            self.assertEqual(accounts[0]["account"], "FLOSSWARE")
+            self.assertEqual(accounts[0]["credential"], "flossware-secret")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 5bad962f5ce7bdd9092653ba09c34cc1ee1cb952 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:41:35 -0500
Subject: [PATCH 107/128] Support package and direct-script imports in
 embedding benchmark

---
 tools/provider_api/embedding_benchmark.py | 35 +++++++++++++++--------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/tools/provider_api/embedding_benchmark.py b/tools/provider_api/embedding_benchmark.py
index 975dffd..f2e28d0 100644
--- a/tools/provider_api/embedding_benchmark.py
+++ b/tools/provider_api/embedding_benchmark.py
@@ -17,16 +17,26 @@
 from pathlib import Path
 from typing import Any
 
-from smoke_test import (
-    EMBEDDING_CASES,
-    PAID_ACCESS,
-    build_headers,
-    classify_http_error,
-    credential_accounts,
-    credential_account_from_env_name,
-    error_summary,
-    sanitize_error_detail,
-)
+try:
+    from .smoke_test import (
+        EMBEDDING_CASES,
+        PAID_ACCESS,
+        build_headers,
+        classify_http_error,
+        credential_accounts,
+        error_summary,
+        sanitize_error_detail,
+    )
+except ImportError:
+    from smoke_test import (
+        EMBEDDING_CASES,
+        PAID_ACCESS,
+        build_headers,
+        classify_http_error,
+        credential_accounts,
+        error_summary,
+        sanitize_error_detail,
+    )
 
 HERE = Path(__file__).resolve().parent
 DATASET_PATH = HERE / "embedding-benchmark.json"
@@ -231,8 +241,9 @@ def post_embeddings(
     except urllib.error.HTTPError as exc:
         detail = sanitize_error_detail(exc.read().decode("utf-8", errors="replace"), key)
         raise ProviderRequestError(exc.code, detail, time.monotonic() - started) from exc
-    except urllib.error.URLError as exc:
-        raise ProviderRequestError(None, str(exc.reason), time.monotonic() - started) from exc
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        reason = getattr(exc, "reason", exc)
+        raise ProviderRequestError(None, str(reason), time.monotonic() - started) from exc
 
 
 class ProviderRequestError(RuntimeError):

From eff9e888872d2d761b5639b956cc0972642a2bd0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:41:47 -0500
Subject: [PATCH 108/128] Correct reciprocal-rank regression fixture

---
 tests/test_embedding_benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_embedding_benchmark.py b/tests/test_embedding_benchmark.py
index 7cdbe00..344b83d 100644
--- a/tests/test_embedding_benchmark.py
+++ b/tests/test_embedding_benchmark.py
@@ -91,7 +91,7 @@ def test_metrics_for_perfect_retrieval(self):
     def test_metrics_penalize_lower_rank_and_missing_relevant_docs(self):
         docs = [[0, 1], [1, 0], [-1, 0]]
         queries = [[1, 0]]
-        relevance = [{"id": "q1", "relevant": ["d1", "d2"]}]
+        relevance = [{"id": "q1", "relevant": ["d1"]}]
         metrics = ranking_metrics(queries, docs, ["d1", "d2", "d3"], relevance, k=2)
         self.assertLess(metrics["mrr_at_2"], 1.0)
         self.assertEqual(metrics["recall_at_2"], 1.0)

From 298003e1fb1fc83bf83b337eedb2a0baeda2e921 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:42:20 -0500
Subject: [PATCH 109/128] Document semantic retrieval benchmark workflow

---
 README.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/README.md b/README.md
index 1f1e3fb..f6b490c 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,29 @@ python3 tools/provider_api/smoke_test.py --all-configured --capability embedding
 
 Access labels are conservative candidate classifications, not a promise that a particular account is free or has quota. The default execution excludes `paid` and `paid-or-account-credit` candidates. The explicit `--include-paid` flag may incur charges. There are no retries. The adapter validates vector shape and numeric values without retaining raw embeddings.
 
+#### Semantic retrieval benchmark
+
+`tools/provider_api/embedding_benchmark.py` evaluates actual retrieval ranking rather than endpoint reachability. Its checked-in dataset contains 16 short passages and 8 natural-language queries with explicit gold-relevant passage IDs. The runner embeds passages with the provider's document task mode and queries with its query task mode, ranks passages using cosine similarity, and reports macro Recall@5, MRR@5, and binary nDCG@5.
+
+```bash
+# Plan only. Defaults to Cohere and Google, one preferred credential per provider.
+python3 tools/provider_api/embedding_benchmark.py
+
+# Execute the two batch requests per provider (one for passages, one for queries).
+python3 tools/provider_api/embedding_benchmark.py --run
+
+# Select providers explicitly.
+python3 tools/provider_api/embedding_benchmark.py --provider COHERE --provider GOOGLE --run
+
+# Include every configured account, or paid/account-credit providers, only deliberately.
+python3 tools/provider_api/embedding_benchmark.py --all-accounts --run
+python3 tools/provider_api/embedding_benchmark.py --provider JINA --include-paid --run
+```
+
+The benchmark requires `--run` before making network requests. By default it selects only Cohere and Google, which passed the initial embedding smoke test, and uses one preferred credential per provider (FlossWare first). Paid/account-credit candidates are excluded unless `--include-paid` is supplied. `--all-accounts` multiplies requests by configured accounts and should be used deliberately. Each provider/account run makes exactly two batch requests when successful. Vectors remain in memory for ranking and are never written to evidence; JSONL records dimensions, latency, usage, per-query top document IDs, and aggregate metrics. No automatic retries are performed.
+
+The dataset is a small engineering-focused diagnostic, not a statistically representative benchmark or a substitute for a production corpus. Gold relevance judgments are hand-curated. Compare provider rankings on this same dataset, then validate promising candidates against representative FlossWare documents and queries before selecting a production embedding model. Vector spaces remain model-specific: do not mix Cohere and Gemini vectors in one fixed-dimension index.
+
 Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/endpoint/embeddings).
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:

From 04d223fb19003cf077ae36f4f163b64a6bca982a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:49:12 -0500
Subject: [PATCH 110/128] Support local corpus datasets and bounded embedding
 batches

---
 tools/provider_api/embedding_benchmark.py | 85 +++++++++++++++++++----
 1 file changed, 73 insertions(+), 12 deletions(-)

diff --git a/tools/provider_api/embedding_benchmark.py b/tools/provider_api/embedding_benchmark.py
index f2e28d0..a01202e 100644
--- a/tools/provider_api/embedding_benchmark.py
+++ b/tools/provider_api/embedding_benchmark.py
@@ -46,18 +46,39 @@
 
 
 def load_dataset(path: Path = DATASET_PATH) -> dict[str, Any]:
-    """Load and validate the fixed benchmark dataset."""
+    """Load and validate a benchmark dataset with binary relevance judgments."""
     data = json.loads(path.read_text(encoding="utf-8"))
     documents = data.get("documents", [])
     queries = data.get("queries", [])
-    doc_ids = [item["id"] for item in documents]
-    if not documents or not queries or len(doc_ids) != len(set(doc_ids)):
-        raise ValueError("Dataset must contain unique document IDs and at least one query")
+    if not isinstance(documents, list) or not isinstance(queries, list):
+        raise ValueError("Dataset documents and queries must be arrays")
+    doc_ids = [item.get("id") for item in documents if isinstance(item, dict)]
+    query_ids = [item.get("id") for item in queries if isinstance(item, dict)]
+    if (
+        not documents
+        or not queries
+        or len(doc_ids) != len(documents)
+        or len(query_ids) != len(queries)
+        or any(not isinstance(value, str) or not value.strip() for value in doc_ids + query_ids)
+        or len(doc_ids) != len(set(doc_ids))
+        or len(query_ids) != len(set(query_ids))
+    ):
+        raise ValueError("Dataset must contain non-empty unique document and query IDs")
+    if any(not isinstance(item.get("text"), str) or not item["text"].strip() for item in documents + queries):
+        raise ValueError("Every document and query must contain non-empty text")
     known = set(doc_ids)
     for query in queries:
         relevant = query.get("relevant", [])
-        if not relevant or not set(relevant).issubset(known):
+        if (
+            not isinstance(relevant, list)
+            or not relevant
+            or any(not isinstance(value, str) for value in relevant)
+            or not set(relevant).issubset(known)
+        ):
             raise ValueError(f"Query {query.get('id')} has missing or unknown relevance IDs")
+    data.setdefault("name", path.stem)
+    data.setdefault("source", "local-dataset")
+    data.setdefault("version", 1)
     return data
 
 
@@ -246,6 +267,33 @@ def post_embeddings(
         raise ProviderRequestError(None, str(reason), time.monotonic() - started) from exc
 
 
+def embed_in_batches(
+    provider: str,
+    model: str,
+    key: str,
+    texts: list[str],
+    mode: str,
+    timeout: float,
+    batch_size: int,
+) -> tuple[list[list[float]], dict[str, Any]]:
+    """Embed a larger corpus in bounded batches without persisting vectors."""
+    if batch_size < 1:
+        raise ValueError("Batch size must be positive")
+    vectors: list[list[float]] = []
+    requests: list[dict[str, Any]] = []
+    for start in range(0, len(texts), batch_size):
+        batch_vectors, metadata = post_embeddings(
+            provider, model, key, texts[start:start + batch_size], mode, timeout
+        )
+        vectors.extend(batch_vectors)
+        requests.append(metadata)
+    return vectors, {
+        "request_count": len(requests),
+        "latency_seconds": round(sum(item["latency_seconds"] for item in requests), 3),
+        "requests": requests,
+    }
+
+
 class ProviderRequestError(RuntimeError):
     def __init__(self, status: int | None, detail: str, elapsed: float):
         super().__init__(detail)
@@ -267,6 +315,7 @@ def run_provider(
     account: dict[str, str],
     dataset: dict[str, Any],
     timeout: float,
+    batch_size: int,
 ) -> dict[str, Any]:
     key = account["credential"]
     docs = dataset["documents"]
@@ -279,8 +328,11 @@ def run_provider(
         "credential_env": account["credential_env"],
         "model": case["model"],
         "access_classification": case["access"],
+        "dataset_name": dataset["name"],
+        "dataset_source": dataset["source"],
         "dataset_version": dataset["version"],
         "top_k": TOP_K,
+        "batch_size": batch_size,
         "document_count": len(docs),
         "query_count": len(queries),
         "status": "failed",
@@ -288,11 +340,13 @@ def run_provider(
         "vectors_persisted": False,
     }
     try:
-        document_vectors, document_meta = post_embeddings(
-            provider, case["model"], key, [item["text"] for item in docs], "document", timeout
+        document_vectors, document_meta = embed_in_batches(
+            provider, case["model"], key, [item["text"] for item in docs],
+            "document", timeout, batch_size
         )
-        query_vectors, query_meta = post_embeddings(
-            provider, case["model"], key, [item["text"] for item in queries], "query", timeout
+        query_vectors, query_meta = embed_in_batches(
+            provider, case["model"], key, [item["text"] for item in queries],
+            "query", timeout, batch_size
         )
         metrics = ranking_metrics(
             query_vectors, document_vectors, [item["id"] for item in docs], queries
@@ -330,11 +384,18 @@ def main() -> int:
     parser.add_argument("--include-paid", action="store_true", help="Explicitly allow paid/account-credit candidates.")
     parser.add_argument("--all-accounts", action="store_true", help="Run once per configured account instead of only the preferred account.")
     parser.add_argument("--timeout", type=float, default=45.0)
+    parser.add_argument("--batch-size", type=int, default=64, help="Maximum texts per provider request.")
+    parser.add_argument("--dataset", type=Path, default=DATASET_PATH, help="Benchmark JSON file; supports local corpus exports.")
     parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
     args = parser.parse_args()
     if args.timeout <= 0:
         parser.error("--timeout must be positive")
-    dataset = load_dataset()
+    if args.batch_size < 1:
+        parser.error("--batch-size must be positive")
+    try:
+        dataset = load_dataset(args.dataset)
+    except (OSError, json.JSONDecodeError, ValueError) as exc:
+        parser.error(f"invalid dataset {args.dataset}: {exc}")
     selected_providers = args.provider or list(DEFAULT_PROVIDERS)
     cases = {item["provider"]: item for item in EMBEDDING_CASES}
     plans: list[tuple[str, dict[str, str], dict[str, str]]] = []
@@ -352,7 +413,7 @@ def main() -> int:
             print(
                 f"{provider:10} {account['account']:10} {case['model']:32} "
                 f"planned ({len(dataset['documents'])} docs + {len(dataset['queries'])} queries; "
-                "2 batch requests)"
+                f"{math.ceil(len(dataset['documents']) / args.batch_size) + math.ceil(len(dataset['queries']) / args.batch_size)} batch requests)"
             )
     if not args.run:
         print("Planning only: no provider requests were made. Add --run to execute.")
@@ -361,7 +422,7 @@ def main() -> int:
         print("No eligible configured provider/account combinations.", file=sys.stderr)
         return 2
     results = [
-        run_provider(provider, case, account, dataset, args.timeout)
+        run_provider(provider, case, account, dataset, args.timeout, args.batch_size)
         for provider, case, account in plans
     ]
     args.output.parent.mkdir(parents=True, exist_ok=True)

From 60b53a9eb5e39a51477741b083771189a8c88033 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:49:52 -0500
Subject: [PATCH 111/128] Test external benchmark datasets and bounded batching

---
 tests/test_embedding_benchmark.py | 54 +++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/test_embedding_benchmark.py b/tests/test_embedding_benchmark.py
index 344b83d..c72633a 100644
--- a/tests/test_embedding_benchmark.py
+++ b/tests/test_embedding_benchmark.py
@@ -8,6 +8,7 @@
 from unittest.mock import patch
 
 from tools.provider_api.embedding_benchmark import (
+    embed_in_batches,
     embedding_batch_spec,
     extract_batch_vectors,
     load_dataset,
@@ -36,6 +37,59 @@ def test_dataset_rejects_unknown_relevance_id(self):
             with self.assertRaises(ValueError):
                 load_dataset(path)
 
+    def test_external_dataset_metadata_and_validation(self):
+        data = {
+            "name": "flossware-corpus-v1",
+            "source": "postgresql-export",
+            "version": 1,
+            "documents": [{"id": "chunk-1", "text": "A representative chunk."}],
+            "queries": [{"id": "query-1", "text": "Find a representative chunk.", "relevant": ["chunk-1"]}],
+        }
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "external.json"
+            path.write_text(json.dumps(data), encoding="utf-8")
+            loaded = load_dataset(path)
+        self.assertEqual(loaded["name"], "flossware-corpus-v1")
+        self.assertEqual(loaded["source"], "postgresql-export")
+
+    def test_dataset_rejects_empty_text_and_duplicate_query_ids(self):
+        bad_datasets = [
+            {
+                "documents": [{"id": "d1", "text": "doc"}],
+                "queries": [{"id": "q1", "text": " ", "relevant": ["d1"]}],
+            },
+            {
+                "documents": [{"id": "d1", "text": "doc"}],
+                "queries": [
+                    {"id": "q1", "text": "query one", "relevant": ["d1"]},
+                    {"id": "q1", "text": "query two", "relevant": ["d1"]},
+                ],
+            },
+        ]
+        with tempfile.TemporaryDirectory() as directory:
+            for index, data in enumerate(bad_datasets):
+                path = Path(directory) / f"bad-{index}.json"
+                path.write_text(json.dumps(data), encoding="utf-8")
+                with self.assertRaises(ValueError):
+                    load_dataset(path)
+
+    def test_embedding_batches_preserve_order_and_aggregate_metadata(self):
+        with patch(
+            "tools.provider_api.embedding_benchmark.post_embeddings",
+            side_effect=[
+                ([[1.0, 0.0], [0.0, 1.0]], {"latency_seconds": 0.2, "http_status": 200}),
+                ([[-1.0, 0.0]], {"latency_seconds": 0.3, "http_status": 200}),
+            ],
+        ) as request:
+            vectors, metadata = embed_in_batches(
+                "COHERE", "embed-v4.0", "secret", ["one", "two", "three"],
+                "document", 10.0, 2,
+            )
+        self.assertEqual(vectors, [[1.0, 0.0], [0.0, 1.0], [-1.0, 0.0]])
+        self.assertEqual(metadata["request_count"], 2)
+        self.assertEqual(metadata["latency_seconds"], 0.5)
+        self.assertEqual(request.call_count, 2)
+
     def test_provider_native_document_and_query_modes(self):
         url, payload = embedding_batch_spec(
             "COHERE", "embed-v4.0", ["text"], "document"

From 283271d3359679f557db66b32e7021e0728b9a36 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:50:15 -0500
Subject: [PATCH 112/128] Document real corpus benchmark dataset input

---
 README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/README.md b/README.md
index f6b490c..fc430d4 100644
--- a/README.md
+++ b/README.md
@@ -119,6 +119,56 @@ The benchmark requires `--run` before making network requests. By default it sel
 
 The dataset is a small engineering-focused diagnostic, not a statistically representative benchmark or a substitute for a production corpus. Gold relevance judgments are hand-curated. Compare provider rankings on this same dataset, then validate promising candidates against representative FlossWare documents and queries before selecting a production embedding model. Vector spaces remain model-specific: do not mix Cohere and Gemini vectors in one fixed-dimension index.
 
+#### Benchmarking an exported FlossWare corpus
+
+The semantic benchmark accepts an external JSON dataset through `--dataset`. This lets the evaluation harness stay independent of the live PostgreSQL schema and avoids coupling this repository to private corpus storage.
+
+Dataset format:
+
+```json
+{
+  "name": "flossware-corpus-v1",
+  "source": "postgresql-export",
+  "version": 1,
+  "documents": [
+    {
+      "id": "stable-chunk-id",
+      "text": "The canonical chunk text used for retrieval.",
+      "metadata": {
+        "document_id": "source-document-id",
+        "collection": "collection-name"
+      }
+    }
+  ],
+  "queries": [
+    {
+      "id": "q-001",
+      "text": "A realistic search query.",
+      "relevant": ["stable-chunk-id"]
+    }
+  ]
+}
+```
+
+Document metadata is optional and is not sent to embedding providers or written to result evidence. Document IDs are used to map relevance judgments and ranked results. Each query must have at least one manually reviewed relevant chunk ID. Keep this dataset local and out of Git when it contains private corpus text.
+
+Run planning and execution:
+
+```bash
+# Validate and plan only; no provider requests.
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json
+
+# Execute after reviewing the dataset and provider data-sharing implications.
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json --run
+
+# Larger datasets are sent in bounded batches (default 64 texts per request).
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json --batch-size 32
+```
+
+The benchmark sends the actual document chunks and query text to each selected external embedding provider when `--run` is supplied. Do not include credentials, secrets, personal data, Red Hat confidential material, or content that is not approved for that provider. Only aggregate metrics, stable IDs, and request metadata are written to JSONL evidence; raw vectors and source text are not persisted by the runner. The dataset file itself remains under the caller's control.
+
+For a credible corpus benchmark, sample representative chunks from the PostgreSQL knowledge corpus, create realistic queries, and have a human review the relevant chunk IDs. Do not infer relevance from an existing embedding model's nearest-neighbor results alone; that would bake the incumbent model's bias into the gold labels. Use a stable corpus snapshot and keep document/query sets identical across providers.
+
 Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/endpoint/embeddings).
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:

From 91a7b4e783ebae205b0b983548b15cdf9288e882 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:54:12 -0500
Subject: [PATCH 113/128] Add local private kbase corpus exporter

---
 tools/provider_api/export_kbase.py | 133 +++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 tools/provider_api/export_kbase.py

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
new file mode 100644
index 0000000..25ee22b
--- /dev/null
+++ b/tools/provider_api/export_kbase.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Export a private Obsidian kbase vault into a local embedding benchmark corpus.
+
+The exporter reads Markdown files and uses FlossWare's canonical chunking
+capability from a sibling checkout. It never calls a provider or writes vault
+content into this repository.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+DEFAULT_CHUNKING_REPO = Path(__file__).resolve().parents[3] / "chunking"
+DEFAULT_OUTPUT = Path("kbase-corpus-export.json")
+TITLE_RE = re.compile(r"^#\\s+(.+?)\\s*#*\\s*$", re.MULTILINE)
+
+
+def load_chunker(chunking_repo: Path):
+    """Load TokenChunker from a local FlossWare/chunking checkout."""
+    package_parent = chunking_repo.expanduser().resolve().parent
+    if not (package_parent / "chunking" / "chunker.py").is_file():
+        raise FileNotFoundError(
+            f"Cannot find chunking package under {package_parent}; "
+            "provide --chunking-repo pointing to the FlossWare/chunking checkout"
+        )
+    sys.path.insert(0, str(package_parent))
+    try:
+        from chunking import TokenChunker
+    except ImportError as exc:
+        raise RuntimeError(f"Could not import TokenChunker from {package_parent}") from exc
+    return TokenChunker
+
+
+def note_title(content: str, relative_path: Path) -> str:
+    match = TITLE_RE.search(content)
+    return match.group(1).strip() if match else relative_path.stem
+
+
+def export_vault(
+    vault: Path,
+    chunker: Any,
+    *,
+    max_tokens: int = 512,
+    overlap: int = 50,
+    exclude_dirs: tuple[str, ...] = (".obsidian", ".git", ".trash"),
+) -> dict[str, Any]:
+    """Create stable chunk records from Markdown notes without modifying the vault."""
+    root = vault.expanduser().resolve()
+    if not root.is_dir():
+        raise NotADirectoryError(f"Vault directory does not exist: {root}")
+    excluded = set(exclude_dirs)
+    documents: list[dict[str, Any]] = []
+    for source in sorted(root.rglob("*.md")):
+        if source.is_symlink() or any(part in excluded or part.startswith(".") for part in source.relative_to(root).parts):
+            continue
+        relative = source.relative_to(root)
+        content = source.read_text(encoding="utf-8")
+        if not content.strip():
+            continue
+        relative_name = relative.as_posix()
+        document_id = "note-" + hashlib.sha256(relative_name.encode("utf-8")).hexdigest()[:24]
+        title = note_title(content, relative)
+        chunks = chunker.chunk(
+            content,
+            document_id=document_id,
+            max_tokens=max_tokens,
+            overlap=overlap,
+            metadata={"source": "kbase", "relative_path": relative_name, "title": title},
+            provenance={"repository": "FlossWare/kbase", "format": "obsidian-markdown"},
+        )
+        for chunk in chunks:
+            documents.append({
+                "id": chunk.id,
+                "text": chunk.content,
+                "metadata": {
+                    "document_id": chunk.document_id,
+                    "sequence": chunk.sequence,
+                    "relative_path": relative_name,
+                    "title": title,
+                    "token_count": chunk.token_count,
+                    "start_offset": chunk.start_offset,
+                    "end_offset": chunk.end_offset,
+                },
+            })
+    return {
+        "name": "flossware-kbase-corpus",
+        "source": "private-kbase-local-export",
+        "version": 1,
+        "documents": documents,
+        "queries": [],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--vault", type=Path, required=True, help="Path to the local private kbase Obsidian vault.")
+    parser.add_argument("--chunking-repo", type=Path, default=DEFAULT_CHUNKING_REPO, help="Local FlossWare/chunking checkout.")
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Local output JSON; keep private corpus exports out of Git.")
+    parser.add_argument("--max-tokens", type=int, default=512)
+    parser.add_argument("--overlap", type=int, default=50)
+    parser.add_argument("--exclude-dir", action="append", default=[], help="Additional directory name to exclude; may be repeated.")
+    args = parser.parse_args()
+    if args.max_tokens < 1 or args.overlap < 0 or args.overlap >= args.max_tokens:
+        parser.error("require max-tokens > 0 and 0 <= overlap < max-tokens")
+    try:
+        TokenChunker = load_chunker(args.chunking_repo)
+        data = export_vault(
+            args.vault,
+            TokenChunker(),
+            max_tokens=args.max_tokens,
+            overlap=args.overlap,
+            exclude_dirs=(".obsidian", ".git", ".trash", *args.exclude_dir),
+        )
+    except (OSError, UnicodeError, RuntimeError, ValueError) as exc:
+        parser.error(str(exc))
+    if not data["documents"]:
+        parser.error("no Markdown chunks found; check the vault path and exclusions")
+    output = args.output.expanduser()
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\\n", encoding="utf-8")
+    print(f"Exported {len(data['documents'])} chunks from {len({item['metadata']['document_id'] for item in data['documents']})} notes")
+    print(f"Local corpus export: {output}")
+    print("No provider requests were made. Queries and relevance judgments are not included.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From d582027b077f5c571861792813157c42945dc7b0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:54:26 -0500
Subject: [PATCH 114/128] Support separate reviewed query and relevance files

---
 tools/provider_api/embedding_benchmark.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tools/provider_api/embedding_benchmark.py b/tools/provider_api/embedding_benchmark.py
index a01202e..7668c92 100644
--- a/tools/provider_api/embedding_benchmark.py
+++ b/tools/provider_api/embedding_benchmark.py
@@ -45,9 +45,12 @@
 TOP_K = 5
 
 
-def load_dataset(path: Path = DATASET_PATH) -> dict[str, Any]:
+def load_dataset(path: Path = DATASET_PATH, queries_path: Path | None = None) -> dict[str, Any]:
     """Load and validate a benchmark dataset with binary relevance judgments."""
     data = json.loads(path.read_text(encoding="utf-8"))
+    if queries_path is not None:
+        query_data = json.loads(queries_path.read_text(encoding="utf-8"))
+        data["queries"] = query_data.get("queries", query_data) if isinstance(query_data, dict) else query_data
     documents = data.get("documents", [])
     queries = data.get("queries", [])
     if not isinstance(documents, list) or not isinstance(queries, list):
@@ -386,6 +389,7 @@ def main() -> int:
     parser.add_argument("--timeout", type=float, default=45.0)
     parser.add_argument("--batch-size", type=int, default=64, help="Maximum texts per provider request.")
     parser.add_argument("--dataset", type=Path, default=DATASET_PATH, help="Benchmark JSON file; supports local corpus exports.")
+    parser.add_argument("--queries", type=Path, help="Separate human-reviewed query/relevance JSON file; overrides queries in the dataset.")
     parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
     args = parser.parse_args()
     if args.timeout <= 0:
@@ -393,7 +397,7 @@ def main() -> int:
     if args.batch_size < 1:
         parser.error("--batch-size must be positive")
     try:
-        dataset = load_dataset(args.dataset)
+        dataset = load_dataset(args.dataset, args.queries)
     except (OSError, json.JSONDecodeError, ValueError) as exc:
         parser.error(f"invalid dataset {args.dataset}: {exc}")
     selected_providers = args.provider or list(DEFAULT_PROVIDERS)

From 64e4f1902e5950a7915bd42749ee2a12a312922a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:55:19 -0500
Subject: [PATCH 115/128] Document local kbase export and reviewed query
 workflow

---
 README.md | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index fc430d4..f23abd3 100644
--- a/README.md
+++ b/README.md
@@ -121,14 +121,14 @@ The dataset is a small engineering-focused diagnostic, not a statistically repre
 
 #### Benchmarking an exported FlossWare corpus
 
-The semantic benchmark accepts an external JSON dataset through `--dataset`. This lets the evaluation harness stay independent of the live PostgreSQL schema and avoids coupling this repository to private corpus storage.
+The semantic benchmark accepts an external JSON dataset through `--dataset`. This keeps the evaluation harness independent of the corpus storage implementation.
 
 Dataset format:
 
 ```json
 {
   "name": "flossware-corpus-v1",
-  "source": "postgresql-export",
+  "source": "local-corpus-export",
   "version": 1,
   "documents": [
     {
@@ -156,7 +156,7 @@ Run planning and execution:
 
 ```bash
 # Validate and plan only; no provider requests.
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json
 
 # Execute after reviewing the dataset and provider data-sharing implications.
 python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json --run
@@ -169,6 +169,28 @@ The benchmark sends the actual document chunks and query text to each selected e
 
 For a credible corpus benchmark, sample representative chunks from the PostgreSQL knowledge corpus, create realistic queries, and have a human review the relevant chunk IDs. Do not infer relevance from an existing embedding model's nearest-neighbor results alone; that would bake the incumbent model's bias into the gold labels. Use a stable corpus snapshot and keep document/query sets identical across providers.
 
+
+#### Exporting the private Obsidian kbase
+
+The canonical FlossWare corpus is the private, Git-backed Obsidian vault in `FlossWare/kbase`. Do not copy its content into this public repository, fixtures, or release artifacts.
+
+The local exporter reads Markdown files from a vault path and delegates segmentation to the canonical `FlossWare/chunking` implementation. It excludes hidden directories plus `.obsidian`, `.git`, and `.trash` by default. The output contains stable note/chunk IDs, relative source paths, offsets, and title metadata. It makes no provider requests and does not modify the vault.
+
+```bash
+python3 tools/provider_api/export_kbase.py \\
+  --vault ~/Development/github/FlossWare/kbase \\
+  --output ~/kbase-corpus-export.json
+
+# Validate and plan only after preparing reviewed query judgments.
+python3 tools/provider_api/embedding_benchmark.py \\
+  --dataset ~/kbase-corpus-export.json \\
+  --queries ~/kbase-reviewed-queries.json
+```
+
+The query file may be either a JSON array or an object containing a `queries` array. Each entry has a unique query ID, query text, and a non-empty `relevant` list of exported chunk IDs. Relevance judgments must be reviewed by a human. Do not use an existing embedding model alone to generate the gold labels.
+
+Keep both exported corpus and query files outside Git. The benchmark transmits their text to selected external providers only when `--run` is supplied. Review the export and provider data-sharing rules before executing.
+
 Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/endpoint/embeddings).
 
 The non-paid candidate set includes Google Gemini 3.5 Flash-Lite, Mistral Small (free API mode), Cohere Command R7B (trial key), catalogue-marked OpenRouter `:free` models (including DeepSeek V4 Flash 0731), and the DeepInfra-hosted DeepSeek V4 Flash 0731 route through Hugging Face's monthly included credits. The explicit paid-comparison batch includes Cerebras, DeepSeek, Groq, and a DeepInfra representative (`deepseek-ai/DeepSeek-V4-Flash-0731`) using DeepInfra's OpenAI-compatible chat endpoint. Hugging Face Inference Providers is also tested through its routed chat-completions endpoint. NVIDIA NIM's previously tested `openai/gpt-oss-120b` endpoint is marked deprecated and is excluded from execution. Paid or promotional-credit comparison models require explicit opt-in:

From d89e2732285b3797ed23a993fa8c0ef26eee551e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:56:05 -0500
Subject: [PATCH 116/128] Fix Markdown title parsing in kbase exporter

---
 tools/provider_api/export_kbase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
index 25ee22b..57edf64 100644
--- a/tools/provider_api/export_kbase.py
+++ b/tools/provider_api/export_kbase.py
@@ -17,7 +17,7 @@
 
 DEFAULT_CHUNKING_REPO = Path(__file__).resolve().parents[3] / "chunking"
 DEFAULT_OUTPUT = Path("kbase-corpus-export.json")
-TITLE_RE = re.compile(r"^#\\s+(.+?)\\s*#*\\s*$", re.MULTILINE)
+TITLE_RE = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.MULTILINE)
 
 
 def load_chunker(chunking_repo: Path):

From 746f7cec1cfaa132b159842c8fdb8218117424a0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:56:35 -0500
Subject: [PATCH 117/128] Write valid newline-terminated corpus JSON

---
 tools/provider_api/export_kbase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
index 57edf64..cec8d5c 100644
--- a/tools/provider_api/export_kbase.py
+++ b/tools/provider_api/export_kbase.py
@@ -122,7 +122,7 @@ def main() -> int:
         parser.error("no Markdown chunks found; check the vault path and exclusions")
     output = args.output.expanduser()
     output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\\n", encoding="utf-8")
+    output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
     print(f"Exported {len(data['documents'])} chunks from {len({item['metadata']['document_id'] for item in data['documents']})} notes")
     print(f"Local corpus export: {output}")
     print("No provider requests were made. Queries and relevance judgments are not included.")

From a31c3f9401f7efdfc2234905a43fe9f4bdda13d0 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:56:48 -0500
Subject: [PATCH 118/128] Align benchmark examples with separate kbase
 judgments

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index f23abd3..491cb1b 100644
--- a/README.md
+++ b/README.md
@@ -159,10 +159,10 @@ Run planning and execution:
 python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json
 
 # Execute after reviewing the dataset and provider data-sharing implications.
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json --run
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json --run
 
 # Larger datasets are sent in bounded batches (default 64 texts per request).
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/flossware-benchmark.json --batch-size 32
+python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json --batch-size 32
 ```
 
 The benchmark sends the actual document chunks and query text to each selected external embedding provider when `--run` is supplied. Do not include credentials, secrets, personal data, Red Hat confidential material, or content that is not approved for that provider. Only aggregate metrics, stable IDs, and request metadata are written to JSONL evidence; raw vectors and source text are not persisted by the runner. The dataset file itself remains under the caller's control.

From 0909099fdd1037325cbc3c3ff3cdb6c291014f9f Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 03:57:07 -0500
Subject: [PATCH 119/128] Format kbase exporter and annotate loader

---
 tools/provider_api/export_kbase.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
index cec8d5c..21c8e70 100644
--- a/tools/provider_api/export_kbase.py
+++ b/tools/provider_api/export_kbase.py
@@ -20,7 +20,7 @@
 TITLE_RE = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.MULTILINE)
 
 
-def load_chunker(chunking_repo: Path):
+def load_chunker(chunking_repo: Path) -> Any:
     """Load TokenChunker from a local FlossWare/chunking checkout."""
     package_parent = chunking_repo.expanduser().resolve().parent
     if not (package_parent / "chunking" / "chunker.py").is_file():
@@ -56,7 +56,10 @@ def export_vault(
     excluded = set(exclude_dirs)
     documents: list[dict[str, Any]] = []
     for source in sorted(root.rglob("*.md")):
-        if source.is_symlink() or any(part in excluded or part.startswith(".") for part in source.relative_to(root).parts):
+        relative_parts = source.relative_to(root).parts
+        if source.is_symlink() or any(
+            part in excluded or part.startswith(".") for part in relative_parts
+        ):
             continue
         relative = source.relative_to(root)
         content = source.read_text(encoding="utf-8")
@@ -98,12 +101,24 @@ def export_vault(
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--vault", type=Path, required=True, help="Path to the local private kbase Obsidian vault.")
-    parser.add_argument("--chunking-repo", type=Path, default=DEFAULT_CHUNKING_REPO, help="Local FlossWare/chunking checkout.")
-    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Local output JSON; keep private corpus exports out of Git.")
+    parser.add_argument(
+        "--vault", type=Path, required=True,
+        help="Path to the local private kbase Obsidian vault.",
+    )
+    parser.add_argument(
+        "--chunking-repo", type=Path, default=DEFAULT_CHUNKING_REPO,
+        help="Local FlossWare/chunking checkout.",
+    )
+    parser.add_argument(
+        "--output", type=Path, default=DEFAULT_OUTPUT,
+        help="Local output JSON; keep corpus exports out of Git.",
+    )
     parser.add_argument("--max-tokens", type=int, default=512)
     parser.add_argument("--overlap", type=int, default=50)
-    parser.add_argument("--exclude-dir", action="append", default=[], help="Additional directory name to exclude; may be repeated.")
+    parser.add_argument(
+        "--exclude-dir", action="append", default=[],
+        help="Additional directory name to exclude; may be repeated.",
+    )
     args = parser.parse_args()
     if args.max_tokens < 1 or args.overlap < 0 or args.overlap >= args.max_tokens:
         parser.error("require max-tokens > 0 and 0 <= overlap < max-tokens")
@@ -123,7 +138,8 @@ def main() -> int:
     output = args.output.expanduser()
     output.parent.mkdir(parents=True, exist_ok=True)
     output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    print(f"Exported {len(data['documents'])} chunks from {len({item['metadata']['document_id'] for item in data['documents']})} notes")
+    note_count = len({item["metadata"]["document_id"] for item in data["documents"]})
+    print(f"Exported {len(data['documents'])} chunks from {note_count} notes")
     print(f"Local corpus export: {output}")
     print("No provider requests were made. Queries and relevance judgments are not included.")
     return 0

From 3c0f3bd95ecdca248e7e120e6aac4f8e9c104b15 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:09:56 -0500
Subject: [PATCH 120/128] Fix canonical chunking checkout path resolution

---
 tools/provider_api/export_kbase.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
index 21c8e70..c4e171d 100644
--- a/tools/provider_api/export_kbase.py
+++ b/tools/provider_api/export_kbase.py
@@ -22,17 +22,18 @@
 
 def load_chunker(chunking_repo: Path) -> Any:
     """Load TokenChunker from a local FlossWare/chunking checkout."""
-    package_parent = chunking_repo.expanduser().resolve().parent
-    if not (package_parent / "chunking" / "chunker.py").is_file():
+    repo_root = chunking_repo.expanduser().resolve()
+    package_dir = repo_root / "chunking"
+    if not (package_dir / "chunker.py").is_file():
         raise FileNotFoundError(
-            f"Cannot find chunking package under {package_parent}; "
+            f"Cannot find chunking package at {package_dir}; "
             "provide --chunking-repo pointing to the FlossWare/chunking checkout"
         )
-    sys.path.insert(0, str(package_parent))
+    sys.path.insert(0, str(repo_root))
     try:
         from chunking import TokenChunker
     except ImportError as exc:
-        raise RuntimeError(f"Could not import TokenChunker from {package_parent}") from exc
+        raise RuntimeError(f"Could not import TokenChunker from {repo_root}") from exc
     return TokenChunker
 
 

From 9d15209f975b99fafa7ac0ae25842f06b11a188e Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:16:21 -0500
Subject: [PATCH 121/128] Make corpus exporter independent of FlossWare
 chunking repo

---
 tools/provider_api/export_kbase.py | 103 +++++++++++++++++------------
 1 file changed, 61 insertions(+), 42 deletions(-)

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
index c4e171d..1989a69 100644
--- a/tools/provider_api/export_kbase.py
+++ b/tools/provider_api/export_kbase.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-"""Export a private Obsidian kbase vault into a local embedding benchmark corpus.
+"""Export a private Obsidian vault into an experimental retrieval corpus.
 
-The exporter reads Markdown files and uses FlossWare's canonical chunking
-capability from a sibling checkout. It never calls a provider or writes vault
-content into this repository.
+This research utility uses a deliberately simple whitespace-window baseline.
+It has no runtime dependency on another FlossWare repository and makes no
+provider requests. It never modifies the source vault.
 """
 from __future__ import annotations
 
@@ -11,30 +11,13 @@
 import hashlib
 import json
 import re
-import sys
 from pathlib import Path
 from typing import Any
 
-DEFAULT_CHUNKING_REPO = Path(__file__).resolve().parents[3] / "chunking"
 DEFAULT_OUTPUT = Path("kbase-corpus-export.json")
 TITLE_RE = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.MULTILINE)
-
-
-def load_chunker(chunking_repo: Path) -> Any:
-    """Load TokenChunker from a local FlossWare/chunking checkout."""
-    repo_root = chunking_repo.expanduser().resolve()
-    package_dir = repo_root / "chunking"
-    if not (package_dir / "chunker.py").is_file():
-        raise FileNotFoundError(
-            f"Cannot find chunking package at {package_dir}; "
-            "provide --chunking-repo pointing to the FlossWare/chunking checkout"
-        )
-    sys.path.insert(0, str(repo_root))
-    try:
-        from chunking import TokenChunker
-    except ImportError as exc:
-        raise RuntimeError(f"Could not import TokenChunker from {repo_root}") from exc
-    return TokenChunker
+TOKEN_RE = re.compile(r"\S+")
+CHUNKING_STRATEGY = "whitespace-window-baseline-v1"
 
 
 def note_title(content: str, relative_path: Path) -> str:
@@ -42,9 +25,49 @@ def note_title(content: str, relative_path: Path) -> str:
     return match.group(1).strip() if match else relative_path.stem
 
 
+def chunk_text(
+    content: str,
+    *,
+    document_id: str,
+    max_tokens: int = 512,
+    overlap: int = 50,
+    metadata: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    """Split text on whitespace-token boundaries while preserving source offsets.
+
+    Token counts are approximate word counts, not model-tokenizer counts. This
+    intentionally modest baseline exists for comparative research only.
+    """
+    if max_tokens < 1 or overlap < 0 or overlap >= max_tokens:
+        raise ValueError("require max_tokens > 0 and 0 <= overlap < max_tokens")
+
+    matches = list(TOKEN_RE.finditer(content))
+    chunks: list[dict[str, Any]] = []
+    step = max_tokens - overlap
+    for sequence, start_index in enumerate(range(0, len(matches), step)):
+        window = matches[start_index:start_index + max_tokens]
+        if not window:
+            break
+        start_offset = window[0].start()
+        end_offset = window[-1].end()
+        chunks.append({
+            "id": f"{document_id}:{sequence}",
+            "document_id": document_id,
+            "sequence": sequence,
+            "content": content[start_offset:end_offset],
+            "token_count": len(window),
+            "start_offset": start_offset,
+            "end_offset": end_offset,
+            "metadata": dict(metadata or {}),
+            "provenance": {"chunking_strategy": CHUNKING_STRATEGY},
+        })
+        if start_index + max_tokens >= len(matches):
+            break
+    return chunks
+
+
 def export_vault(
     vault: Path,
-    chunker: Any,
     *,
     max_tokens: int = 512,
     overlap: int = 50,
@@ -69,32 +92,33 @@ def export_vault(
         relative_name = relative.as_posix()
         document_id = "note-" + hashlib.sha256(relative_name.encode("utf-8")).hexdigest()[:24]
         title = note_title(content, relative)
-        chunks = chunker.chunk(
+        chunks = chunk_text(
             content,
             document_id=document_id,
             max_tokens=max_tokens,
             overlap=overlap,
-            metadata={"source": "kbase", "relative_path": relative_name, "title": title},
-            provenance={"repository": "FlossWare/kbase", "format": "obsidian-markdown"},
+            metadata={"source": "local-kbase-export", "relative_path": relative_name, "title": title},
         )
         for chunk in chunks:
             documents.append({
-                "id": chunk.id,
-                "text": chunk.content,
+                "id": chunk["id"],
+                "text": chunk["content"],
                 "metadata": {
-                    "document_id": chunk.document_id,
-                    "sequence": chunk.sequence,
+                    "document_id": chunk["document_id"],
+                    "sequence": chunk["sequence"],
                     "relative_path": relative_name,
                     "title": title,
-                    "token_count": chunk.token_count,
-                    "start_offset": chunk.start_offset,
-                    "end_offset": chunk.end_offset,
+                    "token_count": chunk["token_count"],
+                    "start_offset": chunk["start_offset"],
+                    "end_offset": chunk["end_offset"],
+                    "chunking_strategy": CHUNKING_STRATEGY,
                 },
             })
     return {
-        "name": "flossware-kbase-corpus",
+        "name": "kbase-corpus-export",
         "source": "private-kbase-local-export",
         "version": 1,
+        "chunking_strategy": CHUNKING_STRATEGY,
         "documents": documents,
         "queries": [],
     }
@@ -106,10 +130,6 @@ def main() -> int:
         "--vault", type=Path, required=True,
         help="Path to the local private kbase Obsidian vault.",
     )
-    parser.add_argument(
-        "--chunking-repo", type=Path, default=DEFAULT_CHUNKING_REPO,
-        help="Local FlossWare/chunking checkout.",
-    )
     parser.add_argument(
         "--output", type=Path, default=DEFAULT_OUTPUT,
         help="Local output JSON; keep corpus exports out of Git.",
@@ -124,15 +144,13 @@ def main() -> int:
     if args.max_tokens < 1 or args.overlap < 0 or args.overlap >= args.max_tokens:
         parser.error("require max-tokens > 0 and 0 <= overlap < max-tokens")
     try:
-        TokenChunker = load_chunker(args.chunking_repo)
         data = export_vault(
             args.vault,
-            TokenChunker(),
             max_tokens=args.max_tokens,
             overlap=args.overlap,
             exclude_dirs=(".obsidian", ".git", ".trash", *args.exclude_dir),
         )
-    except (OSError, UnicodeError, RuntimeError, ValueError) as exc:
+    except (OSError, UnicodeError, ValueError) as exc:
         parser.error(str(exc))
     if not data["documents"]:
         parser.error("no Markdown chunks found; check the vault path and exclusions")
@@ -141,6 +159,7 @@ def main() -> int:
     output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
     note_count = len({item["metadata"]["document_id"] for item in data["documents"]})
     print(f"Exported {len(data['documents'])} chunks from {note_count} notes")
+    print(f"Chunking strategy: {CHUNKING_STRATEGY} (approximate word counts)")
     print(f"Local corpus export: {output}")
     print("No provider requests were made. Queries and relevance judgments are not included.")
     return 0

From 5385ca1eeae3690333692a34289400c091887235 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:16:44 -0500
Subject: [PATCH 122/128] Test standalone corpus export baseline

---
 tests/test_export_kbase.py | 87 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 tests/test_export_kbase.py

diff --git a/tests/test_export_kbase.py b/tests/test_export_kbase.py
new file mode 100644
index 0000000..d1d7d60
--- /dev/null
+++ b/tests/test_export_kbase.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from tools.provider_api.export_kbase import (
+    CHUNKING_STRATEGY,
+    chunk_text,
+    export_vault,
+)
+
+
+class ChunkTextTests(unittest.TestCase):
+    def test_chunks_preserve_text_offsets_and_overlap(self) -> None:
+        source = "one two three four five six"
+        chunks = chunk_text(
+            source,
+            document_id="doc-1",
+            max_tokens=3,
+            overlap=1,
+        )
+
+        self.assertEqual([item["id"] for item in chunks], ["doc-1:0", "doc-1:1"])
+        self.assertEqual([item["content"] for item in chunks], [
+            "one two three",
+            "three four five",
+        ])
+        self.assertEqual([item["token_count"] for item in chunks], [3, 3])
+        for item in chunks:
+            self.assertEqual(
+                source[item["start_offset"]:item["end_offset"]],
+                item["content"],
+            )
+            self.assertEqual(
+                item["provenance"]["chunking_strategy"],
+                CHUNKING_STRATEGY,
+            )
+
+    def test_rejects_invalid_window(self) -> None:
+        with self.assertRaises(ValueError):
+            chunk_text("one two", document_id="doc-1", max_tokens=2, overlap=2)
+
+
+class ExportVaultTests(unittest.TestCase):
+    def test_exports_markdown_and_excludes_hidden_directories(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            vault = Path(temporary)
+            (vault / "topic.md").write_text(
+                "# Topic\n\nalpha beta gamma delta", encoding="utf-8"
+            )
+            (vault / ".obsidian").mkdir()
+            (vault / ".obsidian" / "settings.md").write_text(
+                "must not export", encoding="utf-8"
+            )
+            (vault / "subdir").mkdir()
+            (vault / "subdir" / "empty.md").write_text("  \n", encoding="utf-8")
+
+            result = export_vault(vault, max_tokens=3, overlap=1)
+
+        self.assertEqual(result["chunking_strategy"], CHUNKING_STRATEGY)
+        self.assertEqual(len(result["documents"]), 2)
+        self.assertEqual(
+            [item["text"] for item in result["documents"]],
+            ["# Topic\n\nalpha beta", "beta gamma delta"],
+        )
+        self.assertTrue(all(
+            item["metadata"]["relative_path"] == "topic.md"
+            for item in result["documents"]
+        ))
+
+    def test_note_ids_are_stable_for_same_relative_path(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            vault = Path(temporary)
+            note = vault / "topic.md"
+            note.write_text("one two three", encoding="utf-8")
+            first = export_vault(vault, max_tokens=10, overlap=0)
+            second = export_vault(vault, max_tokens=10, overlap=0)
+
+        self.assertEqual(
+            [item["id"] for item in first["documents"]],
+            [item["id"] for item in second["documents"]],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

From bf079dd7827a05dd17c10c5bdfd5b4be9b3178f6 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:17:27 -0500
Subject: [PATCH 123/128] Fix and verify exporter baseline tests

---
 tests/test_export_kbase.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/test_export_kbase.py b/tests/test_export_kbase.py
index d1d7d60..51df945 100644
--- a/tests/test_export_kbase.py
+++ b/tests/test_export_kbase.py
@@ -21,12 +21,15 @@ def test_chunks_preserve_text_offsets_and_overlap(self) -> None:
             overlap=1,
         )
 
-        self.assertEqual([item["id"] for item in chunks], ["doc-1:0", "doc-1:1"])
-        self.assertEqual([item["content"] for item in chunks], [
-            "one two three",
-            "three four five",
-        ])
-        self.assertEqual([item["token_count"] for item in chunks], [3, 3])
+        self.assertEqual(
+            [item["id"] for item in chunks],
+            ["doc-1:0", "doc-1:1", "doc-1:2"],
+        )
+        self.assertEqual(
+            [item["content"] for item in chunks],
+            ["one two three", "three four five", "five six"],
+        )
+        self.assertEqual([item["token_count"] for item in chunks], [3, 3, 2])
         for item in chunks:
             self.assertEqual(
                 source[item["start_offset"]:item["end_offset"]],
@@ -59,10 +62,10 @@ def test_exports_markdown_and_excludes_hidden_directories(self) -> None:
             result = export_vault(vault, max_tokens=3, overlap=1)
 
         self.assertEqual(result["chunking_strategy"], CHUNKING_STRATEGY)
-        self.assertEqual(len(result["documents"]), 2)
+        self.assertEqual(len(result["documents"]), 3)
         self.assertEqual(
             [item["text"] for item in result["documents"]],
-            ["# Topic\n\nalpha beta", "beta gamma delta"],
+            ["# Topic\n\nalpha", "alpha beta gamma", "gamma delta"],
         )
         self.assertTrue(all(
             item["metadata"]["relative_path"] == "topic.md"

From 114a9f94dea0c8b96be318bdb39aeb46a64c845a Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:17:47 -0500
Subject: [PATCH 124/128] Clarify evaluation as independent Ensemble research

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 491cb1b..0897b00 100644
--- a/README.md
+++ b/README.md
@@ -167,14 +167,14 @@ python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpu
 
 The benchmark sends the actual document chunks and query text to each selected external embedding provider when `--run` is supplied. Do not include credentials, secrets, personal data, Red Hat confidential material, or content that is not approved for that provider. Only aggregate metrics, stable IDs, and request metadata are written to JSONL evidence; raw vectors and source text are not persisted by the runner. The dataset file itself remains under the caller's control.
 
-For a credible corpus benchmark, sample representative chunks from the PostgreSQL knowledge corpus, create realistic queries, and have a human review the relevant chunk IDs. Do not infer relevance from an existing embedding model's nearest-neighbor results alone; that would bake the incumbent model's bias into the gold labels. Use a stable corpus snapshot and keep document/query sets identical across providers.
+For a credible corpus benchmark, sample representative material from the private kbase corpus, create realistic queries, and have a human review the relevant chunk IDs. Do not infer relevance from an existing embedding model's nearest-neighbor results alone; that would bake the incumbent model's bias into the gold labels. Use a stable corpus snapshot and keep document/query sets identical across providers. Chunking is an experimental variable in this research phase, not an assumed shared FlossWare dependency.
 
 
 #### Exporting the private Obsidian kbase
 
 The canonical FlossWare corpus is the private, Git-backed Obsidian vault in `FlossWare/kbase`. Do not copy its content into this public repository, fixtures, or release artifacts.
 
-The local exporter reads Markdown files from a vault path and delegates segmentation to the canonical `FlossWare/chunking` implementation. It excludes hidden directories plus `.obsidian`, `.git`, and `.trash` by default. The output contains stable note/chunk IDs, relative source paths, offsets, and title metadata. It makes no provider requests and does not modify the vault.
+The local exporter reads Markdown files from a vault path and applies a deliberately simple whitespace-window baseline (`whitespace-window-baseline-v1`). This is an experimental comparison point, not a canonical tokenizer or a production chunking decision. Token counts are approximate word counts. The exporter has no dependency on another FlossWare repository. It excludes hidden directories plus `.obsidian`, `.git`, and `.trash` by default. The output contains stable note/chunk IDs, relative source paths, offsets, and title metadata. It makes no provider requests and does not modify the vault. Alternative chunkers can be evaluated independently against the same corpus and judgments.
 
 ```bash
 python3 tools/provider_api/export_kbase.py \\

From 8aad03b36c9d17001db9115979c8ffdeebe0965c Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:18:27 -0500
Subject: [PATCH 125/128] Document Evaluation to Ensemble research progression

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 0897b00..f7f521a 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,8 @@ Evaluation results should distinguish measured facts, evaluator judgments, confi
 
 Experimental provider-discovery and smoke-test scripts live under `tools/provider_api/`. They support the cross-project investigation documented in [FlossWare/research/provider-api-inference](https://github.com/FlossWare/research/tree/research/provider-api-inference-2026-11/provider-api-inference).
 
+This provider/model/capability work is **exploratory research for Ensemble**. Evaluation records what works and why; Ensemble will be the more formal implementation used to test and integrate those lessons. Durable production FlossWare capabilities come later. These scripts are not Ensemble runtime components, and experiments must not assume that an existing FlossWare repository is a required dependency or the predetermined winner.
+
 These scripts are **research utilities, not part of the published `evaluation-ai` package**:
 
 - `catalogue_inventory.sh` reads authenticated provider model catalogues and writes a normalized local JSON report. It does not invoke models.

From 8a67b8e4e623184f6d484e6be8c2a5eab90f4613 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:23:41 -0500
Subject: [PATCH 126/128] Remove private corpus and kbase workflow from
 evaluation research docs

---
 README.md | 72 ++-----------------------------------------------------
 1 file changed, 2 insertions(+), 70 deletions(-)

diff --git a/README.md b/README.md
index f7f521a..d389a81 100644
--- a/README.md
+++ b/README.md
@@ -121,77 +121,9 @@ The benchmark requires `--run` before making network requests. By default it sel
 
 The dataset is a small engineering-focused diagnostic, not a statistically representative benchmark or a substitute for a production corpus. Gold relevance judgments are hand-curated. Compare provider rankings on this same dataset, then validate promising candidates against representative FlossWare documents and queries before selecting a production embedding model. Vector spaces remain model-specific: do not mix Cohere and Gemini vectors in one fixed-dimension index.
 
-#### Benchmarking an exported FlossWare corpus
-
-The semantic benchmark accepts an external JSON dataset through `--dataset`. This keeps the evaluation harness independent of the corpus storage implementation.
-
-Dataset format:
-
-```json
-{
-  "name": "flossware-corpus-v1",
-  "source": "local-corpus-export",
-  "version": 1,
-  "documents": [
-    {
-      "id": "stable-chunk-id",
-      "text": "The canonical chunk text used for retrieval.",
-      "metadata": {
-        "document_id": "source-document-id",
-        "collection": "collection-name"
-      }
-    }
-  ],
-  "queries": [
-    {
-      "id": "q-001",
-      "text": "A realistic search query.",
-      "relevant": ["stable-chunk-id"]
-    }
-  ]
-}
-```
-
-Document metadata is optional and is not sent to embedding providers or written to result evidence. Document IDs are used to map relevance judgments and ranked results. Each query must have at least one manually reviewed relevant chunk ID. Keep this dataset local and out of Git when it contains private corpus text.
-
-Run planning and execution:
-
-```bash
-# Validate and plan only; no provider requests.
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json
-
-# Execute after reviewing the dataset and provider data-sharing implications.
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json --run
-
-# Larger datasets are sent in bounded batches (default 64 texts per request).
-python3 tools/provider_api/embedding_benchmark.py --dataset /path/to/kbase-corpus-export.json --queries /path/to/reviewed-queries.json --batch-size 32
-```
-
-The benchmark sends the actual document chunks and query text to each selected external embedding provider when `--run` is supplied. Do not include credentials, secrets, personal data, Red Hat confidential material, or content that is not approved for that provider. Only aggregate metrics, stable IDs, and request metadata are written to JSONL evidence; raw vectors and source text are not persisted by the runner. The dataset file itself remains under the caller's control.
-
-For a credible corpus benchmark, sample representative material from the private kbase corpus, create realistic queries, and have a human review the relevant chunk IDs. Do not infer relevance from an existing embedding model's nearest-neighbor results alone; that would bake the incumbent model's bias into the gold labels. Use a stable corpus snapshot and keep document/query sets identical across providers. Chunking is an experimental variable in this research phase, not an assumed shared FlossWare dependency.
-
-
-#### Exporting the private Obsidian kbase
-
-The canonical FlossWare corpus is the private, Git-backed Obsidian vault in `FlossWare/kbase`. Do not copy its content into this public repository, fixtures, or release artifacts.
-
-The local exporter reads Markdown files from a vault path and applies a deliberately simple whitespace-window baseline (`whitespace-window-baseline-v1`). This is an experimental comparison point, not a canonical tokenizer or a production chunking decision. Token counts are approximate word counts. The exporter has no dependency on another FlossWare repository. It excludes hidden directories plus `.obsidian`, `.git`, and `.trash` by default. The output contains stable note/chunk IDs, relative source paths, offsets, and title metadata. It makes no provider requests and does not modify the vault. Alternative chunkers can be evaluated independently against the same corpus and judgments.
-
-```bash
-python3 tools/provider_api/export_kbase.py \\
-  --vault ~/Development/github/FlossWare/kbase \\
-  --output ~/kbase-corpus-export.json
-
-# Validate and plan only after preparing reviewed query judgments.
-python3 tools/provider_api/embedding_benchmark.py \\
-  --dataset ~/kbase-corpus-export.json \\
-  --queries ~/kbase-reviewed-queries.json
-```
-
-The query file may be either a JSON array or an object containing a `queries` array. Each entry has a unique query ID, query text, and a non-empty `relevant` list of exported chunk IDs. Relevance judgments must be reviewed by a human. Do not use an existing embedding model alone to generate the gold labels.
+#### Optional external benchmark datasets
 
-Keep both exported corpus and query files outside Git. The benchmark transmits their text to selected external providers only when `--run` is supplied. Review the export and provider data-sharing rules before executing.
+The benchmark accepts a separate JSON dataset through `--dataset`, allowing public or synthetic corpora to be evaluated without coupling the harness to a storage system. This is optional; the checked-in synthetic dataset is self-contained and is sufficient to run the initial comparisons. Any external dataset should use stable document IDs and manually reviewed relevance judgments. Keep private or restricted material out of this repository and do not transmit it to external providers unless explicitly approved.
 
 Provider API references: [Voyage embeddings](https://docs.voyageai.com/docs/embeddings), [Jina embeddings](https://docs.jina.ai/), [Cohere Embed v2](https://docs.cohere.com/reference/embed), [Google embedContent](https://ai.google.dev/api/embeddings), [OpenAI embeddings](https://developers.openai.com/api/reference/resources/embeddings/methods/create), [Mistral embeddings](https://docs.mistral.ai/api/endpoint/embeddings).
 

From faf59d48b87ef50fc1af42768a6ca9eb5cc8b9c7 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:23:53 -0500
Subject: [PATCH 127/128] Remove unnecessary private corpus exporter

---
 tools/provider_api/export_kbase.py | 169 -----------------------------
 1 file changed, 169 deletions(-)
 delete mode 100644 tools/provider_api/export_kbase.py

diff --git a/tools/provider_api/export_kbase.py b/tools/provider_api/export_kbase.py
deleted file mode 100644
index 1989a69..0000000
--- a/tools/provider_api/export_kbase.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-"""Export a private Obsidian vault into an experimental retrieval corpus.
-
-This research utility uses a deliberately simple whitespace-window baseline.
-It has no runtime dependency on another FlossWare repository and makes no
-provider requests. It never modifies the source vault.
-"""
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import re
-from pathlib import Path
-from typing import Any
-
-DEFAULT_OUTPUT = Path("kbase-corpus-export.json")
-TITLE_RE = re.compile(r"^#\s+(.+?)\s*#*\s*$", re.MULTILINE)
-TOKEN_RE = re.compile(r"\S+")
-CHUNKING_STRATEGY = "whitespace-window-baseline-v1"
-
-
-def note_title(content: str, relative_path: Path) -> str:
-    match = TITLE_RE.search(content)
-    return match.group(1).strip() if match else relative_path.stem
-
-
-def chunk_text(
-    content: str,
-    *,
-    document_id: str,
-    max_tokens: int = 512,
-    overlap: int = 50,
-    metadata: dict[str, Any] | None = None,
-) -> list[dict[str, Any]]:
-    """Split text on whitespace-token boundaries while preserving source offsets.
-
-    Token counts are approximate word counts, not model-tokenizer counts. This
-    intentionally modest baseline exists for comparative research only.
-    """
-    if max_tokens < 1 or overlap < 0 or overlap >= max_tokens:
-        raise ValueError("require max_tokens > 0 and 0 <= overlap < max_tokens")
-
-    matches = list(TOKEN_RE.finditer(content))
-    chunks: list[dict[str, Any]] = []
-    step = max_tokens - overlap
-    for sequence, start_index in enumerate(range(0, len(matches), step)):
-        window = matches[start_index:start_index + max_tokens]
-        if not window:
-            break
-        start_offset = window[0].start()
-        end_offset = window[-1].end()
-        chunks.append({
-            "id": f"{document_id}:{sequence}",
-            "document_id": document_id,
-            "sequence": sequence,
-            "content": content[start_offset:end_offset],
-            "token_count": len(window),
-            "start_offset": start_offset,
-            "end_offset": end_offset,
-            "metadata": dict(metadata or {}),
-            "provenance": {"chunking_strategy": CHUNKING_STRATEGY},
-        })
-        if start_index + max_tokens >= len(matches):
-            break
-    return chunks
-
-
-def export_vault(
-    vault: Path,
-    *,
-    max_tokens: int = 512,
-    overlap: int = 50,
-    exclude_dirs: tuple[str, ...] = (".obsidian", ".git", ".trash"),
-) -> dict[str, Any]:
-    """Create stable chunk records from Markdown notes without modifying the vault."""
-    root = vault.expanduser().resolve()
-    if not root.is_dir():
-        raise NotADirectoryError(f"Vault directory does not exist: {root}")
-    excluded = set(exclude_dirs)
-    documents: list[dict[str, Any]] = []
-    for source in sorted(root.rglob("*.md")):
-        relative_parts = source.relative_to(root).parts
-        if source.is_symlink() or any(
-            part in excluded or part.startswith(".") for part in relative_parts
-        ):
-            continue
-        relative = source.relative_to(root)
-        content = source.read_text(encoding="utf-8")
-        if not content.strip():
-            continue
-        relative_name = relative.as_posix()
-        document_id = "note-" + hashlib.sha256(relative_name.encode("utf-8")).hexdigest()[:24]
-        title = note_title(content, relative)
-        chunks = chunk_text(
-            content,
-            document_id=document_id,
-            max_tokens=max_tokens,
-            overlap=overlap,
-            metadata={"source": "local-kbase-export", "relative_path": relative_name, "title": title},
-        )
-        for chunk in chunks:
-            documents.append({
-                "id": chunk["id"],
-                "text": chunk["content"],
-                "metadata": {
-                    "document_id": chunk["document_id"],
-                    "sequence": chunk["sequence"],
-                    "relative_path": relative_name,
-                    "title": title,
-                    "token_count": chunk["token_count"],
-                    "start_offset": chunk["start_offset"],
-                    "end_offset": chunk["end_offset"],
-                    "chunking_strategy": CHUNKING_STRATEGY,
-                },
-            })
-    return {
-        "name": "kbase-corpus-export",
-        "source": "private-kbase-local-export",
-        "version": 1,
-        "chunking_strategy": CHUNKING_STRATEGY,
-        "documents": documents,
-        "queries": [],
-    }
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "--vault", type=Path, required=True,
-        help="Path to the local private kbase Obsidian vault.",
-    )
-    parser.add_argument(
-        "--output", type=Path, default=DEFAULT_OUTPUT,
-        help="Local output JSON; keep corpus exports out of Git.",
-    )
-    parser.add_argument("--max-tokens", type=int, default=512)
-    parser.add_argument("--overlap", type=int, default=50)
-    parser.add_argument(
-        "--exclude-dir", action="append", default=[],
-        help="Additional directory name to exclude; may be repeated.",
-    )
-    args = parser.parse_args()
-    if args.max_tokens < 1 or args.overlap < 0 or args.overlap >= args.max_tokens:
-        parser.error("require max-tokens > 0 and 0 <= overlap < max-tokens")
-    try:
-        data = export_vault(
-            args.vault,
-            max_tokens=args.max_tokens,
-            overlap=args.overlap,
-            exclude_dirs=(".obsidian", ".git", ".trash", *args.exclude_dir),
-        )
-    except (OSError, UnicodeError, ValueError) as exc:
-        parser.error(str(exc))
-    if not data["documents"]:
-        parser.error("no Markdown chunks found; check the vault path and exclusions")
-    output = args.output.expanduser()
-    output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-    note_count = len({item["metadata"]["document_id"] for item in data["documents"]})
-    print(f"Exported {len(data['documents'])} chunks from {note_count} notes")
-    print(f"Chunking strategy: {CHUNKING_STRATEGY} (approximate word counts)")
-    print(f"Local corpus export: {output}")
-    print("No provider requests were made. Queries and relevance judgments are not included.")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

From 5ef5c1309662a8b005e22934d874842a7da25901 Mon Sep 17 00:00:00 2001
From: "Scot P. Floess" <scot.floess@gmail.com>
Date: Sun, 1 Nov 2026 04:24:00 -0500
Subject: [PATCH 128/128] Remove tests for deleted private corpus exporter

---
 tests/test_export_kbase.py | 90 --------------------------------------
 1 file changed, 90 deletions(-)
 delete mode 100644 tests/test_export_kbase.py

diff --git a/tests/test_export_kbase.py b/tests/test_export_kbase.py
deleted file mode 100644
index 51df945..0000000
--- a/tests/test_export_kbase.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from __future__ import annotations
-
-import tempfile
-import unittest
-from pathlib import Path
-
-from tools.provider_api.export_kbase import (
-    CHUNKING_STRATEGY,
-    chunk_text,
-    export_vault,
-)
-
-
-class ChunkTextTests(unittest.TestCase):
-    def test_chunks_preserve_text_offsets_and_overlap(self) -> None:
-        source = "one two three four five six"
-        chunks = chunk_text(
-            source,
-            document_id="doc-1",
-            max_tokens=3,
-            overlap=1,
-        )
-
-        self.assertEqual(
-            [item["id"] for item in chunks],
-            ["doc-1:0", "doc-1:1", "doc-1:2"],
-        )
-        self.assertEqual(
-            [item["content"] for item in chunks],
-            ["one two three", "three four five", "five six"],
-        )
-        self.assertEqual([item["token_count"] for item in chunks], [3, 3, 2])
-        for item in chunks:
-            self.assertEqual(
-                source[item["start_offset"]:item["end_offset"]],
-                item["content"],
-            )
-            self.assertEqual(
-                item["provenance"]["chunking_strategy"],
-                CHUNKING_STRATEGY,
-            )
-
-    def test_rejects_invalid_window(self) -> None:
-        with self.assertRaises(ValueError):
-            chunk_text("one two", document_id="doc-1", max_tokens=2, overlap=2)
-
-
-class ExportVaultTests(unittest.TestCase):
-    def test_exports_markdown_and_excludes_hidden_directories(self) -> None:
-        with tempfile.TemporaryDirectory() as temporary:
-            vault = Path(temporary)
-            (vault / "topic.md").write_text(
-                "# Topic\n\nalpha beta gamma delta", encoding="utf-8"
-            )
-            (vault / ".obsidian").mkdir()
-            (vault / ".obsidian" / "settings.md").write_text(
-                "must not export", encoding="utf-8"
-            )
-            (vault / "subdir").mkdir()
-            (vault / "subdir" / "empty.md").write_text("  \n", encoding="utf-8")
-
-            result = export_vault(vault, max_tokens=3, overlap=1)
-
-        self.assertEqual(result["chunking_strategy"], CHUNKING_STRATEGY)
-        self.assertEqual(len(result["documents"]), 3)
-        self.assertEqual(
-            [item["text"] for item in result["documents"]],
-            ["# Topic\n\nalpha", "alpha beta gamma", "gamma delta"],
-        )
-        self.assertTrue(all(
-            item["metadata"]["relative_path"] == "topic.md"
-            for item in result["documents"]
-        ))
-
-    def test_note_ids_are_stable_for_same_relative_path(self) -> None:
-        with tempfile.TemporaryDirectory() as temporary:
-            vault = Path(temporary)
-            note = vault / "topic.md"
-            note.write_text("one two three", encoding="utf-8")
-            first = export_vault(vault, max_tokens=10, overlap=0)
-            second = export_vault(vault, max_tokens=10, overlap=0)
-
-        self.assertEqual(
-            [item["id"] for item in first["documents"]],
-            [item["id"] for item in second["documents"]],
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()