From 6b62e95f89ee7731cf664cd51097c46e2ede9426 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Sun, 25 Oct 2026 12:50:11 +0530 Subject: [PATCH] feat(roleplay): let DeepSeek hand pivotal turns to GLM-5.3 Chain entries named in ROLEPLAY_HANDOFF_FROM receive an internal function, multillm_hand_off_turn. When the source calls it instead of writing, the Worker discards that attempt and the first available ROLEPLAY_HANDOFF_TO entry writes the turn from the same conversation plus a private note with the source's notes and draft. If no target writes visible text, the draft is the reply. The call never reaches the client: frames are held until visible text appears, and a stray call after real content is removed. At most one handoff runs per ROLEPLAY_HANDOFF_MIN_TURNS turns (default 2); between handoffs the tool stays in the request with tool_choice "none" so the prompt prefix stays cacheable. Callers asking for xhigh or max reasoning start on the targets. Production lets DeepSeek V4.1 Flash on OpenCode Go hand off to GLM-5.3 on OpenCode Go, then on ClinePass. With the function offered, DeepSeek wrote all 12 ordinary test turns itself and handed off 4 of 6 climax and planning turns, each with a full draft; through the Worker code a handed-off turn took about 20 s. The target keeps its configured effort: at max, GLM-5.3 spent a 4,096-token budget reasoning and wrote nothing. A reply that finishes with no visible text now starts over on the next candidate, at most twice, including when the caller sets max_tokens, and fails with a retry message if none writes anything. ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED=false turns this off. --- docs/roleplay.md | 45 ++ package.json | 2 +- tests/fixtures/secret_firewall_coverage.json | 1 + tests/helpers/load_cloudflare_worker.mjs | 1 + tests/outbound_inventory.json | 21 +- tests/test_roleplay_config_worker.mjs | 19 + tests/test_roleplay_handoff_worker.mjs | 472 +++++++++++++++++++ tests/test_runtime_feature_flags_worker.mjs | 2 +- worker/roleplay/completion-result.mjs | 14 + worker/roleplay/config.mjs | 15 + worker/roleplay/continuation.mjs | 117 ++++- worker/roleplay/endpoint.mjs | 19 +- worker/roleplay/handoff.mjs | 249 ++++++++++ worker/roleplay/nonstream-repair.mjs | 106 ++++- worker/roleplay/sse-collector.mjs | 28 ++ worker/roleplay/streaming.mjs | 100 +++- worker/roleplay/transport.mjs | 21 +- wrangler.jsonc | 2 + 18 files changed, 1195 insertions(+), 39 deletions(-) create mode 100644 tests/test_roleplay_handoff_worker.mjs create mode 100644 worker/roleplay/handoff.mjs diff --git a/docs/roleplay.md b/docs/roleplay.md index 2707f125..f0370d4f 100644 --- a/docs/roleplay.md +++ b/docs/roleplay.md @@ -180,6 +180,47 @@ without a roleplay model) use the same chain. Turns that set `routing.mode` or `routing.model` keep the adaptive pool, and so does any turn whose chain has no configured key. Leave the variable unset for the adaptive behaviour above. +### Handing pivotal turns to a stronger model + +`ROLEPLAY_HANDOFF_FROM` names chain entries that may pass a turn on, and +`ROLEPLAY_HANDOFF_TO` the entries that take it; both use the +`ROLEPLAY_INTELLIGENCE_MODELS` format and are unset by default. Production lets +DeepSeek V4.1 Flash on OpenCode Go hand off to GLM-5.3 on OpenCode Go, then on +ClinePass. A source request carries one internal function, `multillm_hand_off_turn`, +whose arguments are a reason (`deeper_reasoning` or `stronger_prose`), short notes +and an optional draft. When the source calls it instead of writing, the Worker +discards that attempt (its frames are held until visible text appears, so nothing +reaches the client) and the first available target writes the turn from the same +conversation plus a private note with the source's notes and draft, at its usual +reasoning effort (at `max`, GLM-5.3 spent a 4,096-token budget reasoning and wrote +nothing). If no target accepts the request or writes visible text, the draft is the +reply; without a draft the rest of the chain follows. +A source that writes the reply and also calls the function keeps its reply, and the +call is removed from the stream. + +`ROLEPLAY_HANDOFF_MIN_TURNS` (default `2`) allows at most one handoff per that many +turns in a session. Between handoffs, and whenever no target is available, the +source still receives the function with `tool_choice: "none"`, so its prompt prefix +and the provider's cache of it stay the same. Callers that ask for `xhigh` or `max` +reasoning start on the targets. Turns with `routing.fallback: "none"` or a session +tier skip the handoff. + +In a test on 2026-10-25 with the function offered, DeepSeek wrote all 12 ordinary +turns itself (scene continuation, conflict, out-of-character recall and small talk) +and handed off 4 of 6 climax and multi-faction planning turns, each with a full +draft of 950 to 1,600 characters. A handoff costs the source's 7 to 15 seconds +before the target starts. Each handoff logs `roleplay_handoff`, and the completion +log counts them in `handoffCount`. + +### Empty replies + +A reply that finishes with no visible text, only reasoning or nothing at all, is +treated as a failure: the turn starts over on the next chain candidate, at most +twice, and fails with a retry message if none writes anything. This also applies +when the caller sets `max_tokens`. The image-prompt repair keeps its own retry when +it can run. Set `ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED=false` to pass empty replies +through. Each retry logs `roleplay_empty_reply_fallback`. + Roleplay generation uses `ROLEPLAY_DEFAULT_REASONING_EFFORT`; the deployment configuration sets the general default to `high`. OpenCode `glm-5.3-flash` defaults to `reasoning_effort: max`. Go enables reasoning through that field; @@ -609,6 +650,10 @@ Non-secret tuning variables: | `ROLEPLAY_UPSTREAM_HEADER_TIMEOUT_MS` | `90000` | Maximum wait for provider response headers before advancing to the next tier | | `ROLEPLAY_PRE_RESPONSE_FALLBACK_ENABLED` | `true` | Advance after a provider transport rejection or header timeout; client aborts never advance | | `ROLEPLAY_PROVIDER_ERROR_FALLBACK_ENABLED` | `true` | Advance only when a 500/502/504 JSON response explicitly identifies both its error code and type as `provider_error`; unknown 5xx responses remain fail-closed | +| `ROLEPLAY_HANDOFF_FROM` | unset | Chain entries offered `multillm_hand_off_turn`; production uses `opencode:deepseek-v4.1-flash` | +| `ROLEPLAY_HANDOFF_TO` | unset | Chain entries that take a handed-off turn, in chain order; production uses GLM-5.3 on OpenCode Go and ClinePass | +| `ROLEPLAY_HANDOFF_MIN_TURNS` | `2` | At most one handoff per this many turns in a session; `0` or `1` lifts the cap | +| `ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED` | `true` | Start a reply with no visible text over on the next candidate, at most twice | | `ROLEPLAY_REFUSAL_FALLBACK_ENABLED` | `true` | Privately regenerate an exact GLM-5.3 Flash refusal trigger with the configured Uncensored Flash candidate; rejected output is neither streamed nor stored | | `ROLEPLAY_DEFAULT_REASONING_EFFORT` | `high` | Deliberate generation default with lower latency and hidden-token use than each provider's maximum; callers can still request `max` per turn | | `ROLEPLAY_STREAM_HEARTBEAT_MS` | `10000` | SSE keepalive interval during upstream silence | diff --git a/package.json b/package.json index 830b7785..6142b403 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --import ./tests/helpers/network_guard.mjs --test tests/test_cloudflare_worker.mjs tests/test_child_safety_worker.mjs tests/test_child_safety_dashboard.mjs tests/test_runtime_feature_flags_worker.mjs tests/test_anthropic_worker.mjs tests/test_haiku_intelligence_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_wave7_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs tests/test_frontend_build.mjs tests/test_deploy_pipeline.mjs tests/test_free_routes_worker.mjs tests/test_client_headers_worker.mjs tests/test_key_controls_expand_worker.mjs tests/test_key_data_guards_worker.mjs tests/test_usage_token_details_worker.mjs tests/test_provider_dispatch_worker.mjs tests/test_dispatch_network_worker.mjs tests/test_key_data_preflight.mjs tests/test_account_contract_worker.mjs tests/test_key_controls_contract_worker.mjs", + "test:worker": "node --import ./tests/helpers/network_guard.mjs --test tests/test_cloudflare_worker.mjs tests/test_child_safety_worker.mjs tests/test_child_safety_dashboard.mjs tests/test_runtime_feature_flags_worker.mjs tests/test_anthropic_worker.mjs tests/test_haiku_intelligence_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_handoff_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_wave7_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs tests/test_frontend_build.mjs tests/test_deploy_pipeline.mjs tests/test_free_routes_worker.mjs tests/test_client_headers_worker.mjs tests/test_key_controls_expand_worker.mjs tests/test_key_data_guards_worker.mjs tests/test_usage_token_details_worker.mjs tests/test_provider_dispatch_worker.mjs tests/test_dispatch_network_worker.mjs tests/test_key_data_preflight.mjs tests/test_account_contract_worker.mjs tests/test_key_controls_contract_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index a83ec1e3..c4ec48c6 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -420,6 +420,7 @@ "worker/roleplay/edge.mjs", "worker/roleplay/endpoint.mjs", "worker/roleplay/fallback-memory.mjs", + "worker/roleplay/handoff.mjs", "worker/roleplay/history-reconciliation.mjs", "worker/roleplay/memory.mjs", "worker/roleplay/message-fragments.mjs", diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 4579d652..da1b35e2 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -7,6 +7,7 @@ function dataModuleUrl(source) { source = source.replaceAll('from "./worker/provider-dispatch.mjs"', `from "${new URL("../../worker/provider-dispatch.mjs", import.meta.url)}"`); source = source.replaceAll('from "./worker/key-controls-capability.mjs"', `from "${new URL("../../worker/key-controls-capability.mjs", import.meta.url)}"`); source = source.replaceAll('from "../provider-dispatch.mjs"', `from "${new URL("../../worker/provider-dispatch.mjs", import.meta.url)}"`); + source = source.replaceAll('from "./handoff.mjs"', `from "${new URL("../../worker/roleplay/handoff.mjs", import.meta.url)}"`); return `data:text/javascript;base64,${Buffer.from(source, "utf8").toString("base64")}`; } diff --git a/tests/outbound_inventory.json b/tests/outbound_inventory.json index f097df2c..010bc018 100644 --- a/tests/outbound_inventory.json +++ b/tests/outbound_inventory.json @@ -2122,17 +2122,17 @@ "worker/roleplay/completion-result.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "924b5dd596ac8ef89f8bd13f04bc7e1067546ed092e47435e3417b66728f1761" + "sha256": "af8ba094f15608f9cb6a47cc13042bb1a89de0dbeb97255a70ec0ad8628b608b" }, "worker/roleplay/config.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "ea0007c314192ae4718fee801465e8678cf5ec8edd7ffd84e9a70480cc10d1ee" + "sha256": "df8aca19da17bf36a87b3c2c9501f7c950033490288b79a866d75f1fd323c121" }, "worker/roleplay/continuation.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "0a3abcb3358db0112e4b6e22cf07695947f8bd2d4cde253de033b7f55c29e5b5" + "sha256": "a17e4e907349783bf740f6a240223c488bdb15504f4c44eb4d876c9aadf12fdd" }, "worker/roleplay/credential-health.mjs": { "kind": "guarded", @@ -2152,7 +2152,7 @@ "worker/roleplay/endpoint.mjs": { "kind": "internal", "reason": "Private session handler; generation and compaction use guarded transport.", - "sha256": "9e460b4d4cd420c7692406a7ba03248b73413c830be54423fdb57fe208a6d3cb" + "sha256": "ec959ea02c60bd85a0457f077ee2ce180adc47a5f27ac0167d053394af5d50d7" }, "worker/roleplay/fallback-memory.mjs": { "kind": "non-provider", @@ -2192,7 +2192,7 @@ "worker/roleplay/nonstream-repair.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "8d3166d81f7be6d0e6f25760a7c05866c4cd5c23eaad36300b8d7f2e3a03e196" + "sha256": "5fea43fd7411c2a5a78ed905333c41750e85c9a5686c0e07a314460bc6a85cf4" }, "worker/roleplay/operator-edge.mjs": { "kind": "internal", @@ -2277,7 +2277,7 @@ "worker/roleplay/sse-collector.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "d609605183dd3f8f2b90f2f7402ba2f45afb797f88297d11460de31ed36e9997" + "sha256": "5ea6fe478d2158914122d60fb2c0152764dc17de5bfe9ce13c34392d55d2db1f" }, "worker/roleplay/state-runtime.mjs": { "kind": "non-provider", @@ -2292,12 +2292,12 @@ "worker/roleplay/streaming.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "cfa1ae35acfbddf5d6f8cd362b13c24247b38df78cbca3fd0bc545924a89b372" + "sha256": "edfd06ed6bb1632833f73119a2c6f9d8e3d814d5793edfc81e02fdc00188734e" }, "worker/roleplay/transport.mjs": { "kind": "guarded", "reason": "Every candidate, compaction and fallback uses providerFetch, including Container egress.", - "sha256": "fb1f50e58dd75196d6920a542df1c262e811a1c4e86d701d21366aa1d1540eb3" + "sha256": "28a8b0f142476e847381ecffbcd356158de4431c29fe9a9680078aa5ad6d4e70" }, "worker/roleplay/turn-runtime.mjs": { "kind": "non-provider", @@ -2453,5 +2453,10 @@ "kind": "internal", "reason": "Content-free capability probe of the actual fenced expansion Container before legacy private dispatch.", "sha256": "1bc8f20ae715d46ceed6a32d90d549688d9479ee87fe25a110235d03c390059e" + }, + "worker/roleplay/handoff.mjs": { + "kind": "non-provider", + "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", + "sha256": "f681974644eb5340debf6d4c0b072bb8dfd5ffcad18a652611d9bdf888d595c3" } } diff --git a/tests/test_roleplay_config_worker.mjs b/tests/test_roleplay_config_worker.mjs index 6fb7a111..2ddea206 100644 --- a/tests/test_roleplay_config_worker.mjs +++ b/tests/test_roleplay_config_worker.mjs @@ -272,6 +272,25 @@ test("deployment starts roleplay on OpenCode Go, then Cline Pass, and keeps Nano assert.equal(config.vars?.NANOGPT_SPEED_ROUTING, ""); }); +test("deployment lets DeepSeek hand pivotal turns to GLM 5.3, at most every other turn", async () => { + const configUrl = new URL("../wrangler.jsonc", import.meta.url); + const config = JSON.parse(await readFile(configUrl, "utf8")); + + // Measured 2026-10-25: with the tool offered, DeepSeek V4.1 Flash wrote all 12 ordinary turns + // itself and handed off 4 of 6 climax and multi-faction planning turns, each with a full draft. + assert.equal(config.vars?.ROLEPLAY_HANDOFF_FROM, "opencode:deepseek-v4.1-flash"); + assert.deepEqual(config.vars?.ROLEPLAY_HANDOFF_TO.split(","), [ + "opencode:glm-5.3", + "cline-pass:cline-pass/glm-5.3", + ]); + // ROLEPLAY_HANDOFF_MIN_TURNS keeps its default of 2. + assert.equal(config.vars?.ROLEPLAY_HANDOFF_MIN_TURNS, undefined); + const chain = config.vars?.ROLEPLAY_INTELLIGENCE_MODELS.split(","); + for (const target of config.vars.ROLEPLAY_HANDOFF_TO.split(",")) { + assert.ok(chain.indexOf(target) > chain.indexOf(config.vars.ROLEPLAY_HANDOFF_FROM), target); + } +}); + test("deployment routes GLM through LinkAPI before NavyAI", async () => { const configUrl = new URL("../wrangler.jsonc", import.meta.url); const config = JSON.parse(await readFile(configUrl, "utf8")); diff --git a/tests/test_roleplay_handoff_worker.mjs b/tests/test_roleplay_handoff_worker.mjs new file mode 100644 index 00000000..4a7e2c87 --- /dev/null +++ b/tests/test_roleplay_handoff_worker.mjs @@ -0,0 +1,472 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + completionResponse, + handleRoleplayEdgeRequest, + makeRoleplayEnv, + roleplayRequest, + withGlobalFetch, +} from "./helpers/roleplay_fixture.mjs"; +import { + ROLEPLAY_HANDOFF_TOOL_NAME, + createRoleplayHandoffFrameFilter, + isEmptyRoleplayReply, + preferRoleplayHandoffTargets, + roleplayHandoffCall, + roleplayHandoffMessages, +} from "../worker/roleplay/handoff.mjs"; +import { getRoleplaySettings } from "../worker/roleplay/config.mjs"; + +const SOURCE = "deepseek-v4.1-flash"; +const TARGET = "glm-5.3"; +const DRAFT = "*Isolde draws the pistol slowly.* \"You'll have neither of them, Commodore.\""; +const TARGET_REPLY = "*Isolde lets the silence stretch until the gun crews flinch.* \"No.\""; + +function handoffEnv(overrides = {}) { + return makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:${TARGET},opencode:mimo-v2.6-pro`, + ROLEPLAY_HANDOFF_FROM: `opencode:${SOURCE}`, + ROLEPLAY_HANDOFF_TO: `opencode:${TARGET}`, + ROLEPLAY_DEFAULT_REASONING_EFFORT: "high", + ...overrides, + }); +} + +function frame(model, delta, finishReason = null) { + return `data: ${JSON.stringify({ model, choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`; +} + +function sse(frames) { + return new Response([...frames, "data: [DONE]\n\n"].join(""), { + headers: { "Content-Type": "text/event-stream" }, + }); +} + +function handoffCallFrames(model, args) { + const text = JSON.stringify(args); + return [ + frame(model, { reasoning_content: "This is the turning point of the story." }), + frame(model, { + tool_calls: [{ + index: 0, + id: "call_handoff", + type: "function", + function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "" }, + }], + }), + frame(model, { tool_calls: [{ index: 0, function: { arguments: text.slice(0, 25) } }] }), + frame(model, { tool_calls: [{ index: 0, function: { arguments: text.slice(25) } }] }), + frame(model, {}, "tool_calls"), + ]; +} + +function contentFrames(model, content) { + const midpoint = Math.ceil(content.length / 2); + return [ + frame(model, { content: content.slice(0, midpoint) }), + frame(model, { content: content.slice(midpoint) }), + frame(model, {}, "stop"), + ]; +} + +function streamedPayloads(body) { + return body + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6))); +} + +function streamedContent(body) { + return streamedPayloads(body) + .map((payload) => payload.choices?.[0]?.delta?.content ?? "") + .join(""); +} + +function visibleContent(body) { + return streamedContent(body).replace(/[\s\S]*?<\/think>/g, "").trim(); +} + +function turn(sessionId, overrides = {}) { + return roleplayRequest({ + session_id: sessionId, + input: "Harrow's frigate runs out its guns. He offers to spare the crew for Rook and the crate.", + model: "roleplay:intelligence", + max_tokens: 1024, + stream: true, + ...overrides, + }); +} + +async function sessionState(fixture) { + const stored = [...fixture.storageBySession.values()][0]?.storage; + return { + state: (await stored?.get("roleplay-session")) ?? {}, + messages: (await stored?.get("roleplay-messages")) ?? [], + }; +} + +function recordingFetch(calls, respond) { + return async (input, init) => { + const payload = JSON.parse(init.body); + calls.push(payload); + return respond(payload, calls.length, new URL(input)); + }; +} + +test("a streamed source turn hands off to the target, which rewrites the draft", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { + reason: "deeper_reasoning", + notes: "Isolde must refuse without softening.", + draft: DRAFT, + })) + : sse(contentFrames(TARGET, TARGET_REPLY)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-stream"), fixture.env); + return { status: result.status, body: await result.text() }; + }); + await fixture.waitForBackgroundWork(); + + assert.equal(response.status, 200); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(calls[0].tools?.[0]?.function?.name, ROLEPLAY_HANDOFF_TOOL_NAME); + assert.equal(calls[0].tool_choice, "auto"); + assert.equal(calls[1].tools, undefined); + // The target keeps the configured effort: at max, GLM-5.3 spent a 4,096-token budget reasoning. + assert.equal(calls[1].reasoning_effort, "high"); + const lastUser = calls[1].messages.findLastIndex((message) => message.role === "user"); + const note = calls[1].messages[lastUser - 1]; + assert.equal(note.role, "system"); + assert.match(note.content, /Private handoff note/); + assert.match(note.content, /careful reasoning/); + assert.ok(note.content.includes(DRAFT)); + assert.ok(note.content.includes("Isolde must refuse without softening.")); + + assert.equal(visibleContent(response.body), TARGET_REPLY); + assert.doesNotMatch(response.body, /tool_calls|multillm_hand_off_turn|turning point/); + + const { state, messages } = await sessionState(fixture); + assert.equal(state.lastHandoffTurn, 0); + assert.equal(state.handoffs, 1); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, TARGET_REPLY); +}); + +// Production reaches OpenCode through the container, which turns reasoning into content +// and drops the closing frame that carries only finish_reason. +function containerHandoffCallFrames(model, args) { + return handoffCallFrames(model, args) + .slice(0, -1) + .flatMap((entry, index) => index === 0 + ? [ + frame(model, { content: "" }), + frame(model, { content: "This is the turning point of the story." }), + entry, + ] + : [entry]) + .concat(frame(model, { content: "\n\n" })); +} + +test("a handoff still happens when the container rewrites reasoning and drops the finish frame", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(containerHandoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Make it land.", draft: DRAFT })) + : sse(contentFrames(TARGET, TARGET_REPLY)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-container"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), TARGET_REPLY); + assert.doesNotMatch(response, /tool_calls|multillm_hand_off_turn/); +}); + +test("a target that writes nothing visible leaves the source's draft as the reply", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { reason: "deeper_reasoning", notes: "Plan.", draft: DRAFT })) + : sse([frame(TARGET, { reasoning_content: "Weighing every faction." }), frame(TARGET, {}, "length")]), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-target-empty"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), DRAFT); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, DRAFT); +}); + +test("a source that writes the reply keeps it, and its stray handoff call is removed", async () => { + const fixture = handoffEnv(); + const calls = []; + const reply = "*Isolde laughs and turns the wheel hard to port.*"; + const response = await withGlobalFetch(recordingFetch(calls, () => sse([ + ...contentFrames(SOURCE, reply).slice(0, 2), + frame(SOURCE, { + tool_calls: [{ + index: 0, + id: "call_late", + type: "function", + function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "{\"reason\":\"stronger_prose\",\"notes\":\"x\"}" }, + }], + }), + frame(SOURCE, {}, "tool_calls"), + ])), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-late-call"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); + assert.equal(visibleContent(response), reply); + assert.doesNotMatch(response, /tool_calls|multillm_hand_off_turn/); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); +}); + +test("the next turn after a handoff withholds the tool until the turn gap passes", async () => { + const fixture = handoffEnv(); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE && calls.length === 1 + ? sse(handoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Pivotal." })) + : sse(contentFrames(payload.model, "*The guns fall silent.*")), + ), async () => { + for (let index = 0; index < 3; index += 1) { + const result = await handleRoleplayEdgeRequest( + turn("session-handoff-gap", { input: `Turn ${index + 1}.` }), + fixture.env, + ); + await result.text(); + await fixture.waitForBackgroundWork(); + } + }); + + const sourceCalls = calls.filter((call) => call.model === SOURCE); + assert.deepEqual(sourceCalls.map((call) => call.tool_choice), ["auto", "none", "auto"]); + // The tool definition stays on every source call so the cached prompt prefix does not change. + assert.ok(sourceCalls.every((call) => call.tools?.[0]?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME)); +}); + +test("when every target fails, the source's draft becomes the reply", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Pivotal.", draft: DRAFT })) + : new Response('{"error":"rate limited"}', { status: 429, headers: { "Content-Type": "application/json" } }), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-draft"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), DRAFT); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, DRAFT); +}); + +test("a non-streamed handoff returns the target's reply without the tool call", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => { + if (payload.model !== SOURCE) return completionResponse(payload.model, TARGET_REPLY); + return new Response(JSON.stringify({ + id: "chatcmpl-source", + object: "chat.completion", + model: SOURCE, + choices: [{ + index: 0, + message: { + role: "assistant", + content: "", + tool_calls: [{ + id: "call_handoff", + type: "function", + function: { + name: ROLEPLAY_HANDOFF_TOOL_NAME, + arguments: JSON.stringify({ reason: "stronger_prose", notes: "Pivotal.", draft: DRAFT }), + }, + }], + }, + finish_reason: "tool_calls", + }], + }), { headers: { "Content-Type": "application/json" } }); + }), () => handleRoleplayEdgeRequest(turn("session-handoff-json", { stream: false }), fixture.env)); + await fixture.waitForBackgroundWork(); + + assert.equal(response.status, 200); + const body = await response.json(); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(body.choices[0].message.content, TARGET_REPLY); + assert.equal(body.choices[0].message.tool_calls, undefined); + assert.equal(body.choices[0].finish_reason, "stop"); + assert.equal(response.headers.get("X-Roleplay-Model"), TARGET); +}); + +test("callers asking for xhigh or max reasoning start on the target", async () => { + const fixture = handoffEnv(); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => sse(contentFrames(payload.model, TARGET_REPLY))), async () => { + const result = await handleRoleplayEdgeRequest( + turn("session-handoff-xhigh", { reasoning_effort: "xhigh" }), + fixture.env, + ); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [TARGET]); + assert.equal(calls[0].tools, undefined); +}); + +test("without handoff settings no tool is offered", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:${TARGET}`, + }); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => sse(contentFrames(payload.model, TARGET_REPLY))), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-off"), fixture.env); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); + assert.equal(calls[0].tools, undefined); + assert.equal(calls[0].tool_choice, undefined); +}); + +test("a streamed reply with only reasoning falls through to the next candidate", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const calls = []; + const reply = "*Isolde finally answers.*"; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + calls.length === 1 + ? sse([frame(payload.model, { reasoning_content: "Thinking it over." }), frame(payload.model, {}, "stop")]) + : sse(contentFrames(payload.model, reply)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-empty-stream"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, "mimo-v2.6-pro"]); + assert.equal(visibleContent(response), reply); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, reply); +}); + +test("a non-streamed empty reply falls through, and an all-empty turn says so", async () => { + const empty = (model) => new Response(JSON.stringify({ + id: "chatcmpl-empty", + object: "chat.completion", + model, + choices: [{ index: 0, message: { role: "assistant", content: "", reasoning_content: "Hmm." }, finish_reason: "stop" }], + }), { headers: { "Content-Type": "application/json" } }); + + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + calls.length === 1 ? empty(payload.model) : completionResponse(payload.model, "*Isolde answers.*"), + ), () => handleRoleplayEdgeRequest(turn("session-empty-json", { stream: false }), fixture.env)); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, "mimo-v2.6-pro"]); + assert.equal((await response.json()).choices[0].message.content, "*Isolde answers.*"); + + const allEmpty = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const failed = await withGlobalFetch(async (_input, init) => empty(JSON.parse(init.body).model), () => + handleRoleplayEdgeRequest(turn("session-empty-json-all", { stream: false }), allEmpty.env)); + const content = (await failed.json()).choices[0].message.content; + assert.match(content, /could not complete this response/); +}); + +test("the empty-reply retry can be switched off", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED: "false", + }); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => + sse([frame(payload.model, { reasoning_content: "Thinking." }), frame(payload.model, {}, "stop")]), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-empty-off"), fixture.env); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); +}); + +test("handoff helpers parse calls, place the note and order candidates", () => { + assert.deepEqual( + roleplayHandoffCall([{ function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "{not json" } }]), + { reason: "stronger_prose", notes: "", draft: "" }, + ); + assert.equal(roleplayHandoffCall([{ function: { name: "other", arguments: "{}" } }]), null); + + const messages = roleplayHandoffMessages( + [{ role: "system", content: "Card." }, { role: "user", content: "Go." }], + { reason: "stronger_prose", notes: "", draft: "Draft text." }, + ); + assert.deepEqual(messages.map((message) => message.role), ["system", "system", "user"]); + assert.match(messages[1].content, /Draft text\./); + + const settings = getRoleplaySettings({ + ROLEPLAY_HANDOFF_FROM: `opencode:${SOURCE}`, + ROLEPLAY_HANDOFF_TO: `opencode:${TARGET},cline-pass:cline-pass/glm-5.3`, + }); + const chain = [ + { provider: "opencode", model: SOURCE }, + { provider: "opencode", model: TARGET }, + { provider: "opencode", model: "mimo-v2.6-pro" }, + { provider: "cline-pass", model: "cline-pass/glm-5.3" }, + ]; + assert.deepEqual( + preferRoleplayHandoffTargets(chain, { forwarded: { reasoning_effort: "max" } }, settings) + .map((candidate) => candidate.model), + [TARGET, "cline-pass/glm-5.3", SOURCE, "mimo-v2.6-pro"], + ); + assert.equal(preferRoleplayHandoffTargets(chain, { forwarded: { reasoning_effort: "high" } }, settings), chain); + + assert.equal(isEmptyRoleplayReply({ assistant: " ", finishReason: "stop" }), true); + assert.equal(isEmptyRoleplayReply({ assistant: "", finishReason: "stop", toolCalls: [{ function: { name: "lookup" } }] }), false); + assert.equal(isEmptyRoleplayReply({ assistant: "", finishReason: "stop", terminated: false }), false); +}); + +test("the frame filter drops the handoff call and keeps other tool calls", () => { + const filter = createRoleplayHandoffFrameFilter(); + const output = filter.filter([ + frame("m", { tool_calls: [{ index: 0, function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "" } }] }), + frame("m", { tool_calls: [{ index: 0, function: { arguments: "{\"reason\":" } }] }), + frame("m", { tool_calls: [{ index: 1, function: { name: "lookup", arguments: "{}" } }] }), + frame("m", {}, "tool_calls"), + ]); + const payloads = output.map((entry) => JSON.parse(entry.slice(6))); + assert.equal(payloads.length, 2); + assert.equal(payloads[0].choices[0].delta.tool_calls[0].function.name, "lookup"); + assert.equal(payloads[1].choices[0].finish_reason, "tool_calls"); +}); diff --git a/tests/test_runtime_feature_flags_worker.mjs b/tests/test_runtime_feature_flags_worker.mjs index 8ea62950..eaabf8b6 100644 --- a/tests/test_runtime_feature_flags_worker.mjs +++ b/tests/test_runtime_feature_flags_worker.mjs @@ -11,7 +11,7 @@ const { MultiLLMProxyContainer } = await loadWorkerModule(); test("deployment groups feature settings while preserving Container flags and secrets", () => { assert.equal(Object.keys(config.vars.RUNTIME_FEATURE_FLAGS).length, 7); - assert.equal(Object.keys(config.vars).length, 65); + assert.equal(Object.keys(config.vars).length, 67); const source = { ...config.vars, ANTHROPIC_API_KEY: "synthetic-provider-secret" }; const container = new MultiLLMProxyContainer({}, source); assert.equal(container.envVars.MODEL_COOLDOWN_ENABLED, "true"); diff --git a/worker/roleplay/completion-result.mjs b/worker/roleplay/completion-result.mjs index 9dacd553..8e5fdd64 100644 --- a/worker/roleplay/completion-result.mjs +++ b/worker/roleplay/completion-result.mjs @@ -76,6 +76,12 @@ export function roleplayCompletionDisposition({ }; } +function diagnosticCount(diagnostics, reason) { + return Array.isArray(diagnostics) + ? diagnostics.filter((entry) => entry?.reason === reason).length + : 0; +} + export function logRoleplayStreamCompletion({ candidate, parsed, @@ -116,6 +122,10 @@ export function logRoleplayStreamCompletion({ heartbeatCount: completion.heartbeatCount, continuationCount: completion.continuationCount, refusalFallbackCount: completion.refusalFallbackCount, + handoffCount: + diagnosticCount(completion.continuationDiagnostics, "handoff") || undefined, + emptyReplyCount: + diagnosticCount(completion.continuationDiagnostics, "empty_reply") || undefined, upstreamCallCount: completion.upstreamCallCount, ...contractTelemetry( parsed.outputContract, @@ -147,6 +157,10 @@ export function logRoleplayNonStreamCompletion({ finishReason: completion.finishReason || undefined, continuationCount: completion.continuationCount, refusalFallbackCount: completion.refusalFallbackCount, + handoffCount: + diagnosticCount(completion.continuationDiagnostics, "handoff") || undefined, + emptyReplyCount: + diagnosticCount(completion.continuationDiagnostics, "empty_reply") || undefined, upstreamCallCount: completion.upstreamCallCount, queueMs: Math.round(timings.queueMs ?? 0), stateLoadMs: Math.round(timings.stateLoadMs ?? 0), diff --git a/worker/roleplay/config.mjs b/worker/roleplay/config.mjs index f3e0cd1f..771059bb 100644 --- a/worker/roleplay/config.mjs +++ b/worker/roleplay/config.mjs @@ -454,6 +454,14 @@ function parseIntelligenceChain(value) { return chain; } +// ROLEPLAY_HANDOFF_FROM and ROLEPLAY_HANDOFF_TO name chain entries the same way; unset means none. +function parseHandoffEntries(value) { + if (typeof value !== "string" || !value.trim()) return new Set(); + return new Set( + parseIntelligenceChain(value).map(({ provider, model }) => `${provider}:${model}`), + ); +} + export function intelligenceModelFamily(model) { const normalized = String(model ?? "").toLowerCase(); if (/(?:^|\/)glm[-_]?[0-9]/.test(normalized)) return "glm"; @@ -735,6 +743,13 @@ export function getRoleplaySettings(env) { env.ROLEPLAY_PROVIDER_LIMITS, ), intelligenceChain: parseIntelligenceChain(env.ROLEPLAY_INTELLIGENCE_MODELS), + handoffSources: parseHandoffEntries(env.ROLEPLAY_HANDOFF_FROM), + handoffTargets: parseHandoffEntries(env.ROLEPLAY_HANDOFF_TO), + handoffMinTurns: boundedInteger(env.ROLEPLAY_HANDOFF_MIN_TURNS, 2, 0, 1_000), + emptyReplyFallbackEnabled: booleanSetting( + env.ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED, + true, + ), autoRoute: String(env.ROLEPLAY_AUTO_ROUTE ?? "").trim().toLowerCase() === "intelligence" ? "intelligence" diff --git a/worker/roleplay/continuation.mjs b/worker/roleplay/continuation.mjs index afe835f8..9fa2560c 100644 --- a/worker/roleplay/continuation.mjs +++ b/worker/roleplay/continuation.mjs @@ -14,6 +14,15 @@ import { recordModelRefusal, } from "./transport.mjs"; import { classifyRoleplayRefusal } from "./refusal-fallback.mjs"; +import { + isEmptyRoleplayReply, + recordRoleplayHandoff, + roleplayHandoffCall, + roleplayHandoffMessages, +} from "./handoff.mjs"; + +// Legs that start the turn over on other candidates instead of continuing the current one. +const FRESH_LEG_REASONS = new Set(["semantic_refusal", "handoff", "empty_reply"]); const LENGTH_CONTINUATION_INSTRUCTION = [ "[Automatic continuation of an incomplete response]", @@ -103,9 +112,19 @@ export function createRoleplayContinuation({ signal, idempotencyKey, refusalFallbackCandidates = [], + handoffTargets = [], + laterCandidates = [], }) { let currentState = state; let currentCandidate = candidate; + // A handoff adds its note to the prompt of every later leg, and its draft stands in for the + // reply if no later leg writes one. + let legMessages = messages; + let handoffDraft = ""; + let remainingHandoffTargets = [...handoffTargets]; + let remainingLaterCandidates = [...laterCandidates]; + let handoffCount = 0; + let emptyReplyCount = 0; let remainingRefusalFallbackCandidates = settings.refusalFallbackEnabled ? [...refusalFallbackCandidates] : []; @@ -116,7 +135,10 @@ export function createRoleplayContinuation({ (settings.maxAutoContinuations > 0 || settings.maxOutputContractRepairs > 0); const canOpen = - enabled || remainingRefusalFallbackCandidates.length > 0; + enabled || + remainingRefusalFallbackCandidates.length > 0 || + remainingHandoffTargets.length > 0 || + (settings.emptyReplyFallbackEnabled === true && remainingLaterCandidates.length > 0); const assess = ({ assistant, finishReason }) => { const cleaned = cleanRoleplayOutput( @@ -158,38 +180,61 @@ export function createRoleplayContinuation({ return { reason: "", limit: 0, contractAnalysis, cleaned }; }; + const freshLegCandidates = (reason, handoff) => { + if (reason === "semantic_refusal") return remainingRefusalFallbackCandidates; + if (reason === "empty_reply") { + return settings.emptyReplyFallbackEnabled === true ? remainingLaterCandidates : []; + } + if (!remainingHandoffTargets.length) return []; + // A draft can stand in for the reply if every target fails; without one the rest of the + // chain follows the targets. + return handoff?.draft + ? remainingHandoffTargets + : [ + ...remainingHandoffTargets, + ...remainingLaterCandidates.filter((entry) => !remainingHandoffTargets.includes(entry)), + ]; + }; + const openResponse = async ({ assistant, continuationCount, reason, contractAnalysis, + handoff = null, signal: nextSignal = signal, }) => { const semanticRefusal = reason === "semantic_refusal"; + const freshLeg = FRESH_LEG_REASONS.has(reason); const retriesFromBeginning = reason === "empty_eof" || reason === "empty_story"; + const freshCandidates = freshLeg ? freshLegCandidates(reason, handoff) : []; if ( - semanticRefusal - ? !remainingRefusalFallbackCandidates.length + freshLeg + ? !freshCandidates.length : !enabled || (!assistant.trim() && !retriesFromBeginning) ) { return null; } - const nextMessages = semanticRefusal - ? messages + if (reason === "handoff") { + legMessages = roleplayHandoffMessages(legMessages, handoff); + handoffDraft = handoff?.draft ?? ""; + } + const nextMessages = freshLeg + ? legMessages : continuationMessages( - messages, + legMessages, assistant, reason, contractAnalysis, ); const nextEstimatedInputTokens = estimateTokens(nextMessages); const nextCandidates = prepareRoleplayCandidates( - semanticRefusal - ? remainingRefusalFallbackCandidates + freshLeg + ? freshCandidates : [currentCandidate], nextEstimatedInputTokens, - semanticRefusal ? parsed.maxTokens : null, + freshLeg ? parsed.maxTokens : null, settings, ); if (semanticRefusal) { @@ -200,6 +245,27 @@ export function createRoleplayContinuation({ remainingRefusalFallbackCandidates = []; refusalFallbackCount += 1; } + if (reason === "handoff") { + currentState = recordRoleplayHandoff(currentState); + remainingHandoffTargets = []; + handoffCount += 1; + console.log(JSON.stringify({ + event: "roleplay_handoff", + provider: currentCandidate.provider, + model: currentCandidate.model, + handoffReason: handoff?.reason, + notesCharacters: handoff?.notes?.length ?? 0, + draftCharacters: handoff?.draft?.length ?? 0, + })); + } + if (reason === "empty_reply") { + emptyReplyCount += 1; + console.log(JSON.stringify({ + event: "roleplay_empty_reply_fallback", + provider: currentCandidate.provider, + model: currentCandidate.model, + })); + } if (!nextCandidates.length) { logRoleplayError( "roleplay_continuation_context_exhausted", @@ -281,6 +347,10 @@ export function createRoleplayContinuation({ return null; } currentCandidate = attempted.candidate; + if (freshLeg) { + remainingLaterCandidates = attempted.laterCandidates ?? []; + if (reason !== "handoff") remainingHandoffTargets = attempted.handoffTargets ?? []; + } return attempted; }; @@ -299,6 +369,23 @@ export function createRoleplayContinuation({ get refusalFallbackEnabled() { return remainingRefusalFallbackCandidates.length > 0; }, + // The stream holds a leg's frames until it shows visible text whenever that leg may still be + // replaced: by a refusal fallback or by a handoff. + get holdUntilVisible() { + return ( + remainingRefusalFallbackCandidates.length > 0 || + remainingHandoffTargets.length > 0 + ); + }, + get handoffCount() { + return handoffCount; + }, + get handoffDraft() { + return handoffDraft; + }, + get emptyReplyCount() { + return emptyReplyCount; + }, get upstreamCallCount() { return upstreamCallCount; }, @@ -310,6 +397,18 @@ export function createRoleplayContinuation({ return assess(input).reason; }, classifyRefusal: classifyRoleplayRefusal, + detectHandoff(toolCalls) { + return remainingHandoffTargets.length > 0 ? roleplayHandoffCall(toolCalls) : null; + }, + detectEmptyReply(input) { + // The output-contract repair already retries an image-only reply when it may run. + if (input?.reason === "empty_story" && enabled) return false; + return ( + settings.emptyReplyFallbackEnabled === true && + remainingLaterCandidates.length > 0 && + isEmptyRoleplayReply(input) + ); + }, openResponse, async open(input) { const attempted = await openResponse(input); diff --git a/worker/roleplay/endpoint.mjs b/worker/roleplay/endpoint.mjs index 521d6ecd..cb37836f 100644 --- a/worker/roleplay/endpoint.mjs +++ b/worker/roleplay/endpoint.mjs @@ -46,6 +46,7 @@ import { } from "./completion-result.mjs"; import { createExtractiveCompactionDigest } from "./fallback-memory.mjs"; import { createRoleplayContinuation } from "./continuation.mjs"; +import { preferRoleplayHandoffTargets } from "./handoff.mjs"; import { buildConfiguredCandidates, autoRoutePreference, @@ -99,6 +100,8 @@ export { isRoleplayPath, scopePublicRoleplaySessionId }; import { handleRoleplayEdgeRequest as roleplayEdgeRequest } from "./edge.mjs"; export const handleRoleplayEdgeRequest = (request, env) => withRoleplayGatewayAuthority(request, env, roleplayEdgeRequest); const INTERNAL_OUTPUT_MODE_HEADER = "X-MultiLLM-Roleplay-Output-Mode"; +// Turns without fallbacks also skip the handoff and the empty-reply retry. +const WITHOUT_LEG_HANDOFFS = Object.freeze({ handoffSources: new Set(), emptyReplyFallbackEnabled: false }); async function preparePromptInjection(payload, env, record = true) { const mode = env.PROMPT_INJECTION_MODE; @@ -299,7 +302,7 @@ export class RoleplaySession extends DurableObject { } parsedInitial.routing = parseRoutingPolicy(payload.routing, { sessionTierEnabled: this.sessionTiers.settings.enabled }); if (parsedInitial.routing.fallback === "none") { - settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0 }; + settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0, ...WITHOUT_LEG_HANDOFFS }; } const { profile, parsed: parsedWithProfile } = effectiveRoleplayCharacter( state, @@ -396,7 +399,7 @@ export class RoleplaySession extends DurableObject { tierScope.turn = tierTurn; if (tierTurn) { candidates = tierTurn.candidates; - settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0 }; + settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0, ...WITHOUT_LEG_HANDOFFS }; } if (parsed.routing.fallback === "none") candidates.splice(1); if (!candidates.length) { @@ -634,6 +637,7 @@ export class RoleplaySession extends DurableObject { ); generationCandidates = prepareCanaryCandidates(generationCandidates, canary, roleplayMessages, estimateTokens, settings); if (tierTurn) generationCandidates = tierTurn.select(generationCandidates); + generationCandidates = preferRoleplayHandoffTargets(generationCandidates, parsed, settings); if (!generationCandidates.length) { state = markRoleplayRequest(state, idempotencyKey, "context_too_large"); await stateRepository.save(state); @@ -693,6 +697,8 @@ export class RoleplaySession extends DurableObject { cleanup, promptCache, refusalFallbackCandidates, + handoffTargets, + laterCandidates, } = attempted; // Dispatch and continuation share the selected view. Persistence and // recovery above retain the original conversation independently. @@ -716,6 +722,8 @@ export class RoleplaySession extends DurableObject { signal: request.signal, idempotencyKey, refusalFallbackCandidates, + handoffTargets, + laterCandidates, }); protectRoleplayContinuation(continuation, canary, request.signal); const selectionReason = candidate.selectionReason || (parsed.routing.mode !== "provider-priority" ? parsed.routing.mode : @@ -799,7 +807,10 @@ export class RoleplaySession extends DurableObject { model: candidate.model, }, detectRefusal: continuation.classifyRefusal, - refusalFallbackEnabled: continuation.refusalFallbackEnabled, + refusalFallbackEnabled: continuation.holdUntilVisible, + detectHandoff: continuation.detectHandoff, + detectEmptyReply: continuation.detectEmptyReply, + getHandoffDraft: () => continuation.handoffDraft, onComplete: async (completion) => { completion = canaryCompletion(completion, canary); await preserveRecovery(this.ctx.storage, recovery, completion, trace?.id, retention); @@ -913,7 +924,7 @@ export class RoleplaySession extends DurableObject { applyRoleplayRouteHeaders( responseHeaders, continuation.candidate, - fallbackCount + continuation.refusalFallbackCount, + fallbackCount + continuation.refusalFallbackCount + continuation.emptyReplyCount, ); responseBytes = new TextEncoder().encode( JSON.stringify(completionResult.payload), diff --git a/worker/roleplay/handoff.mjs b/worker/roleplay/handoff.mjs new file mode 100644 index 00000000..6977809a --- /dev/null +++ b/worker/roleplay/handoff.mjs @@ -0,0 +1,249 @@ +// A fast first-pass model may hand a roleplay turn to a stronger one. Configured source models +// (ROLEPLAY_HANDOFF_FROM) receive an internal function; when one calls it instead of writing, +// the Worker discards that leg and the first available target (ROLEPLAY_HANDOFF_TO) writes the +// turn from the source's notes and draft. Clients never see the function or its call. + +export const ROLEPLAY_HANDOFF_TOOL_NAME = "multillm_hand_off_turn"; + +const HANDOFF_REASONS = new Set(["deeper_reasoning", "stronger_prose"]); +const MAX_NOTES_CHARACTERS = 2_000; +const MAX_DRAFT_CHARACTERS = 32_000; +const MAX_ARGUMENT_CHARACTERS = 64_000; +const REASONING_EFFORTS_THAT_PREFER_TARGETS = new Set(["xhigh", "max"]); + +export const ROLEPLAY_HANDOFF_TOOL = Object.freeze({ + type: "function", + function: { + name: ROLEPLAY_HANDOFF_TOOL_NAME, + description: [ + "Pass this turn to a senior writer instead of answering it yourself.", + "Call it only when the turn clearly needs more than you can deliver well:", + "deeper_reasoning for tangled plot logic, many characters or factions to keep consistent,", + "or a plan or mystery that must add up; stronger_prose for a pivotal scene such as a", + "climax, a major reveal, a death or a turning point in a relationship.", + "Most turns are ordinary: write those yourself and do not call this.", + "Never call it for short, simple or out-of-character replies.", + ].join(" "), + parameters: { + type: "object", + properties: { + reason: { type: "string", enum: ["deeper_reasoning", "stronger_prose"] }, + notes: { + type: "string", + description: "What the reply must achieve: key beats, tone and continuity facts, in under 80 words.", + }, + draft: { + type: "string", + description: "Optional: your draft of the reply. The senior writer keeps what works and rewrites the rest.", + }, + }, + required: ["reason", "notes"], + }, + }, +}); + +function entryKey(candidate) { + return `${String(candidate?.provider ?? "").toLowerCase()}:${candidate?.model ?? ""}`; +} + +export function isRoleplayHandoffSource(candidate, settings) { + return Boolean(settings?.handoffTargets?.size && settings?.handoffSources?.has(entryKey(candidate))); +} + +function isRoleplayHandoffTarget(candidate, settings) { + return Boolean(settings?.handoffTargets?.has(entryKey(candidate))); +} + +/** Later candidates a source may hand off to: the configured targets, in chain order. */ +export function roleplayHandoffTargets(candidates, candidateIndex, settings) { + if (!isRoleplayHandoffSource(candidates?.[candidateIndex], settings)) return []; + return candidates + .slice(candidateIndex + 1) + .filter((candidate) => isRoleplayHandoffTarget(candidate, settings)); +} + +/** At most one handoff every `handoffMinTurns` turns per session; 0 or 1 lifts the cap. */ +export function roleplayHandoffAllowed(state, settings) { + const last = state?.lastHandoffTurn; + if (!Number.isSafeInteger(last)) return true; + return (state?.turns ?? 0) - last >= (settings?.handoffMinTurns ?? 0); +} + +export function recordRoleplayHandoff(state) { + return { + ...state, + lastHandoffTurn: state?.turns ?? 0, + handoffs: (state?.handoffs ?? 0) + 1, + }; +} + +// The tool stays on every source leg so the prompt prefix (and the provider's cache of it) does +// not change between turns; `tool_choice: "none"` withholds it when no handoff may happen. +export function injectRoleplayHandoffTool(payload, available) { + if (!payload || !Array.isArray(payload.messages) || payload.tools !== undefined) { + return payload; + } + return { + ...payload, + tools: [ROLEPLAY_HANDOFF_TOOL], + tool_choice: available ? "auto" : "none", + }; +} + +/** Callers asking for xhigh or max reasoning start on the targets instead of the sources. */ +export function preferRoleplayHandoffTargets(candidates, parsed, settings) { + const effort = parsed?.forwarded?.reasoning_effort; + if (!REASONING_EFFORTS_THAT_PREFER_TARGETS.has(effort) || !settings?.handoffTargets?.size) { + return candidates; + } + const firstSource = candidates.findIndex((candidate) => isRoleplayHandoffSource(candidate, settings)); + if (firstSource < 0) return candidates; + const promoted = candidates + .slice(firstSource) + .filter((candidate) => isRoleplayHandoffTarget(candidate, settings)); + if (!promoted.length) return candidates; + return [ + ...candidates.slice(0, firstSource), + ...promoted, + ...candidates.slice(firstSource).filter((candidate) => !promoted.includes(candidate)), + ]; +} + +function clipped(value, limit) { + return typeof value === "string" ? value.trim().slice(0, limit) : ""; +} + +export function roleplayHandoffCall(toolCalls) { + const call = Array.isArray(toolCalls) + ? toolCalls.find((entry) => entry?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME) + : null; + if (!call) return null; + let args = {}; + try { + const parsed = JSON.parse(call.function.arguments || "{}"); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) args = parsed; + } catch { + // A malformed call still asks for the handoff; it only loses its notes. + } + return { + reason: HANDOFF_REASONS.has(args.reason) ? args.reason : "stronger_prose", + notes: clipped(args.notes, MAX_NOTES_CHARACTERS), + draft: clipped(args.draft, MAX_DRAFT_CHARACTERS), + }; +} + +/** The target's prompt: the original turn plus a private note placed before the last user message. */ +export function roleplayHandoffMessages(messages, handoff) { + if (!Array.isArray(messages) || !handoff) return messages; + const lines = [ + "[Private handoff note]", + handoff.reason === "deeper_reasoning" + ? "A first-pass writer passed the next turn to you because it needs careful reasoning: keep every character, plan and continuity fact consistent." + : "A first-pass writer passed the next turn to you because it needs your strongest prose.", + "Write the reply yourself and follow every roleplay instruction above. Never mention this note, the handoff or the draft.", + ]; + if (handoff.notes) lines.push(`Their notes: ${handoff.notes}`); + if (handoff.draft) { + lines.push("Their draft of the reply follows. Keep what works, fix what does not, and rewrite freely:"); + lines.push(handoff.draft); + } + const next = [...messages]; + let insertionIndex = next.length; + for (let index = next.length - 1; index >= 0; index -= 1) { + if (next[index]?.role === "user") { + insertionIndex = index; + break; + } + } + next.splice(insertionIndex, 0, { role: "system", content: lines.join("\n") }); + return next; +} + +export function withoutRoleplayHandoffToolCalls(message) { + if (!message || !Array.isArray(message.tool_calls)) return message; + const remaining = message.tool_calls.filter( + (call) => call?.function?.name !== ROLEPLAY_HANDOFF_TOOL_NAME, + ); + if (remaining.length === message.tool_calls.length) return message; + const next = { ...message }; + if (remaining.length) next.tool_calls = remaining; + else delete next.tool_calls; + return next; +} + +/** A finished reply with no visible text and no caller tool call gave the reader nothing. */ +export function isEmptyRoleplayReply({ assistant = "", finishReason = "", toolCalls = [], terminated = true } = {}) { + if (!terminated || String(assistant).trim()) return false; + const callerToolCalls = Array.isArray(toolCalls) + ? toolCalls.filter((call) => call?.function?.name !== ROLEPLAY_HANDOFF_TOOL_NAME) + : []; + if (callerToolCalls.length) return false; + return ["", "stop", "length", "eos", "end_turn", "tool_calls"].includes(String(finishReason).trim().toLowerCase()); +} + +function frameJson(frame) { + const data = String(frame ?? "") + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trimStart()) + .join("\n") + .trim(); + if (!data || data === "[DONE]") return null; + try { + return JSON.parse(data); + } catch { + return null; + } +} + +// Removes the handoff call from a source leg's frames before anything reaches the client. Later +// deltas of a call carry only its index, so the filter remembers which indexes are the handoff's. +export function createRoleplayHandoffFrameFilter() { + let handoffIndexes = new Set(); + let otherToolCalls = false; + + const rewrite = (frame) => { + if (typeof frame !== "string" || !frame.includes("tool_calls")) return [frame]; + const payload = frameJson(frame); + const choice = payload?.choices?.[0]; + if (!choice) return [frame]; + const delta = choice.delta && typeof choice.delta === "object" ? { ...choice.delta } : null; + let changed = false; + if (delta && Array.isArray(delta.tool_calls)) { + const kept = []; + for (const call of delta.tool_calls) { + const index = Number.isSafeInteger(call?.index) ? call.index : 0; + if (call?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME) handoffIndexes.add(index); + if (handoffIndexes.has(index)) { + changed = true; + continue; + } + otherToolCalls = true; + kept.push(call); + } + if (kept.length) delta.tool_calls = kept; + else delete delta.tool_calls; + } + let finishReason = choice.finish_reason; + if (finishReason === "tool_calls" && handoffIndexes.size && !otherToolCalls) { + finishReason = "stop"; + changed = true; + } + if (!changed) return [frame]; + const hasDelta = delta && Object.keys(delta).some((key) => key !== "role"); + if (!hasDelta && (finishReason === null || finishReason === undefined) && !payload.usage) return []; + const choices = [...payload.choices]; + choices[0] = { ...choice, ...(delta ? { delta } : {}), finish_reason: finishReason ?? null }; + return [`data: ${JSON.stringify({ ...payload, choices })}\n\n`]; + }; + + return { + filter(frames) { + return frames.flatMap(rewrite); + }, + reset() { + handoffIndexes = new Set(); + otherToolCalls = false; + }, + }; +} diff --git a/worker/roleplay/nonstream-repair.mjs b/worker/roleplay/nonstream-repair.mjs index 0fae7c2c..516af2fd 100644 --- a/worker/roleplay/nonstream-repair.mjs +++ b/worker/roleplay/nonstream-repair.mjs @@ -4,6 +4,7 @@ import { } from "./memory.mjs"; import { normalizeRoleplayCompletionPayload } from "./reasoning-output.mjs"; import { ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE } from "./refusal-fallback.mjs"; +import { withoutRoleplayHandoffToolCalls } from "./handoff.mjs"; import { MAX_RESPONSE_BYTES, logRoleplayError, @@ -54,7 +55,7 @@ function completionPayload(payload, content, finishReason) { ? { ...choices[0] } : { index: 0 }; const message = first.message && typeof first.message === "object" - ? { ...first.message } + ? withoutRoleplayHandoffToolCalls({ ...first.message }) : { role: "assistant" }; delete message.refusal; message.content = content; @@ -64,6 +65,11 @@ function completionPayload(payload, content, finishReason) { return { ...payload, choices }; } +function completionToolCalls(payload) { + const calls = payload?.choices?.[0]?.message?.tool_calls; + return Array.isArray(calls) ? calls : []; +} + function completionRefusal(payload) { const refusal = payload?.choices?.[0]?.message?.refusal; return typeof refusal === "string" ? refusal : ""; @@ -115,6 +121,50 @@ export async function repairNonStreamingCompletion({ let terminalReason = ""; let contractAnalysis = null; let refusalFallbackFailed = false; + let handoffDraftUsed = false; + + // Starts the turn over on other candidates; null when none returned a readable completion. + const openFreshLeg = async (reason, handoff = null) => { + const attempted = await continuation.openResponse({ + assistant, + continuationCount: continuationCount + 1, + reason, + contractAnalysis: null, + handoff, + }); + const payload = attempted + ? await readContinuationPayload(attempted, signal) + : null; + if (!payload) return null; + metadata = { + provider: continuation.candidate.provider, + model: continuation.candidate.model, + }; + return { payload, leg: normalizedLeg(payload, metadata) }; + }; + const acceptFreshLeg = (next, reason) => { + basePayload = next.leg.payload; + assistant = next.leg.assistant; + clientContent = next.leg.clientContent; + finishReason = next.leg.finishReason; + continuationCount += 1; + continuationsByReason[reason] = (continuationsByReason[reason] ?? 0) + 1; + }; + const failTurn = (reason) => { + assistant = ""; + clientContent = ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE; + finishReason = "stop"; + terminalReason = reason; + refusalFallbackFailed = true; + }; + const usableLeg = (next) => + next && + next.leg.assistant.trim() && + !continuation.classifyRefusal({ + assistant: next.leg.assistant, + refusal: completionRefusal(next.payload), + finishReason: next.leg.finishReason, + }).refused; const refusal = continuation.classifyRefusal({ assistant, @@ -167,6 +217,56 @@ export async function repairNonStreamingCompletion({ finishReason = "stop"; terminalReason = "refusal_fallback_failed"; } + } else { + const handoff = assistant.trim() + ? null + : continuation.detectHandoff(completionToolCalls(initialPayload)); + if (handoff) { + continuationDiagnostics.push({ + reason: "handoff", + handoffReason: handoff.reason, + draftCharacters: handoff.draft.length, + charactersDiscarded: clientContent.length, + accepted: false, + }); + const next = await openFreshLeg("handoff", handoff); + if (usableLeg(next)) { + acceptFreshLeg(next, "handoff"); + } else if (handoff.draft) { + // No target wrote the turn, so the source's own draft is the reply. + assistant = handoff.draft; + clientContent = handoff.draft; + finishReason = "stop"; + handoffDraftUsed = true; + } else { + failTurn("handoff_failed"); + } + } else { + let toolCalls = completionToolCalls(initialPayload); + let retried = false; + while ( + (continuationsByReason.empty_reply ?? 0) < 2 && + continuation.detectEmptyReply({ + assistant, + finishReason, + toolCalls, + terminated: true, + reason: continuation.assess({ assistant, finishReason }).reason, + }) + ) { + retried = true; + continuationDiagnostics.push({ + reason: "empty_reply", + charactersDiscarded: clientContent.length, + accepted: false, + }); + const next = await openFreshLeg("empty_reply"); + if (!next) break; + acceptFreshLeg(next, "empty_reply"); + toolCalls = completionToolCalls(next.payload); + } + if (retried && !assistant.trim() && !toolCalls.length) failTurn("empty_reply_failed"); + } } while (!refusalFallbackFailed) { @@ -284,6 +384,8 @@ export async function repairNonStreamingCompletion({ const success = !terminalReason; const finalFinishReason = terminalReason === "output_limit" ? "length" : finishReason || "stop"; + const completionReason = + terminalReason || (handoffDraftUsed ? "handoff_draft" : "complete"); return { payload: completionPayload( @@ -294,7 +396,7 @@ export async function repairNonStreamingCompletion({ assistant: continuation.cleanOutput(assistant).content, finishReason: finalFinishReason, success, - reason: terminalReason || "complete", + reason: completionReason, continuationCount, upstreamCallCount: continuation.upstreamCallCount, refusalFallbackCount: continuation.refusalFallbackCount, diff --git a/worker/roleplay/sse-collector.mjs b/worker/roleplay/sse-collector.mjs index a28a5f80..fdcdddb9 100644 --- a/worker/roleplay/sse-collector.mjs +++ b/worker/roleplay/sse-collector.mjs @@ -4,6 +4,8 @@ import { } from "./reasoning-output.mjs"; const MAX_COLLECTED_ASSISTANT_CHARACTERS = 1_000_000; +const MAX_TOOL_CALLS = 16; +const MAX_TOOL_ARGUMENT_CHARACTERS = 64_000; function splitSseFrames(text, flush = false) { const frames = []; @@ -147,6 +149,9 @@ export function createSseAssistantCollector({ let truncated = false; let template = null; let completionTokens = 0; + // Streamed tool calls arrive as deltas keyed by index: the first carries the name, the rest + // append argument text. + const toolCalls = []; const reasoningNormalizer = reasoningMetadata ? createRoleplayReasoningFrameNormalizer(reasoningMetadata) : null; @@ -167,6 +172,26 @@ export function createSseAssistantCollector({ return current + content.slice(0, remaining); }; + const appendToolCalls = (deltas) => { + if (!Array.isArray(deltas)) return; + for (const delta of deltas) { + const index = Number.isSafeInteger(delta?.index) ? delta.index : 0; + if (index < 0 || index >= MAX_TOOL_CALLS) continue; + const call = toolCalls[index] ??= { + id: "", + type: "function", + function: { name: "", arguments: "" }, + }; + if (typeof delta.id === "string" && delta.id) call.id = delta.id; + const name = delta.function?.name; + if (typeof name === "string" && name && !call.function.name) call.function.name = name; + const args = delta.function?.arguments; + if (typeof args === "string" && args) { + call.function.arguments = (call.function.arguments + args).slice(0, MAX_TOOL_ARGUMENT_CHARACTERS); + } + } + }; + const appendContent = (content) => { clientContent = appendBounded(clientContent, content); const visible = visibleCollector @@ -204,6 +229,7 @@ export function createSseAssistantCollector({ } template = parsed.payload; appendContent(choice?.delta?.content); + appendToolCalls(choice?.delta?.tool_calls); refusal = appendBounded(refusal, choice?.delta?.refusal); const terminal = typeof choice.finish_reason === "string" @@ -247,6 +273,7 @@ export function createSseAssistantCollector({ if (parsed?.done || choice?.finish_reason || [delta?.content, delta?.reasoning_content, delta?.reasoning, delta?.refusal] .some((value) => typeof value === "string" && value.length > 0) || + (Array.isArray(delta?.tool_calls) && delta.tool_calls.length > 0) || (Array.isArray(delta?.reasoning_details) && delta.reasoning_details.some((detail) => [typeof detail === "string" ? detail : "", detail?.text, detail?.reasoning, detail?.reasoning_content] .some((value) => typeof value === "string" && value.length > 0)))) { @@ -280,6 +307,7 @@ export function createSseAssistantCollector({ truncated, template, completionTokens, + toolCalls: toolCalls.filter(Boolean), output, withheld: [...withheld], }; diff --git a/worker/roleplay/streaming.mjs b/worker/roleplay/streaming.mjs index 066c42c5..9b6ed504 100644 --- a/worker/roleplay/streaming.mjs +++ b/worker/roleplay/streaming.mjs @@ -8,6 +8,7 @@ import { createRoleplayRefusalStreamGate, ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE, } from "./refusal-fallback.mjs"; +import { createRoleplayHandoffFrameFilter } from "./handoff.mjs"; const SSE_ENCODER = new TextEncoder(); const HEARTBEAT_COMMENT = SSE_ENCODER.encode( @@ -88,6 +89,9 @@ export function createObservedStream({ reasoningMetadata = null, detectRefusal = null, refusalFallbackEnabled = false, + detectHandoff = null, + detectEmptyReply = null, + getHandoffDraft = null, }) { let resolveCompletion; const completion = new Promise((resolve) => { @@ -126,6 +130,7 @@ export function createObservedStream({ const refusalGate = createRoleplayRefusalStreamGate( refusalFallbackEnabled, ); + let handoffFilter = createRoleplayHandoffFrameFilter(); let settled = false; let released = false; let pendingRead = null; @@ -169,6 +174,7 @@ export function createObservedStream({ currentReasoningMetadata = next.reasoningMetadata ?? currentReasoningMetadata; collector = newCollector(); + handoffFilter = createRoleplayHandoffFrameFilter(); refusalGate.reset(Boolean(next.refusalFallbackEnabled)); released = false; pendingRead = null; @@ -353,10 +359,8 @@ export function createObservedStream({ }) : { refused: false, reason: "" }; const semanticRefusal = refusal.refused === true; - const completedOutputFrames = [ - ...refusalGate.consume(collected.output), - ...refusalGate.complete(semanticRefusal), - ]; + collected.output = handoffFilter.filter(collected.output); + collected.withheld = handoffFilter.filter(collected.withheld); const outputLimited = collected.finishReason === "length"; const assistantAddition = activeContinuationReason === "output_contract" @@ -399,6 +403,23 @@ export function createObservedStream({ candidateClientContent = cleanOutput(candidateClientContent).content; } + const handoff = + !semanticRefusal && + !candidateAssistant.trim() && + typeof detectHandoff === "function" + ? detectHandoff(collected.toolCalls) + : null; + const emptyReply = + !semanticRefusal && + !handoff && + typeof detectEmptyReply === "function" && + detectEmptyReply({ + assistant: candidateAssistant, + finishReason: collected.finishReason, + toolCalls: collected.toolCalls, + terminated: collected.terminated, + reason: decision.reason, + }); if (semanticRefusal) { decision = { reason: "semantic_refusal", @@ -406,6 +427,21 @@ export function createObservedStream({ contractAnalysis: null, cleaned: { content: assistant }, }; + } else if (handoff) { + decision = { + reason: "handoff", + limit: 1, + contractAnalysis: null, + cleaned: { content: assistant }, + handoff, + }; + } else if (emptyReply) { + decision = { + reason: "empty_reply", + limit: 2, + contractAnalysis: null, + cleaned: { content: assistant }, + }; } else if (outputLimited) { decision = { ...decision, @@ -426,6 +462,13 @@ export function createObservedStream({ }; } + const completedOutputFrames = [ + ...refusalGate.consume(collected.output), + ...refusalGate.complete( + semanticRefusal || Boolean(handoff) || emptyReply, + ), + ]; + let accepted = true; if (activeContinuationReason === "output_contract") { accepted = missingFieldsStrictlyDecreased( @@ -451,7 +494,9 @@ export function createObservedStream({ const discardForFreshRetry = decision.reason === "empty_eof" || decision.reason === "empty_story" || - decision.reason === "semantic_refusal"; + decision.reason === "semantic_refusal" || + decision.reason === "handoff" || + decision.reason === "empty_reply"; if (discardForFreshRetry) { continuationDiagnostics.push({ reason: decision.reason, @@ -459,10 +504,14 @@ export function createObservedStream({ decision.reason === "semantic_refusal" ? refusal.reason : undefined, + handoffReason: decision.handoff?.reason, + draftCharacters: decision.handoff + ? decision.handoff.draft.length + : undefined, charactersDiscarded: collected.clientContent.length, accepted: false, }); - if (decision.reason === "empty_eof") { + if (decision.reason === "empty_eof" || decision.reason === "empty_reply") { // Reasoning frames may already be visible downstream. Flush // the leg's normalizer output so its envelope closes // before a retry starts, while still discarding it from the @@ -528,6 +577,7 @@ export function createObservedStream({ continuationCount: continuationCount + 1, reason: incompleteReason, contractAnalysis: decision.contractAnalysis, + handoff: decision.handoff ?? null, }), ).then( (next) => { @@ -580,9 +630,29 @@ export function createObservedStream({ continue; } } + // When no handoff target wrote any visible text, the source's own draft is the reply. + const handoffDraft = + incompleteReason === "handoff" + ? decision.handoff?.draft ?? "" + : !assistant.trim() && typeof getHandoffDraft === "function" + ? getHandoffDraft() || "" + : ""; const refusalFallbackFailed = - incompleteReason === "semantic_refusal"; - if (refusalFallbackFailed) { + !handoffDraft && + (incompleteReason === "semantic_refusal" || + incompleteReason === "empty_reply" || + incompleteReason === "handoff"); + if (handoffDraft) { + assistant = handoffDraft; + enqueueFrames( + controller, + bufferedCompletionFrames( + template ?? collected.template, + handoffDraft, + "stop", + ), + ); + } else if (refusalFallbackFailed) { enqueueFrames( controller, bufferedCompletionFrames( @@ -609,12 +679,16 @@ export function createObservedStream({ void settle({ success: collected.terminated && - !incompleteReason && + (!incompleteReason || Boolean(handoffDraft)) && !refusalFallbackFailed, assistant, - finishReason: collected.finishReason, - reason: refusalFallbackFailed - ? "refusal_fallback_failed" + finishReason: handoffDraft ? "stop" : collected.finishReason, + reason: handoffDraft + ? "handoff_draft" + : refusalFallbackFailed + ? incompleteReason === "semantic_refusal" + ? "refusal_fallback_failed" + : `${incompleteReason}_failed` : incompleteReason ? incompleteReason : collected.terminated @@ -632,7 +706,7 @@ export function createObservedStream({ firstByteAt = now; } const decoded = decoder.decode(value, { stream: true }); - const frames = collector.consume(decoded); + const frames = handoffFilter.filter(collector.consume(decoded)); if (!frames.length) { continue; } diff --git a/worker/roleplay/transport.mjs b/worker/roleplay/transport.mjs index 5d0b957e..f1013661 100644 --- a/worker/roleplay/transport.mjs +++ b/worker/roleplay/transport.mjs @@ -23,6 +23,13 @@ import { injectRoleplayRefusalControl, roleplayRefusalFallbackCandidates, } from "./refusal-fallback.mjs"; +import { + ROLEPLAY_HANDOFF_TOOL, + injectRoleplayHandoffTool, + isRoleplayHandoffSource, + roleplayHandoffAllowed, + roleplayHandoffTargets, +} from "./handoff.mjs"; const RESPONSE_HEADER_WHITELIST = new Set([ "cache-control", @@ -874,11 +881,21 @@ export async function attemptRoleplayCandidates( }; } - const upstreamPayload = refusalFallbackCandidates.length + const controlledPayload = refusalFallbackCandidates.length ? injectRoleplayRefusalControl( preparedPayload?.payload ?? preparedPayload, ) : preparedPayload?.payload ?? preparedPayload; + const handoffTargets = roleplayHandoffTargets(candidates, candidateIndex, settings); + const upstreamPayload = isRoleplayHandoffSource(candidate, settings) + ? injectRoleplayHandoffTool( + controlledPayload, + handoffTargets.length > 0 && roleplayHandoffAllowed(nextState, settings), + ) + : controlledPayload; + const handoffOffered = + upstreamPayload?.tool_choice === "auto" && + upstreamPayload.tools?.[0] === ROLEPLAY_HANDOFF_TOOL; try { attempted = await fetchCandidateWithPaygoFallback( @@ -922,6 +939,8 @@ export async function attemptRoleplayCandidates( fallbackCount, promptCache: preparedPayload?.promptCache, refusalFallbackCandidates, + handoffTargets: handoffOffered ? handoffTargets : [], + laterCandidates: candidates.slice(candidateIndex + 1), }; } diff --git a/wrangler.jsonc b/wrangler.jsonc index 5428ca70..087bbb1c 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -111,6 +111,8 @@ "ROLEPLAY_PROVIDER_LIMITS": "{}", "ROLEPLAY_AUTO_ROUTE": "intelligence", "ROLEPLAY_INTELLIGENCE_MODELS": "opencode:deepseek-v4.1-flash,opencode:glm-5.3,opencode:mimo-v2.6-pro,cline-pass:cline-pass/glm-5.3,nanogpt:xiaomi/mimo-v2.6-pro,nanogpt:z-ai/glm-5.3,nanogpt:z-ai/glm-5.3-flash,nanogpt:z-ai/glm-5.2,cline-pass:cline-pass/glm-5.3-flash,cline-pass:cline-pass/mimo-v2.6-pro", + "ROLEPLAY_HANDOFF_FROM": "opencode:deepseek-v4.1-flash", + "ROLEPLAY_HANDOFF_TO": "opencode:glm-5.3,cline-pass:cline-pass/glm-5.3", "GLM_AUTO_OPTIMIZE": "true", "GLM_AUTO_OPTIMIZE_TRIGGER_TOKENS": "8000", "GLM_AUTO_OPTIMIZE_KEEP_RECENT_TURNS": "8",