diff --git a/docs/roleplay.md b/docs/roleplay.md index 2707f125..f0370d4f 100644 --- a/docs/roleplay.md +++ b/docs/roleplay.md @@ -180,6 +180,47 @@ without a roleplay model) use the same chain. Turns that set `routing.mode` or `routing.model` keep the adaptive pool, and so does any turn whose chain has no configured key. Leave the variable unset for the adaptive behaviour above. +### Handing pivotal turns to a stronger model + +`ROLEPLAY_HANDOFF_FROM` names chain entries that may pass a turn on, and +`ROLEPLAY_HANDOFF_TO` the entries that take it; both use the +`ROLEPLAY_INTELLIGENCE_MODELS` format and are unset by default. Production lets +DeepSeek V4.1 Flash on OpenCode Go hand off to GLM-5.3 on OpenCode Go, then on +ClinePass. A source request carries one internal function, `multillm_hand_off_turn`, +whose arguments are a reason (`deeper_reasoning` or `stronger_prose`), short notes +and an optional draft. When the source calls it instead of writing, the Worker +discards that attempt (its frames are held until visible text appears, so nothing +reaches the client) and the first available target writes the turn from the same +conversation plus a private note with the source's notes and draft, at its usual +reasoning effort (at `max`, GLM-5.3 spent a 4,096-token budget reasoning and wrote +nothing). If no target accepts the request or writes visible text, the draft is the +reply; without a draft the rest of the chain follows. +A source that writes the reply and also calls the function keeps its reply, and the +call is removed from the stream. + +`ROLEPLAY_HANDOFF_MIN_TURNS` (default `2`) allows at most one handoff per that many +turns in a session. Between handoffs, and whenever no target is available, the +source still receives the function with `tool_choice: "none"`, so its prompt prefix +and the provider's cache of it stay the same. Callers that ask for `xhigh` or `max` +reasoning start on the targets. Turns with `routing.fallback: "none"` or a session +tier skip the handoff. + +In a test on 2026-10-25 with the function offered, DeepSeek wrote all 12 ordinary +turns itself (scene continuation, conflict, out-of-character recall and small talk) +and handed off 4 of 6 climax and multi-faction planning turns, each with a full +draft of 950 to 1,600 characters. A handoff costs the source's 7 to 15 seconds +before the target starts. Each handoff logs `roleplay_handoff`, and the completion +log counts them in `handoffCount`. + +### Empty replies + +A reply that finishes with no visible text, only reasoning or nothing at all, is +treated as a failure: the turn starts over on the next chain candidate, at most +twice, and fails with a retry message if none writes anything. This also applies +when the caller sets `max_tokens`. The image-prompt repair keeps its own retry when +it can run. Set `ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED=false` to pass empty replies +through. Each retry logs `roleplay_empty_reply_fallback`. + Roleplay generation uses `ROLEPLAY_DEFAULT_REASONING_EFFORT`; the deployment configuration sets the general default to `high`. OpenCode `glm-5.3-flash` defaults to `reasoning_effort: max`. Go enables reasoning through that field; @@ -609,6 +650,10 @@ Non-secret tuning variables: | `ROLEPLAY_UPSTREAM_HEADER_TIMEOUT_MS` | `90000` | Maximum wait for provider response headers before advancing to the next tier | | `ROLEPLAY_PRE_RESPONSE_FALLBACK_ENABLED` | `true` | Advance after a provider transport rejection or header timeout; client aborts never advance | | `ROLEPLAY_PROVIDER_ERROR_FALLBACK_ENABLED` | `true` | Advance only when a 500/502/504 JSON response explicitly identifies both its error code and type as `provider_error`; unknown 5xx responses remain fail-closed | +| `ROLEPLAY_HANDOFF_FROM` | unset | Chain entries offered `multillm_hand_off_turn`; production uses `opencode:deepseek-v4.1-flash` | +| `ROLEPLAY_HANDOFF_TO` | unset | Chain entries that take a handed-off turn, in chain order; production uses GLM-5.3 on OpenCode Go and ClinePass | +| `ROLEPLAY_HANDOFF_MIN_TURNS` | `2` | At most one handoff per this many turns in a session; `0` or `1` lifts the cap | +| `ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED` | `true` | Start a reply with no visible text over on the next candidate, at most twice | | `ROLEPLAY_REFUSAL_FALLBACK_ENABLED` | `true` | Privately regenerate an exact GLM-5.3 Flash refusal trigger with the configured Uncensored Flash candidate; rejected output is neither streamed nor stored | | `ROLEPLAY_DEFAULT_REASONING_EFFORT` | `high` | Deliberate generation default with lower latency and hidden-token use than each provider's maximum; callers can still request `max` per turn | | `ROLEPLAY_STREAM_HEARTBEAT_MS` | `10000` | SSE keepalive interval during upstream silence | diff --git a/package.json b/package.json index 830b7785..6142b403 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --import ./tests/helpers/network_guard.mjs --test tests/test_cloudflare_worker.mjs tests/test_child_safety_worker.mjs tests/test_child_safety_dashboard.mjs tests/test_runtime_feature_flags_worker.mjs tests/test_anthropic_worker.mjs tests/test_haiku_intelligence_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_wave7_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs tests/test_frontend_build.mjs tests/test_deploy_pipeline.mjs tests/test_free_routes_worker.mjs tests/test_client_headers_worker.mjs tests/test_key_controls_expand_worker.mjs tests/test_key_data_guards_worker.mjs tests/test_usage_token_details_worker.mjs tests/test_provider_dispatch_worker.mjs tests/test_dispatch_network_worker.mjs tests/test_key_data_preflight.mjs tests/test_account_contract_worker.mjs tests/test_key_controls_contract_worker.mjs", + "test:worker": "node --import ./tests/helpers/network_guard.mjs --test tests/test_cloudflare_worker.mjs tests/test_child_safety_worker.mjs tests/test_child_safety_dashboard.mjs tests/test_runtime_feature_flags_worker.mjs tests/test_anthropic_worker.mjs tests/test_haiku_intelligence_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_handoff_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_wave7_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs tests/test_frontend_build.mjs tests/test_deploy_pipeline.mjs tests/test_free_routes_worker.mjs tests/test_client_headers_worker.mjs tests/test_key_controls_expand_worker.mjs tests/test_key_data_guards_worker.mjs tests/test_usage_token_details_worker.mjs tests/test_provider_dispatch_worker.mjs tests/test_dispatch_network_worker.mjs tests/test_key_data_preflight.mjs tests/test_account_contract_worker.mjs tests/test_key_controls_contract_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index a83ec1e3..c4ec48c6 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -420,6 +420,7 @@ "worker/roleplay/edge.mjs", "worker/roleplay/endpoint.mjs", "worker/roleplay/fallback-memory.mjs", + "worker/roleplay/handoff.mjs", "worker/roleplay/history-reconciliation.mjs", "worker/roleplay/memory.mjs", "worker/roleplay/message-fragments.mjs", diff --git a/tests/helpers/load_cloudflare_worker.mjs b/tests/helpers/load_cloudflare_worker.mjs index 4579d652..da1b35e2 100644 --- a/tests/helpers/load_cloudflare_worker.mjs +++ b/tests/helpers/load_cloudflare_worker.mjs @@ -7,6 +7,7 @@ function dataModuleUrl(source) { source = source.replaceAll('from "./worker/provider-dispatch.mjs"', `from "${new URL("../../worker/provider-dispatch.mjs", import.meta.url)}"`); source = source.replaceAll('from "./worker/key-controls-capability.mjs"', `from "${new URL("../../worker/key-controls-capability.mjs", import.meta.url)}"`); source = source.replaceAll('from "../provider-dispatch.mjs"', `from "${new URL("../../worker/provider-dispatch.mjs", import.meta.url)}"`); + source = source.replaceAll('from "./handoff.mjs"', `from "${new URL("../../worker/roleplay/handoff.mjs", import.meta.url)}"`); return `data:text/javascript;base64,${Buffer.from(source, "utf8").toString("base64")}`; } diff --git a/tests/outbound_inventory.json b/tests/outbound_inventory.json index f097df2c..010bc018 100644 --- a/tests/outbound_inventory.json +++ b/tests/outbound_inventory.json @@ -2122,17 +2122,17 @@ "worker/roleplay/completion-result.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "924b5dd596ac8ef89f8bd13f04bc7e1067546ed092e47435e3417b66728f1761" + "sha256": "af8ba094f15608f9cb6a47cc13042bb1a89de0dbeb97255a70ec0ad8628b608b" }, "worker/roleplay/config.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "ea0007c314192ae4718fee801465e8678cf5ec8edd7ffd84e9a70480cc10d1ee" + "sha256": "df8aca19da17bf36a87b3c2c9501f7c950033490288b79a866d75f1fd323c121" }, "worker/roleplay/continuation.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "0a3abcb3358db0112e4b6e22cf07695947f8bd2d4cde253de033b7f55c29e5b5" + "sha256": "a17e4e907349783bf740f6a240223c488bdb15504f4c44eb4d876c9aadf12fdd" }, "worker/roleplay/credential-health.mjs": { "kind": "guarded", @@ -2152,7 +2152,7 @@ "worker/roleplay/endpoint.mjs": { "kind": "internal", "reason": "Private session handler; generation and compaction use guarded transport.", - "sha256": "9e460b4d4cd420c7692406a7ba03248b73413c830be54423fdb57fe208a6d3cb" + "sha256": "ec959ea02c60bd85a0457f077ee2ce180adc47a5f27ac0167d053394af5d50d7" }, "worker/roleplay/fallback-memory.mjs": { "kind": "non-provider", @@ -2192,7 +2192,7 @@ "worker/roleplay/nonstream-repair.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "8d3166d81f7be6d0e6f25760a7c05866c4cd5c23eaad36300b8d7f2e3a03e196" + "sha256": "5fea43fd7411c2a5a78ed905333c41750e85c9a5686c0e07a314460bc6a85cf4" }, "worker/roleplay/operator-edge.mjs": { "kind": "internal", @@ -2277,7 +2277,7 @@ "worker/roleplay/sse-collector.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "d609605183dd3f8f2b90f2f7402ba2f45afb797f88297d11460de31ed36e9997" + "sha256": "5ea6fe478d2158914122d60fb2c0152764dc17de5bfe9ce13c34392d55d2db1f" }, "worker/roleplay/state-runtime.mjs": { "kind": "non-provider", @@ -2292,12 +2292,12 @@ "worker/roleplay/streaming.mjs": { "kind": "non-provider", "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", - "sha256": "cfa1ae35acfbddf5d6f8cd362b13c24247b38df78cbca3fd0bc545924a89b372" + "sha256": "edfd06ed6bb1632833f73119a2c6f9d8e3d814d5793edfc81e02fdc00188734e" }, "worker/roleplay/transport.mjs": { "kind": "guarded", "reason": "Every candidate, compaction and fallback uses providerFetch, including Container egress.", - "sha256": "fb1f50e58dd75196d6920a542df1c262e811a1c4e86d701d21366aa1d1540eb3" + "sha256": "28a8b0f142476e847381ecffbcd356158de4431c29fe9a9680078aa5ad6d4e70" }, "worker/roleplay/turn-runtime.mjs": { "kind": "non-provider", @@ -2453,5 +2453,10 @@ "kind": "internal", "reason": "Content-free capability probe of the actual fenced expansion Container before legacy private dispatch.", "sha256": "1bc8f20ae715d46ceed6a32d90d549688d9479ee87fe25a110235d03c390059e" + }, + "worker/roleplay/handoff.mjs": { + "kind": "non-provider", + "reason": "Runtime module pinned independently of HTTP client spelling; network attempts require runtime admission.", + "sha256": "f681974644eb5340debf6d4c0b072bb8dfd5ffcad18a652611d9bdf888d595c3" } } diff --git a/tests/test_roleplay_config_worker.mjs b/tests/test_roleplay_config_worker.mjs index 6fb7a111..2ddea206 100644 --- a/tests/test_roleplay_config_worker.mjs +++ b/tests/test_roleplay_config_worker.mjs @@ -272,6 +272,25 @@ test("deployment starts roleplay on OpenCode Go, then Cline Pass, and keeps Nano assert.equal(config.vars?.NANOGPT_SPEED_ROUTING, ""); }); +test("deployment lets DeepSeek hand pivotal turns to GLM 5.3, at most every other turn", async () => { + const configUrl = new URL("../wrangler.jsonc", import.meta.url); + const config = JSON.parse(await readFile(configUrl, "utf8")); + + // Measured 2026-10-25: with the tool offered, DeepSeek V4.1 Flash wrote all 12 ordinary turns + // itself and handed off 4 of 6 climax and multi-faction planning turns, each with a full draft. + assert.equal(config.vars?.ROLEPLAY_HANDOFF_FROM, "opencode:deepseek-v4.1-flash"); + assert.deepEqual(config.vars?.ROLEPLAY_HANDOFF_TO.split(","), [ + "opencode:glm-5.3", + "cline-pass:cline-pass/glm-5.3", + ]); + // ROLEPLAY_HANDOFF_MIN_TURNS keeps its default of 2. + assert.equal(config.vars?.ROLEPLAY_HANDOFF_MIN_TURNS, undefined); + const chain = config.vars?.ROLEPLAY_INTELLIGENCE_MODELS.split(","); + for (const target of config.vars.ROLEPLAY_HANDOFF_TO.split(",")) { + assert.ok(chain.indexOf(target) > chain.indexOf(config.vars.ROLEPLAY_HANDOFF_FROM), target); + } +}); + test("deployment routes GLM through LinkAPI before NavyAI", async () => { const configUrl = new URL("../wrangler.jsonc", import.meta.url); const config = JSON.parse(await readFile(configUrl, "utf8")); diff --git a/tests/test_roleplay_handoff_worker.mjs b/tests/test_roleplay_handoff_worker.mjs new file mode 100644 index 00000000..4a7e2c87 --- /dev/null +++ b/tests/test_roleplay_handoff_worker.mjs @@ -0,0 +1,472 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + completionResponse, + handleRoleplayEdgeRequest, + makeRoleplayEnv, + roleplayRequest, + withGlobalFetch, +} from "./helpers/roleplay_fixture.mjs"; +import { + ROLEPLAY_HANDOFF_TOOL_NAME, + createRoleplayHandoffFrameFilter, + isEmptyRoleplayReply, + preferRoleplayHandoffTargets, + roleplayHandoffCall, + roleplayHandoffMessages, +} from "../worker/roleplay/handoff.mjs"; +import { getRoleplaySettings } from "../worker/roleplay/config.mjs"; + +const SOURCE = "deepseek-v4.1-flash"; +const TARGET = "glm-5.3"; +const DRAFT = "*Isolde draws the pistol slowly.* \"You'll have neither of them, Commodore.\""; +const TARGET_REPLY = "*Isolde lets the silence stretch until the gun crews flinch.* \"No.\""; + +function handoffEnv(overrides = {}) { + return makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:${TARGET},opencode:mimo-v2.6-pro`, + ROLEPLAY_HANDOFF_FROM: `opencode:${SOURCE}`, + ROLEPLAY_HANDOFF_TO: `opencode:${TARGET}`, + ROLEPLAY_DEFAULT_REASONING_EFFORT: "high", + ...overrides, + }); +} + +function frame(model, delta, finishReason = null) { + return `data: ${JSON.stringify({ model, choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`; +} + +function sse(frames) { + return new Response([...frames, "data: [DONE]\n\n"].join(""), { + headers: { "Content-Type": "text/event-stream" }, + }); +} + +function handoffCallFrames(model, args) { + const text = JSON.stringify(args); + return [ + frame(model, { reasoning_content: "This is the turning point of the story." }), + frame(model, { + tool_calls: [{ + index: 0, + id: "call_handoff", + type: "function", + function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "" }, + }], + }), + frame(model, { tool_calls: [{ index: 0, function: { arguments: text.slice(0, 25) } }] }), + frame(model, { tool_calls: [{ index: 0, function: { arguments: text.slice(25) } }] }), + frame(model, {}, "tool_calls"), + ]; +} + +function contentFrames(model, content) { + const midpoint = Math.ceil(content.length / 2); + return [ + frame(model, { content: content.slice(0, midpoint) }), + frame(model, { content: content.slice(midpoint) }), + frame(model, {}, "stop"), + ]; +} + +function streamedPayloads(body) { + return body + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6))); +} + +function streamedContent(body) { + return streamedPayloads(body) + .map((payload) => payload.choices?.[0]?.delta?.content ?? "") + .join(""); +} + +function visibleContent(body) { + return streamedContent(body).replace(/[\s\S]*?<\/think>/g, "").trim(); +} + +function turn(sessionId, overrides = {}) { + return roleplayRequest({ + session_id: sessionId, + input: "Harrow's frigate runs out its guns. He offers to spare the crew for Rook and the crate.", + model: "roleplay:intelligence", + max_tokens: 1024, + stream: true, + ...overrides, + }); +} + +async function sessionState(fixture) { + const stored = [...fixture.storageBySession.values()][0]?.storage; + return { + state: (await stored?.get("roleplay-session")) ?? {}, + messages: (await stored?.get("roleplay-messages")) ?? [], + }; +} + +function recordingFetch(calls, respond) { + return async (input, init) => { + const payload = JSON.parse(init.body); + calls.push(payload); + return respond(payload, calls.length, new URL(input)); + }; +} + +test("a streamed source turn hands off to the target, which rewrites the draft", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { + reason: "deeper_reasoning", + notes: "Isolde must refuse without softening.", + draft: DRAFT, + })) + : sse(contentFrames(TARGET, TARGET_REPLY)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-stream"), fixture.env); + return { status: result.status, body: await result.text() }; + }); + await fixture.waitForBackgroundWork(); + + assert.equal(response.status, 200); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(calls[0].tools?.[0]?.function?.name, ROLEPLAY_HANDOFF_TOOL_NAME); + assert.equal(calls[0].tool_choice, "auto"); + assert.equal(calls[1].tools, undefined); + // The target keeps the configured effort: at max, GLM-5.3 spent a 4,096-token budget reasoning. + assert.equal(calls[1].reasoning_effort, "high"); + const lastUser = calls[1].messages.findLastIndex((message) => message.role === "user"); + const note = calls[1].messages[lastUser - 1]; + assert.equal(note.role, "system"); + assert.match(note.content, /Private handoff note/); + assert.match(note.content, /careful reasoning/); + assert.ok(note.content.includes(DRAFT)); + assert.ok(note.content.includes("Isolde must refuse without softening.")); + + assert.equal(visibleContent(response.body), TARGET_REPLY); + assert.doesNotMatch(response.body, /tool_calls|multillm_hand_off_turn|turning point/); + + const { state, messages } = await sessionState(fixture); + assert.equal(state.lastHandoffTurn, 0); + assert.equal(state.handoffs, 1); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, TARGET_REPLY); +}); + +// Production reaches OpenCode through the container, which turns reasoning into content +// and drops the closing frame that carries only finish_reason. +function containerHandoffCallFrames(model, args) { + return handoffCallFrames(model, args) + .slice(0, -1) + .flatMap((entry, index) => index === 0 + ? [ + frame(model, { content: "" }), + frame(model, { content: "This is the turning point of the story." }), + entry, + ] + : [entry]) + .concat(frame(model, { content: "\n\n" })); +} + +test("a handoff still happens when the container rewrites reasoning and drops the finish frame", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(containerHandoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Make it land.", draft: DRAFT })) + : sse(contentFrames(TARGET, TARGET_REPLY)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-container"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), TARGET_REPLY); + assert.doesNotMatch(response, /tool_calls|multillm_hand_off_turn/); +}); + +test("a target that writes nothing visible leaves the source's draft as the reply", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { reason: "deeper_reasoning", notes: "Plan.", draft: DRAFT })) + : sse([frame(TARGET, { reasoning_content: "Weighing every faction." }), frame(TARGET, {}, "length")]), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-target-empty"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), DRAFT); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, DRAFT); +}); + +test("a source that writes the reply keeps it, and its stray handoff call is removed", async () => { + const fixture = handoffEnv(); + const calls = []; + const reply = "*Isolde laughs and turns the wheel hard to port.*"; + const response = await withGlobalFetch(recordingFetch(calls, () => sse([ + ...contentFrames(SOURCE, reply).slice(0, 2), + frame(SOURCE, { + tool_calls: [{ + index: 0, + id: "call_late", + type: "function", + function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "{\"reason\":\"stronger_prose\",\"notes\":\"x\"}" }, + }], + }), + frame(SOURCE, {}, "tool_calls"), + ])), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-late-call"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); + assert.equal(visibleContent(response), reply); + assert.doesNotMatch(response, /tool_calls|multillm_hand_off_turn/); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); +}); + +test("the next turn after a handoff withholds the tool until the turn gap passes", async () => { + const fixture = handoffEnv(); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE && calls.length === 1 + ? sse(handoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Pivotal." })) + : sse(contentFrames(payload.model, "*The guns fall silent.*")), + ), async () => { + for (let index = 0; index < 3; index += 1) { + const result = await handleRoleplayEdgeRequest( + turn("session-handoff-gap", { input: `Turn ${index + 1}.` }), + fixture.env, + ); + await result.text(); + await fixture.waitForBackgroundWork(); + } + }); + + const sourceCalls = calls.filter((call) => call.model === SOURCE); + assert.deepEqual(sourceCalls.map((call) => call.tool_choice), ["auto", "none", "auto"]); + // The tool definition stays on every source call so the cached prompt prefix does not change. + assert.ok(sourceCalls.every((call) => call.tools?.[0]?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME)); +}); + +test("when every target fails, the source's draft becomes the reply", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + payload.model === SOURCE + ? sse(handoffCallFrames(SOURCE, { reason: "stronger_prose", notes: "Pivotal.", draft: DRAFT })) + : new Response('{"error":"rate limited"}', { status: 429, headers: { "Content-Type": "application/json" } }), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-draft"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(visibleContent(response), DRAFT); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, DRAFT); +}); + +test("a non-streamed handoff returns the target's reply without the tool call", async () => { + const fixture = handoffEnv(); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => { + if (payload.model !== SOURCE) return completionResponse(payload.model, TARGET_REPLY); + return new Response(JSON.stringify({ + id: "chatcmpl-source", + object: "chat.completion", + model: SOURCE, + choices: [{ + index: 0, + message: { + role: "assistant", + content: "", + tool_calls: [{ + id: "call_handoff", + type: "function", + function: { + name: ROLEPLAY_HANDOFF_TOOL_NAME, + arguments: JSON.stringify({ reason: "stronger_prose", notes: "Pivotal.", draft: DRAFT }), + }, + }], + }, + finish_reason: "tool_calls", + }], + }), { headers: { "Content-Type": "application/json" } }); + }), () => handleRoleplayEdgeRequest(turn("session-handoff-json", { stream: false }), fixture.env)); + await fixture.waitForBackgroundWork(); + + assert.equal(response.status, 200); + const body = await response.json(); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, TARGET]); + assert.equal(body.choices[0].message.content, TARGET_REPLY); + assert.equal(body.choices[0].message.tool_calls, undefined); + assert.equal(body.choices[0].finish_reason, "stop"); + assert.equal(response.headers.get("X-Roleplay-Model"), TARGET); +}); + +test("callers asking for xhigh or max reasoning start on the target", async () => { + const fixture = handoffEnv(); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => sse(contentFrames(payload.model, TARGET_REPLY))), async () => { + const result = await handleRoleplayEdgeRequest( + turn("session-handoff-xhigh", { reasoning_effort: "xhigh" }), + fixture.env, + ); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [TARGET]); + assert.equal(calls[0].tools, undefined); +}); + +test("without handoff settings no tool is offered", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:${TARGET}`, + }); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => sse(contentFrames(payload.model, TARGET_REPLY))), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-handoff-off"), fixture.env); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); + assert.equal(calls[0].tools, undefined); + assert.equal(calls[0].tool_choice, undefined); +}); + +test("a streamed reply with only reasoning falls through to the next candidate", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const calls = []; + const reply = "*Isolde finally answers.*"; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + calls.length === 1 + ? sse([frame(payload.model, { reasoning_content: "Thinking it over." }), frame(payload.model, {}, "stop")]) + : sse(contentFrames(payload.model, reply)), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-empty-stream"), fixture.env); + return await result.text(); + }); + await fixture.waitForBackgroundWork(); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE, "mimo-v2.6-pro"]); + assert.equal(visibleContent(response), reply); + const finishReasons = streamedPayloads(response) + .map((payload) => payload.choices?.[0]?.finish_reason) + .filter(Boolean); + assert.deepEqual(finishReasons, ["stop"]); + const { messages } = await sessionState(fixture); + assert.equal(messages.findLast((message) => message.role === "assistant")?.content, reply); +}); + +test("a non-streamed empty reply falls through, and an all-empty turn says so", async () => { + const empty = (model) => new Response(JSON.stringify({ + id: "chatcmpl-empty", + object: "chat.completion", + model, + choices: [{ index: 0, message: { role: "assistant", content: "", reasoning_content: "Hmm." }, finish_reason: "stop" }], + }), { headers: { "Content-Type": "application/json" } }); + + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const calls = []; + const response = await withGlobalFetch(recordingFetch(calls, (payload) => + calls.length === 1 ? empty(payload.model) : completionResponse(payload.model, "*Isolde answers.*"), + ), () => handleRoleplayEdgeRequest(turn("session-empty-json", { stream: false }), fixture.env)); + assert.deepEqual(calls.map((call) => call.model), [SOURCE, "mimo-v2.6-pro"]); + assert.equal((await response.json()).choices[0].message.content, "*Isolde answers.*"); + + const allEmpty = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + }); + const failed = await withGlobalFetch(async (_input, init) => empty(JSON.parse(init.body).model), () => + handleRoleplayEdgeRequest(turn("session-empty-json-all", { stream: false }), allEmpty.env)); + const content = (await failed.json()).choices[0].message.content; + assert.match(content, /could not complete this response/); +}); + +test("the empty-reply retry can be switched off", async () => { + const fixture = makeRoleplayEnv({ + ROLEPLAY_INTELLIGENCE_MODELS: `opencode:${SOURCE},opencode:mimo-v2.6-pro`, + ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED: "false", + }); + const calls = []; + await withGlobalFetch(recordingFetch(calls, (payload) => + sse([frame(payload.model, { reasoning_content: "Thinking." }), frame(payload.model, {}, "stop")]), + ), async () => { + const result = await handleRoleplayEdgeRequest(turn("session-empty-off"), fixture.env); + await result.text(); + }); + + assert.deepEqual(calls.map((call) => call.model), [SOURCE]); +}); + +test("handoff helpers parse calls, place the note and order candidates", () => { + assert.deepEqual( + roleplayHandoffCall([{ function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "{not json" } }]), + { reason: "stronger_prose", notes: "", draft: "" }, + ); + assert.equal(roleplayHandoffCall([{ function: { name: "other", arguments: "{}" } }]), null); + + const messages = roleplayHandoffMessages( + [{ role: "system", content: "Card." }, { role: "user", content: "Go." }], + { reason: "stronger_prose", notes: "", draft: "Draft text." }, + ); + assert.deepEqual(messages.map((message) => message.role), ["system", "system", "user"]); + assert.match(messages[1].content, /Draft text\./); + + const settings = getRoleplaySettings({ + ROLEPLAY_HANDOFF_FROM: `opencode:${SOURCE}`, + ROLEPLAY_HANDOFF_TO: `opencode:${TARGET},cline-pass:cline-pass/glm-5.3`, + }); + const chain = [ + { provider: "opencode", model: SOURCE }, + { provider: "opencode", model: TARGET }, + { provider: "opencode", model: "mimo-v2.6-pro" }, + { provider: "cline-pass", model: "cline-pass/glm-5.3" }, + ]; + assert.deepEqual( + preferRoleplayHandoffTargets(chain, { forwarded: { reasoning_effort: "max" } }, settings) + .map((candidate) => candidate.model), + [TARGET, "cline-pass/glm-5.3", SOURCE, "mimo-v2.6-pro"], + ); + assert.equal(preferRoleplayHandoffTargets(chain, { forwarded: { reasoning_effort: "high" } }, settings), chain); + + assert.equal(isEmptyRoleplayReply({ assistant: " ", finishReason: "stop" }), true); + assert.equal(isEmptyRoleplayReply({ assistant: "", finishReason: "stop", toolCalls: [{ function: { name: "lookup" } }] }), false); + assert.equal(isEmptyRoleplayReply({ assistant: "", finishReason: "stop", terminated: false }), false); +}); + +test("the frame filter drops the handoff call and keeps other tool calls", () => { + const filter = createRoleplayHandoffFrameFilter(); + const output = filter.filter([ + frame("m", { tool_calls: [{ index: 0, function: { name: ROLEPLAY_HANDOFF_TOOL_NAME, arguments: "" } }] }), + frame("m", { tool_calls: [{ index: 0, function: { arguments: "{\"reason\":" } }] }), + frame("m", { tool_calls: [{ index: 1, function: { name: "lookup", arguments: "{}" } }] }), + frame("m", {}, "tool_calls"), + ]); + const payloads = output.map((entry) => JSON.parse(entry.slice(6))); + assert.equal(payloads.length, 2); + assert.equal(payloads[0].choices[0].delta.tool_calls[0].function.name, "lookup"); + assert.equal(payloads[1].choices[0].finish_reason, "tool_calls"); +}); diff --git a/tests/test_runtime_feature_flags_worker.mjs b/tests/test_runtime_feature_flags_worker.mjs index 8ea62950..eaabf8b6 100644 --- a/tests/test_runtime_feature_flags_worker.mjs +++ b/tests/test_runtime_feature_flags_worker.mjs @@ -11,7 +11,7 @@ const { MultiLLMProxyContainer } = await loadWorkerModule(); test("deployment groups feature settings while preserving Container flags and secrets", () => { assert.equal(Object.keys(config.vars.RUNTIME_FEATURE_FLAGS).length, 7); - assert.equal(Object.keys(config.vars).length, 65); + assert.equal(Object.keys(config.vars).length, 67); const source = { ...config.vars, ANTHROPIC_API_KEY: "synthetic-provider-secret" }; const container = new MultiLLMProxyContainer({}, source); assert.equal(container.envVars.MODEL_COOLDOWN_ENABLED, "true"); diff --git a/worker/roleplay/completion-result.mjs b/worker/roleplay/completion-result.mjs index 9dacd553..8e5fdd64 100644 --- a/worker/roleplay/completion-result.mjs +++ b/worker/roleplay/completion-result.mjs @@ -76,6 +76,12 @@ export function roleplayCompletionDisposition({ }; } +function diagnosticCount(diagnostics, reason) { + return Array.isArray(diagnostics) + ? diagnostics.filter((entry) => entry?.reason === reason).length + : 0; +} + export function logRoleplayStreamCompletion({ candidate, parsed, @@ -116,6 +122,10 @@ export function logRoleplayStreamCompletion({ heartbeatCount: completion.heartbeatCount, continuationCount: completion.continuationCount, refusalFallbackCount: completion.refusalFallbackCount, + handoffCount: + diagnosticCount(completion.continuationDiagnostics, "handoff") || undefined, + emptyReplyCount: + diagnosticCount(completion.continuationDiagnostics, "empty_reply") || undefined, upstreamCallCount: completion.upstreamCallCount, ...contractTelemetry( parsed.outputContract, @@ -147,6 +157,10 @@ export function logRoleplayNonStreamCompletion({ finishReason: completion.finishReason || undefined, continuationCount: completion.continuationCount, refusalFallbackCount: completion.refusalFallbackCount, + handoffCount: + diagnosticCount(completion.continuationDiagnostics, "handoff") || undefined, + emptyReplyCount: + diagnosticCount(completion.continuationDiagnostics, "empty_reply") || undefined, upstreamCallCount: completion.upstreamCallCount, queueMs: Math.round(timings.queueMs ?? 0), stateLoadMs: Math.round(timings.stateLoadMs ?? 0), diff --git a/worker/roleplay/config.mjs b/worker/roleplay/config.mjs index f3e0cd1f..771059bb 100644 --- a/worker/roleplay/config.mjs +++ b/worker/roleplay/config.mjs @@ -454,6 +454,14 @@ function parseIntelligenceChain(value) { return chain; } +// ROLEPLAY_HANDOFF_FROM and ROLEPLAY_HANDOFF_TO name chain entries the same way; unset means none. +function parseHandoffEntries(value) { + if (typeof value !== "string" || !value.trim()) return new Set(); + return new Set( + parseIntelligenceChain(value).map(({ provider, model }) => `${provider}:${model}`), + ); +} + export function intelligenceModelFamily(model) { const normalized = String(model ?? "").toLowerCase(); if (/(?:^|\/)glm[-_]?[0-9]/.test(normalized)) return "glm"; @@ -735,6 +743,13 @@ export function getRoleplaySettings(env) { env.ROLEPLAY_PROVIDER_LIMITS, ), intelligenceChain: parseIntelligenceChain(env.ROLEPLAY_INTELLIGENCE_MODELS), + handoffSources: parseHandoffEntries(env.ROLEPLAY_HANDOFF_FROM), + handoffTargets: parseHandoffEntries(env.ROLEPLAY_HANDOFF_TO), + handoffMinTurns: boundedInteger(env.ROLEPLAY_HANDOFF_MIN_TURNS, 2, 0, 1_000), + emptyReplyFallbackEnabled: booleanSetting( + env.ROLEPLAY_EMPTY_REPLY_FALLBACK_ENABLED, + true, + ), autoRoute: String(env.ROLEPLAY_AUTO_ROUTE ?? "").trim().toLowerCase() === "intelligence" ? "intelligence" diff --git a/worker/roleplay/continuation.mjs b/worker/roleplay/continuation.mjs index afe835f8..9fa2560c 100644 --- a/worker/roleplay/continuation.mjs +++ b/worker/roleplay/continuation.mjs @@ -14,6 +14,15 @@ import { recordModelRefusal, } from "./transport.mjs"; import { classifyRoleplayRefusal } from "./refusal-fallback.mjs"; +import { + isEmptyRoleplayReply, + recordRoleplayHandoff, + roleplayHandoffCall, + roleplayHandoffMessages, +} from "./handoff.mjs"; + +// Legs that start the turn over on other candidates instead of continuing the current one. +const FRESH_LEG_REASONS = new Set(["semantic_refusal", "handoff", "empty_reply"]); const LENGTH_CONTINUATION_INSTRUCTION = [ "[Automatic continuation of an incomplete response]", @@ -103,9 +112,19 @@ export function createRoleplayContinuation({ signal, idempotencyKey, refusalFallbackCandidates = [], + handoffTargets = [], + laterCandidates = [], }) { let currentState = state; let currentCandidate = candidate; + // A handoff adds its note to the prompt of every later leg, and its draft stands in for the + // reply if no later leg writes one. + let legMessages = messages; + let handoffDraft = ""; + let remainingHandoffTargets = [...handoffTargets]; + let remainingLaterCandidates = [...laterCandidates]; + let handoffCount = 0; + let emptyReplyCount = 0; let remainingRefusalFallbackCandidates = settings.refusalFallbackEnabled ? [...refusalFallbackCandidates] : []; @@ -116,7 +135,10 @@ export function createRoleplayContinuation({ (settings.maxAutoContinuations > 0 || settings.maxOutputContractRepairs > 0); const canOpen = - enabled || remainingRefusalFallbackCandidates.length > 0; + enabled || + remainingRefusalFallbackCandidates.length > 0 || + remainingHandoffTargets.length > 0 || + (settings.emptyReplyFallbackEnabled === true && remainingLaterCandidates.length > 0); const assess = ({ assistant, finishReason }) => { const cleaned = cleanRoleplayOutput( @@ -158,38 +180,61 @@ export function createRoleplayContinuation({ return { reason: "", limit: 0, contractAnalysis, cleaned }; }; + const freshLegCandidates = (reason, handoff) => { + if (reason === "semantic_refusal") return remainingRefusalFallbackCandidates; + if (reason === "empty_reply") { + return settings.emptyReplyFallbackEnabled === true ? remainingLaterCandidates : []; + } + if (!remainingHandoffTargets.length) return []; + // A draft can stand in for the reply if every target fails; without one the rest of the + // chain follows the targets. + return handoff?.draft + ? remainingHandoffTargets + : [ + ...remainingHandoffTargets, + ...remainingLaterCandidates.filter((entry) => !remainingHandoffTargets.includes(entry)), + ]; + }; + const openResponse = async ({ assistant, continuationCount, reason, contractAnalysis, + handoff = null, signal: nextSignal = signal, }) => { const semanticRefusal = reason === "semantic_refusal"; + const freshLeg = FRESH_LEG_REASONS.has(reason); const retriesFromBeginning = reason === "empty_eof" || reason === "empty_story"; + const freshCandidates = freshLeg ? freshLegCandidates(reason, handoff) : []; if ( - semanticRefusal - ? !remainingRefusalFallbackCandidates.length + freshLeg + ? !freshCandidates.length : !enabled || (!assistant.trim() && !retriesFromBeginning) ) { return null; } - const nextMessages = semanticRefusal - ? messages + if (reason === "handoff") { + legMessages = roleplayHandoffMessages(legMessages, handoff); + handoffDraft = handoff?.draft ?? ""; + } + const nextMessages = freshLeg + ? legMessages : continuationMessages( - messages, + legMessages, assistant, reason, contractAnalysis, ); const nextEstimatedInputTokens = estimateTokens(nextMessages); const nextCandidates = prepareRoleplayCandidates( - semanticRefusal - ? remainingRefusalFallbackCandidates + freshLeg + ? freshCandidates : [currentCandidate], nextEstimatedInputTokens, - semanticRefusal ? parsed.maxTokens : null, + freshLeg ? parsed.maxTokens : null, settings, ); if (semanticRefusal) { @@ -200,6 +245,27 @@ export function createRoleplayContinuation({ remainingRefusalFallbackCandidates = []; refusalFallbackCount += 1; } + if (reason === "handoff") { + currentState = recordRoleplayHandoff(currentState); + remainingHandoffTargets = []; + handoffCount += 1; + console.log(JSON.stringify({ + event: "roleplay_handoff", + provider: currentCandidate.provider, + model: currentCandidate.model, + handoffReason: handoff?.reason, + notesCharacters: handoff?.notes?.length ?? 0, + draftCharacters: handoff?.draft?.length ?? 0, + })); + } + if (reason === "empty_reply") { + emptyReplyCount += 1; + console.log(JSON.stringify({ + event: "roleplay_empty_reply_fallback", + provider: currentCandidate.provider, + model: currentCandidate.model, + })); + } if (!nextCandidates.length) { logRoleplayError( "roleplay_continuation_context_exhausted", @@ -281,6 +347,10 @@ export function createRoleplayContinuation({ return null; } currentCandidate = attempted.candidate; + if (freshLeg) { + remainingLaterCandidates = attempted.laterCandidates ?? []; + if (reason !== "handoff") remainingHandoffTargets = attempted.handoffTargets ?? []; + } return attempted; }; @@ -299,6 +369,23 @@ export function createRoleplayContinuation({ get refusalFallbackEnabled() { return remainingRefusalFallbackCandidates.length > 0; }, + // The stream holds a leg's frames until it shows visible text whenever that leg may still be + // replaced: by a refusal fallback or by a handoff. + get holdUntilVisible() { + return ( + remainingRefusalFallbackCandidates.length > 0 || + remainingHandoffTargets.length > 0 + ); + }, + get handoffCount() { + return handoffCount; + }, + get handoffDraft() { + return handoffDraft; + }, + get emptyReplyCount() { + return emptyReplyCount; + }, get upstreamCallCount() { return upstreamCallCount; }, @@ -310,6 +397,18 @@ export function createRoleplayContinuation({ return assess(input).reason; }, classifyRefusal: classifyRoleplayRefusal, + detectHandoff(toolCalls) { + return remainingHandoffTargets.length > 0 ? roleplayHandoffCall(toolCalls) : null; + }, + detectEmptyReply(input) { + // The output-contract repair already retries an image-only reply when it may run. + if (input?.reason === "empty_story" && enabled) return false; + return ( + settings.emptyReplyFallbackEnabled === true && + remainingLaterCandidates.length > 0 && + isEmptyRoleplayReply(input) + ); + }, openResponse, async open(input) { const attempted = await openResponse(input); diff --git a/worker/roleplay/endpoint.mjs b/worker/roleplay/endpoint.mjs index 521d6ecd..cb37836f 100644 --- a/worker/roleplay/endpoint.mjs +++ b/worker/roleplay/endpoint.mjs @@ -46,6 +46,7 @@ import { } from "./completion-result.mjs"; import { createExtractiveCompactionDigest } from "./fallback-memory.mjs"; import { createRoleplayContinuation } from "./continuation.mjs"; +import { preferRoleplayHandoffTargets } from "./handoff.mjs"; import { buildConfiguredCandidates, autoRoutePreference, @@ -99,6 +100,8 @@ export { isRoleplayPath, scopePublicRoleplaySessionId }; import { handleRoleplayEdgeRequest as roleplayEdgeRequest } from "./edge.mjs"; export const handleRoleplayEdgeRequest = (request, env) => withRoleplayGatewayAuthority(request, env, roleplayEdgeRequest); const INTERNAL_OUTPUT_MODE_HEADER = "X-MultiLLM-Roleplay-Output-Mode"; +// Turns without fallbacks also skip the handoff and the empty-reply retry. +const WITHOUT_LEG_HANDOFFS = Object.freeze({ handoffSources: new Set(), emptyReplyFallbackEnabled: false }); async function preparePromptInjection(payload, env, record = true) { const mode = env.PROMPT_INJECTION_MODE; @@ -299,7 +302,7 @@ export class RoleplaySession extends DurableObject { } parsedInitial.routing = parseRoutingPolicy(payload.routing, { sessionTierEnabled: this.sessionTiers.settings.enabled }); if (parsedInitial.routing.fallback === "none") { - settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0 }; + settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0, ...WITHOUT_LEG_HANDOFFS }; } const { profile, parsed: parsedWithProfile } = effectiveRoleplayCharacter( state, @@ -396,7 +399,7 @@ export class RoleplaySession extends DurableObject { tierScope.turn = tierTurn; if (tierTurn) { candidates = tierTurn.candidates; - settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0 }; + settings = { ...settings, refusalFallbackEnabled: false, maxAutoContinuations: 0, maxOutputContractRepairs: 0, ...WITHOUT_LEG_HANDOFFS }; } if (parsed.routing.fallback === "none") candidates.splice(1); if (!candidates.length) { @@ -634,6 +637,7 @@ export class RoleplaySession extends DurableObject { ); generationCandidates = prepareCanaryCandidates(generationCandidates, canary, roleplayMessages, estimateTokens, settings); if (tierTurn) generationCandidates = tierTurn.select(generationCandidates); + generationCandidates = preferRoleplayHandoffTargets(generationCandidates, parsed, settings); if (!generationCandidates.length) { state = markRoleplayRequest(state, idempotencyKey, "context_too_large"); await stateRepository.save(state); @@ -693,6 +697,8 @@ export class RoleplaySession extends DurableObject { cleanup, promptCache, refusalFallbackCandidates, + handoffTargets, + laterCandidates, } = attempted; // Dispatch and continuation share the selected view. Persistence and // recovery above retain the original conversation independently. @@ -716,6 +722,8 @@ export class RoleplaySession extends DurableObject { signal: request.signal, idempotencyKey, refusalFallbackCandidates, + handoffTargets, + laterCandidates, }); protectRoleplayContinuation(continuation, canary, request.signal); const selectionReason = candidate.selectionReason || (parsed.routing.mode !== "provider-priority" ? parsed.routing.mode : @@ -799,7 +807,10 @@ export class RoleplaySession extends DurableObject { model: candidate.model, }, detectRefusal: continuation.classifyRefusal, - refusalFallbackEnabled: continuation.refusalFallbackEnabled, + refusalFallbackEnabled: continuation.holdUntilVisible, + detectHandoff: continuation.detectHandoff, + detectEmptyReply: continuation.detectEmptyReply, + getHandoffDraft: () => continuation.handoffDraft, onComplete: async (completion) => { completion = canaryCompletion(completion, canary); await preserveRecovery(this.ctx.storage, recovery, completion, trace?.id, retention); @@ -913,7 +924,7 @@ export class RoleplaySession extends DurableObject { applyRoleplayRouteHeaders( responseHeaders, continuation.candidate, - fallbackCount + continuation.refusalFallbackCount, + fallbackCount + continuation.refusalFallbackCount + continuation.emptyReplyCount, ); responseBytes = new TextEncoder().encode( JSON.stringify(completionResult.payload), diff --git a/worker/roleplay/handoff.mjs b/worker/roleplay/handoff.mjs new file mode 100644 index 00000000..6977809a --- /dev/null +++ b/worker/roleplay/handoff.mjs @@ -0,0 +1,249 @@ +// A fast first-pass model may hand a roleplay turn to a stronger one. Configured source models +// (ROLEPLAY_HANDOFF_FROM) receive an internal function; when one calls it instead of writing, +// the Worker discards that leg and the first available target (ROLEPLAY_HANDOFF_TO) writes the +// turn from the source's notes and draft. Clients never see the function or its call. + +export const ROLEPLAY_HANDOFF_TOOL_NAME = "multillm_hand_off_turn"; + +const HANDOFF_REASONS = new Set(["deeper_reasoning", "stronger_prose"]); +const MAX_NOTES_CHARACTERS = 2_000; +const MAX_DRAFT_CHARACTERS = 32_000; +const MAX_ARGUMENT_CHARACTERS = 64_000; +const REASONING_EFFORTS_THAT_PREFER_TARGETS = new Set(["xhigh", "max"]); + +export const ROLEPLAY_HANDOFF_TOOL = Object.freeze({ + type: "function", + function: { + name: ROLEPLAY_HANDOFF_TOOL_NAME, + description: [ + "Pass this turn to a senior writer instead of answering it yourself.", + "Call it only when the turn clearly needs more than you can deliver well:", + "deeper_reasoning for tangled plot logic, many characters or factions to keep consistent,", + "or a plan or mystery that must add up; stronger_prose for a pivotal scene such as a", + "climax, a major reveal, a death or a turning point in a relationship.", + "Most turns are ordinary: write those yourself and do not call this.", + "Never call it for short, simple or out-of-character replies.", + ].join(" "), + parameters: { + type: "object", + properties: { + reason: { type: "string", enum: ["deeper_reasoning", "stronger_prose"] }, + notes: { + type: "string", + description: "What the reply must achieve: key beats, tone and continuity facts, in under 80 words.", + }, + draft: { + type: "string", + description: "Optional: your draft of the reply. The senior writer keeps what works and rewrites the rest.", + }, + }, + required: ["reason", "notes"], + }, + }, +}); + +function entryKey(candidate) { + return `${String(candidate?.provider ?? "").toLowerCase()}:${candidate?.model ?? ""}`; +} + +export function isRoleplayHandoffSource(candidate, settings) { + return Boolean(settings?.handoffTargets?.size && settings?.handoffSources?.has(entryKey(candidate))); +} + +function isRoleplayHandoffTarget(candidate, settings) { + return Boolean(settings?.handoffTargets?.has(entryKey(candidate))); +} + +/** Later candidates a source may hand off to: the configured targets, in chain order. */ +export function roleplayHandoffTargets(candidates, candidateIndex, settings) { + if (!isRoleplayHandoffSource(candidates?.[candidateIndex], settings)) return []; + return candidates + .slice(candidateIndex + 1) + .filter((candidate) => isRoleplayHandoffTarget(candidate, settings)); +} + +/** At most one handoff every `handoffMinTurns` turns per session; 0 or 1 lifts the cap. */ +export function roleplayHandoffAllowed(state, settings) { + const last = state?.lastHandoffTurn; + if (!Number.isSafeInteger(last)) return true; + return (state?.turns ?? 0) - last >= (settings?.handoffMinTurns ?? 0); +} + +export function recordRoleplayHandoff(state) { + return { + ...state, + lastHandoffTurn: state?.turns ?? 0, + handoffs: (state?.handoffs ?? 0) + 1, + }; +} + +// The tool stays on every source leg so the prompt prefix (and the provider's cache of it) does +// not change between turns; `tool_choice: "none"` withholds it when no handoff may happen. +export function injectRoleplayHandoffTool(payload, available) { + if (!payload || !Array.isArray(payload.messages) || payload.tools !== undefined) { + return payload; + } + return { + ...payload, + tools: [ROLEPLAY_HANDOFF_TOOL], + tool_choice: available ? "auto" : "none", + }; +} + +/** Callers asking for xhigh or max reasoning start on the targets instead of the sources. */ +export function preferRoleplayHandoffTargets(candidates, parsed, settings) { + const effort = parsed?.forwarded?.reasoning_effort; + if (!REASONING_EFFORTS_THAT_PREFER_TARGETS.has(effort) || !settings?.handoffTargets?.size) { + return candidates; + } + const firstSource = candidates.findIndex((candidate) => isRoleplayHandoffSource(candidate, settings)); + if (firstSource < 0) return candidates; + const promoted = candidates + .slice(firstSource) + .filter((candidate) => isRoleplayHandoffTarget(candidate, settings)); + if (!promoted.length) return candidates; + return [ + ...candidates.slice(0, firstSource), + ...promoted, + ...candidates.slice(firstSource).filter((candidate) => !promoted.includes(candidate)), + ]; +} + +function clipped(value, limit) { + return typeof value === "string" ? value.trim().slice(0, limit) : ""; +} + +export function roleplayHandoffCall(toolCalls) { + const call = Array.isArray(toolCalls) + ? toolCalls.find((entry) => entry?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME) + : null; + if (!call) return null; + let args = {}; + try { + const parsed = JSON.parse(call.function.arguments || "{}"); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) args = parsed; + } catch { + // A malformed call still asks for the handoff; it only loses its notes. + } + return { + reason: HANDOFF_REASONS.has(args.reason) ? args.reason : "stronger_prose", + notes: clipped(args.notes, MAX_NOTES_CHARACTERS), + draft: clipped(args.draft, MAX_DRAFT_CHARACTERS), + }; +} + +/** The target's prompt: the original turn plus a private note placed before the last user message. */ +export function roleplayHandoffMessages(messages, handoff) { + if (!Array.isArray(messages) || !handoff) return messages; + const lines = [ + "[Private handoff note]", + handoff.reason === "deeper_reasoning" + ? "A first-pass writer passed the next turn to you because it needs careful reasoning: keep every character, plan and continuity fact consistent." + : "A first-pass writer passed the next turn to you because it needs your strongest prose.", + "Write the reply yourself and follow every roleplay instruction above. Never mention this note, the handoff or the draft.", + ]; + if (handoff.notes) lines.push(`Their notes: ${handoff.notes}`); + if (handoff.draft) { + lines.push("Their draft of the reply follows. Keep what works, fix what does not, and rewrite freely:"); + lines.push(handoff.draft); + } + const next = [...messages]; + let insertionIndex = next.length; + for (let index = next.length - 1; index >= 0; index -= 1) { + if (next[index]?.role === "user") { + insertionIndex = index; + break; + } + } + next.splice(insertionIndex, 0, { role: "system", content: lines.join("\n") }); + return next; +} + +export function withoutRoleplayHandoffToolCalls(message) { + if (!message || !Array.isArray(message.tool_calls)) return message; + const remaining = message.tool_calls.filter( + (call) => call?.function?.name !== ROLEPLAY_HANDOFF_TOOL_NAME, + ); + if (remaining.length === message.tool_calls.length) return message; + const next = { ...message }; + if (remaining.length) next.tool_calls = remaining; + else delete next.tool_calls; + return next; +} + +/** A finished reply with no visible text and no caller tool call gave the reader nothing. */ +export function isEmptyRoleplayReply({ assistant = "", finishReason = "", toolCalls = [], terminated = true } = {}) { + if (!terminated || String(assistant).trim()) return false; + const callerToolCalls = Array.isArray(toolCalls) + ? toolCalls.filter((call) => call?.function?.name !== ROLEPLAY_HANDOFF_TOOL_NAME) + : []; + if (callerToolCalls.length) return false; + return ["", "stop", "length", "eos", "end_turn", "tool_calls"].includes(String(finishReason).trim().toLowerCase()); +} + +function frameJson(frame) { + const data = String(frame ?? "") + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trimStart()) + .join("\n") + .trim(); + if (!data || data === "[DONE]") return null; + try { + return JSON.parse(data); + } catch { + return null; + } +} + +// Removes the handoff call from a source leg's frames before anything reaches the client. Later +// deltas of a call carry only its index, so the filter remembers which indexes are the handoff's. +export function createRoleplayHandoffFrameFilter() { + let handoffIndexes = new Set(); + let otherToolCalls = false; + + const rewrite = (frame) => { + if (typeof frame !== "string" || !frame.includes("tool_calls")) return [frame]; + const payload = frameJson(frame); + const choice = payload?.choices?.[0]; + if (!choice) return [frame]; + const delta = choice.delta && typeof choice.delta === "object" ? { ...choice.delta } : null; + let changed = false; + if (delta && Array.isArray(delta.tool_calls)) { + const kept = []; + for (const call of delta.tool_calls) { + const index = Number.isSafeInteger(call?.index) ? call.index : 0; + if (call?.function?.name === ROLEPLAY_HANDOFF_TOOL_NAME) handoffIndexes.add(index); + if (handoffIndexes.has(index)) { + changed = true; + continue; + } + otherToolCalls = true; + kept.push(call); + } + if (kept.length) delta.tool_calls = kept; + else delete delta.tool_calls; + } + let finishReason = choice.finish_reason; + if (finishReason === "tool_calls" && handoffIndexes.size && !otherToolCalls) { + finishReason = "stop"; + changed = true; + } + if (!changed) return [frame]; + const hasDelta = delta && Object.keys(delta).some((key) => key !== "role"); + if (!hasDelta && (finishReason === null || finishReason === undefined) && !payload.usage) return []; + const choices = [...payload.choices]; + choices[0] = { ...choice, ...(delta ? { delta } : {}), finish_reason: finishReason ?? null }; + return [`data: ${JSON.stringify({ ...payload, choices })}\n\n`]; + }; + + return { + filter(frames) { + return frames.flatMap(rewrite); + }, + reset() { + handoffIndexes = new Set(); + otherToolCalls = false; + }, + }; +} diff --git a/worker/roleplay/nonstream-repair.mjs b/worker/roleplay/nonstream-repair.mjs index 0fae7c2c..516af2fd 100644 --- a/worker/roleplay/nonstream-repair.mjs +++ b/worker/roleplay/nonstream-repair.mjs @@ -4,6 +4,7 @@ import { } from "./memory.mjs"; import { normalizeRoleplayCompletionPayload } from "./reasoning-output.mjs"; import { ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE } from "./refusal-fallback.mjs"; +import { withoutRoleplayHandoffToolCalls } from "./handoff.mjs"; import { MAX_RESPONSE_BYTES, logRoleplayError, @@ -54,7 +55,7 @@ function completionPayload(payload, content, finishReason) { ? { ...choices[0] } : { index: 0 }; const message = first.message && typeof first.message === "object" - ? { ...first.message } + ? withoutRoleplayHandoffToolCalls({ ...first.message }) : { role: "assistant" }; delete message.refusal; message.content = content; @@ -64,6 +65,11 @@ function completionPayload(payload, content, finishReason) { return { ...payload, choices }; } +function completionToolCalls(payload) { + const calls = payload?.choices?.[0]?.message?.tool_calls; + return Array.isArray(calls) ? calls : []; +} + function completionRefusal(payload) { const refusal = payload?.choices?.[0]?.message?.refusal; return typeof refusal === "string" ? refusal : ""; @@ -115,6 +121,50 @@ export async function repairNonStreamingCompletion({ let terminalReason = ""; let contractAnalysis = null; let refusalFallbackFailed = false; + let handoffDraftUsed = false; + + // Starts the turn over on other candidates; null when none returned a readable completion. + const openFreshLeg = async (reason, handoff = null) => { + const attempted = await continuation.openResponse({ + assistant, + continuationCount: continuationCount + 1, + reason, + contractAnalysis: null, + handoff, + }); + const payload = attempted + ? await readContinuationPayload(attempted, signal) + : null; + if (!payload) return null; + metadata = { + provider: continuation.candidate.provider, + model: continuation.candidate.model, + }; + return { payload, leg: normalizedLeg(payload, metadata) }; + }; + const acceptFreshLeg = (next, reason) => { + basePayload = next.leg.payload; + assistant = next.leg.assistant; + clientContent = next.leg.clientContent; + finishReason = next.leg.finishReason; + continuationCount += 1; + continuationsByReason[reason] = (continuationsByReason[reason] ?? 0) + 1; + }; + const failTurn = (reason) => { + assistant = ""; + clientContent = ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE; + finishReason = "stop"; + terminalReason = reason; + refusalFallbackFailed = true; + }; + const usableLeg = (next) => + next && + next.leg.assistant.trim() && + !continuation.classifyRefusal({ + assistant: next.leg.assistant, + refusal: completionRefusal(next.payload), + finishReason: next.leg.finishReason, + }).refused; const refusal = continuation.classifyRefusal({ assistant, @@ -167,6 +217,56 @@ export async function repairNonStreamingCompletion({ finishReason = "stop"; terminalReason = "refusal_fallback_failed"; } + } else { + const handoff = assistant.trim() + ? null + : continuation.detectHandoff(completionToolCalls(initialPayload)); + if (handoff) { + continuationDiagnostics.push({ + reason: "handoff", + handoffReason: handoff.reason, + draftCharacters: handoff.draft.length, + charactersDiscarded: clientContent.length, + accepted: false, + }); + const next = await openFreshLeg("handoff", handoff); + if (usableLeg(next)) { + acceptFreshLeg(next, "handoff"); + } else if (handoff.draft) { + // No target wrote the turn, so the source's own draft is the reply. + assistant = handoff.draft; + clientContent = handoff.draft; + finishReason = "stop"; + handoffDraftUsed = true; + } else { + failTurn("handoff_failed"); + } + } else { + let toolCalls = completionToolCalls(initialPayload); + let retried = false; + while ( + (continuationsByReason.empty_reply ?? 0) < 2 && + continuation.detectEmptyReply({ + assistant, + finishReason, + toolCalls, + terminated: true, + reason: continuation.assess({ assistant, finishReason }).reason, + }) + ) { + retried = true; + continuationDiagnostics.push({ + reason: "empty_reply", + charactersDiscarded: clientContent.length, + accepted: false, + }); + const next = await openFreshLeg("empty_reply"); + if (!next) break; + acceptFreshLeg(next, "empty_reply"); + toolCalls = completionToolCalls(next.payload); + } + if (retried && !assistant.trim() && !toolCalls.length) failTurn("empty_reply_failed"); + } } while (!refusalFallbackFailed) { @@ -284,6 +384,8 @@ export async function repairNonStreamingCompletion({ const success = !terminalReason; const finalFinishReason = terminalReason === "output_limit" ? "length" : finishReason || "stop"; + const completionReason = + terminalReason || (handoffDraftUsed ? "handoff_draft" : "complete"); return { payload: completionPayload( @@ -294,7 +396,7 @@ export async function repairNonStreamingCompletion({ assistant: continuation.cleanOutput(assistant).content, finishReason: finalFinishReason, success, - reason: terminalReason || "complete", + reason: completionReason, continuationCount, upstreamCallCount: continuation.upstreamCallCount, refusalFallbackCount: continuation.refusalFallbackCount, diff --git a/worker/roleplay/sse-collector.mjs b/worker/roleplay/sse-collector.mjs index a28a5f80..fdcdddb9 100644 --- a/worker/roleplay/sse-collector.mjs +++ b/worker/roleplay/sse-collector.mjs @@ -4,6 +4,8 @@ import { } from "./reasoning-output.mjs"; const MAX_COLLECTED_ASSISTANT_CHARACTERS = 1_000_000; +const MAX_TOOL_CALLS = 16; +const MAX_TOOL_ARGUMENT_CHARACTERS = 64_000; function splitSseFrames(text, flush = false) { const frames = []; @@ -147,6 +149,9 @@ export function createSseAssistantCollector({ let truncated = false; let template = null; let completionTokens = 0; + // Streamed tool calls arrive as deltas keyed by index: the first carries the name, the rest + // append argument text. + const toolCalls = []; const reasoningNormalizer = reasoningMetadata ? createRoleplayReasoningFrameNormalizer(reasoningMetadata) : null; @@ -167,6 +172,26 @@ export function createSseAssistantCollector({ return current + content.slice(0, remaining); }; + const appendToolCalls = (deltas) => { + if (!Array.isArray(deltas)) return; + for (const delta of deltas) { + const index = Number.isSafeInteger(delta?.index) ? delta.index : 0; + if (index < 0 || index >= MAX_TOOL_CALLS) continue; + const call = toolCalls[index] ??= { + id: "", + type: "function", + function: { name: "", arguments: "" }, + }; + if (typeof delta.id === "string" && delta.id) call.id = delta.id; + const name = delta.function?.name; + if (typeof name === "string" && name && !call.function.name) call.function.name = name; + const args = delta.function?.arguments; + if (typeof args === "string" && args) { + call.function.arguments = (call.function.arguments + args).slice(0, MAX_TOOL_ARGUMENT_CHARACTERS); + } + } + }; + const appendContent = (content) => { clientContent = appendBounded(clientContent, content); const visible = visibleCollector @@ -204,6 +229,7 @@ export function createSseAssistantCollector({ } template = parsed.payload; appendContent(choice?.delta?.content); + appendToolCalls(choice?.delta?.tool_calls); refusal = appendBounded(refusal, choice?.delta?.refusal); const terminal = typeof choice.finish_reason === "string" @@ -247,6 +273,7 @@ export function createSseAssistantCollector({ if (parsed?.done || choice?.finish_reason || [delta?.content, delta?.reasoning_content, delta?.reasoning, delta?.refusal] .some((value) => typeof value === "string" && value.length > 0) || + (Array.isArray(delta?.tool_calls) && delta.tool_calls.length > 0) || (Array.isArray(delta?.reasoning_details) && delta.reasoning_details.some((detail) => [typeof detail === "string" ? detail : "", detail?.text, detail?.reasoning, detail?.reasoning_content] .some((value) => typeof value === "string" && value.length > 0)))) { @@ -280,6 +307,7 @@ export function createSseAssistantCollector({ truncated, template, completionTokens, + toolCalls: toolCalls.filter(Boolean), output, withheld: [...withheld], }; diff --git a/worker/roleplay/streaming.mjs b/worker/roleplay/streaming.mjs index 066c42c5..9b6ed504 100644 --- a/worker/roleplay/streaming.mjs +++ b/worker/roleplay/streaming.mjs @@ -8,6 +8,7 @@ import { createRoleplayRefusalStreamGate, ROLEPLAY_REFUSAL_FALLBACK_FAILURE_MESSAGE, } from "./refusal-fallback.mjs"; +import { createRoleplayHandoffFrameFilter } from "./handoff.mjs"; const SSE_ENCODER = new TextEncoder(); const HEARTBEAT_COMMENT = SSE_ENCODER.encode( @@ -88,6 +89,9 @@ export function createObservedStream({ reasoningMetadata = null, detectRefusal = null, refusalFallbackEnabled = false, + detectHandoff = null, + detectEmptyReply = null, + getHandoffDraft = null, }) { let resolveCompletion; const completion = new Promise((resolve) => { @@ -126,6 +130,7 @@ export function createObservedStream({ const refusalGate = createRoleplayRefusalStreamGate( refusalFallbackEnabled, ); + let handoffFilter = createRoleplayHandoffFrameFilter(); let settled = false; let released = false; let pendingRead = null; @@ -169,6 +174,7 @@ export function createObservedStream({ currentReasoningMetadata = next.reasoningMetadata ?? currentReasoningMetadata; collector = newCollector(); + handoffFilter = createRoleplayHandoffFrameFilter(); refusalGate.reset(Boolean(next.refusalFallbackEnabled)); released = false; pendingRead = null; @@ -353,10 +359,8 @@ export function createObservedStream({ }) : { refused: false, reason: "" }; const semanticRefusal = refusal.refused === true; - const completedOutputFrames = [ - ...refusalGate.consume(collected.output), - ...refusalGate.complete(semanticRefusal), - ]; + collected.output = handoffFilter.filter(collected.output); + collected.withheld = handoffFilter.filter(collected.withheld); const outputLimited = collected.finishReason === "length"; const assistantAddition = activeContinuationReason === "output_contract" @@ -399,6 +403,23 @@ export function createObservedStream({ candidateClientContent = cleanOutput(candidateClientContent).content; } + const handoff = + !semanticRefusal && + !candidateAssistant.trim() && + typeof detectHandoff === "function" + ? detectHandoff(collected.toolCalls) + : null; + const emptyReply = + !semanticRefusal && + !handoff && + typeof detectEmptyReply === "function" && + detectEmptyReply({ + assistant: candidateAssistant, + finishReason: collected.finishReason, + toolCalls: collected.toolCalls, + terminated: collected.terminated, + reason: decision.reason, + }); if (semanticRefusal) { decision = { reason: "semantic_refusal", @@ -406,6 +427,21 @@ export function createObservedStream({ contractAnalysis: null, cleaned: { content: assistant }, }; + } else if (handoff) { + decision = { + reason: "handoff", + limit: 1, + contractAnalysis: null, + cleaned: { content: assistant }, + handoff, + }; + } else if (emptyReply) { + decision = { + reason: "empty_reply", + limit: 2, + contractAnalysis: null, + cleaned: { content: assistant }, + }; } else if (outputLimited) { decision = { ...decision, @@ -426,6 +462,13 @@ export function createObservedStream({ }; } + const completedOutputFrames = [ + ...refusalGate.consume(collected.output), + ...refusalGate.complete( + semanticRefusal || Boolean(handoff) || emptyReply, + ), + ]; + let accepted = true; if (activeContinuationReason === "output_contract") { accepted = missingFieldsStrictlyDecreased( @@ -451,7 +494,9 @@ export function createObservedStream({ const discardForFreshRetry = decision.reason === "empty_eof" || decision.reason === "empty_story" || - decision.reason === "semantic_refusal"; + decision.reason === "semantic_refusal" || + decision.reason === "handoff" || + decision.reason === "empty_reply"; if (discardForFreshRetry) { continuationDiagnostics.push({ reason: decision.reason, @@ -459,10 +504,14 @@ export function createObservedStream({ decision.reason === "semantic_refusal" ? refusal.reason : undefined, + handoffReason: decision.handoff?.reason, + draftCharacters: decision.handoff + ? decision.handoff.draft.length + : undefined, charactersDiscarded: collected.clientContent.length, accepted: false, }); - if (decision.reason === "empty_eof") { + if (decision.reason === "empty_eof" || decision.reason === "empty_reply") { // Reasoning frames may already be visible downstream. Flush // the leg's normalizer output so its envelope closes // before a retry starts, while still discarding it from the @@ -528,6 +577,7 @@ export function createObservedStream({ continuationCount: continuationCount + 1, reason: incompleteReason, contractAnalysis: decision.contractAnalysis, + handoff: decision.handoff ?? null, }), ).then( (next) => { @@ -580,9 +630,29 @@ export function createObservedStream({ continue; } } + // When no handoff target wrote any visible text, the source's own draft is the reply. + const handoffDraft = + incompleteReason === "handoff" + ? decision.handoff?.draft ?? "" + : !assistant.trim() && typeof getHandoffDraft === "function" + ? getHandoffDraft() || "" + : ""; const refusalFallbackFailed = - incompleteReason === "semantic_refusal"; - if (refusalFallbackFailed) { + !handoffDraft && + (incompleteReason === "semantic_refusal" || + incompleteReason === "empty_reply" || + incompleteReason === "handoff"); + if (handoffDraft) { + assistant = handoffDraft; + enqueueFrames( + controller, + bufferedCompletionFrames( + template ?? collected.template, + handoffDraft, + "stop", + ), + ); + } else if (refusalFallbackFailed) { enqueueFrames( controller, bufferedCompletionFrames( @@ -609,12 +679,16 @@ export function createObservedStream({ void settle({ success: collected.terminated && - !incompleteReason && + (!incompleteReason || Boolean(handoffDraft)) && !refusalFallbackFailed, assistant, - finishReason: collected.finishReason, - reason: refusalFallbackFailed - ? "refusal_fallback_failed" + finishReason: handoffDraft ? "stop" : collected.finishReason, + reason: handoffDraft + ? "handoff_draft" + : refusalFallbackFailed + ? incompleteReason === "semantic_refusal" + ? "refusal_fallback_failed" + : `${incompleteReason}_failed` : incompleteReason ? incompleteReason : collected.terminated @@ -632,7 +706,7 @@ export function createObservedStream({ firstByteAt = now; } const decoded = decoder.decode(value, { stream: true }); - const frames = collector.consume(decoded); + const frames = handoffFilter.filter(collector.consume(decoded)); if (!frames.length) { continue; } diff --git a/worker/roleplay/transport.mjs b/worker/roleplay/transport.mjs index 5d0b957e..f1013661 100644 --- a/worker/roleplay/transport.mjs +++ b/worker/roleplay/transport.mjs @@ -23,6 +23,13 @@ import { injectRoleplayRefusalControl, roleplayRefusalFallbackCandidates, } from "./refusal-fallback.mjs"; +import { + ROLEPLAY_HANDOFF_TOOL, + injectRoleplayHandoffTool, + isRoleplayHandoffSource, + roleplayHandoffAllowed, + roleplayHandoffTargets, +} from "./handoff.mjs"; const RESPONSE_HEADER_WHITELIST = new Set([ "cache-control", @@ -874,11 +881,21 @@ export async function attemptRoleplayCandidates( }; } - const upstreamPayload = refusalFallbackCandidates.length + const controlledPayload = refusalFallbackCandidates.length ? injectRoleplayRefusalControl( preparedPayload?.payload ?? preparedPayload, ) : preparedPayload?.payload ?? preparedPayload; + const handoffTargets = roleplayHandoffTargets(candidates, candidateIndex, settings); + const upstreamPayload = isRoleplayHandoffSource(candidate, settings) + ? injectRoleplayHandoffTool( + controlledPayload, + handoffTargets.length > 0 && roleplayHandoffAllowed(nextState, settings), + ) + : controlledPayload; + const handoffOffered = + upstreamPayload?.tool_choice === "auto" && + upstreamPayload.tools?.[0] === ROLEPLAY_HANDOFF_TOOL; try { attempted = await fetchCandidateWithPaygoFallback( @@ -922,6 +939,8 @@ export async function attemptRoleplayCandidates( fallbackCount, promptCache: preparedPayload?.promptCache, refusalFallbackCandidates, + handoffTargets: handoffOffered ? handoffTargets : [], + laterCandidates: candidates.slice(candidateIndex + 1), }; } diff --git a/wrangler.jsonc b/wrangler.jsonc index 5428ca70..087bbb1c 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -111,6 +111,8 @@ "ROLEPLAY_PROVIDER_LIMITS": "{}", "ROLEPLAY_AUTO_ROUTE": "intelligence", "ROLEPLAY_INTELLIGENCE_MODELS": "opencode:deepseek-v4.1-flash,opencode:glm-5.3,opencode:mimo-v2.6-pro,cline-pass:cline-pass/glm-5.3,nanogpt:xiaomi/mimo-v2.6-pro,nanogpt:z-ai/glm-5.3,nanogpt:z-ai/glm-5.3-flash,nanogpt:z-ai/glm-5.2,cline-pass:cline-pass/glm-5.3-flash,cline-pass:cline-pass/mimo-v2.6-pro", + "ROLEPLAY_HANDOFF_FROM": "opencode:deepseek-v4.1-flash", + "ROLEPLAY_HANDOFF_TO": "opencode:glm-5.3,cline-pass:cline-pass/glm-5.3", "GLM_AUTO_OPTIMIZE": "true", "GLM_AUTO_OPTIMIZE_TRIGGER_TOKENS": "8000", "GLM_AUTO_OPTIMIZE_KEEP_RECENT_TURNS": "8",