From f682c905ef475390c06f7dbe285a9579bf71afa7 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Sep 2026 15:00:14 +0000 Subject: [PATCH 1/2] feat: add neutral native provider IR Signed-off-by: Paul Querna --- crates/protocol/src/llm.rs | 473 ++++++++++++++++ crates/protocol/src/stream.rs | 333 +++++++++++- crates/protocol/tests/codex_neutral_ir.rs | 513 ++++++++++++++++++ .../tests/fixtures/codex_neutral_ir.json | 318 +++++++++++ .../src/codecs/anthropic/buffered.rs | 40 +- .../src/codecs/anthropic/stream.rs | 17 +- .../src/codecs/bedrock/stream.rs | 17 +- .../src/codecs/openai_chat/buffered.rs | 18 + .../src/codecs/openai_chat/stream.rs | 17 +- .../src/codecs/responses/buffered.rs | 18 + .../src/codecs/responses/stream.rs | 19 +- 11 files changed, 1764 insertions(+), 19 deletions(-) create mode 100644 crates/protocol/tests/codex_neutral_ir.rs create mode 100644 crates/protocol/tests/fixtures/codex_neutral_ir.json diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index c452a4ed3..04948f52c 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -7,6 +7,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use thiserror::Error; use crate::format::FormatId; @@ -140,6 +141,26 @@ pub enum ContentBlock { ToolCall(ToolCall), /// Result of an earlier tool invocation. ToolResult(ToolResult), + /// Free-form input tool invocation requested by the assistant. + CustomToolCall(CustomToolCall), + /// Result of an earlier free-form input tool invocation. + CustomToolResult(CustomToolResult), + /// Computer interaction requested by the assistant. + ComputerToolCall(ComputerToolCall), + /// Screenshot and safety acknowledgements returned after computer interaction. + ComputerToolResult(ComputerToolResult), + /// Provider-hosted tool invocation requested by the assistant. + HostedToolCall(HostedToolCall), + /// Result produced by a provider-hosted tool. + HostedToolResult(HostedToolResult), + /// Bounded opaque state retained for a later request. + OpaqueState(OpaqueState), + /// Provider-created conversation compaction state. + Compaction(CompactionItem), + /// A provider-native generated image. + GeneratedImage(GeneratedImage), + /// A provider pause that requires another turn instead of ordinary completion. + PauseTurn(PauseTurn), /// Provider refusal content. Refusal { /// Human-readable refusal text. @@ -237,6 +258,260 @@ pub struct ToolResult { pub is_error: Option, } +/// Free-form input tool invocation. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CustomToolCall { + /// Provider tool-call identifier used to pair the result. + pub id: String, + /// Provider item identifier, when the protocol distinguishes items from calls. + pub item_id: Option, + /// Tool name. + pub name: String, + /// Unparsed text input supplied to the tool. + pub input: String, +} + +/// Result of an earlier [`CustomToolCall`]. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CustomToolResult { + /// Identifier of the custom tool call this result answers. + pub tool_call_id: String, + /// Text returned by the tool. + pub output: String, + /// Whether tool execution failed, when reported. + pub is_error: Option, +} + +/// Screen geometry and environment exposed to a computer tool. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ComputerToolConfig { + /// Width of the interactive surface in pixels. + pub display_width: u32, + /// Height of the interactive surface in pixels. + pub display_height: u32, + /// Kind of interactive surface. + pub environment: ComputerEnvironment, +} + +/// Kind of interactive surface exposed to a computer tool. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ComputerEnvironment { + /// Browser viewport. + Browser, + /// Desktop session. + Desktop, + /// Mobile-device session. + Mobile, +} + +/// Coordinate on a computer tool's interactive surface. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct ComputerPoint { + /// Horizontal coordinate. + pub x: i64, + /// Vertical coordinate. + pub y: i64, +} + +/// Mouse button used by a computer action. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ComputerMouseButton { + /// Primary mouse button. + Left, + /// Middle mouse button. + Middle, + /// Secondary mouse button. + Right, +} + +/// One provider-neutral computer interaction. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ComputerAction { + /// Click one point. + Click { + /// Click location. + point: ComputerPoint, + /// Mouse button to click. + button: ComputerMouseButton, + }, + /// Double-click one point. + DoubleClick { + /// Click location. + point: ComputerPoint, + /// Mouse button to click. + button: ComputerMouseButton, + }, + /// Drag through an ordered path. + Drag { + /// Ordered drag path. + path: Vec, + }, + /// Press one or more keys together. + KeyPress { + /// Provider-neutral key names. + keys: Vec, + }, + /// Move the pointer without clicking. + Move { + /// Destination. + point: ComputerPoint, + }, + /// Request a screenshot without another interaction. + Screenshot, + /// Scroll at a point. + Scroll { + /// Pointer location for the scroll. + point: ComputerPoint, + /// Horizontal scroll distance. + delta_x: i64, + /// Vertical scroll distance. + delta_y: i64, + }, + /// Enter literal text. + Type { + /// Text to enter. + text: String, + }, + /// Wait for the remote surface. + Wait, +} + +/// One safety check attached to a computer action. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct ComputerSafetyCheck { + /// Stable identifier echoed when the check is acknowledged. + pub id: String, + /// Human-readable check description. + pub description: String, +} + +/// Computer actions requested by the assistant. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ComputerToolCall { + /// Provider tool-call identifier used to pair the result. + pub id: String, + /// Provider item identifier, when supplied. + pub item_id: Option, + /// Ordered actions to perform. + pub actions: Vec, + /// Checks that must be acknowledged before executing the actions. + pub pending_safety_checks: Vec, +} + +/// Result of an earlier [`ComputerToolCall`]. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ComputerToolResult { + /// Identifier of the computer tool call this result answers. + pub tool_call_id: String, + /// Screenshot or other native image returned after executing the action. + pub output: ImageSource, + /// Safety checks acknowledged by the executor. + pub acknowledged_safety_checks: Vec, +} + +/// A provider-hosted tool capability. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostedTool { + /// Search public web content. + WebSearch { + /// Optional allowlist of domains. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + allowed_domains: Vec, + }, + /// Search provider-managed files. + FileSearch { + /// Provider-managed collection identifiers. + vector_store_ids: Vec, + /// Maximum result count, when constrained. + max_results: Option, + }, + /// Execute code in a provider-managed container. + CodeInterpreter { + /// Existing container identifier, when one is reused. + container_id: Option, + }, + /// Generate an image. + ImageGeneration, +} + +/// Invocation of a provider-hosted tool. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct HostedToolCall { + /// Provider call identifier. + pub id: String, + /// Provider item identifier, when supplied. + pub item_id: Option, + /// Hosted capability being invoked. + pub tool: HostedTool, + /// Capability arguments expressed by the neutral JSON input contract. + pub arguments: Value, +} + +/// Result of an earlier [`HostedToolCall`]. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct HostedToolResult { + /// Identifier of the hosted tool call this result answers. + pub tool_call_id: String, + /// Ordered typed output content. + pub content: Vec, + /// Whether hosted execution failed, when reported. + pub is_error: Option, +} + +/// Grammar accepted by a free-form input tool. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CustomToolFormat { + /// Unconstrained text. + Text, + /// Input constrained by a grammar. + Grammar { + /// Grammar language. + syntax: GrammarSyntax, + /// Grammar definition. + definition: String, + }, +} + +/// Supported grammar languages for free-form tools. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GrammarSyntax { + /// Lark grammar. + Lark, + /// Regular expression. + Regex, +} + +/// Provider-neutral declaration for a non-function tool. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SpecializedToolDefinition { + /// Free-form input tool. + Custom { + /// Tool name exposed to the model. + name: String, + /// Human-readable tool description. + description: Option, + /// Accepted free-form input. + format: CustomToolFormat, + }, + /// Computer interaction tool. + Computer { + /// Screen and environment exposed to the model. + configuration: ComputerToolConfig, + }, + /// Provider-hosted capability. + Hosted { + /// Capability exposed to the model. + tool: HostedTool, + }, +} + /// Normalized tool definition. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ToolDefinition { @@ -298,6 +573,192 @@ pub struct ReasoningParams { pub raw: Option, } +/// Purpose of encrypted or otherwise opaque provider state. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpaqueStatePurpose { + /// Encrypted reasoning continuation state. + Reasoning, + /// Conversation continuation state. + Conversation, + /// Remote compaction state. + Compaction, + /// Within-turn routing state. + TurnRouting, +} + +/// Error returned when opaque provider state exceeds its protocol bound. +#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)] +#[error("opaque provider state exceeds {MAX_OPAQUE_STATE_BYTES} bytes")] +pub struct OpaqueStateTooLarge; + +/// Maximum encoded size of one opaque provider state value. +pub const MAX_OPAQUE_STATE_BYTES: usize = 1024 * 1024; + +/// Bounded opaque state that can be replayed without exposing provider spellings. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct OpaqueState { + purpose: OpaqueStatePurpose, + data: String, +} + +impl OpaqueState { + /// Creates bounded opaque state. + pub fn new( + purpose: OpaqueStatePurpose, + data: impl Into, + ) -> Result { + let data = data.into(); + if data.len() > MAX_OPAQUE_STATE_BYTES { + return Err(OpaqueStateTooLarge); + } + Ok(Self { purpose, data }) + } + + /// Purpose of this state. + pub fn purpose(&self) -> OpaqueStatePurpose { + self.purpose + } + + /// Encoded state value. + pub fn data(&self) -> &str { + &self.data + } +} + +impl<'de> Deserialize<'de> for OpaqueState { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct Repr { + purpose: OpaqueStatePurpose, + data: String, + } + + let repr = Repr::deserialize(deserializer)?; + Self::new(repr.purpose, repr.data).map_err(serde::de::Error::custom) + } +} + +/// Provider-created conversation compaction output. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct CompactionItem { + /// Provider item identifier, when supplied. + pub id: Option, + /// Human-readable summary, when this compaction form exposes one. + pub summary: Option, + /// Opaque continuation state, when this compaction form is encrypted. + pub state: Option, +} + +/// Provider-native generated image output. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct GeneratedImage { + /// Provider item identifier, when supplied. + pub id: Option, + /// Generated image location or inline payload. + pub source: ImageSource, +} + +/// Update to a turn's generation configuration. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct TurnConfigurationUpdate { + /// New reasoning effort, when changed. + pub reasoning_effort: Option, + /// New response verbosity, when changed. + pub text_verbosity: Option, +} + +/// A provider pause that requires another request to continue the turn. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct PauseTurn { + /// Stable continuation identifier, when supplied. + pub continuation_id: Option, + /// Safe provider-neutral reason, when supplied. + pub reason: Option, +} + +/// Non-message input appended to a turn in order. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case")] +pub enum TurnInputItem { + /// Change turn configuration after the preceding conversation items. + ConfigurationUpdate(TurnConfigurationUpdate), + /// Request provider-native remote compaction as the final turn item. + RemoteCompaction, +} + +/// Safe provider error classification. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProviderErrorClass { + /// Credential was not accepted. + Authentication, + /// Credential was valid but lacked authority. + Authorization, + /// Provider rate limit. + RateLimited, + /// Request violated the provider contract. + InvalidRequest, + /// Request exceeded the model context window. + ContextWindow, + /// Provider safety policy rejected content. + ContentFiltered, + /// Provider or dependency was unavailable. + Unavailable, + /// Provider response violated the expected protocol. + Protocol, + /// Provider reported an internal failure. + Internal, +} + +/// Structured, body-free provider error. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct StructuredProviderError { + /// Stable provider-neutral classification. + pub class: ProviderErrorClass, + /// HTTP status, when the failure came from HTTP. + pub status: Option, + /// Bounded safe provider code, when allowlisted by an adapter. + pub code: Option, + /// Provider-advertised retry delay. + pub retry_after_ms: Option, +} + +/// Provider-neutral response terminal state. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case")] +pub enum ResponseTerminal { + /// Response completed successfully. + Completed, + /// Response ended without completing. + Incomplete { + /// Safe provider-neutral reason, when supplied. + reason: Option, + }, + /// Response failed with a structured body-free error. + Failed(StructuredProviderError), + /// Provider paused the turn for explicit continuation. + Paused(PauseTurn), +} + +/// Nonsemantic response metadata retained across compatible requests. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct ResponseMetadata { + /// Model-catalog entity tag returned by the provider. + pub model_etag: Option, + /// Bounded within-turn routing state. + pub turn_state: Option, +} + +impl ResponseMetadata { + pub(crate) fn is_empty(&self) -> bool { + self.model_etag.is_none() && self.turn_state.is_none() + } +} + /// Provider-specific fields that do not have first-class conversation fields. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] @@ -337,8 +798,14 @@ pub struct LlmRequest { pub instructions: Vec, /// Ordered conversation messages. pub messages: Vec, + /// Ordered non-message items appended after the conversation. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub trailing_items: Vec, /// Tools available to the model. pub tools: Vec, + /// Non-function tools available to the model. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub specialized_tools: Vec, /// Policy controlling model tool selection. pub tool_choice: Option, /// Common sampling controls. @@ -472,6 +939,12 @@ pub struct AggLlmResponse { pub outputs: Vec, /// Normalized token usage. pub usage: Usage, + /// Nonsemantic response metadata. + #[serde(default, skip_serializing_if = "ResponseMetadata::is_empty")] + pub metadata: ResponseMetadata, + /// Provider-neutral terminal state, when explicitly reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub terminal: Option, /// Provider response fields without normalized equivalents. pub extensions: ProviderExtensions, /// Exact provider bodies retained for lossless round trips. diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index 6d51079b7..c8b2faced 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -17,7 +17,11 @@ use thiserror::Error; use crate::{ LlmClientError, format::FormatId, - llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage}, + llm::{ + AggLlmResponse, CompactionItem, ComputerToolCall, ContentBlock, CustomToolCall, + GeneratedImage, HostedToolCall, OpaqueState, PauseTurn, ResponseMetadata, ResponseOutput, + ResponseTerminal, Role, StopReason, ToolCall, Usage, + }, }; /// Status reported for an upstream error delivered inside a streaming body. The @@ -212,9 +216,9 @@ impl AggLlmResponse { /// downstream expects `LlmResponse::Stream` — for instance when `stream: true` was /// requested and the algorithm had to aggregate before it could return. /// - /// This conversion is lossy: only text, reasoning, and tool-call content has - /// a synthetic chunk representation. Refusals, tool results, media, files, - /// unknown blocks, response extensions, and preservation metadata are omitted. + /// This conversion retains semantic output, typed continuation state, + /// response metadata, and explicit terminal state. Tool results, input media, + /// files, unknown blocks, response extensions, and preservation metadata are omitted. pub fn into_stream(self) -> LlmResponseStream { let mut chunks: Vec = Vec::new(); chunks.push(LlmResponseChunk::MessageStart { @@ -245,19 +249,79 @@ impl AggLlmResponse { }); } } - ContentBlock::ToolCall(tool) => { - let args = serde_json::to_string(&tool.arguments).unwrap_or_default(); + ContentBlock::ToolCall(call) => { chunks.push(LlmResponseChunk::ToolCallDelta { index: tool_call_index, - id: Some(tool.id), - name: Some(tool.name), - arguments_delta: Some(args), + id: Some(call.id.clone()), + name: Some(call.name.clone()), + arguments_delta: serde_json::to_string(&call.arguments).ok(), + }); + chunks.push(LlmResponseChunk::ToolCallDone { + index: tool_call_index, + call, + }); + tool_call_index += 1; + } + ContentBlock::CustomToolCall(call) => { + chunks.push(LlmResponseChunk::CustomToolCallDelta { + index: tool_call_index, + id: Some(call.id.clone()), + item_id: call.item_id.clone(), + name: Some(call.name.clone()), + input_delta: call.input.clone(), + }); + chunks.push(LlmResponseChunk::CustomToolCallDone { + index: tool_call_index, + call, + }); + tool_call_index += 1; + } + ContentBlock::ComputerToolCall(call) => { + chunks.push(LlmResponseChunk::ComputerToolCallDone { + index: tool_call_index, + call, + }); + tool_call_index += 1; + } + ContentBlock::HostedToolCall(call) => { + chunks.push(LlmResponseChunk::HostedToolCallDone { + index: tool_call_index, + call, }); tool_call_index += 1; } - // Other variants (Image, ToolResult, Refusal, etc.) don't have - // a streaming chunk representation and don't appear in assistant outputs. - _ => {} + ContentBlock::Refusal { text } => { + chunks.push(LlmResponseChunk::RefusalDelta { + index: output_index, + text: text.clone(), + }); + chunks.push(LlmResponseChunk::RefusalDone { + index: output_index, + text, + }); + } + ContentBlock::OpaqueState(state) => { + chunks.push(LlmResponseChunk::OpaqueState(state)); + } + ContentBlock::Compaction(item) => { + chunks.push(LlmResponseChunk::CompactionDone(item)); + } + ContentBlock::GeneratedImage(image) => { + chunks.push(LlmResponseChunk::GeneratedImageDone(image)); + } + ContentBlock::PauseTurn(pause) => { + chunks.push(LlmResponseChunk::PauseTurn(pause)); + } + // Results and input media do not appear in assistant output streams. + ContentBlock::Image { .. } + | ContentBlock::Audio { .. } + | ContentBlock::Video { .. } + | ContentBlock::File { .. } + | ContentBlock::ToolResult(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::Unknown { .. } => {} } } chunks.push(LlmResponseChunk::MessageStop { @@ -268,6 +332,12 @@ impl AggLlmResponse { }), }); } + if !self.metadata.is_empty() { + chunks.push(LlmResponseChunk::ResponseMetadata(self.metadata)); + } + if let Some(terminal) = self.terminal { + chunks.push(LlmResponseChunk::ResponseTerminal(terminal)); + } chunks.push(LlmResponseChunk::Usage(self.usage)); Box::pin(futures::stream::iter( chunks.into_iter().map(|chunk| Ok(chunk.into())), @@ -338,6 +408,94 @@ pub enum LlmResponseChunk { /// Fragment of the serialized tool arguments. arguments_delta: Option, }, + /// Finalizes a JSON-schema function tool call. + ToolCallDone { + /// Tool-call index within the response. + index: usize, + /// Complete tool call. + call: ToolCall, + }, + /// Adds free-form input to a custom tool call. + CustomToolCallDelta { + /// Tool-call index within the response. + index: usize, + /// Provider call identifier, normally supplied by the first delta. + id: Option, + /// Provider item identifier, normally supplied by the first delta. + item_id: Option, + /// Tool name, normally supplied by the first delta. + name: Option, + /// Free-form input fragment. + input_delta: String, + }, + /// Finalizes a free-form input tool call. + CustomToolCallDone { + /// Tool-call index within the response. + index: usize, + /// Complete custom tool call. + call: CustomToolCall, + }, + /// Finalizes a computer tool call. + ComputerToolCallDone { + /// Tool-call index within the response. + index: usize, + /// Complete computer call. + call: ComputerToolCall, + }, + /// Finalizes a provider-hosted tool call. + HostedToolCallDone { + /// Tool-call index within the response. + index: usize, + /// Complete hosted call. + call: HostedToolCall, + }, + /// Starts one reasoning output item. + ReasoningStarted { + /// Provider output index. + index: usize, + }, + /// Finalizes one reasoning output item and its continuation state. + ReasoningDone { + /// Provider output index. + index: usize, + /// Complete reasoning text when the provider reports it only at completion. + text: Option, + /// Bounded opaque state associated with the reasoning item. + state: Vec, + }, + /// Adds refusal text to one output index. + RefusalDelta { + /// Provider output index. + index: usize, + /// Refusal text fragment. + text: String, + }, + /// Finalizes a refusal. + RefusalDone { + /// Provider output index. + index: usize, + /// Complete refusal text. + text: String, + }, + /// Final text for providers that deliver completion text atomically. + TextDone { + /// Provider output index. + index: usize, + /// Complete text. + text: String, + }, + /// Bounded opaque provider state that does not itself commit semantic output. + OpaqueState(OpaqueState), + /// Nonsemantic response metadata. + ResponseMetadata(ResponseMetadata), + /// Final provider-native generated image. + GeneratedImageDone(GeneratedImage), + /// Final remote compaction result. + CompactionDone(CompactionItem), + /// Final pause-turn result. + PauseTurn(PauseTurn), + /// Explicit response terminal state. + ResponseTerminal(ResponseTerminal), /// Reports token usage, normally near the end of the stream. Usage(Usage), /// Ends a response message. @@ -357,6 +515,52 @@ pub enum LlmResponseChunk { }, } +/// Whether a stream chunk commits downstream-visible response semantics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StreamSemanticClass { + /// Metadata, lifecycle, or partial tool state that is safe to discard before commitment. + NonSemantic, + /// Output that prohibits replay or provider fallback. + Semantic, +} + +impl LlmResponseChunk { + /// Classifies the chunk for the shared semantic-commitment boundary. + pub fn semantic_class(&self) -> StreamSemanticClass { + match self { + Self::TextDelta { text, .. } + | Self::ReasoningDelta { text, .. } + | Self::ReasoningDetailsDelta { text, .. } + | Self::RefusalDelta { text, .. } + | Self::TextDone { text, .. } + if !text.is_empty() => + { + StreamSemanticClass::Semantic + } + Self::ReasoningDone { + text: Some(text), .. + } if !text.is_empty() => StreamSemanticClass::Semantic, + Self::RefusalDone { .. } + | Self::ToolCallDone { .. } + | Self::CustomToolCallDone { .. } + | Self::ComputerToolCallDone { .. } + | Self::HostedToolCallDone { .. } + | Self::GeneratedImageDone(_) + | Self::CompactionDone(_) + | Self::PauseTurn(_) + | Self::MessageStop { .. } + | Self::ResponseTerminal(ResponseTerminal::Completed) + | Self::ResponseTerminal(ResponseTerminal::Paused(_)) => StreamSemanticClass::Semantic, + _ => StreamSemanticClass::NonSemantic, + } + } + + /// Returns whether this chunk crosses the semantic-commitment boundary. + pub fn commits_semantics(&self) -> bool { + self.semantic_class() == StreamSemanticClass::Semantic + } +} + /// Folds a sequence of [`LlmResponseChunk`]s into the terminal [`AggLlmResponse`]. /// /// Text and reasoning deltas concatenate; tool-call deltas assemble by index (name, @@ -379,8 +583,14 @@ pub struct ResponseAccumulator { reasoning: Option, reasoning_details: Vec, tool_calls: BTreeMap, + custom_tool_calls: BTreeMap, usage: Usage, stop_reason: Option, + refusal: Option, + opaque_state: Vec, + finalized_blocks: Vec, + metadata: ResponseMetadata, + terminal: Option, } /// A tool call being assembled from streamed [`LlmResponseChunk::ToolCallDelta`]s. @@ -391,6 +601,15 @@ struct PartialToolCall { arguments: String, } +/// A custom tool call being assembled from streamed input deltas. +#[derive(Default)] +struct PartialCustomToolCall { + id: Option, + item_id: Option, + name: Option, + input: String, +} + impl ResponseAccumulator { /// A fresh accumulator with no chunks applied. pub fn new() -> Self { @@ -410,6 +629,11 @@ impl ResponseAccumulator { } } LlmResponseChunk::TextDelta { text, .. } => self.text.push_str(&text), + LlmResponseChunk::TextDone { text, .. } => { + if self.text.is_empty() { + self.text = text; + } + } LlmResponseChunk::ReasoningDelta { text, .. } => { self.reasoning .get_or_insert_with(String::new) @@ -425,6 +649,19 @@ impl ResponseAccumulator { .push_str(&text); } } + LlmResponseChunk::ReasoningStarted { .. } => {} + LlmResponseChunk::ReasoningDone { text, state, .. } => { + if self.reasoning.as_deref().is_none_or(str::is_empty) + && let Some(text) = text + { + self.reasoning = Some(text); + } + self.opaque_state.extend(state); + } + LlmResponseChunk::RefusalDelta { text, .. } => { + self.refusal.get_or_insert_with(String::new).push_str(&text); + } + LlmResponseChunk::RefusalDone { text, .. } => self.refusal = Some(text), LlmResponseChunk::ToolCallDelta { index, id, @@ -442,6 +679,59 @@ impl ResponseAccumulator { call.arguments.push_str(&delta); } } + LlmResponseChunk::ToolCallDone { index, call } => { + self.tool_calls.remove(&index); + self.finalized_blocks.push(ContentBlock::ToolCall(call)); + } + LlmResponseChunk::CustomToolCallDelta { + index, + id, + item_id, + name, + input_delta, + } => { + let call = self.custom_tool_calls.entry(index).or_default(); + if id.is_some() { + call.id = id; + } + if item_id.is_some() { + call.item_id = item_id; + } + if name.is_some() { + call.name = name; + } + call.input.push_str(&input_delta); + } + LlmResponseChunk::CustomToolCallDone { index, call } => { + self.custom_tool_calls.remove(&index); + self.finalized_blocks + .push(ContentBlock::CustomToolCall(call)); + } + LlmResponseChunk::ComputerToolCallDone { call, .. } => self + .finalized_blocks + .push(ContentBlock::ComputerToolCall(call)), + LlmResponseChunk::HostedToolCallDone { call, .. } => self + .finalized_blocks + .push(ContentBlock::HostedToolCall(call)), + LlmResponseChunk::OpaqueState(state) => self.opaque_state.push(state), + LlmResponseChunk::ResponseMetadata(metadata) => { + if metadata.model_etag.is_some() { + self.metadata.model_etag = metadata.model_etag; + } + if metadata.turn_state.is_some() { + self.metadata.turn_state = metadata.turn_state; + } + } + LlmResponseChunk::GeneratedImageDone(image) => self + .finalized_blocks + .push(ContentBlock::GeneratedImage(image)), + LlmResponseChunk::CompactionDone(item) => { + self.finalized_blocks.push(ContentBlock::Compaction(item)); + } + LlmResponseChunk::PauseTurn(pause) => { + self.finalized_blocks.push(ContentBlock::PauseTurn(pause)); + } + LlmResponseChunk::ResponseTerminal(terminal) => self.terminal = Some(terminal), LlmResponseChunk::Usage(usage) => self.usage = usage, LlmResponseChunk::MessageStop { reason } => { self.stop_reason = Some(stop_reason_from_str(reason.as_deref())); @@ -450,8 +740,8 @@ impl ResponseAccumulator { } } - /// Build the buffered response. Content is ordered reasoning, then text, then - /// tool calls (by ascending delta index) — a single assistant output. + /// Build the buffered response. Content is ordered reasoning, text, refusal, + /// partial calls by index, then finalized typed items. pub fn finish(self) -> AggLlmResponse { let mut content = Vec::new(); if self.reasoning.is_some() || !self.reasoning_details.is_empty() { @@ -470,6 +760,9 @@ impl ResponseAccumulator { if !self.text.is_empty() { content.push(ContentBlock::Text { text: self.text }); } + if let Some(text) = self.refusal { + content.push(ContentBlock::Refusal { text }); + } for call in self.tool_calls.into_values() { content.push(ContentBlock::ToolCall(ToolCall { id: call.id.unwrap_or_default(), @@ -477,6 +770,16 @@ impl ResponseAccumulator { arguments: parse_tool_arguments(&call.arguments), })); } + for call in self.custom_tool_calls.into_values() { + content.push(ContentBlock::CustomToolCall(CustomToolCall { + id: call.id.unwrap_or_default(), + item_id: call.item_id, + name: call.name.unwrap_or_default(), + input: call.input, + })); + } + content.extend(self.opaque_state.into_iter().map(ContentBlock::OpaqueState)); + content.extend(self.finalized_blocks); AggLlmResponse { id: self.id, model: self.model, @@ -486,6 +789,8 @@ impl ResponseAccumulator { stop_reason: self.stop_reason, }], usage: self.usage, + metadata: self.metadata, + terminal: self.terminal, ..AggLlmResponse::default() } } diff --git a/crates/protocol/tests/codex_neutral_ir.rs b/crates/protocol/tests/codex_neutral_ir.rs new file mode 100644 index 000000000..70f3c769d --- /dev/null +++ b/crates/protocol/tests/codex_neutral_ir.rs @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Stable serialization and exhaustive-match coverage for native provider-neutral IR. + +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use switchyard_protocol::{ + AggLlmResponse, ComputerAction, ComputerEnvironment, ComputerMouseButton, ContentBlock, + CustomToolFormat, FileSource, GrammarSyntax, HostedTool, ImageSource, LlmRequest, + LlmResponseChunk, MAX_OPAQUE_STATE_BYTES, MediaSource, Message, OpaqueState, + OpaqueStatePurpose, ProviderErrorClass, ResponseMetadata, ResponseOutput, ResponseTerminal, + Role, SpecializedToolDefinition, StreamSemanticClass, TurnInputItem, +}; + +type TestResult = Result<(), Box>; + +fn decode_round_trip(value: &Value) -> Result +where + T: DeserializeOwned + Serialize, +{ + let decoded = serde_json::from_value(value.clone())?; + assert_eq!(serde_json::to_value(&decoded)?, *value); + Ok(decoded) +} + +fn content_block_kind(block: &ContentBlock) -> &'static str { + match block { + ContentBlock::Text { .. } => "text", + ContentBlock::Reasoning { .. } => "reasoning", + ContentBlock::Image { .. } => "image", + ContentBlock::Audio { .. } => "audio", + ContentBlock::Video { .. } => "video", + ContentBlock::File { .. } => "file", + ContentBlock::ToolCall(_) => "tool_call", + ContentBlock::ToolResult(_) => "tool_result", + ContentBlock::CustomToolCall(_) => "custom_tool_call", + ContentBlock::CustomToolResult(_) => "custom_tool_result", + ContentBlock::ComputerToolCall(_) => "computer_tool_call", + ContentBlock::ComputerToolResult(_) => "computer_tool_result", + ContentBlock::HostedToolCall(_) => "hosted_tool_call", + ContentBlock::HostedToolResult(_) => "hosted_tool_result", + ContentBlock::OpaqueState(_) => "opaque_state", + ContentBlock::Compaction(_) => "compaction", + ContentBlock::GeneratedImage(_) => "generated_image", + ContentBlock::PauseTurn(_) => "pause_turn", + ContentBlock::Refusal { .. } => "refusal", + ContentBlock::Unknown { .. } => "unknown", + } +} + +fn chunk_kind(chunk: &LlmResponseChunk) -> &'static str { + match chunk { + LlmResponseChunk::MessageStart { .. } => "message_start", + LlmResponseChunk::TextDelta { .. } => "text_delta", + LlmResponseChunk::ReasoningDelta { .. } => "reasoning_delta", + LlmResponseChunk::ReasoningDetailsDelta { .. } => "reasoning_details_delta", + LlmResponseChunk::ToolCallDelta { .. } => "tool_call_delta", + LlmResponseChunk::ToolCallDone { .. } => "tool_call_done", + LlmResponseChunk::CustomToolCallDelta { .. } => "custom_tool_call_delta", + LlmResponseChunk::CustomToolCallDone { .. } => "custom_tool_call_done", + LlmResponseChunk::ComputerToolCallDone { .. } => "computer_tool_call_done", + LlmResponseChunk::HostedToolCallDone { .. } => "hosted_tool_call_done", + LlmResponseChunk::ReasoningStarted { .. } => "reasoning_started", + LlmResponseChunk::ReasoningDone { .. } => "reasoning_done", + LlmResponseChunk::RefusalDelta { .. } => "refusal_delta", + LlmResponseChunk::RefusalDone { .. } => "refusal_done", + LlmResponseChunk::TextDone { .. } => "text_done", + LlmResponseChunk::OpaqueState(_) => "opaque_state", + LlmResponseChunk::ResponseMetadata(_) => "response_metadata", + LlmResponseChunk::GeneratedImageDone(_) => "generated_image_done", + LlmResponseChunk::CompactionDone(_) => "compaction_done", + LlmResponseChunk::PauseTurn(_) => "pause_turn", + LlmResponseChunk::ResponseTerminal(_) => "response_terminal", + LlmResponseChunk::Usage(_) => "usage", + LlmResponseChunk::MessageStop { .. } => "message_stop", + LlmResponseChunk::DecodeError { .. } => "decode_error", + LlmResponseChunk::StreamError { .. } => "stream_error", + } +} + +fn specialized_tool_kind(tool: &SpecializedToolDefinition) -> &'static str { + match tool { + SpecializedToolDefinition::Custom { .. } => "custom", + SpecializedToolDefinition::Computer { .. } => "computer", + SpecializedToolDefinition::Hosted { .. } => "hosted", + } +} + +fn turn_input_kind(item: &TurnInputItem) -> &'static str { + match item { + TurnInputItem::ConfigurationUpdate(_) => "configuration_update", + TurnInputItem::RemoteCompaction => "remote_compaction", + } +} + +fn terminal_kind(terminal: &ResponseTerminal) -> &'static str { + match terminal { + ResponseTerminal::Completed => "completed", + ResponseTerminal::Incomplete { .. } => "incomplete", + ResponseTerminal::Failed(_) => "failed", + ResponseTerminal::Paused(_) => "paused", + } +} + +fn provider_error_kind(class: ProviderErrorClass) -> &'static str { + match class { + ProviderErrorClass::Authentication => "authentication", + ProviderErrorClass::Authorization => "authorization", + ProviderErrorClass::RateLimited => "rate_limited", + ProviderErrorClass::InvalidRequest => "invalid_request", + ProviderErrorClass::ContextWindow => "context_window", + ProviderErrorClass::ContentFiltered => "content_filtered", + ProviderErrorClass::Unavailable => "unavailable", + ProviderErrorClass::Protocol => "protocol", + ProviderErrorClass::Internal => "internal", + } +} + +fn computer_action_kind(action: &ComputerAction) -> &'static str { + match action { + ComputerAction::Click { .. } => "click", + ComputerAction::DoubleClick { .. } => "double_click", + ComputerAction::Drag { .. } => "drag", + ComputerAction::KeyPress { .. } => "key_press", + ComputerAction::Move { .. } => "move", + ComputerAction::Screenshot => "screenshot", + ComputerAction::Scroll { .. } => "scroll", + ComputerAction::Type { .. } => "type", + ComputerAction::Wait => "wait", + } +} + +fn hosted_tool_kind(tool: &HostedTool) -> &'static str { + match tool { + HostedTool::WebSearch { .. } => "web_search", + HostedTool::FileSearch { .. } => "file_search", + HostedTool::CodeInterpreter { .. } => "code_interpreter", + HostedTool::ImageGeneration => "image_generation", + } +} + +fn custom_format_kind(format: &CustomToolFormat) -> &'static str { + match format { + CustomToolFormat::Text => "text", + CustomToolFormat::Grammar { + syntax: GrammarSyntax::Lark, + .. + } => "lark", + CustomToolFormat::Grammar { + syntax: GrammarSyntax::Regex, + .. + } => "regex", + } +} + +fn environment_kind(environment: ComputerEnvironment) -> &'static str { + match environment { + ComputerEnvironment::Browser => "browser", + ComputerEnvironment::Desktop => "desktop", + ComputerEnvironment::Mobile => "mobile", + } +} + +fn mouse_button_kind(button: ComputerMouseButton) -> &'static str { + match button { + ComputerMouseButton::Left => "left", + ComputerMouseButton::Middle => "middle", + ComputerMouseButton::Right => "right", + } +} + +fn opaque_state_kind(purpose: OpaqueStatePurpose) -> &'static str { + match purpose { + OpaqueStatePurpose::Reasoning => "reasoning", + OpaqueStatePurpose::Conversation => "conversation", + OpaqueStatePurpose::Compaction => "compaction", + OpaqueStatePurpose::TurnRouting => "turn_routing", + } +} + +fn block_uses_escape_hatch(block: &ContentBlock) -> bool { + match block { + ContentBlock::Image { + source: ImageSource::Raw(_), + } + | ContentBlock::GeneratedImage(switchyard_protocol::GeneratedImage { + source: ImageSource::Raw(_), + .. + }) + | ContentBlock::ComputerToolResult(switchyard_protocol::ComputerToolResult { + output: ImageSource::Raw(_), + .. + }) + | ContentBlock::File { + source: FileSource::Raw(_), + } + | ContentBlock::Audio { + source: MediaSource::Raw(_), + } + | ContentBlock::Video { + source: MediaSource::Raw(_), + } + | ContentBlock::Unknown { .. } => true, + ContentBlock::ToolResult(result) => result.content.iter().any(block_uses_escape_hatch), + ContentBlock::HostedToolResult(result) => { + result.content.iter().any(block_uses_escape_hatch) + } + ContentBlock::Text { .. } + | ContentBlock::Reasoning { .. } + | ContentBlock::Image { .. } + | ContentBlock::Audio { .. } + | ContentBlock::Video { .. } + | ContentBlock::File { .. } + | ContentBlock::ToolCall(_) + | ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) + | ContentBlock::Refusal { .. } => false, + } +} +#[test] +fn codex_neutral_ir_fixture_round_trips() -> TestResult { + let fixture: Value = serde_json::from_str(include_str!("fixtures/codex_neutral_ir.json"))?; + + let blocks: Vec = decode_round_trip(&fixture["content_blocks"])?; + assert_eq!( + blocks.iter().map(content_block_kind).collect::>(), + [ + "custom_tool_call", + "custom_tool_result", + "computer_tool_call", + "computer_tool_result", + "hosted_tool_call", + "hosted_tool_result", + "opaque_state", + "compaction", + "generated_image", + "pause_turn", + ] + ); + + let tools: Vec = decode_round_trip(&fixture["specialized_tools"])?; + assert_eq!( + tools.iter().map(specialized_tool_kind).collect::>(), + [ + "custom", "custom", "custom", "computer", "computer", "computer", "hosted", "hosted", + "hosted", "hosted", + ] + ); + + let trailing: Vec = decode_round_trip(&fixture["trailing_items"])?; + assert_eq!( + trailing.iter().map(turn_input_kind).collect::>(), + ["configuration_update", "remote_compaction"] + ); + + let _: switchyard_protocol::ResponseMetadata = decode_round_trip(&fixture["metadata"])?; + let terminals: Vec = decode_round_trip(&fixture["terminals"])?; + assert_eq!( + terminals.iter().map(terminal_kind).collect::>(), + ["completed", "incomplete", "failed", "paused"] + ); + + let chunks: Vec = decode_round_trip(&fixture["chunks"])?; + assert_eq!( + chunks.iter().map(chunk_kind).collect::>(), + [ + "tool_call_done", + "custom_tool_call_delta", + "custom_tool_call_done", + "computer_tool_call_done", + "hosted_tool_call_done", + "reasoning_started", + "reasoning_done", + "refusal_delta", + "refusal_done", + "text_done", + "opaque_state", + "response_metadata", + "generated_image_done", + "compaction_done", + "pause_turn", + "response_terminal", + ] + ); + assert_eq!( + chunks + .iter() + .map(LlmResponseChunk::semantic_class) + .collect::>(), + [ + StreamSemanticClass::Semantic, + StreamSemanticClass::NonSemantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::NonSemantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::NonSemantic, + StreamSemanticClass::NonSemantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + StreamSemanticClass::Semantic, + ] + ); + + Ok(()) +} + +#[test] +fn nested_enum_variants_remain_exhaustive() -> TestResult { + let fixture: Value = serde_json::from_str(include_str!("fixtures/codex_neutral_ir.json"))?; + let blocks: Vec = serde_json::from_value(fixture["content_blocks"].clone())?; + let ContentBlock::ComputerToolCall(call) = &blocks[2] else { + panic!("fixture must contain the computer call at index 2"); + }; + assert_eq!( + call.actions + .iter() + .map(computer_action_kind) + .collect::>(), + [ + "click", + "double_click", + "click", + "drag", + "key_press", + "move", + "screenshot", + "scroll", + "type", + "wait", + ] + ); + + let tools: Vec = + serde_json::from_value(fixture["specialized_tools"].clone())?; + let custom_formats = tools.iter().filter_map(|tool| match tool { + SpecializedToolDefinition::Custom { format, .. } => Some(custom_format_kind(format)), + SpecializedToolDefinition::Computer { .. } | SpecializedToolDefinition::Hosted { .. } => { + None + } + }); + assert_eq!( + custom_formats.collect::>(), + ["text", "lark", "regex"] + ); + + let environments = tools.iter().filter_map(|tool| match tool { + SpecializedToolDefinition::Computer { configuration } => { + Some(environment_kind(configuration.environment)) + } + SpecializedToolDefinition::Custom { .. } | SpecializedToolDefinition::Hosted { .. } => None, + }); + assert_eq!( + environments.collect::>(), + ["browser", "desktop", "mobile"] + ); + + let hosted = tools.iter().filter_map(|tool| match tool { + SpecializedToolDefinition::Hosted { tool } => Some(hosted_tool_kind(tool)), + SpecializedToolDefinition::Custom { .. } | SpecializedToolDefinition::Computer { .. } => { + None + } + }); + assert_eq!( + hosted.collect::>(), + [ + "web_search", + "file_search", + "code_interpreter", + "image_generation" + ] + ); + + assert_eq!( + [ + ComputerMouseButton::Left, + ComputerMouseButton::Middle, + ComputerMouseButton::Right, + ] + .map(mouse_button_kind), + ["left", "middle", "right"] + ); + assert_eq!( + [ + OpaqueStatePurpose::Reasoning, + OpaqueStatePurpose::Conversation, + OpaqueStatePurpose::Compaction, + OpaqueStatePurpose::TurnRouting, + ] + .map(opaque_state_kind), + ["reasoning", "conversation", "compaction", "turn_routing"] + ); + assert_eq!( + [ + ProviderErrorClass::Authentication, + ProviderErrorClass::Authorization, + ProviderErrorClass::RateLimited, + ProviderErrorClass::InvalidRequest, + ProviderErrorClass::ContextWindow, + ProviderErrorClass::ContentFiltered, + ProviderErrorClass::Unavailable, + ProviderErrorClass::Protocol, + ProviderErrorClass::Internal, + ] + .map(provider_error_kind), + [ + "authentication", + "authorization", + "rate_limited", + "invalid_request", + "context_window", + "content_filtered", + "unavailable", + "protocol", + "internal", + ] + ); + + Ok(()) +} + +#[test] +fn new_default_fields_preserve_existing_serialized_forms() -> TestResult { + let request = serde_json::to_value(LlmRequest::default())?; + assert!(request.get("trailing_items").is_none()); + assert!(request.get("specialized_tools").is_none()); + + let response = serde_json::to_value(switchyard_protocol::AggLlmResponse::default())?; + assert!(response.get("metadata").is_none()); + assert!(response.get("terminal").is_none()); + Ok(()) +} + +#[test] +fn opaque_state_enforces_its_serialized_size_bound() -> TestResult { + let at_limit = "x".repeat(MAX_OPAQUE_STATE_BYTES); + let state = OpaqueState::new(OpaqueStatePurpose::Conversation, at_limit)?; + let _: OpaqueState = serde_json::from_value(serde_json::to_value(state)?)?; + + let over_limit = "x".repeat(MAX_OPAQUE_STATE_BYTES + 1); + assert!(OpaqueState::new(OpaqueStatePurpose::Conversation, over_limit.clone()).is_err()); + assert!( + serde_json::from_value::(serde_json::json!({ + "purpose": "conversation", + "data": over_limit, + })) + .is_err() + ); + Ok(()) +} + +#[test] +fn codex_parity_fixture_needs_no_escape_hatches() -> TestResult { + let fixture: Value = serde_json::from_str(include_str!("fixtures/codex_neutral_ir.json"))?; + let blocks: Vec = serde_json::from_value(fixture["content_blocks"].clone())?; + let specialized_tools: Vec = + serde_json::from_value(fixture["specialized_tools"].clone())?; + let trailing_items: Vec = + serde_json::from_value(fixture["trailing_items"].clone())?; + let metadata: ResponseMetadata = serde_json::from_value(fixture["metadata"].clone())?; + let terminals: Vec = serde_json::from_value(fixture["terminals"].clone())?; + + assert!(!blocks.iter().any(block_uses_escape_hatch)); + let request = LlmRequest { + messages: vec![Message { + role: Role::User, + content: blocks.clone(), + }], + trailing_items, + specialized_tools, + ..LlmRequest::default() + }; + assert!(request.extensions.fields.is_empty()); + assert!(request.preservation.requests.is_empty()); + assert!(request.reasoning.raw.is_none()); + let request: LlmRequest = serde_json::from_value(serde_json::to_value(request)?)?; + assert!(request.extensions.fields.is_empty()); + + let response = AggLlmResponse { + outputs: vec![ResponseOutput { + role: Role::Assistant, + content: blocks, + stop_reason: None, + }], + metadata, + terminal: terminals.into_iter().next(), + ..AggLlmResponse::default() + }; + assert!(response.extensions.fields.is_empty()); + assert!(response.preservation.responses.is_empty()); + let response: AggLlmResponse = serde_json::from_value(serde_json::to_value(response)?)?; + assert!(response.extensions.fields.is_empty()); + assert!( + !response.outputs[0] + .content + .iter() + .any(block_uses_escape_hatch) + ); + Ok(()) +} diff --git a/crates/protocol/tests/fixtures/codex_neutral_ir.json b/crates/protocol/tests/fixtures/codex_neutral_ir.json new file mode 100644 index 000000000..04120fdcb --- /dev/null +++ b/crates/protocol/tests/fixtures/codex_neutral_ir.json @@ -0,0 +1,318 @@ +{ + "content_blocks": [ + { + "type": "custom_tool_call", + "id": "custom-call-1", + "item_id": "item-custom-1", + "name": "shell", + "input": "printf hello" + }, + { + "type": "custom_tool_result", + "tool_call_id": "custom-call-1", + "output": "hello", + "is_error": false + }, + { + "type": "computer_tool_call", + "id": "computer-call-1", + "item_id": "item-computer-1", + "actions": [ + { + "type": "click", + "point": { "x": 10, "y": 20 }, + "button": "left" + }, + { + "type": "double_click", + "point": { "x": 30, "y": 40 }, + "button": "middle" + }, + { + "type": "click", + "point": { "x": 50, "y": 60 }, + "button": "right" + }, + { + "type": "drag", + "path": [{ "x": 1, "y": 2 }, { "x": 3, "y": 4 }] + }, + { + "type": "key_press", + "keys": ["CTRL", "L"] + }, + { + "type": "move", + "point": { "x": 70, "y": 80 } + }, + { "type": "screenshot" }, + { + "type": "scroll", + "point": { "x": 90, "y": 100 }, + "delta_x": 0, + "delta_y": 500 + }, + { + "type": "type", + "text": "query" + }, + { "type": "wait" } + ], + "pending_safety_checks": [ + { "id": "safe-1", "description": "Confirm navigation" } + ] + }, + { + "type": "computer_tool_result", + "tool_call_id": "computer-call-1", + "output": { + "type": "base64", + "data": { "media_type": "image/png", "data": "aW1hZ2U=" } + }, + "acknowledged_safety_checks": ["safe-1"] + }, + { + "type": "hosted_tool_call", + "id": "hosted-call-1", + "item_id": "item-hosted-1", + "tool": { + "type": "web_search", + "allowed_domains": ["example.com"] + }, + "arguments": { "query": "switchyard" } + }, + { + "type": "hosted_tool_result", + "tool_call_id": "hosted-call-1", + "content": [{ "type": "text", "text": "result" }], + "is_error": null + }, + { + "type": "opaque_state", + "purpose": "reasoning", + "data": "encrypted-reasoning" + }, + { + "type": "compaction", + "id": "compaction-1", + "summary": "Earlier context", + "state": { "purpose": "compaction", "data": "compact-state" } + }, + { + "type": "generated_image", + "id": "image-1", + "source": { + "type": "url", + "data": { "url": "https://example.com/image.png", "detail": "high" } + } + }, + { + "type": "pause_turn", + "continuation_id": "continue-1", + "reason": "awaiting_input" + } + ], + "specialized_tools": [ + { + "type": "custom", + "name": "free_text", + "description": null, + "format": { "type": "text" } + }, + { + "type": "custom", + "name": "lark_tool", + "description": "Lark input", + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: WORD" + } + }, + { + "type": "custom", + "name": "regex_tool", + "description": "Regex input", + "format": { + "type": "grammar", + "syntax": "regex", + "definition": "[a-z]+" + } + }, + { + "type": "computer", + "configuration": { + "display_width": 1280, + "display_height": 720, + "environment": "browser" + } + }, + { + "type": "computer", + "configuration": { + "display_width": 1920, + "display_height": 1080, + "environment": "desktop" + } + }, + { + "type": "computer", + "configuration": { + "display_width": 390, + "display_height": 844, + "environment": "mobile" + } + }, + { + "type": "hosted", + "tool": { + "type": "web_search", + "allowed_domains": ["example.com"] + } + }, + { + "type": "hosted", + "tool": { + "type": "file_search", + "vector_store_ids": ["vs-1"], + "max_results": 5 + } + }, + { + "type": "hosted", + "tool": { + "type": "code_interpreter", + "container_id": "container-1" + } + }, + { + "type": "hosted", + "tool": { "type": "image_generation" } + } + ], + "trailing_items": [ + { + "type": "configuration_update", + "data": { + "reasoning_effort": "high", + "text_verbosity": "concise" + } + }, + { "type": "remote_compaction" } + ], + "metadata": { + "model_etag": "etag-1", + "turn_state": { "purpose": "turn_routing", "data": "route-state" } + }, + "terminals": [ + { "type": "completed" }, + { "type": "incomplete", "data": { "reason": "max_output" } }, + { + "type": "failed", + "data": { + "class": "authentication", + "status": 401, + "code": "invalid_credential", + "retry_after_ms": null + } + }, + { + "type": "paused", + "data": { + "continuation_id": "continue-2", + "reason": "awaiting_tool" + } + } + ], + "chunks": [ + { + "ToolCallDone": { + "index": 0, + "call": { "id": "call-1", "name": "lookup", "arguments": { "q": "rust" } } + } + }, + { + "CustomToolCallDelta": { + "index": 1, + "id": "custom-call-1", + "item_id": "item-custom-1", + "name": "shell", + "input_delta": "printf" + } + }, + { + "CustomToolCallDone": { + "index": 1, + "call": { + "id": "custom-call-1", + "item_id": "item-custom-1", + "name": "shell", + "input": "printf hello" + } + } + }, + { + "ComputerToolCallDone": { + "index": 2, + "call": { + "id": "computer-call-2", + "item_id": null, + "actions": [{ "type": "screenshot" }], + "pending_safety_checks": [] + } + } + }, + { + "HostedToolCallDone": { + "index": 3, + "call": { + "id": "hosted-call-2", + "item_id": null, + "tool": { "type": "image_generation" }, + "arguments": {} + } + } + }, + { "ReasoningStarted": { "index": 0 } }, + { + "ReasoningDone": { + "index": 0, + "text": "reasoned", + "state": [{ "purpose": "reasoning", "data": "reason-state" }] + } + }, + { "RefusalDelta": { "index": 0, "text": "cannot" } }, + { "RefusalDone": { "index": 0, "text": "cannot comply" } }, + { "TextDone": { "index": 0, "text": "answer" } }, + { "OpaqueState": { "purpose": "conversation", "data": "conversation-state" } }, + { + "ResponseMetadata": { + "model_etag": "etag-2", + "turn_state": { "purpose": "turn_routing", "data": "turn-state" } + } + }, + { + "GeneratedImageDone": { + "id": "image-2", + "source": { + "type": "base64", + "data": { "media_type": "image/png", "data": "aW1hZ2U=" } + } + } + }, + { + "CompactionDone": { + "id": "compaction-2", + "summary": null, + "state": { "purpose": "compaction", "data": "compact-state-2" } + } + }, + { + "PauseTurn": { + "continuation_id": "continue-3", + "reason": "awaiting_input" + } + }, + { "ResponseTerminal": { "type": "completed" } } + ] +} diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 86ab45026..90a74c69d 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -303,6 +303,8 @@ impl FormatCodec for AnthropicMessagesCodec { )), }], usage: decode_anthropic_usage(body.get("usage")), + metadata: Default::default(), + terminal: None, extensions: ProviderExtensions { fields: provider_extensions( body, @@ -832,6 +834,22 @@ fn encode_anthropic_content_with_policy( )?; blocks.push(json!({"type": "text", "text": json_string(raw)})); } + ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) => { + push_lossy( + diagnostics, + policy, + "typed content block is not representable by the Anthropic adapter", + )?; + } other => blocks.extend(encode_one_anthropic_block(other)), } } @@ -974,6 +992,16 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec { }), MediaSource::Raw(raw) => raw.clone(), }], + ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) => Vec::new(), ContentBlock::Unknown { raw, .. } => vec![raw.clone()], } } @@ -997,7 +1025,17 @@ fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec { | ContentBlock::Audio { .. } | ContentBlock::Video { .. } | ContentBlock::ToolCall(_) - | ContentBlock::ToolResult(_) => Vec::new(), + | ContentBlock::ToolResult(_) + | ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) => Vec::new(), } } diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 7095b2456..e4b64ab5c 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -179,7 +179,7 @@ fn encode_anthropic_stream( })] } } - LlmResponseChunk::TextDelta { text, .. } => { + LlmResponseChunk::TextDelta { text, .. } | LlmResponseChunk::RefusalDelta { text, .. } => { state.output_tokens_seen += 1; let mut out = ensure_anthropic_text_block(state); out.push(json!({ @@ -221,6 +221,21 @@ fn encode_anthropic_stream( state.saw_backend_usage = true; Vec::new() } + LlmResponseChunk::ToolCallDone { .. } + | LlmResponseChunk::CustomToolCallDelta { .. } + | LlmResponseChunk::CustomToolCallDone { .. } + | LlmResponseChunk::ComputerToolCallDone { .. } + | LlmResponseChunk::HostedToolCallDone { .. } + | LlmResponseChunk::ReasoningStarted { .. } + | LlmResponseChunk::ReasoningDone { .. } + | LlmResponseChunk::RefusalDone { .. } + | LlmResponseChunk::TextDone { .. } + | LlmResponseChunk::OpaqueState(_) + | LlmResponseChunk::ResponseMetadata(_) + | LlmResponseChunk::GeneratedImageDone(_) + | LlmResponseChunk::CompactionDone(_) + | LlmResponseChunk::PauseTurn(_) + | LlmResponseChunk::ResponseTerminal(_) => Vec::new(), LlmResponseChunk::MessageStop { reason } => { state.stop_reason = reason.or_else(|| state.stop_reason.clone()); Vec::new() diff --git a/crates/switchyard-translation/src/codecs/bedrock/stream.rs b/crates/switchyard-translation/src/codecs/bedrock/stream.rs index bc01eb710..635f7f54b 100644 --- a/crates/switchyard-translation/src/codecs/bedrock/stream.rs +++ b/crates/switchyard-translation/src/codecs/bedrock/stream.rs @@ -163,7 +163,7 @@ fn encode_bedrock_event(state: &mut StreamTranslationState, event: LlmResponseCh vec![json!({"messageStart": {"role": "assistant"}})] } } - LlmResponseChunk::TextDelta { text, .. } => { + LlmResponseChunk::TextDelta { text, .. } | LlmResponseChunk::RefusalDelta { text, .. } => { state.output_tokens_seen += 1; let index = ensure_text_block(state); vec![json!({ @@ -199,6 +199,21 @@ fn encode_bedrock_event(state: &mut StreamTranslationState, event: LlmResponseCh state.saw_backend_usage = true; Vec::new() } + LlmResponseChunk::ToolCallDone { .. } + | LlmResponseChunk::CustomToolCallDelta { .. } + | LlmResponseChunk::CustomToolCallDone { .. } + | LlmResponseChunk::ComputerToolCallDone { .. } + | LlmResponseChunk::HostedToolCallDone { .. } + | LlmResponseChunk::ReasoningStarted { .. } + | LlmResponseChunk::ReasoningDone { .. } + | LlmResponseChunk::RefusalDone { .. } + | LlmResponseChunk::TextDone { .. } + | LlmResponseChunk::OpaqueState(_) + | LlmResponseChunk::ResponseMetadata(_) + | LlmResponseChunk::GeneratedImageDone(_) + | LlmResponseChunk::CompactionDone(_) + | LlmResponseChunk::PauseTurn(_) + | LlmResponseChunk::ResponseTerminal(_) => Vec::new(), LlmResponseChunk::MessageStop { reason } => { state.stop_reason = reason.or_else(|| state.stop_reason.clone()); Vec::new() diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 12ff3bb3d..5131f20bb 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -265,6 +265,8 @@ impl FormatCodec for OpenAiChatCodec { .map(ToOwned::to_owned), outputs: Vec::new(), usage: decode_openai_usage(object.get("usage")), + metadata: Default::default(), + terminal: None, extensions: ProviderExtensions { fields: provider_extensions(object, &["id", "model", "choices", "usage"]), }, @@ -1088,6 +1090,22 @@ pub(crate) fn encode_openai_content( ContentBlock::Reasoning { .. } | ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_) => {} + ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) => { + push_lossy( + diagnostics, + policy, + "typed content block is not representable by the OpenAI Chat adapter", + )?; + } } } Ok(Value::Array(blocks)) diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index e640d196b..801938630 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -197,7 +197,7 @@ fn encode_openai_chat_stream( )] } } - LlmResponseChunk::TextDelta { text, .. } => { + LlmResponseChunk::TextDelta { text, .. } | LlmResponseChunk::RefusalDelta { text, .. } => { vec![openai_stream_chunk( state, json!({"content": text}), @@ -254,6 +254,21 @@ fn encode_openai_chat_stream( Vec::new() } } + LlmResponseChunk::ToolCallDone { .. } + | LlmResponseChunk::CustomToolCallDelta { .. } + | LlmResponseChunk::CustomToolCallDone { .. } + | LlmResponseChunk::ComputerToolCallDone { .. } + | LlmResponseChunk::HostedToolCallDone { .. } + | LlmResponseChunk::ReasoningStarted { .. } + | LlmResponseChunk::ReasoningDone { .. } + | LlmResponseChunk::RefusalDone { .. } + | LlmResponseChunk::TextDone { .. } + | LlmResponseChunk::OpaqueState(_) + | LlmResponseChunk::ResponseMetadata(_) + | LlmResponseChunk::GeneratedImageDone(_) + | LlmResponseChunk::CompactionDone(_) + | LlmResponseChunk::PauseTurn(_) + | LlmResponseChunk::ResponseTerminal(_) => Vec::new(), LlmResponseChunk::MessageStop { reason } => { if state.finished { return Vec::new(); diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 3aad15b75..12c9fcc3b 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -313,6 +313,8 @@ impl FormatCodec for OpenAiResponsesCodec { .map(ToOwned::to_owned), outputs, usage: decode_responses_usage(body.get("usage")), + metadata: Default::default(), + terminal: None, extensions: ProviderExtensions { fields: provider_extensions( body, @@ -1451,6 +1453,22 @@ fn encode_responses_content( ContentBlock::Reasoning { .. } | ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_) => {} + ContentBlock::CustomToolCall(_) + | ContentBlock::CustomToolResult(_) + | ContentBlock::ComputerToolCall(_) + | ContentBlock::ComputerToolResult(_) + | ContentBlock::HostedToolCall(_) + | ContentBlock::HostedToolResult(_) + | ContentBlock::OpaqueState(_) + | ContentBlock::Compaction(_) + | ContentBlock::GeneratedImage(_) + | ContentBlock::PauseTurn(_) => { + push_lossy( + diagnostics, + policy, + "typed content block is not representable by the Responses adapter", + )?; + } } } Ok(Value::Array(blocks)) diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index db5da15de..0c6d3d8f4 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -313,7 +313,9 @@ fn encode_responses_stream( record_source_identity(state, id, model); ensure_responses_created(state) } - LlmResponseChunk::TextDelta { text, .. } => encode_responses_text_delta(state, text), + LlmResponseChunk::TextDelta { text, .. } | LlmResponseChunk::RefusalDelta { text, .. } => { + encode_responses_text_delta(state, text) + } LlmResponseChunk::ReasoningDelta { index, text } => { encode_responses_reasoning_delta(state, index, text) } @@ -361,6 +363,21 @@ fn encode_responses_stream( state.saw_backend_usage = true; Vec::new() } + LlmResponseChunk::ToolCallDone { .. } + | LlmResponseChunk::CustomToolCallDelta { .. } + | LlmResponseChunk::CustomToolCallDone { .. } + | LlmResponseChunk::ComputerToolCallDone { .. } + | LlmResponseChunk::HostedToolCallDone { .. } + | LlmResponseChunk::ReasoningStarted { .. } + | LlmResponseChunk::ReasoningDone { .. } + | LlmResponseChunk::RefusalDone { .. } + | LlmResponseChunk::TextDone { .. } + | LlmResponseChunk::OpaqueState(_) + | LlmResponseChunk::ResponseMetadata(_) + | LlmResponseChunk::GeneratedImageDone(_) + | LlmResponseChunk::CompactionDone(_) + | LlmResponseChunk::PauseTurn(_) + | LlmResponseChunk::ResponseTerminal(_) => Vec::new(), LlmResponseChunk::MessageStop { reason } => { state.stop_reason = reason.or_else(|| state.stop_reason.clone()); Vec::new() From 68770e1d6d1c982b9306c632564908afeb5bc9fc Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Sep 2026 16:31:23 +0000 Subject: [PATCH 2/2] Add typed credential routing failures Co-authored-by: c1-squire-dev[bot] --- crates/protocol/src/client.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs index 9c5d9d787..963b389e3 100644 --- a/crates/protocol/src/client.rs +++ b/crates/protocol/src/client.rs @@ -188,6 +188,10 @@ pub enum RoutedCallFailureClass { ProviderRejected, /// Host policy denied the call. PolicyDenied, + /// No credential source applies to this exact target and trusted identity. + CredentialNotApplicable, + /// The credential source recognized the request but denied it. + CredentialDenied, /// The credential for the target could not be obtained or used. CredentialUnavailable, /// The target's configuration is invalid. @@ -220,6 +224,8 @@ impl RoutedCallFailureClass { Self::InvalidResponse => "invalid_response", Self::ProviderRejected => "provider_rejected", Self::PolicyDenied => "policy_denied", + Self::CredentialNotApplicable => "credential_not_applicable", + Self::CredentialDenied => "credential_denied", Self::CredentialUnavailable => "credential_unavailable", Self::Configuration => "configuration", Self::WorkBudget => "work_budget", @@ -558,7 +564,7 @@ mod tests { use super::*; /// Every class, so the tag table below cannot silently miss one. - const ALL_CLASSES: [RoutedCallFailureClass; 18] = [ + const ALL_CLASSES: [RoutedCallFailureClass; 20] = [ RoutedCallFailureClass::CircuitOpen, RoutedCallFailureClass::ProviderTargetsExhausted, RoutedCallFailureClass::TargetIncompatible, @@ -572,6 +578,8 @@ mod tests { RoutedCallFailureClass::InvalidResponse, RoutedCallFailureClass::ProviderRejected, RoutedCallFailureClass::PolicyDenied, + RoutedCallFailureClass::CredentialNotApplicable, + RoutedCallFailureClass::CredentialDenied, RoutedCallFailureClass::CredentialUnavailable, RoutedCallFailureClass::Configuration, RoutedCallFailureClass::WorkBudget, @@ -597,6 +605,8 @@ mod tests { "invalid_response", "provider_rejected", "policy_denied", + "credential_not_applicable", + "credential_denied", "credential_unavailable", "configuration", "work_budget",