diff --git a/Readme.md b/Readme.md index 6cab360..4aff243 100644 --- a/Readme.md +++ b/Readme.md @@ -179,6 +179,8 @@ $response = $agentClient->run($client, new LLMRequest( **→ [Tools Documentation](docs/guides/tools.md)** +**→ [Agent Runs: progress, limits and cancellation](docs/guides/agent-runs.md)** + ### ✅ Feedback Loops Build self-correcting agents that validate and improve their outputs: @@ -271,7 +273,7 @@ use Soukicz\Llm\Client\OpenAI\Model\GPT54; $request = new LLMRequest( model: new GPT54(GPT54::VERSION_2026_03_05), conversation: $conversation, - reasoningConfig: ReasoningEffort::HIGH // NONE, MINIMAL, LOW, MEDIUM, HIGH, or EXTRA_HIGH + reasoningConfig: ReasoningEffort::HIGH // NONE, MINIMAL, LOW, MEDIUM, HIGH, XHIGH, EXTRA_HIGH or MAX ); // Or use token-based budget control (Anthropic only) diff --git a/docs/guides/agent-runs.md b/docs/guides/agent-runs.md new file mode 100644 index 0000000..a40ea06 --- /dev/null +++ b/docs/guides/agent-runs.md @@ -0,0 +1,72 @@ +# Agent Runs: Progress, Limits and Cancellation + +`LLMAgentClient::run()` loops while the model calls tools. Pass `AgentRunOptions` as the fourth argument to watch +the run and to stop it safely. Without options a run still differs from older versions in four ways: a refusal throws `AgentRunInterruptedException` (reason `refusal`) instead of returning the refused text, a `pause_turn` stop continues the run, a tool that throws, rejects or returns the wrong type becomes an error result for the model instead of aborting the run, and error results are sent to Claude with `is_error`. + +```php +getStepUsage(), $stepResponse->getTotalCostUsd() (cumulative) + } + + public function onToolCallStarted(string $toolName, array $input): void { + echo "running $toolName\n"; + } + + public function onToolCallFinished(string $toolName, array $input, LLMMessageContents $result, bool $isError, int $durationMs): void { + echo "$toolName " . ($isError ? 'failed' : 'done') . " in {$durationMs} ms\n"; + } +}; + +try { + $response = (new LLMAgentClient())->run($client, $request, null, new AgentRunOptions( + listener: $listener, + maxIterations: 40, // model calls + maxCostUsd: 5.0, // cumulative, cache reads and writes included + deadline: microtime(true) + 480, // unix time + shouldCancel: fn(): bool => $stopRequested(), + )); +} catch (AgentRunInterruptedException $e) { + $e->getReason(); // 'iterations' | 'cost' | 'deadline' | 'cancelled' | 'refusal' + $e->getConversation(); // everything so far, safe to store and continue later + $e->getUsage(); // TokenUsage of the run so far + $e->getCostUsd(); +} +``` + +## When the limits are checked + +| Option | Checked | Effect | +|---|---|---| +| `maxIterations` | before each model call | N calls run, the next one throws `iterations` | +| `maxCostUsd` | after each model response | throws `cost`; the tools of that response do not run | +| `deadline` | before each model call and each tool call | throws `deadline` | +| `shouldCancel` | before each model call and each tool call (before the deadline) | throws `cancelled` | + +When a run stops while tool calls are open, the stored conversation gets one closing user message. That message holds the results that finished, plus `ERROR: not run, the agent run was interrupted ()` for the rest. Every `tool_use` therefore has a `tool_result`, and the conversation replays as a valid request. + +## Tool errors + +A tool may fail in three ways: +- it throws; +- its promise is rejected; +- it returns something that is not `LLMMessageContents`. + +Each one becomes an error result such as `ERROR: RuntimeException: timeout`, and the run continues. Results built with `LLMMessageContents::fromErrorString()` are sent to Claude with `is_error: true`. + +## Refusals and paused turns + +- `StopReason::REFUSAL`: a safety classifier declined (HTTP 200), with details in `LLMResponse::getStopDetails()`. In an agent run this throws `AgentRunInterruptedException` with reason `refusal`. The refused turn is left out of `getConversation()`. A plain `sendRequestAsync()` call returns the stop reason instead. +- `StopReason::PAUSE_TURN`: a server-side tool loop paused. The agent loop re-sends the conversation unchanged and goes on. This counts as an iteration. + +## Usage and cost + +`LLMResponse::getUsage()` returns a `Soukicz\Llm\Usage\TokenUsage` for the whole run, and `getStepUsage()` for one API call. Both carry `input` (uncached), `output`, `cacheWrite5m`, `cacheWrite1h`, `cacheRead` and `reasoning`. `getTotalCostUsd()` includes cache writes at their TTL price and cache reads. OpenAI fills `cacheRead` and `reasoning` from the provider's own counters; Gemini fills `reasoning` only, so its `cacheRead` stays 0. diff --git a/docs/guides/caching.md b/docs/guides/caching.md index bff7697..cd41e41 100644 --- a/docs/guides/caching.md +++ b/docs/guides/caching.md @@ -402,6 +402,21 @@ Behavior details: - OpenAI and Gemini cache prompts automatically, so the option is a no-op there. - Prompts shorter than the provider's minimum (e.g. 1024 tokens for Claude Sonnet) are silently not cached. +### System prompt + +Create the system message with `cached: true` to put a cache breakpoint right after it. With a `ConversationCacheConfig` TTL, the same TTL is used there too, because longer TTLs must come first: + +```php + print($event->delta) ), + thinkingDisplay: ThinkingDisplay::UPDATES, // Optional (Anthropic): what thinking blocks contain + contextManagement: new ContextManagement( // Optional (Anthropic): server-side context editing + clearToolUses: new ClearToolUses(triggerInputTokens: 60000, keepToolUses: 3), + ), ); ``` @@ -70,11 +77,15 @@ $conversation = new LLMConversation([ ### temperature -Controls randomness in responses (0.0 to 1.0): +Controls randomness in responses (0.0 to 1.0), or `null` to send no temperature at all: - **0.0** - Deterministic, focused responses (the library default) - **0.5** - Balanced - **1.0** - Creative, varied responses +- **null** - Omit the parameter and use the provider default + +Claude Opus 4.7 and newer, Sonnet 5 and newer, Haiku 5.5 and Fable reject sampling parameters. The Anthropic +encoder never sends `temperature` to them (`AnthropicModel::supportsSamplingParameters()`), whatever you pass. ```php getStepUsage() is that call's usage + */ + public function onModelResponse(LLMResponse $stepResponse): void; + + public function onToolCallStarted(string $toolName, array $input): void; + + public function onToolCallFinished(string $toolName, array $input, LLMMessageContents $result, bool $isError, int $durationMs): void; +} diff --git a/src/Agent/AgentRunGuard.php b/src/Agent/AgentRunGuard.php new file mode 100644 index 0000000..7a20a33 --- /dev/null +++ b/src/Agent/AgentRunGuard.php @@ -0,0 +1,131 @@ +options->maxIterations !== null && $this->modelCalls >= $this->options->maxIterations) { + throw $this->interrupt(AgentRunInterruptedException::REASON_ITERATIONS, $request->getConversation()); + } + $reason = $this->deadlineOrCancelReason(); + if ($reason !== null) { + throw $this->interrupt($reason, $request->getConversation()); + } + } + + public function afterModelResponse(LLMResponse $response): void { + $this->modelCalls++; + $this->lastResponse = $response; + $this->options->listener?->onModelResponse($response); + + if ($response->getStopReason() === StopReason::REFUSAL) { + throw $this->interrupt(AgentRunInterruptedException::REASON_REFUSAL, self::withoutLastMessage($response->getConversation())); + } + if ($this->options->maxCostUsd !== null && $response->getTotalCostUsd() > $this->options->maxCostUsd) { + throw $this->interrupt( + AgentRunInterruptedException::REASON_COST, + self::closeOpenToolUses($response->getConversation(), [], AgentRunInterruptedException::REASON_COST) + ); + } + } + + /** + * @param array $startedResults promises of LLMMessageToolResult already started for this response + */ + public function beforeToolCall(LLMResponse $response, array $startedResults): void { + $reason = $this->deadlineOrCancelReason(); + if ($reason === null) { + return; + } + /** @var LLMMessageToolResult[] $done */ + $done = Utils::unwrap($startedResults); + + throw $this->interrupt($reason, self::closeOpenToolUses($response->getConversation(), $done, $reason)); + } + + public function toolCallStarted(string $toolName, array $input): void { + $this->options->listener?->onToolCallStarted($toolName, $input); + } + + public function toolCallFinished(string $toolName, array $input, LLMMessageContents $result, int $durationMs): void { + $this->options->listener?->onToolCallFinished($toolName, $input, $result, $result->isError(), $durationMs); + } + + private function deadlineOrCancelReason(): ?string { + if ($this->options->shouldCancel !== null && ($this->options->shouldCancel)() === true) { + return AgentRunInterruptedException::REASON_CANCELLED; + } + if ($this->options->deadline !== null && microtime(true) >= $this->options->deadline) { + return AgentRunInterruptedException::REASON_DEADLINE; + } + + return null; + } + + private function interrupt(string $reason, LLMConversation $conversation): AgentRunInterruptedException { + return new AgentRunInterruptedException( + $reason, + $conversation, + $this->lastResponse?->getUsage() ?? new TokenUsage(), + $this->lastResponse?->getTotalCostUsd() ?? 0.0, + ); + } + + /** + * @param LLMMessageToolResult[] $done + */ + private static function closeOpenToolUses(LLMConversation $conversation, array $done, string $reason): LLMConversation { + if ($conversation->getMessages() === []) { + return $conversation; + } + $last = $conversation->getLastMessage(); + if (!$last->isAssistant()) { + return $conversation; + } + + $doneById = []; + foreach ($done as $result) { + $doneById[$result->getId()] = $result; + } + $results = []; + foreach ($last->getContents() as $content) { + if ($content instanceof LLMMessageToolUse) { + $results[] = $doneById[$content->getId()] ?? new LLMMessageToolResult( + $content->getId(), + LLMMessageContents::fromErrorString('ERROR: not run, the agent run was interrupted (' . $reason . ')') + ); + } + } + + return $results === [] ? $conversation : $conversation->withMessage(LLMMessage::createFromUser(new LLMMessageContents($results))); + } + + private static function withoutLastMessage(LLMConversation $conversation): LLMConversation { + $messages = $conversation->getMessages(); + array_pop($messages); + + return new LLMConversation(array_values($messages), $conversation->getThreadId()); + } +} diff --git a/src/Agent/AgentRunInterruptedException.php b/src/Agent/AgentRunInterruptedException.php new file mode 100644 index 0000000..dbeccc7 --- /dev/null +++ b/src/Agent/AgentRunInterruptedException.php @@ -0,0 +1,43 @@ +reason; + } + + public function getConversation(): LLMConversation { + return $this->conversation; + } + + public function getUsage(): TokenUsage { + return $this->usage; + } + + public function getCostUsd(): float { + return $this->costUsd; + } +} diff --git a/src/Agent/AgentRunOptions.php b/src/Agent/AgentRunOptions.php new file mode 100644 index 0000000..4a54224 --- /dev/null +++ b/src/Agent/AgentRunOptions.php @@ -0,0 +1,23 @@ +cachedHttpClient; } - private function getHeaders(): array { + /** + * @param string[] $requestBetas + */ + private function getHeaders(array $requestBetas = []): array { $headers = [ 'accept-encoding' => 'gzip', 'anthropic-version' => '2023-06-01', 'x-api-key' => $this->apiKey, ]; - if (!empty($this->betaFeatures)) { - $headers['anthropic-beta'] = implode(',', $this->betaFeatures); + $betas = array_values(array_unique([...$this->betaFeatures, ...$requestBetas])); + if ($betas !== []) { + $headers['anthropic-beta'] = implode(',', $betas); } return $headers; } - private function invokeModel(array $data): PromiseInterface { + private function invokeModel(array $data, array $betas): PromiseInterface { return $this->getCachedHttpClient()->postAsync('https://api.anthropic.com/v1/messages', [ - 'headers' => $this->getHeaders(), + 'headers' => $this->getHeaders($betas), 'json' => $data, ])->then(function (ResponseInterface $response) { return new ModelResponse(json_decode((string) $response->getBody(), true, 512, JSON_THROW_ON_ERROR), (int) $response->getHeaderLine('X-Request-Duration-ms')); @@ -78,7 +82,7 @@ private function invokeModel(array $data): PromiseInterface { }); } - private function invokeModelStreaming(array $data, StreamListenerInterface $streamListener): PromiseInterface { + private function invokeModelStreaming(array $data, StreamListenerInterface $streamListener, array $betas): PromiseInterface { // Build cache key from non-streaming request (before adding stream:true) $cacheKeyRequest = new Request('POST', 'https://api.anthropic.com/v1/messages', [], json_encode($data, JSON_THROW_ON_ERROR)); @@ -97,7 +101,7 @@ private function invokeModelStreaming(array $data, StreamListenerInterface $stre $data['stream'] = true; $requestStart = microtime(true); - $headers = $this->getHeaders(); + $headers = $this->getHeaders($betas); // Avoid gzip compression on SSE streams $headers['accept-encoding'] = 'identity'; @@ -131,9 +135,10 @@ public function sendRequestAsync(LLMRequest $request): PromiseInterface { $encoded = $this->encodeRequest($request); $streamListener = $request->getStreamListener(); + $betas = $this->getRequestBetas($request); $modelPromise = $streamListener !== null - ? $this->invokeModelStreaming($encoded, $streamListener) - : $this->invokeModel($encoded); + ? $this->invokeModelStreaming($encoded, $streamListener, $betas) + : $this->invokeModel($encoded, $betas); return $modelPromise->then(function (ModelResponse $modelResponse) use ($request): LLMResponse|PromiseInterface { $encodedResponseOrRequest = $this->decodeResponse($request, $modelResponse); @@ -151,15 +156,17 @@ public function sendRequestAsync(LLMRequest $request): PromiseInterface { */ public function createBatch(array $requests): string { $params = []; + $betas = []; foreach ($requests as $customId => $request) { $params[] = [ 'custom_id' => $customId, 'params' => $this->encodeRequest($request), ]; + $betas = [...$betas, ...$this->getRequestBetas($request)]; } $response = $this->getHttpClient()->post('https://api.anthropic.com/v1/messages/batches', [ - 'headers' => $this->getHeaders(), + 'headers' => $this->getHeaders($betas), 'json' => [ 'requests' => $params, ], diff --git a/src/Client/Anthropic/AnthropicEncoder.php b/src/Client/Anthropic/AnthropicEncoder.php index 05fe9d6..fb0e5d7 100644 --- a/src/Client/Anthropic/AnthropicEncoder.php +++ b/src/Client/Anthropic/AnthropicEncoder.php @@ -3,14 +3,18 @@ namespace Soukicz\Llm\Client\Anthropic; use InvalidArgumentException; +use Soukicz\Llm\Client\Anthropic\Model\AnthropicModel; +use Soukicz\Llm\Client\Anthropic\Model\AnthropicPrices; use Soukicz\Llm\Client\Anthropic\Tool\AnthropicNativeTool; use Soukicz\Llm\Client\Anthropic\Tool\AnthropicNativeToolWithOptions; use Soukicz\Llm\Client\ModelEncoder; +use Soukicz\Llm\Client\ModelInterface; use Soukicz\Llm\Client\ModelResponse; use Soukicz\Llm\Client\StopReason; use Soukicz\Llm\Config\ConversationCacheConfig; use Soukicz\Llm\Config\ReasoningBudget; use Soukicz\Llm\Config\ReasoningEffort; +use Soukicz\Llm\Config\ThinkingDisplay; use Soukicz\Llm\LLMRequest; use Soukicz\Llm\LLMResponse; use Soukicz\Llm\Message\LLMMessage; @@ -18,12 +22,15 @@ use Soukicz\Llm\Message\LLMMessageContent; use Soukicz\Llm\Message\LLMMessageContents; use Soukicz\Llm\Message\LLMMessageImage; +use Soukicz\Llm\Message\LLMMessageOpaque; use Soukicz\Llm\Message\LLMMessagePdf; use Soukicz\Llm\Message\LLMMessageReasoning; +use Soukicz\Llm\Message\LLMMessageRedactedReasoning; use Soukicz\Llm\Message\LLMMessageStructuredData; use Soukicz\Llm\Message\LLMMessageText; use Soukicz\Llm\Message\LLMMessageToolResult; use Soukicz\Llm\Message\LLMMessageToolUse; +use Soukicz\Llm\Usage\TokenUsage; class AnthropicEncoder implements ModelEncoder { /** @@ -86,14 +93,18 @@ private static function normalizeSchemaForStrictMode(array $schema): array { * requires longer-TTL cache entries to appear before shorter-TTL ones, so a 5-minute * explicit breakpoint followed by 1-hour moving breakpoints would be rejected. */ - private function addConversationCacheBreakpoints(array $encodedMessages, ConversationCacheConfig $config): array { + private function addConversationCacheBreakpoints(array $encodedMessages, ConversationCacheConfig $config, int $breakpointsInUse = 0): array { $cacheControl = ['type' => 'ephemeral']; if ($config->getTtl() !== null) { $cacheControl['ttl'] = $config->getTtl()->value; } - $breakpointCount = 0; + $breakpointCount = $breakpointsInUse; foreach ($encodedMessages as $messageIndex => $encodedMessage) { + if (!is_array($encodedMessage['content'])) { + // mid-conversation system message sent as a plain string + continue; + } foreach ($encodedMessage['content'] as $contentIndex => $encodedContent) { if (isset($encodedContent['cache_control'])) { $breakpointCount++; @@ -128,6 +139,63 @@ private function addConversationCacheBreakpoints(array $encodedMessages, Convers return $encodedMessages; } + private function encodeSystemContent(LLMMessage $message): array { + if (count($message->getContents()) !== 1) { + throw new \InvalidArgumentException('System message supports only one content block'); + } + $content = $message->getContents()[0]; + if (!$content instanceof LLMMessageText) { + throw new \InvalidArgumentException('Unsupported system message type'); + } + + return $this->addCacheAttribute($content, ['type' => 'text', 'text' => $content->getText()]); + } + + /** + * Sends a later system message as `role: system` where the model and its position allow it (after a user + * message, last or before an assistant message); otherwise appends its text to the previous user message, + * or to the next one - appending keeps tool_result blocks first. + */ + private function placeMidConversationSystemMessages(array $encodedMessages, ModelInterface $model): array { + $native = $model instanceof AnthropicModel && $model->supportsMidConversationSystemMessages(); + $result = []; + $pending = []; + $count = count($encodedMessages); + for ($i = 0; $i < $count; $i++) { + $message = $encodedMessages[$i]; + if ($message['role'] !== 'system') { + if ($pending !== [] && $message['role'] === 'user') { + $message['content'] = [...$message['content'], ...$pending]; + $pending = []; + } + $result[] = $message; + continue; + } + + $previousIndex = array_key_last($result); + $previousRole = $previousIndex === null ? null : $result[$previousIndex]['role']; + $nextRole = $encodedMessages[$i + 1]['role'] ?? null; + if ($native && $previousRole === 'user' && ($nextRole === null || $nextRole === 'assistant')) { + $content = $message['content']; + $result[] = [ + 'role' => 'system', + 'content' => count($content) === 1 && !isset($content[0]['cache_control']) ? $content[0]['text'] : $content, + ]; + continue; + } + if ($previousRole === 'user') { + $result[$previousIndex]['content'] = [...$result[$previousIndex]['content'], ...$message['content']]; + continue; + } + $pending = [...$pending, ...$message['content']]; + } + if ($pending !== []) { + throw new \InvalidArgumentException('A system message after the conversation start needs a user message next to it'); + } + + return $result; + } + private function addCacheAttribute(LLMMessageContent $content, array $data): array { if ($content->isCached()) { $data['cache_control'] = ['type' => 'ephemeral']; @@ -183,31 +251,53 @@ public function encodeMessageContent(LLMMessageContent $messageContent): array { ]); } + if ($messageContent instanceof LLMMessageRedactedReasoning) { + return [ + 'type' => 'redacted_thinking', + 'data' => $messageContent->getData(), + ]; + } + if ($messageContent instanceof LLMMessageOpaque) { + if ($messageContent->getProvider() !== 'anthropic') { + throw new InvalidArgumentException('Opaque content from provider "' . $messageContent->getProvider() . '" cannot be sent to Anthropic'); + } + $block = $messageContent->getBlock(); + // json_decode(..., true) turns an empty JSON object into []; the API expects an object back + if (array_key_exists('input', $block) && $block['input'] === []) { + $block['input'] = new \stdClass(); + } + + return $block; + } throw new InvalidArgumentException('Unsupported message content type: ' . get_class($messageContent)); } public function encodeRequest(LLMRequest $request): array { - $systemPrompt = null; + $systemBlocks = null; $encodedMessages = []; - foreach ($request->getConversation()->getMessages() as $message) { + $messages = $request->getConversation()->getMessages(); + $systemMessageCount = count(array_filter($messages, static fn(LLMMessage $message): bool => $message->isSystem())); + $seenNonSystem = false; + foreach ($messages as $message) { + if ($message->isSystem()) { + $systemBlock = $this->encodeSystemContent($message); + // the opening system message (or the only one anywhere, as before) is the top-level system prompt + if ($systemBlocks === null && (!$seenNonSystem || $systemMessageCount === 1)) { + $systemBlocks = [$systemBlock]; + continue; + } + if (!$seenNonSystem) { + throw new \InvalidArgumentException('Multiple system messages'); + } + $encodedMessages[] = ['role' => 'system', 'content' => [$systemBlock]]; + continue; + } + $seenNonSystem = true; + if ($message->isUser()) { $role = 'user'; } elseif ($message->isAssistant()) { $role = 'assistant'; - } elseif ($message->isSystem()) { - if ($systemPrompt !== null) { - throw new \InvalidArgumentException('Multiple system messages'); - } - if (count($message->getContents()) !== 1) { - throw new \InvalidArgumentException('System message supports only one content block'); - } - $content = $message->getContents()[0]; - if ($content instanceof LLMMessageText) { - $systemPrompt = $content->getText(); - } else { - throw new \InvalidArgumentException('Unsupported system message type'); - } - continue; } else { throw new \InvalidArgumentException('Unsupported message role'); } @@ -239,7 +329,7 @@ public function encodeRequest(LLMRequest $request): array { 'tool_use_id' => $messageContent->getId(), 'content' => $contentParts, ]; - if ($message->getContents()->isError()) { + if ($toolResultContent->isError()) { $toolContent['is_error'] = true; } $contents[] = $this->addCacheAttribute($messageContent, $toolContent); @@ -253,17 +343,27 @@ public function encodeRequest(LLMRequest $request): array { ]; } + $encodedMessages = $this->placeMidConversationSystemMessages($encodedMessages, $request->getModel()); + $conversationCacheConfig = $request->getConversationCacheConfig(); + $systemBreakpoints = $systemBlocks !== null && isset($systemBlocks[0]['cache_control']) ? 1 : 0; if ($conversationCacheConfig !== null) { - $encodedMessages = $this->addConversationCacheBreakpoints($encodedMessages, $conversationCacheConfig); + if ($systemBreakpoints === 1 && $conversationCacheConfig->getTtl() !== null) { + // longer TTLs must come first; the system prompt precedes every message + $systemBlocks[0]['cache_control']['ttl'] = $conversationCacheConfig->getTtl()->value; + } + $encodedMessages = $this->addConversationCacheBreakpoints($encodedMessages, $conversationCacheConfig, $systemBreakpoints); } $options = [ 'max_tokens' => $request->getMaxTokens(), - 'temperature' => $request->getTemperature(), 'messages' => $encodedMessages, 'model' => $request->getModel()->getCode(), ]; + $model = $request->getModel(); + if ($request->getTemperature() !== null && (!$model instanceof AnthropicModel || $model->supportsSamplingParameters())) { + $options['temperature'] = $request->getTemperature(); + } $outputConfig = []; @@ -275,22 +375,33 @@ public function encodeRequest(LLMRequest $request): array { 'budget_tokens' => $reasoningConfig->getMaxTokens(), ]; } elseif ($reasoningConfig instanceof ReasoningEffort) { - if ($reasoningConfig !== ReasoningEffort::NONE) { + if ($reasoningConfig === ReasoningEffort::NONE) { + $model = $request->getModel(); + if ($model instanceof AnthropicModel && $model->hasAdaptiveThinkingByDefault()) { + // thinking runs anyway on these models; without an explicit effort they use their + // default (medium on Opus 5.5), so the lowest effort is the closest to "none" + $outputConfig['effort'] = 'low'; + } + } else { $options['thinking'] = [ 'type' => 'adaptive', ]; - $outputConfig['effort'] = match ($reasoningConfig) { - ReasoningEffort::MINIMAL, ReasoningEffort::LOW => 'low', - ReasoningEffort::MEDIUM => 'medium', - ReasoningEffort::HIGH => 'high', - ReasoningEffort::EXTRA_HIGH => 'max', - }; + $outputConfig['effort'] = self::anthropicEffort($reasoningConfig); } } else { throw new \InvalidArgumentException('Unsupported reasoning config type'); } } + $thinkingDisplay = self::effectiveThinkingDisplay($request); + if ($thinkingDisplay !== null) { + if (isset($options['thinking'])) { + $options['thinking']['display'] = $thinkingDisplay->value; + } elseif ($model instanceof AnthropicModel && $model->hasAdaptiveThinkingByDefault()) { + $options['thinking'] = ['type' => 'adaptive', 'display' => $thinkingDisplay->value]; + } + } + $structuredOutputConfig = $request->getStructuredOutputConfig(); if ($structuredOutputConfig !== null) { $outputConfig['format'] = [ @@ -303,8 +414,13 @@ public function encodeRequest(LLMRequest $request): array { $options['output_config'] = $outputConfig; } - if ($systemPrompt !== null) { - $options['system'] = $systemPrompt; + $contextEdits = $request->getContextManagement()?->toAnthropicEdits() ?? []; + if ($contextEdits !== []) { + $options['context_management'] = ['edits' => $contextEdits]; + } + + if ($systemBlocks !== null) { + $options['system'] = isset($systemBlocks[0]['cache_control']) ? $systemBlocks : $systemBlocks[0]['text']; } if (!empty($request->getStopSequences())) { $options['stop_sequences'] = $request->getStopSequences(); @@ -343,6 +459,46 @@ public function encodeRequest(LLMRequest $request): array { return $options; } + /** + * Beta features (anthropic-beta header values) this request needs on top of the client's own + * + * @return string[] + */ + public function getRequestBetas(LLMRequest $request): array { + $betas = []; + if (self::effectiveThinkingDisplay($request) === ThinkingDisplay::UPDATES) { + $betas[] = 'thinking-display-updates-2026-08-18'; + } + if (($request->getContextManagement()?->toAnthropicEdits() ?? []) !== []) { + $betas[] = 'context-management-2025-06-27'; + } + + return $betas; + } + + /** + * The requested display, minus `updates` on a model that does not accept it (the API rejects it with a 400) + */ + private static function effectiveThinkingDisplay(LLMRequest $request): ?ThinkingDisplay { + $display = $request->getThinkingDisplay(); + $model = $request->getModel(); + if ($display === ThinkingDisplay::UPDATES && $model instanceof AnthropicModel && !$model->supportsThinkingUpdates()) { + return null; + } + + return $display; + } + + private static function anthropicEffort(ReasoningEffort $effort): string { + return match ($effort) { + ReasoningEffort::NONE, ReasoningEffort::MINIMAL, ReasoningEffort::LOW => 'low', + ReasoningEffort::MEDIUM => 'medium', + ReasoningEffort::HIGH => 'high', + ReasoningEffort::XHIGH => 'xhigh', + ReasoningEffort::EXTRA_HIGH, ReasoningEffort::MAX => 'max', + }; + } + public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse): LLMRequest|LLMResponse { $response = $modelResponse->getData(); $responseTimeMs = $modelResponse->getResponseTimeMs(); @@ -360,35 +516,57 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $responseContents[] = new LLMMessageReasoning($content['thinking'], $content['signature']); } elseif ($content['type'] === 'tool_use') { $responseContents[] = new LLMMessageToolUse($content['id'], $content['name'], $content['input']); + } elseif ($content['type'] === 'redacted_thinking') { + $responseContents[] = new LLMMessageRedactedReasoning($content['data']); } else { - throw new \InvalidArgumentException('Unsupported message type'); + $responseContents[] = new LLMMessageOpaque('anthropic', $content); } } $request = $request->withMessage(LLMMessage::createFromAssistant(new LLMMessageContents($responseContents))); - $cacheInputTokens = $response['usage']['cache_creation_input_tokens'] ?? 0; - $cacheReadInputTokens = $response['usage']['cache_read_input_tokens'] ?? 0; - - $inputPrice = $response['usage']['input_tokens'] * ($request->getModel()->getInputPricePerMillionTokens() / 1000 / 1000); - $outputPrice = $response['usage']['output_tokens'] * ($request->getModel()->getOutputPricePerMillionTokens() / 1000 / 1000); + $usage = $response['usage'] ?? []; + $cacheWriteTotal = $usage['cache_creation_input_tokens'] ?? 0; + $cacheWrite1h = $usage['cache_creation']['ephemeral_1h_input_tokens'] ?? 0; + $cacheWrite5m = $usage['cache_creation']['ephemeral_5m_input_tokens'] ?? ($cacheWriteTotal - $cacheWrite1h); + $stepUsage = new TokenUsage( + input: $usage['input_tokens'] ?? 0, + output: $usage['output_tokens'] ?? 0, + cacheWrite5m: $cacheWrite5m, + cacheWrite1h: $cacheWrite1h, + cacheRead: $usage['cache_read_input_tokens'] ?? 0, + ); - $inputPrice += $cacheInputTokens * ($request->getModel()->getCachedInputPricePerMillionTokens() / 1000 / 1000); - // Cache reads are input tokens, so their cost belongs to the input bucket - $inputPrice += $cacheReadInputTokens * ($request->getModel()->getCachedOutputPricePerMillionTokens() / 1000 / 1000); + $model = $request->getModel(); + $prices = $model instanceof AnthropicModel + ? $model->getPrices($stepUsage->promptTokens()) + : new AnthropicPrices( + $model->getInputPricePerMillionTokens(), + $model->getOutputPricePerMillionTokens(), + $model->getCachedInputPricePerMillionTokens(), + $model->getCachedInputPricePerMillionTokens(), + $model->getCachedOutputPricePerMillionTokens(), + ); + + // cache writes and reads are input-side costs + $inputPrice = ( + $stepUsage->input * $prices->input + + $stepUsage->cacheWrite5m * $prices->cacheWrite5m + + $stepUsage->cacheWrite1h * $prices->cacheWrite1h + + $stepUsage->cacheRead * $prices->cacheRead + ) / 1_000_000; + $outputPrice = $stepUsage->output * $prices->output / 1_000_000; $request = $request - ->withCost( - $response['usage']['input_tokens'], - $response['usage']['output_tokens'], - $inputPrice, - $outputPrice - ) + ->withCost($stepUsage->input, $stepUsage->output, $inputPrice, $outputPrice) + ->withUsage($stepUsage) ->withTime($responseTimeMs); $stopReason = match ($response['stop_reason']) { 'end_turn', 'stop_sequence' => StopReason::FINISHED, - 'max_tokens' => StopReason::LENGTH, + 'max_tokens', 'model_context_window_exceeded' => StopReason::LENGTH, 'tool_use' => StopReason::TOOL_USE, + 'refusal' => StopReason::REFUSAL, + 'pause_turn' => StopReason::PAUSE_TURN, default => throw new \InvalidArgumentException('Unsupported stop reason "' . $response['stop_reason'] . '"'), }; @@ -401,6 +579,9 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $request->getPreviousInputCostUSD(), $request->getPreviousOutputCostUSD(), $request->getPreviousTimeMs(), + $request->getPreviousUsage(), + $stepUsage, + $response['stop_details'] ?? null, ); } } diff --git a/src/Client/Anthropic/Model/AnthropicClaude47Opus.php b/src/Client/Anthropic/Model/AnthropicClaude47Opus.php index 037278b..e883a2a 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude47Opus.php +++ b/src/Client/Anthropic/Model/AnthropicClaude47Opus.php @@ -7,6 +7,15 @@ public function getCode(): string { return 'claude-opus-4-7'; } + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + public function getInputPricePerMillionTokens(): float { return 5.0; } diff --git a/src/Client/Anthropic/Model/AnthropicClaude48Opus.php b/src/Client/Anthropic/Model/AnthropicClaude48Opus.php index 9ce00c7..2889694 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude48Opus.php +++ b/src/Client/Anthropic/Model/AnthropicClaude48Opus.php @@ -7,6 +7,19 @@ public function getCode(): string { return 'claude-opus-4-8'; } + public function supportsMidConversationSystemMessages(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + public function getInputPricePerMillionTokens(): float { return 5.0; } diff --git a/src/Client/Anthropic/Model/AnthropicClaude51Fable.php b/src/Client/Anthropic/Model/AnthropicClaude51Fable.php index 22536c8..bf9a31a 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude51Fable.php +++ b/src/Client/Anthropic/Model/AnthropicClaude51Fable.php @@ -7,6 +7,27 @@ public function getCode(): string { return 'claude-fable-5-1'; } + public function supportsThinkingUpdates(): bool { + return true; + } + + public function supportsMidConversationSystemMessages(): bool { + return true; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + /** * Cache reads are billed at 0.025x the base input price. * diff --git a/src/Client/Anthropic/Model/AnthropicClaude55Haiku.php b/src/Client/Anthropic/Model/AnthropicClaude55Haiku.php index 85d691c..8419e3b 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude55Haiku.php +++ b/src/Client/Anthropic/Model/AnthropicClaude55Haiku.php @@ -3,18 +3,19 @@ namespace Soukicz\Llm\Client\Anthropic\Model; /** - * Prices for prompts up to 100k tokens; longer prompts cost $0.50 input / $2.50 output. + * Claude Haiku 5.5: 1M context, 128K output, adaptive thinking on by default (effort low..max, default medium), + * vision and PDF input. Priced by prompt length: a request whose prompt (input + cache reads + cache writes) is + * over 100,000 tokens pays the long-prompt rate card for all of its tokens. + * + * @see https://platform.claude.com/docs/en/about-claude/pricing (read 2026-10-20) */ class AnthropicClaude55Haiku extends AnthropicModel { + private const LONG_PROMPT_THRESHOLD_TOKENS = 100_000; + public function getCode(): string { return 'claude-haiku-5-5'; } - /** - * Cache reads are billed at 0.1x the base input price. - * - * @see https://platform.claude.com/docs/en/about-claude/pricing - */ public function getInputPricePerMillionTokens(): float { return 0.1; } @@ -30,4 +31,27 @@ public function getCachedInputPricePerMillionTokens(): float { public function getCachedOutputPricePerMillionTokens(): float { return 0.01; } + + public function getPrices(int $promptTokens): AnthropicPrices { + if ($promptTokens > self::LONG_PROMPT_THRESHOLD_TOKENS) { + return new AnthropicPrices(input: 0.5, output: 2.5, cacheWrite5m: 0.625, cacheWrite1h: 1.0, cacheRead: 0.05); + } + + return parent::getPrices($promptTokens); + } + + /** + * Non-default sampling values return a 400 on this model + */ + public function supportsSamplingParameters(): bool { + return false; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + public function supportsMidConversationSystemMessages(): bool { + return true; + } } diff --git a/src/Client/Anthropic/Model/AnthropicClaude55Opus.php b/src/Client/Anthropic/Model/AnthropicClaude55Opus.php index c3c16e1..dc48cf1 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude55Opus.php +++ b/src/Client/Anthropic/Model/AnthropicClaude55Opus.php @@ -7,6 +7,27 @@ public function getCode(): string { return 'claude-opus-5-5'; } + public function supportsThinkingUpdates(): bool { + return true; + } + + public function supportsMidConversationSystemMessages(): bool { + return true; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + /** * Cache reads are billed at 0.05x the base input price. * diff --git a/src/Client/Anthropic/Model/AnthropicClaude55Sonnet.php b/src/Client/Anthropic/Model/AnthropicClaude55Sonnet.php index 93aecdd..adb94d3 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude55Sonnet.php +++ b/src/Client/Anthropic/Model/AnthropicClaude55Sonnet.php @@ -2,16 +2,17 @@ namespace Soukicz\Llm\Client\Anthropic\Model; +/** + * Claude Sonnet 5.5: 1M context, 128K output, adaptive thinking on by default (effort low..max, default high), + * vision and PDF input. Cache reads are billed at 0.05x the base input price. + * + * @see https://platform.claude.com/docs/en/about-claude/pricing (read 2026-10-20) + */ class AnthropicClaude55Sonnet extends AnthropicModel { public function getCode(): string { return 'claude-sonnet-5-5'; } - /** - * Same pricing as Claude Sonnet 5. Cache reads are billed at 0.1x the base input price. - * - * @see https://platform.claude.com/docs/en/about-claude/pricing - */ public function getInputPricePerMillionTokens(): float { return 2.0; } @@ -25,6 +26,25 @@ public function getCachedInputPricePerMillionTokens(): float { } public function getCachedOutputPricePerMillionTokens(): float { - return 0.2; + return 0.1; + } + + /** + * Non-default sampling values return a 400 on this model + */ + public function supportsSamplingParameters(): bool { + return false; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + public function supportsThinkingUpdates(): bool { + return true; + } + + public function supportsMidConversationSystemMessages(): bool { + return true; } } diff --git a/src/Client/Anthropic/Model/AnthropicClaude5Fable.php b/src/Client/Anthropic/Model/AnthropicClaude5Fable.php index 8f5aef3..a0a789f 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude5Fable.php +++ b/src/Client/Anthropic/Model/AnthropicClaude5Fable.php @@ -7,6 +7,27 @@ public function getCode(): string { return 'claude-fable-5'; } + public function supportsThinkingUpdates(): bool { + return true; + } + + public function supportsMidConversationSystemMessages(): bool { + return true; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + public function getInputPricePerMillionTokens(): float { return 10.0; } diff --git a/src/Client/Anthropic/Model/AnthropicClaude5Opus.php b/src/Client/Anthropic/Model/AnthropicClaude5Opus.php index eb2d4dd..5770975 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude5Opus.php +++ b/src/Client/Anthropic/Model/AnthropicClaude5Opus.php @@ -7,6 +7,23 @@ public function getCode(): string { return 'claude-opus-5'; } + public function supportsMidConversationSystemMessages(): bool { + return true; + } + + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + public function getInputPricePerMillionTokens(): float { return 5.0; } diff --git a/src/Client/Anthropic/Model/AnthropicClaude5Sonnet.php b/src/Client/Anthropic/Model/AnthropicClaude5Sonnet.php index 2d97ba2..87e8955 100644 --- a/src/Client/Anthropic/Model/AnthropicClaude5Sonnet.php +++ b/src/Client/Anthropic/Model/AnthropicClaude5Sonnet.php @@ -7,6 +7,19 @@ public function getCode(): string { return 'claude-sonnet-5'; } + public function hasAdaptiveThinkingByDefault(): bool { + return true; + } + + /** + * Sampling parameters return a 400 on this model + * + * @see https://platform.claude.com/docs/en/about-claude/models/overview + */ + public function supportsSamplingParameters(): bool { + return false; + } + /** * The $2 / $10 introductory pricing became the standard price; the * previously announced increase to $3 / $15 on September 1, 2026 did not happen. diff --git a/src/Client/Anthropic/Model/AnthropicModel.php b/src/Client/Anthropic/Model/AnthropicModel.php index c7b1cf7..cb18ff0 100644 --- a/src/Client/Anthropic/Model/AnthropicModel.php +++ b/src/Client/Anthropic/Model/AnthropicModel.php @@ -5,5 +5,72 @@ use Soukicz\Llm\Client\ModelInterface; abstract class AnthropicModel implements ModelInterface { + /** + * Whether `role: system` messages are accepted inside `messages` (after a user message). Without it later + * system messages are merged into the neighbouring user message. + */ + public function supportsMidConversationSystemMessages(): bool { + return false; + } + /** + * Whether the model runs adaptive thinking when the request has no `thinking` parameter + * (thinking cannot be turned off by omission; effort is the only control) + */ + public function hasAdaptiveThinkingByDefault(): bool { + return false; + } + + /** + * Whether the model accepts temperature / top_p / top_k. Models that reject them (400) never get them sent. + */ + public function supportsSamplingParameters(): bool { + return true; + } + + /** + * Whether `thinking.display: "updates"` (and its beta) is accepted. Other models would answer with a 400, + * so the encoder falls back to the default display for them. + */ + public function supportsThinkingUpdates(): bool { + return false; + } + + /** + * 5-minute cache write price; the same value as the legacy getCachedInputPricePerMillionTokens() + */ + public function getCacheWrite5mPricePerMillionTokens(): float { + return $this->getCachedInputPricePerMillionTokens(); + } + + /** + * 1-hour cache write price: 2x the base input price on every Claude model + * + * @see https://platform.claude.com/docs/en/about-claude/pricing + */ + public function getCacheWrite1hPricePerMillionTokens(): float { + return $this->getInputPricePerMillionTokens() * 2; + } + + /** + * Cache read (hit and refresh) price; the same value as the legacy getCachedOutputPricePerMillionTokens(), + * whose name is historical + */ + public function getCacheReadPricePerMillionTokens(): float { + return $this->getCachedOutputPricePerMillionTokens(); + } + + /** + * Rate card for a request whose prompt (uncached input + cache writes + cache reads) has the given size. + * Models priced by prompt length override this. + */ + public function getPrices(int $promptTokens): AnthropicPrices { + return new AnthropicPrices( + $this->getInputPricePerMillionTokens(), + $this->getOutputPricePerMillionTokens(), + $this->getCacheWrite5mPricePerMillionTokens(), + $this->getCacheWrite1hPricePerMillionTokens(), + $this->getCacheReadPricePerMillionTokens(), + ); + } } diff --git a/src/Client/Anthropic/Model/AnthropicPrices.php b/src/Client/Anthropic/Model/AnthropicPrices.php new file mode 100644 index 0000000..6620077 --- /dev/null +++ b/src/Client/Anthropic/Model/AnthropicPrices.php @@ -0,0 +1,17 @@ + $contents, 'generationConfig' => [ - 'temperature' => $request->getTemperature(), 'maxOutputTokens' => $request->getMaxTokens(), ], ]; + if ($request->getTemperature() !== null) { + $requestData['generationConfig']['temperature'] = $request->getTemperature(); + } if (!empty($this->safetySettings)) { $requestData['safetySettings'] = $this->safetySettings; } @@ -160,7 +163,7 @@ public function encodeRequest(LLMRequest $request): array { ReasoningEffort::MINIMAL => 512, ReasoningEffort::LOW => 1024, ReasoningEffort::MEDIUM => 8192, - ReasoningEffort::HIGH, ReasoningEffort::EXTRA_HIGH => 24576, + ReasoningEffort::HIGH, ReasoningEffort::XHIGH, ReasoningEffort::EXTRA_HIGH, ReasoningEffort::MAX => 24576, }, ]; } else { @@ -169,7 +172,7 @@ public function encodeRequest(LLMRequest $request): array { ReasoningEffort::MINIMAL => 'minimal', ReasoningEffort::LOW => 'low', ReasoningEffort::MEDIUM => 'medium', - ReasoningEffort::HIGH, ReasoningEffort::EXTRA_HIGH => 'high', + ReasoningEffort::HIGH, ReasoningEffort::XHIGH, ReasoningEffort::EXTRA_HIGH, ReasoningEffort::MAX => 'high', }, ]; } @@ -246,6 +249,7 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $stopReason = StopReason::TOOL_USE; } + $stepUsage = new TokenUsage(); if (isset($response['usageMetadata'])) { $promptTokenCount = $response['usageMetadata']['promptTokenCount']; $outputTokenCount = $response['usageMetadata']['candidatesTokenCount'] ?? 0; @@ -253,12 +257,12 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $inputPrice = $promptTokenCount * ($model->getInputPricePerMillionTokens() / 1_000_000); $outputPrice = $outputTokenCount * ($model->getOutputPricePerMillionTokens() / 1_000_000); - $request = $request->withCost( - $promptTokenCount, - $outputTokenCount, - $inputPrice, - $outputPrice + $stepUsage = new TokenUsage( + input: $promptTokenCount, + output: $outputTokenCount, + reasoning: $response['usageMetadata']['thoughtsTokenCount'] ?? 0, ); + $request = $request->withCost($promptTokenCount, $outputTokenCount, $inputPrice, $outputPrice)->withUsage($stepUsage); } return new LLMResponse( @@ -269,7 +273,9 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $request->getPreviousMaximumOutputTokens(), $request->getPreviousInputCostUSD(), $request->getPreviousOutputCostUSD(), - $request->getPreviousTimeMs() + $request->getPreviousTimeMs(), + $request->getPreviousUsage(), + $stepUsage, ); } diff --git a/src/Client/Gemini/Model/Gemini4Argon.php b/src/Client/Gemini/Model/Gemini4Argon.php new file mode 100644 index 0000000..df820c9 --- /dev/null +++ b/src/Client/Gemini/Model/Gemini4Argon.php @@ -0,0 +1,36 @@ +getInputPricePerMillionTokens(); + } + + public function getCachedOutputPricePerMillionTokens(): float { + return $this->getOutputPricePerMillionTokens(); + } +} diff --git a/src/Client/LLMAgentClient.php b/src/Client/LLMAgentClient.php index cb3103a..fc297b3 100644 --- a/src/Client/LLMAgentClient.php +++ b/src/Client/LLMAgentClient.php @@ -7,113 +7,145 @@ use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\Utils; use InvalidArgumentException; +use Soukicz\Llm\Agent\AgentRunGuard; +use Soukicz\Llm\Agent\AgentRunInterruptedException; +use Soukicz\Llm\Agent\AgentRunOptions; use Soukicz\Llm\LLMRequest; use Soukicz\Llm\LLMResponse; use Soukicz\Llm\Log\LLMLogger; use Soukicz\Llm\Message\LLMMessage; use Soukicz\Llm\Message\LLMMessageContents; -use Soukicz\Llm\Message\LLMMessageText; use Soukicz\Llm\Message\LLMMessageToolResult; use Soukicz\Llm\Message\LLMMessageToolUse; +use Soukicz\Llm\Tool\ToolDefinition; use Swaggest\JsonSchema\Schema; class LLMAgentClient { public function __construct(private readonly ?LLMLogger $logger = null) { } - public function run(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback = null): LLMResponse { - return $this->runAsync($client, $request, $feedbackCallback)->wait(); + /** + * @throws AgentRunInterruptedException when a limit of $options is hit, the run is cancelled or the model refuses + */ + public function run(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback = null, ?AgentRunOptions $options = null): LLMResponse { + return $this->runAsync($client, $request, $feedbackCallback, $options)->wait(); } /** - * @return PromiseInterface + * @return PromiseInterface rejected with AgentRunInterruptedException on a limit, cancel or refusal */ - public function runAsync(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback = null): PromiseInterface { + public function runAsync(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback = null, ?AgentRunOptions $options = null): PromiseInterface { + $guard = new AgentRunGuard($options ?? new AgentRunOptions()); $this->logger?->requestStarted($request); - return $this->sendAndProcessRequest($client, $request, $feedbackCallback); + return $this->sendAndProcessRequest($client, $request, $feedbackCallback, $guard); } /** - * Helper method that handles the full request-response flow including tool use - * * @return PromiseInterface */ - private function sendAndProcessRequest(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback): PromiseInterface { - return $client->sendRequestAsync($request)->then(function (LLMResponse $response) use ($client, $request, $feedbackCallback) { + private function sendAndProcessRequest(LLMClient $client, LLMRequest $request, ?callable $feedbackCallback, AgentRunGuard $guard): PromiseInterface { + try { + $guard->beforeModelCall($request); + } catch (AgentRunInterruptedException $e) { + return Create::rejectionFor($e); + } + + return $client->sendRequestAsync($request)->then(function (LLMResponse $response) use ($client, $feedbackCallback, $guard) { $this->logger?->requestFinished($response); + $guard->afterModelResponse($response); + + if ($response->getStopReason() === StopReason::PAUSE_TURN) { + // a paused server-side tool loop resumes when the same conversation is sent again + $this->logger?->requestStarted($response->getRequest()); + + return $this->sendAndProcessRequest($client, $response->getRequest(), $feedbackCallback, $guard); + } - // First check for and handle any tool use - this has highest priority if ($response->getStopReason() === StopReason::TOOL_USE) { - return $this->processToolUseResponse($response, $client, $request, $feedbackCallback); + return $this->processToolUseResponse($response, $client, $feedbackCallback, $guard); } - // If no tool use, then process other response types (feedback) - return $this->postProcessResponse($response, $client, $feedbackCallback); + return $this->postProcessResponse($response, $client, $feedbackCallback, $guard); }); } /** - * Process a response that contains tool use requests - * * @return PromiseInterface */ - private function processToolUseResponse(LLMResponse $response, LLMClient $client, LLMRequest $request, ?callable $feedbackCallback): PromiseInterface { - $toolResponseContents = []; - + private function processToolUseResponse(LLMResponse $response, LLMClient $client, ?callable $feedbackCallback, AgentRunGuard $guard): PromiseInterface { + $toolResults = []; foreach ($response->getConversation()->getLastMessage()->getContents() as $content) { if (!$content instanceof LLMMessageToolUse) { continue; } + $guard->beforeToolCall($response, $toolResults); + $toolResults[] = $this->callTool($response->getRequest()->getTools(), $content, $guard); + } - $tool = null; - foreach ($request->getTools() as $candidateTool) { - if ($candidateTool->getName() === $content->getName()) { - $tool = $candidateTool; - break; - } - } + $newRequest = $response->getRequest()->withMessage(LLMMessage::createFromUser(new LLMMessageContents(Utils::unwrap($toolResults)))); + $this->logger?->requestStarted($newRequest); - if ($tool === null) { - $toolResponseContents[] = Create::promiseFor(new LLMMessageToolResult( - $content->getId(), - LLMMessageContents::fromErrorString('ERROR: Tool "' . $content->getName() . '" is not available') - )); - continue; - } + return $this->sendAndProcessRequest($client, $newRequest, $feedbackCallback, $guard); + } - $input = $content->getInput(); - $noContent = empty($input) && empty($tool->getInputSchema()['required']); - - if (!$noContent) { - try { - Schema::import(json_decode(json_encode($tool->getInputSchema())))->in(json_decode(json_encode($input))); - } catch (Exception $e) { - $toolResponseContents[] = Create::promiseFor(new LLMMessageToolResult( - $content->getId(), - LLMMessageContents::fromErrorString('ERROR: Input is not matching expected schema: ' . $e->getMessage()) - )); - continue; - } + /** + * Runs one tool call; never rejects - every failure becomes an error result for the model + * + * @param ToolDefinition[] $tools + * @return PromiseInterface + */ + private function callTool(array $tools, LLMMessageToolUse $toolUse, AgentRunGuard $guard): PromiseInterface { + $name = $toolUse->getName(); + $input = $toolUse->getInput(); + $guard->toolCallStarted($name, $input); + $startedAt = hrtime(true); + $finish = static function (LLMMessageContents $result) use ($guard, $toolUse, $name, $input, $startedAt): LLMMessageToolResult { + $guard->toolCallFinished($name, $input, $result, intdiv(hrtime(true) - $startedAt, 1_000_000)); + + return new LLMMessageToolResult($toolUse->getId(), $result); + }; + + $tool = null; + foreach ($tools as $candidate) { + if ($candidate->getName() === $name) { + $tool = $candidate; + break; } + } + if ($tool === null) { + return Create::promiseFor($finish(LLMMessageContents::fromErrorString('ERROR: Tool "' . $name . '" is not available'))); + } - $toolResponse = $tool->handle($input); - if ($toolResponse instanceof LLMMessageContents) { - $toolResponse = Create::promiseFor($toolResponse); + $noContent = empty($input) && empty($tool->getInputSchema()['required']); + if (!$noContent) { + try { + Schema::import(json_decode(json_encode($tool->getInputSchema())))->in(json_decode(json_encode($input))); + } catch (Exception $e) { + return Create::promiseFor($finish(LLMMessageContents::fromErrorString('ERROR: Input is not matching expected schema: ' . $e->getMessage()))); } - $toolResponseContents[] = $toolResponse->then(function (LLMMessageContents $response) use ($content) { - return new LLMMessageToolResult($content->getId(), $response); - }); } - $newRequest = $response->getRequest()->withMessage(LLMMessage::createFromUser(new LLMMessageContents(Utils::unwrap($toolResponseContents)))); - $this->logger?->requestStarted($newRequest); + try { + $handled = $tool->handle($input); + } catch (\Throwable $e) { + $handled = self::toolErrorContents($e); + } + if ($handled instanceof LLMMessageContents) { + // finish synchronous tools immediately so listener events and durations stay per tool + return Create::promiseFor($finish($handled)); + } - // Use sendAndProcessRequest to ensure full processing of the response, including potential nested tool uses - return $this->sendAndProcessRequest($client, $newRequest, $feedbackCallback); + return $handled + ->then(static fn(mixed $value): LLMMessageContents => self::toolResultContents($value)) + ->otherwise(static fn(mixed $reason): LLMMessageContents => self::toolErrorContents($reason)) + ->then($finish); } - private function postProcessResponse(LLMResponse $llmResponse, LLMClient $LLMClient, ?callable $feedbackCallback): PromiseInterface { + /** + * @return PromiseInterface + */ + private function postProcessResponse(LLMResponse $llmResponse, LLMClient $LLMClient, ?callable $feedbackCallback, AgentRunGuard $guard): PromiseInterface { $request = $llmResponse->getRequest(); if ($feedbackCallback) { @@ -124,11 +156,26 @@ private function postProcessResponse(LLMResponse $llmResponse, LLMClient $LLMCli } $request = $request->withMessage($feedback); - return $this->sendAndProcessRequest($LLMClient, $request, $feedbackCallback); + return $this->sendAndProcessRequest($LLMClient, $request, $feedbackCallback, $guard); } } return Create::promiseFor($llmResponse); } + private static function toolResultContents(mixed $value): LLMMessageContents { + if ($value instanceof LLMMessageContents) { + return $value; + } + + return LLMMessageContents::fromErrorString('ERROR: tool returned ' . get_debug_type($value) . ', expected LLMMessageContents'); + } + + private static function toolErrorContents(mixed $reason): LLMMessageContents { + if ($reason instanceof \Throwable) { + return LLMMessageContents::fromErrorString('ERROR: ' . $reason::class . ': ' . $reason->getMessage()); + } + + return LLMMessageContents::fromErrorString('ERROR: tool failed: ' . (is_scalar($reason) ? (string) $reason : get_debug_type($reason))); + } } diff --git a/src/Client/OpenAI/Model/GPT61Sol.php b/src/Client/OpenAI/Model/GPT61Sol.php new file mode 100644 index 0000000..0a9faee --- /dev/null +++ b/src/Client/OpenAI/Model/GPT61Sol.php @@ -0,0 +1,28 @@ + $request->getModel()->getCode(), 'messages' => $encodedMessages, 'max_completion_tokens' => $request->getMaxTokens(), - 'temperature' => $request->getTemperature(), ]; + if ($request->getTemperature() !== null) { + $requestData['temperature'] = $request->getTemperature(); + } $reasoningConfig = $request->getReasoningConfig(); if ($reasoningConfig) { @@ -154,7 +157,7 @@ public function encodeRequest(LLMRequest $request): array { ReasoningEffort::MINIMAL => 'minimal', ReasoningEffort::MEDIUM => 'medium', ReasoningEffort::HIGH => 'high', - ReasoningEffort::EXTRA_HIGH => 'xhigh', + ReasoningEffort::XHIGH, ReasoningEffort::EXTRA_HIGH, ReasoningEffort::MAX => 'xhigh', }; } else { throw new InvalidArgumentException('Unsupported reasoning config type'); @@ -198,6 +201,7 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $response = $modelResponse->getData(); $model = $request->getModel(); + $stepUsage = new TokenUsage(); if (isset($response['usage'])) { $promptTokens = $response['usage']['prompt_tokens']; $completionTokens = $response['usage']['completion_tokens']; @@ -207,7 +211,13 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse + $cachedTokens * ($model->getCachedInputPricePerMillionTokens() / 1_000_000); $outputPrice = $completionTokens * ($model->getOutputPricePerMillionTokens() / 1_000_000); - $request = $request->withCost($promptTokens, $completionTokens, $inputPrice, $outputPrice); + $stepUsage = new TokenUsage( + input: $promptTokens - $cachedTokens, + output: $completionTokens, + cacheRead: $cachedTokens, + reasoning: $response['usage']['completion_tokens_details']['reasoning_tokens'] ?? 0, + ); + $request = $request->withCost($promptTokens, $completionTokens, $inputPrice, $outputPrice)->withUsage($stepUsage); } $request = $request->withTime((int) $modelResponse->getResponseTimeMs()); @@ -258,7 +268,9 @@ public function decodeResponse(LLMRequest $request, ModelResponse $modelResponse $request->getPreviousMaximumOutputTokens(), $request->getPreviousInputCostUSD(), $request->getPreviousOutputCostUSD(), - $request->getPreviousTimeMs() + $request->getPreviousTimeMs(), + $request->getPreviousUsage(), + $stepUsage, ); } } diff --git a/src/Client/StopReason.php b/src/Client/StopReason.php index 212bc2c..aa3ab1b 100644 --- a/src/Client/StopReason.php +++ b/src/Client/StopReason.php @@ -7,4 +7,8 @@ enum StopReason: string { case TOOL_USE = 'tool_use'; case LENGTH = 'length'; case SAFETY = 'safety'; + /** The provider's safety classifier declined the request (HTTP 200); see LLMResponse::getStopDetails() */ + case REFUSAL = 'refusal'; + /** A server-side tool loop paused; re-sending the conversation unchanged resumes it */ + case PAUSE_TURN = 'pause_turn'; } diff --git a/src/Config/ClearToolUses.php b/src/Config/ClearToolUses.php new file mode 100644 index 0000000..bf6e9a4 --- /dev/null +++ b/src/Config/ClearToolUses.php @@ -0,0 +1,48 @@ + 'clear_tool_uses_20250919']; + if ($this->triggerInputTokens !== null) { + $edit['trigger'] = ['type' => 'input_tokens', 'value' => $this->triggerInputTokens]; + } + if ($this->keepToolUses !== null) { + $edit['keep'] = ['type' => 'tool_uses', 'value' => $this->keepToolUses]; + } + if ($this->clearAtLeastInputTokens !== null) { + $edit['clear_at_least'] = ['type' => 'input_tokens', 'value' => $this->clearAtLeastInputTokens]; + } + if ($this->excludeTools !== []) { + $edit['exclude_tools'] = array_values($this->excludeTools); + } + if ($this->clearToolInputs) { + $edit['clear_tool_inputs'] = true; + } + + return $edit; + } +} diff --git a/src/Config/ContextManagement.php b/src/Config/ContextManagement.php new file mode 100644 index 0000000..157eebb --- /dev/null +++ b/src/Config/ContextManagement.php @@ -0,0 +1,21 @@ +> + */ + public function toAnthropicEdits(): array { + return $this->clearToolUses === null ? [] : [$this->clearToolUses->toArray()]; + } +} diff --git a/src/Config/ReasoningEffort.php b/src/Config/ReasoningEffort.php index 18d20fa..d9b27f2 100644 --- a/src/Config/ReasoningEffort.php +++ b/src/Config/ReasoningEffort.php @@ -8,5 +8,10 @@ enum ReasoningEffort: string { case LOW = 'low'; case MEDIUM = 'medium'; case HIGH = 'high'; + /** Between HIGH and MAX (Anthropic `xhigh`, OpenAI `xhigh`) */ + case XHIGH = 'xhigh'; + /** Kept for compatibility: Anthropic `max`, OpenAI `xhigh` */ case EXTRA_HIGH = 'extra_high'; + /** The provider's highest level (Anthropic `max`) */ + case MAX = 'max'; } diff --git a/src/Config/ThinkingDisplay.php b/src/Config/ThinkingDisplay.php new file mode 100644 index 0000000..3b88339 --- /dev/null +++ b/src/Config/ThinkingDisplay.php @@ -0,0 +1,14 @@ +conversation; } @@ -47,7 +54,10 @@ public function getModel(): ModelInterface { return $this->model; } - public function getTemperature(): float { + /** + * null = do not send a temperature (provider default) + */ + public function getTemperature(): ?float { return $this->temperature; } @@ -118,6 +128,20 @@ public function withTime(int $timeMs): self { return $clone; } + /** + * Cumulative usage of every model call made for this request so far (agent loops included) + */ + public function getPreviousUsage(): TokenUsage { + return $this->previousUsage ?? new TokenUsage(); + } + + public function withUsage(TokenUsage $stepUsage): self { + $clone = clone $this; + $clone->previousUsage = $this->getPreviousUsage()->add($stepUsage); + + return $clone; + } + public function getLastMessage(): LLMMessage { @@ -140,4 +164,11 @@ public function getStreamListener(): ?StreamListenerInterface { return $this->streamListener; } + public function getThinkingDisplay(): ?ThinkingDisplay { + return $this->thinkingDisplay; + } + + public function getContextManagement(): ?ContextManagement { + return $this->contextManagement; + } } diff --git a/src/LLMResponse.php b/src/LLMResponse.php index 180ff02..e4d2396 100644 --- a/src/LLMResponse.php +++ b/src/LLMResponse.php @@ -5,9 +5,22 @@ use Soukicz\Llm\Client\StopReason; use Soukicz\Llm\Message\LLMMessageStructuredData; use Soukicz\Llm\Message\LLMMessageText; +use Soukicz\Llm\Usage\TokenUsage; class LLMResponse { - public function __construct(private readonly LLMRequest $request, private readonly StopReason $stopReason, private readonly int $inputTokens, private readonly int $outputTokens, private readonly int $maximumOutputTokens, private readonly ?float $inputPriceUsd, private readonly ?float $outputPriceUsd, private int $totalTimeMs) { + public function __construct( + private readonly LLMRequest $request, + private readonly StopReason $stopReason, + private readonly int $inputTokens, + private readonly int $outputTokens, + private readonly int $maximumOutputTokens, + private readonly ?float $inputPriceUsd, + private readonly ?float $outputPriceUsd, + private int $totalTimeMs, + private readonly ?TokenUsage $usage = null, + private readonly ?TokenUsage $stepUsage = null, + private readonly ?array $stopDetails = null, + ) { } public function getConversation(): LLMConversation { @@ -68,4 +81,32 @@ public function getTotalTimeMs(): int { return $this->totalTimeMs; } + /** + * Provider details of the stop (Anthropic `stop_details`: {type, category, explanation} on a refusal), null otherwise + */ + public function getStopDetails(): ?array { + return $this->stopDetails; + } + + /** + * Cumulative usage of the whole run (every model call of an agent loop) + */ + public function getUsage(): TokenUsage { + return $this->usage ?? new TokenUsage(input: $this->inputTokens, output: $this->outputTokens); + } + + /** + * Usage of the model call that produced this response only + */ + public function getStepUsage(): TokenUsage { + return $this->stepUsage ?? $this->getUsage(); + } + + /** + * Cumulative cost in USD, cache writes and reads included (they are part of the input price) + */ + public function getTotalCostUsd(): float { + return ($this->inputPriceUsd ?? 0.0) + ($this->outputPriceUsd ?? 0.0); + } + } diff --git a/src/MarkdownFormatter.php b/src/MarkdownFormatter.php index 2e4def7..0326051 100644 --- a/src/MarkdownFormatter.php +++ b/src/MarkdownFormatter.php @@ -6,8 +6,10 @@ use Soukicz\Llm\Message\LLMMessageArrayData; use Soukicz\Llm\Message\LLMMessageContent; use Soukicz\Llm\Message\LLMMessageImage; +use Soukicz\Llm\Message\LLMMessageOpaque; use Soukicz\Llm\Message\LLMMessagePdf; use Soukicz\Llm\Message\LLMMessageReasoning; +use Soukicz\Llm\Message\LLMMessageRedactedReasoning; use Soukicz\Llm\Message\LLMMessageStructuredData; use Soukicz\Llm\Message\LLMMessageText; use Soukicz\Llm\Message\LLMMessageToolResult; @@ -43,6 +45,14 @@ private function messageContentToString(LLMMessageContent $content): string { return "**Structured output:**\n\n```json\n" . json_encode($content->getData(), JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "\n```\n"; } + if ($content instanceof LLMMessageRedactedReasoning) { + return '**Redacted reasoning**'; + } + + if ($content instanceof LLMMessageOpaque) { + return '**' . $content->getProvider() . ' block:** `' . $content->getType() . '`'; + } + throw new RuntimeException('Unknown message content type'); } @@ -56,7 +66,7 @@ public function responseToMarkdown(LLMRequest|LLMResponse $requestOrResponse): s } $markdown = ' - **Model:** ' . $request->getModel()->getCode() . "\n"; - $markdown .= ' - **Temperature:** ' . $request->getTemperature() . "\n"; + $markdown .= ' - **Temperature:** ' . ($request->getTemperature() ?? 'provider default') . "\n"; $markdown .= ' - **Max tokens:** ' . $request->getMaxTokens() . "\n"; foreach ($request->getConversation()->getMessages() as $message) { diff --git a/src/Message/LLMMessage.php b/src/Message/LLMMessage.php index e2efde1..1769172 100644 --- a/src/Message/LLMMessage.php +++ b/src/Message/LLMMessage.php @@ -50,8 +50,11 @@ public static function createFromSystem(LLMMessageContents $content): LLMMessage return new self(self::TYPE_SYSTEM, $content); } - public static function createFromSystemString(string $content): LLMMessage { - return new self(self::TYPE_SYSTEM, new LLMMessageContents([new LLMMessageText($content)])); + /** + * @param bool $cached put a prompt-cache breakpoint after this system prompt (Anthropic) + */ + public static function createFromSystemString(string $content, bool $cached = false): LLMMessage { + return new self(self::TYPE_SYSTEM, new LLMMessageContents([new LLMMessageText($content, $cached)])); } diff --git a/src/Message/LLMMessageOpaque.php b/src/Message/LLMMessageOpaque.php new file mode 100644 index 0000000..456c6fe --- /dev/null +++ b/src/Message/LLMMessageOpaque.php @@ -0,0 +1,36 @@ +provider; + } + + public function getBlock(): array { + return $this->block; + } + + public function getType(): string { + return (string) ($this->block['type'] ?? ''); + } + + public function isCached(): bool { + return false; + } + + public function jsonSerialize(): array { + return ['provider' => $this->provider, 'block' => $this->block]; + } + + public static function fromJson(array $data): self { + return new self($data['provider'], $data['block']); + } +} diff --git a/src/Message/LLMMessagePdf.php b/src/Message/LLMMessagePdf.php index b3b22cc..9567cd9 100644 --- a/src/Message/LLMMessagePdf.php +++ b/src/Message/LLMMessagePdf.php @@ -28,6 +28,6 @@ public function jsonSerialize(): array { } public static function fromJson(array $data): self { - return new self($data['encoding'], $data['data'], $data['cached']); + return new self($data['encoding'], $data['data'], $data['cached'] ?? false); } } diff --git a/src/Message/LLMMessageRedactedReasoning.php b/src/Message/LLMMessageRedactedReasoning.php new file mode 100644 index 0000000..668772a --- /dev/null +++ b/src/Message/LLMMessageRedactedReasoning.php @@ -0,0 +1,27 @@ +data; + } + + public function isCached(): bool { + return false; + } + + public function jsonSerialize(): array { + return ['data' => $this->data]; + } + + public static function fromJson(array $data): self { + return new self($data['data']); + } +} diff --git a/src/Stream/AnthropicStreamAccumulator.php b/src/Stream/AnthropicStreamAccumulator.php index d4002e0..147c5ea 100644 --- a/src/Stream/AnthropicStreamAccumulator.php +++ b/src/Stream/AnthropicStreamAccumulator.php @@ -78,6 +78,7 @@ public static function consume(StreamInterface $stream, StreamListenerInterface $content = []; $usage = []; $stopReason = null; + $stopDetails = null; // Per-block accumulators $blockTexts = []; @@ -193,6 +194,16 @@ public static function consume(StreamInterface $stream, StreamListenerInterface 'input' => $inputJson !== '' ? json_decode($inputJson, true, 512, JSON_THROW_ON_ERROR) : [], ]; break; + + default: + // redacted_thinking, server tool blocks and future types: keep the block as started, + // with any streamed input + $block = $meta; + if ($blockToolJsons[$index] !== '') { + $block['input'] = json_decode($blockToolJsons[$index], true, 512, JSON_THROW_ON_ERROR); + } + $content[$index] = $block; + break; } $listener->onStreamEvent(new StreamEvent( @@ -205,6 +216,9 @@ public static function consume(StreamInterface $stream, StreamListenerInterface if (isset($data['delta']['stop_reason'])) { $stopReason = $data['delta']['stop_reason']; } + if (isset($data['delta']['stop_details'])) { + $stopDetails = $data['delta']['stop_details']; + } if (isset($data['usage'])) { $usage = array_merge($usage, $data['usage']); } @@ -230,6 +244,7 @@ public static function consume(StreamInterface $stream, StreamListenerInterface return [ 'content' => array_values($content), 'stop_reason' => $stopReason, + 'stop_details' => $stopDetails, 'usage' => $usage, ]; } diff --git a/src/Usage/TokenUsage.php b/src/Usage/TokenUsage.php new file mode 100644 index 0000000..208225a --- /dev/null +++ b/src/Usage/TokenUsage.php @@ -0,0 +1,51 @@ +input + $other->input, + $this->output + $other->output, + $this->cacheWrite5m + $other->cacheWrite5m, + $this->cacheWrite1h + $other->cacheWrite1h, + $this->cacheRead + $other->cacheRead, + $this->reasoning + $other->reasoning, + ); + } + + public function promptTokens(): int { + return $this->input + $this->cacheWrite5m + $this->cacheWrite1h + $this->cacheRead; + } + + /** + * @return array{input: int, output: int, cacheWrite5m: int, cacheWrite1h: int, cacheRead: int, reasoning: int} + */ + public function jsonSerialize(): array { + return [ + 'input' => $this->input, + 'output' => $this->output, + 'cacheWrite5m' => $this->cacheWrite5m, + 'cacheWrite1h' => $this->cacheWrite1h, + 'cacheRead' => $this->cacheRead, + 'reasoning' => $this->reasoning, + ]; + } +} diff --git a/tests/Agent/AgentRunTest.php b/tests/Agent/AgentRunTest.php new file mode 100644 index 0000000..bcaf477 --- /dev/null +++ b/tests/Agent/AgentRunTest.php @@ -0,0 +1,293 @@ + 'object', 'properties' => ['expression' => ['type' => 'string']], 'required' => ['expression']]; + + /** + * @param CallbackToolDefinition[] $tools + */ + private function request(array $tools): LLMRequest { + return new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: new LLMConversation([LLMMessage::createFromUserString('What is 2+2?')]), + tools: $tools, + ); + } + + private function calculator(int &$calls, ?\Closure $sideEffect = null): CallbackToolDefinition { + return new CallbackToolDefinition('calculator', 'Calculator', self::SCHEMA, function (array $input) use (&$calls, $sideEffect): LLMMessageContents { + $calls++; + if ($sideEffect !== null) { + $sideEffect(); + } + + return LLMMessageContents::fromArrayData(['result' => 4]); + }); + } + + public function testListenerSeesEveryModelCallAndToolCall(): void { + $calls = 0; + $failing = new CallbackToolDefinition('lookup', 'Lookup', self::SCHEMA, function (array $input): LLMMessageContents { + throw new \RuntimeException('offline'); + }); + $client = new ScriptedLLMClient( + ScriptedLLMClient::reply(new LLMMessageContents([ + new LLMMessageToolUse('t1', 'calculator', ['expression' => '2+2']), + new LLMMessageToolUse('t2', 'lookup', ['expression' => 'x']), + ]), StopReason::TOOL_USE), + ScriptedLLMClient::text('4'), + ); + $listener = new RecordingAgentListener(); + + $response = (new LLMAgentClient())->run($client, $this->request([$this->calculator($calls), $failing]), null, new AgentRunOptions(listener: $listener)); + + $this->assertSame('4', $response->getLastText()); + $this->assertSame([ + ['model', 'tool_use', 100], + ['start', 'calculator', ['expression' => '2+2']], + ['finish', 'calculator', false], + ['start', 'lookup', ['expression' => 'x']], + ['finish', 'lookup', true], + ['model', 'finished', 100], + ], $listener->events); + $this->assertCount(2, $listener->durationsMs); + foreach ($listener->durationsMs as $durationMs) { + $this->assertGreaterThanOrEqual(0, $durationMs); + } + } + + public function testIterationLimitStopsBeforeTheNextModelCall(): void { + $calls = 0; + $client = new ScriptedLLMClient(...ScriptedLLMClient::repeat(5, ScriptedLLMClient::toolUse('t', 'calculator', ['expression' => '2+2']))); + + try { + (new LLMAgentClient())->run($client, $this->request([$this->calculator($calls)]), null, new AgentRunOptions(maxIterations: 3)); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(AgentRunInterruptedException::REASON_ITERATIONS, $e->getReason()); + $this->assertCount(3, $client->sent); + $this->assertSame(3, $calls); + $last = $e->getConversation()->getLastMessage(); + $this->assertTrue($last->isUser()); + $this->assertInstanceOf(LLMMessageToolResult::class, $last->getContents()[0]); + $this->assertSame(300, $e->getUsage()->input); + } + } + + public function testCostLimitStopsAfterTheResponseThatCrossesIt(): void { + $calls = 0; + $client = new ScriptedLLMClient( + ScriptedLLMClient::toolUse('t1', 'calculator', ['expression' => '1'], 0.6), + ScriptedLLMClient::toolUse('t2', 'calculator', ['expression' => '2'], 0.6), + ScriptedLLMClient::text('never'), + ); + + try { + (new LLMAgentClient())->run($client, $this->request([$this->calculator($calls)]), null, new AgentRunOptions(maxCostUsd: 1.0)); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(AgentRunInterruptedException::REASON_COST, $e->getReason()); + $this->assertCount(2, $client->sent); + $this->assertSame(1, $calls, 'the tools of the response that crossed the limit must not run'); + $this->assertEqualsWithDelta(1.2, $e->getCostUsd(), 1e-9); + $closing = $e->getConversation()->getLastMessage()->getContents()[0]; + $this->assertSame('t2', $closing->getId()); + $this->assertTrue($closing->getContent()->isError()); + $this->assertSame('ERROR: not run, the agent run was interrupted (cost)', $closing->getContent()->getMessages()[0]->getText()); + } + } + + public function testPassedDeadlineStopsBeforeAnyModelCall(): void { + $client = new ScriptedLLMClient(ScriptedLLMClient::text('never')); + $request = $this->request([]); + + try { + (new LLMAgentClient())->run($client, $request, null, new AgentRunOptions(deadline: microtime(true) - 1)); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(AgentRunInterruptedException::REASON_DEADLINE, $e->getReason()); + $this->assertCount(0, $client->sent); + $this->assertSame($request->getConversation(), $e->getConversation()); + $this->assertSame(0.0, $e->getCostUsd()); + $this->assertEquals(new TokenUsage(), $e->getUsage()); + } + } + + public function testRunAsyncRejectsInsteadOfThrowing(): void { + $promise = (new LLMAgentClient())->runAsync(new ScriptedLLMClient(), $this->request([]), null, new AgentRunOptions(deadline: microtime(true) - 1)); + + $this->assertSame('rejected', $promise->getState()); + } + + /** + * Review focus 1: the stored conversation must replay as a valid request + */ + public function testCancelBetweenToolCallsClosesTheOpenToolUses(): void { + $calls = 0; + $cancel = false; + $tool = $this->calculator($calls, function () use (&$cancel): void { + $cancel = true; + }); + $client = new ScriptedLLMClient( + ScriptedLLMClient::reply(new LLMMessageContents([ + new LLMMessageToolUse('t1', 'calculator', ['expression' => '1']), + new LLMMessageToolUse('t2', 'calculator', ['expression' => '2']), + ]), StopReason::TOOL_USE), + ScriptedLLMClient::text('never'), + ); + $shouldCancel = function () use (&$cancel): bool { + return $cancel; + }; + + try { + (new LLMAgentClient())->run($client, $this->request([$tool]), null, new AgentRunOptions(shouldCancel: $shouldCancel)); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(AgentRunInterruptedException::REASON_CANCELLED, $e->getReason()); + $this->assertCount(1, $client->sent); + $this->assertSame(1, $calls); + $results = $e->getConversation()->getLastMessage()->getContents(); + $this->assertSame('t1', $results[0]->getId()); + $this->assertFalse($results[0]->getContent()->isError()); + $this->assertSame('t2', $results[1]->getId()); + $this->assertSame('ERROR: not run, the agent run was interrupted (cancelled)', $results[1]->getContent()->getMessages()[0]->getText()); + + $encoded = (new AnthropicEncoder())->encodeRequest(new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: $e->getConversation()->withMessage(LLMMessage::createFromUserString('continue')), + )); + $toolUseIds = array_column(array_filter($encoded['messages'][1]['content'], static fn(array $b): bool => $b['type'] === 'tool_use'), 'id'); + $toolResultIds = array_column(array_filter($encoded['messages'][2]['content'], static fn(array $b): bool => $b['type'] === 'tool_result'), 'tool_use_id'); + $this->assertSame(['t1', 't2'], $toolUseIds); + $this->assertSame($toolUseIds, $toolResultIds); + } + } + + public function testInterruptedConversationReplaysAfterJsonRoundTrip(): void { + $calls = 0; + $cancel = false; + $tool = $this->calculator($calls, function () use (&$cancel): void { + $cancel = true; + }); + $opaque = ['type' => 'web_search_tool_result', 'tool_use_id' => 'srv1', 'content' => [['type' => 'web_search_result', 'url' => 'https://example.com']]]; + $client = new ScriptedLLMClient( + ScriptedLLMClient::reply(new LLMMessageContents([ + new LLMMessageReasoning('thinking it over', 'sig-1'), + new LLMMessageRedactedReasoning('opaque-redacted-data'), + new LLMMessageOpaque('anthropic', ['type' => 'server_tool_use', 'id' => 'srv1', 'name' => 'web_search', 'input' => []]), + new LLMMessageOpaque('anthropic', $opaque), + new LLMMessageToolUse('t1', 'calculator', ['expression' => '1']), + new LLMMessageToolUse('t2', 'calculator', ['expression' => '2']), + ]), StopReason::TOOL_USE), + ScriptedLLMClient::text('never'), + ); + + try { + (new LLMAgentClient())->run($client, $this->request([$tool]), null, new AgentRunOptions(shouldCancel: static function () use (&$cancel): bool { + return $cancel; + })); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(1, $calls); + $restored = LLMConversation::fromJson(json_decode(json_encode($e->getConversation(), JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR)); + } + + $encoded = (new AnthropicEncoder())->encodeRequest(new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: $restored->withMessage(LLMMessage::createFromUserString('continue')), + )); + + $this->assertSame(['user', 'assistant', 'user', 'user'], array_column($encoded['messages'], 'role')); + $assistant = $encoded['messages'][1]['content']; + $this->assertSame(['thinking', 'redacted_thinking', 'server_tool_use', 'web_search_tool_result', 'tool_use', 'tool_use'], array_column($assistant, 'type')); + $this->assertSame('sig-1', $assistant[0]['signature']); + $this->assertSame('opaque-redacted-data', $assistant[1]['data']); + $this->assertEquals((object) [], $assistant[2]['input']); + $this->assertSame($opaque, $assistant[3]); + $toolUseIds = array_column(array_filter($assistant, static fn(array $b): bool => $b['type'] === 'tool_use'), 'id'); + $results = array_values(array_filter($encoded['messages'][2]['content'], static fn(array $b): bool => $b['type'] === 'tool_result')); + $this->assertSame(['t1', 't2'], $toolUseIds); + $this->assertSame($toolUseIds, array_column($results, 'tool_use_id')); + $this->assertArrayNotHasKey('is_error', $results[0]); + $this->assertTrue($results[1]['is_error']); + } + + public function testRefusalEndsTheRunWithoutTheRefusedTurn(): void { + $client = new ScriptedLLMClient( + ScriptedLLMClient::reply(new LLMMessageContents([]), StopReason::REFUSAL, 0.01, ['type' => 'refusal', 'category' => 'cyber', 'explanation' => null]), + ); + + try { + (new LLMAgentClient())->run($client, $this->request([]), null, new AgentRunOptions()); + $this->fail('Expected AgentRunInterruptedException'); + } catch (AgentRunInterruptedException $e) { + $this->assertSame(AgentRunInterruptedException::REASON_REFUSAL, $e->getReason()); + $this->assertCount(1, $e->getConversation()->getMessages()); + $this->assertEqualsWithDelta(0.01, $e->getCostUsd(), 1e-12); + } + } + + public function testRefusalWithoutOptionsAlsoEndsTheRun(): void { + $client = new ScriptedLLMClient(ScriptedLLMClient::reply(new LLMMessageContents([]), StopReason::REFUSAL)); + + $this->expectException(AgentRunInterruptedException::class); + + (new LLMAgentClient())->run($client, $this->request([])); + } + + public function testPauseTurnIsResumedWithTheSameConversation(): void { + $client = new ScriptedLLMClient( + ScriptedLLMClient::reply(new LLMMessageContents([ + new LLMMessageOpaque('anthropic', ['type' => 'server_tool_use', 'id' => 's1', 'name' => 'web_search', 'input' => ['query' => 'x']]), + ]), StopReason::PAUSE_TURN), + ScriptedLLMClient::text('done'), + ); + + $response = (new LLMAgentClient())->run($client, $this->request([])); + + $this->assertSame('done', $response->getLastText()); + $this->assertCount(2, $client->sent); + $this->assertTrue($client->sent[1]->getLastMessage()->isAssistant()); + $this->assertEquals(new TokenUsage(input: 200, output: 100), $response->getUsage()); + } + + public function testToolMayReturnAPdfThatReachesTheNextModelCall(): void { + $uploads = new CallbackToolDefinition('uploads', 'Uploads', ['type' => 'object', 'properties' => ['fileId' => ['type' => 'string']], 'required' => ['fileId']], + fn(array $input): LLMMessageContents => new LLMMessageContents([new LLMMessageText('manual.pdf'), new LLMMessagePdf('base64', 'JVBERi0x')])); + $client = new ScriptedLLMClient( + ScriptedLLMClient::toolUse('t1', 'uploads', ['fileId' => 'abc']), + ScriptedLLMClient::text('Read it.'), + ); + + (new LLMAgentClient())->run($client, $this->request([$uploads])); + + $encoded = (new AnthropicEncoder())->encodeRequest($client->sent[1]); + $this->assertSame('document', $encoded['messages'][2]['content'][0]['content'][1]['type']); + } +} diff --git a/tests/Agent/RecordingAgentListener.php b/tests/Agent/RecordingAgentListener.php new file mode 100644 index 0000000..1d94486 --- /dev/null +++ b/tests/Agent/RecordingAgentListener.php @@ -0,0 +1,30 @@ + */ + public array $events = []; + + /** @var list */ + public array $durationsMs = []; + + public function onModelResponse(LLMResponse $stepResponse): void { + $this->events[] = ['model', $stepResponse->getStopReason()->value, $stepResponse->getStepUsage()->input]; + } + + public function onToolCallStarted(string $toolName, array $input): void { + $this->events[] = ['start', $toolName, $input]; + } + + public function onToolCallFinished(string $toolName, array $input, LLMMessageContents $result, bool $isError, int $durationMs): void { + $this->events[] = ['finish', $toolName, $isError]; + $this->durationsMs[] = $durationMs; + } +} diff --git a/tests/Client/Anthropic/AnthropicClientBetaHeadersTest.php b/tests/Client/Anthropic/AnthropicClientBetaHeadersTest.php new file mode 100644 index 0000000..c52907f --- /dev/null +++ b/tests/Client/Anthropic/AnthropicClientBetaHeadersTest.php @@ -0,0 +1,74 @@ + */ + private array $history = []; + + private function client(array $clientBetas, Response $response): AnthropicClient { + $stack = HandlerStack::create(new MockHandler([$response])); + $this->history = []; + $stack->push(Middleware::history($this->history)); + + $client = new AnthropicClient('test-key', null, null, $clientBetas); + (new \ReflectionProperty(AnthropicClient::class, 'httpClient'))->setValue($client, new Client(['handler' => $stack])); + + return $client; + } + + private static function messageResponse(): Response { + return new Response(200, [], json_encode([ + 'content' => [['type' => 'text', 'text' => 'ok']], + 'stop_reason' => 'end_turn', + 'usage' => ['input_tokens' => 1, 'output_tokens' => 1], + ], JSON_THROW_ON_ERROR)); + } + + private static function request(?ThinkingDisplay $display): LLMRequest { + return new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + thinkingDisplay: $display, + ); + } + + public function testRequestBetasAreMergedWithClientBetas(): void { + $client = $this->client(['files-api-2025-04-14'], self::messageResponse()); + + $client->sendRequestAsync(self::request(ThinkingDisplay::UPDATES))->wait(); + + $this->assertSame('files-api-2025-04-14,thinking-display-updates-2026-08-18', $this->history[0]['request']->getHeaderLine('anthropic-beta')); + } + + public function testNoBetaHeaderWithoutBetas(): void { + $client = $this->client([], self::messageResponse()); + + $client->sendRequestAsync(self::request(null))->wait(); + + $this->assertFalse($this->history[0]['request']->hasHeader('anthropic-beta')); + } + + public function testBatchSendsTheBetasOfItsRequests(): void { + $client = $this->client([], new Response(200, [], json_encode(['id' => 'msgbatch_1'], JSON_THROW_ON_ERROR))); + + $client->createBatch(['a' => self::request(null), 'b' => self::request(ThinkingDisplay::UPDATES)]); + + $this->assertSame('thinking-display-updates-2026-08-18', $this->history[0]['request']->getHeaderLine('anthropic-beta')); + } +} diff --git a/tests/Client/Anthropic/AnthropicDecoderBlocksTest.php b/tests/Client/Anthropic/AnthropicDecoderBlocksTest.php new file mode 100644 index 0000000..4da119a --- /dev/null +++ b/tests/Client/Anthropic/AnthropicDecoderBlocksTest.php @@ -0,0 +1,121 @@ + 'server_tool_use', 'id' => 'srvtoolu_1', 'name' => 'web_search', 'input' => []]; + + private function decode(array $content, string $stopReason, array $extra = []): LLMResponse { + $request = new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hello')]), + ); + $response = (new AnthropicEncoder())->decodeResponse($request, new ModelResponse([ + 'content' => $content, + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + 'stop_reason' => $stopReason, + ] + $extra, 100)); + $this->assertInstanceOf(LLMResponse::class, $response); + + return $response; + } + + private function encodeWithNextUserMessage(LLMConversation $conversation): array { + return (new AnthropicEncoder())->encodeRequest(new LLMRequest( + model: new AnthropicClaude55Opus(), + conversation: $conversation->withMessage(LLMMessage::createFromUserString('Next')), + )); + } + + public function testRefusalIsAStopReasonWithDetails(): void { + $response = $this->decode([], 'refusal', [ + 'stop_details' => ['type' => 'refusal', 'category' => 'cyber', 'explanation' => 'Declined.'], + ]); + + $this->assertSame(StopReason::REFUSAL, $response->getStopReason()); + $this->assertSame('cyber', $response->getStopDetails()['category']); + } + + public function testPauseTurnIsAStopReason(): void { + $response = $this->decode([self::SERVER_TOOL_USE], 'pause_turn'); + + $this->assertSame(StopReason::PAUSE_TURN, $response->getStopReason()); + $this->assertNull($response->getStopDetails()); + } + + public function testContextWindowExceededIsLength(): void { + $this->assertSame(StopReason::LENGTH, $this->decode([['type' => 'text', 'text' => 'Partial']], 'model_context_window_exceeded')->getStopReason()); + } + + public function testRedactedThinkingRoundTripsUnchanged(): void { + $response = $this->decode([['type' => 'redacted_thinking', 'data' => 'EmwKAhgB'], ['type' => 'text', 'text' => 'Done']], 'end_turn'); + + $blocks = $response->getConversation()->getLastMessage()->getContents(); + $this->assertInstanceOf(LLMMessageRedactedReasoning::class, $blocks[0]); + $this->assertSame('EmwKAhgB', $blocks[0]->getData()); + + $encoded = $this->encodeWithNextUserMessage($response->getConversation()); + $this->assertSame(['type' => 'redacted_thinking', 'data' => 'EmwKAhgB'], $encoded['messages'][1]['content'][0]); + } + + public function testUnknownBlocksArePassedThroughUnchanged(): void { + $result = [ + 'type' => 'web_search_tool_result', + 'tool_use_id' => 'srvtoolu_1', + 'content' => [['type' => 'web_search_result', 'url' => 'https://example.com', 'title' => 'Example']], + ]; + $response = $this->decode([self::SERVER_TOOL_USE, $result, ['type' => 'text', 'text' => 'Found it']], 'end_turn'); + + $blocks = $response->getConversation()->getLastMessage()->getContents(); + $this->assertInstanceOf(LLMMessageOpaque::class, $blocks[0]); + $this->assertSame('server_tool_use', $blocks[0]->getType()); + $this->assertSame('anthropic', $blocks[0]->getProvider()); + + $assistant = $this->encodeWithNextUserMessage($response->getConversation())['messages'][1]['content']; + // an empty tool input goes back as a JSON object, never as a list + $this->assertSame('{"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{}}', json_encode($assistant[0], JSON_THROW_ON_ERROR)); + $this->assertSame($result, $assistant[1]); + } + + /** + * Review focus 4: transcripts are stored as JSON and replayed on the next turn + */ + public function testUnknownBlocksSurviveJsonPersistence(): void { + $response = $this->decode([self::SERVER_TOOL_USE, ['type' => 'redacted_thinking', 'data' => 'abc']], 'end_turn'); + + $json = json_encode($response->getConversation(), JSON_THROW_ON_ERROR); + $restored = LLMConversation::fromJson(json_decode($json, true, 512, JSON_THROW_ON_ERROR)); + + $assistant = $this->encodeWithNextUserMessage($restored)['messages'][1]['content']; + $this->assertSame('{"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{}}', json_encode($assistant[0], JSON_THROW_ON_ERROR)); + $this->assertSame(['type' => 'redacted_thinking', 'data' => 'abc'], $assistant[1]); + } + + public function testOpaqueContentOfAnotherProviderIsRejected(): void { + $conversation = new LLMConversation([ + LLMMessage::createFromUserString('Hi'), + LLMMessage::createFromAssistant(new LLMMessageContents([new LLMMessageOpaque('openai', ['type' => 'reasoning'])])), + ]); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Opaque content from provider "openai" cannot be sent to Anthropic'); + + $this->encodeWithNextUserMessage($conversation); + } +} diff --git a/tests/Client/Anthropic/AnthropicDecoderPricingTest.php b/tests/Client/Anthropic/AnthropicDecoderPricingTest.php index a272a3e..f1b3aab 100644 --- a/tests/Client/Anthropic/AnthropicDecoderPricingTest.php +++ b/tests/Client/Anthropic/AnthropicDecoderPricingTest.php @@ -7,12 +7,14 @@ use PHPUnit\Framework\TestCase; use Soukicz\Llm\Client\Anthropic\AnthropicEncoder; use Soukicz\Llm\Client\Anthropic\Model\AnthropicClaude45Sonnet; +use Soukicz\Llm\Client\Anthropic\Model\AnthropicClaude55Haiku; use Soukicz\Llm\Client\ModelResponse; use Soukicz\Llm\Client\StopReason; use Soukicz\Llm\LLMConversation; use Soukicz\Llm\LLMRequest; use Soukicz\Llm\LLMResponse; use Soukicz\Llm\Message\LLMMessage; +use Soukicz\Llm\Usage\TokenUsage; class AnthropicDecoderPricingTest extends TestCase { private AnthropicEncoder $encoder; @@ -71,4 +73,73 @@ public function testCacheTokensAreChargedToInputBucket(): void { // Output stays pure output: 100k * $15/M $this->assertEqualsWithDelta(1.5, $response->getOutputPriceUsd(), 1e-9); } + + public function testOneHourCacheWritesUseTheOneHourPrice(): void { + $response = $this->decode([ + 'input_tokens' => 0, + 'output_tokens' => 0, + 'cache_creation_input_tokens' => 2_000_000, + 'cache_creation' => ['ephemeral_5m_input_tokens' => 1_000_000, 'ephemeral_1h_input_tokens' => 1_000_000], + ]); + + // 1M 5-minute writes ($3.75) + 1M 1-hour writes (2x base = $6.00) + $this->assertEqualsWithDelta(9.75, $response->getInputPriceUsd(), 1e-9); + $this->assertEquals(new TokenUsage(cacheWrite5m: 1_000_000, cacheWrite1h: 1_000_000), $response->getStepUsage()); + } + + public function testWritesWithoutTtlBreakdownCountAsFiveMinute(): void { + $response = $this->decode([ + 'input_tokens' => 10, + 'output_tokens' => 5, + 'cache_creation_input_tokens' => 500, + 'cache_read_input_tokens' => 200, + ]); + + $this->assertEquals(new TokenUsage(input: 10, output: 5, cacheWrite5m: 500, cacheRead: 200), $response->getStepUsage()); + } + + /** + * Review focus 3: a step served from cache still costs money, so a loop of cheap steps reaches maxCostUsd + */ + public function testCacheReadOnlyStepStillCosts(): void { + $response = $this->decode([ + 'input_tokens' => 0, + 'output_tokens' => 0, + 'cache_read_input_tokens' => 1_000_000, + ]); + + $this->assertEqualsWithDelta(0.30, $response->getTotalCostUsd(), 1e-9); + } + + public function testUsageAccumulatesAcrossSteps(): void { + $first = $this->decode(['input_tokens' => 100, 'output_tokens' => 10, 'cache_read_input_tokens' => 1000]); + $second = $this->encoder->decodeResponse($first->getRequest(), new ModelResponse([ + 'content' => [['type' => 'text', 'text' => 'Again']], + 'usage' => ['input_tokens' => 50, 'output_tokens' => 5, 'cache_read_input_tokens' => 2000], + 'stop_reason' => 'end_turn', + ], 300)); + + $this->assertInstanceOf(LLMResponse::class, $second); + $this->assertEquals(new TokenUsage(input: 50, output: 5, cacheRead: 2000), $second->getStepUsage()); + $this->assertEquals(new TokenUsage(input: 150, output: 15, cacheRead: 3000), $second->getUsage()); + // Sonnet 4.5: $3 input, $0.30 cache read, $15 output + $this->assertEqualsWithDelta((150 * 3.0 + 3000 * 0.30 + 15 * 15.0) / 1_000_000, $second->getTotalCostUsd(), 1e-12); + } + + public function testHaiku55UsesTheLongPromptRateCardAbove100kPromptTokens(): void { + $request = new LLMRequest( + model: new AnthropicClaude55Haiku(), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hello')]), + ); + $decode = fn(int $input): LLMResponse => $this->encoder->decodeResponse($request, new ModelResponse([ + 'content' => [['type' => 'text', 'text' => 'ok']], + 'usage' => ['input_tokens' => $input, 'output_tokens' => 1_000, 'cache_read_input_tokens' => 50_000], + 'stop_reason' => 'end_turn', + ], 10)); + + // exactly 100,000 prompt tokens is not "over 100,000": short-prompt prices + $this->assertEqualsWithDelta((50_000 * 0.10 + 50_000 * 0.01 + 1_000 * 0.50) / 1_000_000, $decode(50_000)->getTotalCostUsd(), 1e-12); + // one token more and the whole request pays the long-prompt prices, cache reads included + $this->assertEqualsWithDelta((50_001 * 0.50 + 50_000 * 0.05 + 1_000 * 2.50) / 1_000_000, $decode(50_001)->getTotalCostUsd(), 1e-12); + } } diff --git a/tests/Client/Anthropic/AnthropicEncoderContextManagementTest.php b/tests/Client/Anthropic/AnthropicEncoderContextManagementTest.php new file mode 100644 index 0000000..f1c2d9e --- /dev/null +++ b/tests/Client/Anthropic/AnthropicEncoderContextManagementTest.php @@ -0,0 +1,71 @@ +request(new ContextManagement(clearToolUses: new ClearToolUses(triggerInputTokens: 60000, keepToolUses: 3, clearToolInputs: false, excludeTools: []))); + $encoder = new AnthropicEncoder(); + + $this->assertSame(['edits' => [[ + 'type' => 'clear_tool_uses_20250919', + 'trigger' => ['type' => 'input_tokens', 'value' => 60000], + 'keep' => ['type' => 'tool_uses', 'value' => 3], + ]]], $encoder->encodeRequest($request)['context_management']); + $this->assertSame(['context-management-2025-06-27'], $encoder->getRequestBetas($request)); + } + + public function testEveryField(): void { + $encoded = (new AnthropicEncoder())->encodeRequest($this->request(new ContextManagement(new ClearToolUses( + triggerInputTokens: 30000, + keepToolUses: 2, + clearToolInputs: true, + excludeTools: ['str_replace_based_edit_tool'], + clearAtLeastInputTokens: 5000, + )))); + + $this->assertSame([ + 'type' => 'clear_tool_uses_20250919', + 'trigger' => ['type' => 'input_tokens', 'value' => 30000], + 'keep' => ['type' => 'tool_uses', 'value' => 2], + 'clear_at_least' => ['type' => 'input_tokens', 'value' => 5000], + 'exclude_tools' => ['str_replace_based_edit_tool'], + 'clear_tool_inputs' => true, + ], $encoded['context_management']['edits'][0]); + } + + public function testDefaultsOnlySendTheType(): void { + $encoded = (new AnthropicEncoder())->encodeRequest($this->request(new ContextManagement(new ClearToolUses()))); + + $this->assertSame(['edits' => [['type' => 'clear_tool_uses_20250919']]], $encoded['context_management']); + } + + public function testNothingWithoutEdits(): void { + $encoder = new AnthropicEncoder(); + + foreach ([null, new ContextManagement()] as $contextManagement) { + $request = $this->request($contextManagement); + $this->assertArrayNotHasKey('context_management', $encoder->encodeRequest($request)); + $this->assertSame([], $encoder->getRequestBetas($request)); + } + } +} diff --git a/tests/Client/Anthropic/AnthropicEncoderEffortTest.php b/tests/Client/Anthropic/AnthropicEncoderEffortTest.php new file mode 100644 index 0000000..65e93bf --- /dev/null +++ b/tests/Client/Anthropic/AnthropicEncoderEffortTest.php @@ -0,0 +1,68 @@ +encodeRequest(new LLMRequest( + model: $model, + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + reasoningConfig: $effort, + )); + } + + /** + * @return array + */ + public static function efforts(): array { + return [ + 'minimal' => [ReasoningEffort::MINIMAL, 'low'], + 'low' => [ReasoningEffort::LOW, 'low'], + 'medium' => [ReasoningEffort::MEDIUM, 'medium'], + 'high' => [ReasoningEffort::HIGH, 'high'], + 'xhigh' => [ReasoningEffort::XHIGH, 'xhigh'], + 'extra_high keeps max' => [ReasoningEffort::EXTRA_HIGH, 'max'], + 'max' => [ReasoningEffort::MAX, 'max'], + ]; + } + + /** + * @dataProvider efforts + */ + public function testEffortIsAlwaysSent(ReasoningEffort $effort, string $expected): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), $effort); + + $this->assertSame(['type' => 'adaptive'], $encoded['thinking']); + $this->assertSame($expected, $encoded['output_config']['effort']); + } + + public function testNoneSendsTheLowestEffortWhereThinkingRunsByDefault(): void { + $model = new AnthropicClaude55Opus(); + $encoded = $this->encode($model, ReasoningEffort::NONE); + + $this->assertTrue($model->hasAdaptiveThinkingByDefault()); + $this->assertArrayNotHasKey('thinking', $encoded); + $this->assertSame(['effort' => 'low'], $encoded['output_config']); + } + + public function testNoneSendsNothingWhereThinkingIsOffByDefault(): void { + $model = new AnthropicClaude45Sonnet(AnthropicClaude45Sonnet::VERSION_20250929); + $encoded = $this->encode($model, ReasoningEffort::NONE); + + $this->assertFalse($model->hasAdaptiveThinkingByDefault()); + $this->assertArrayNotHasKey('thinking', $encoded); + $this->assertArrayNotHasKey('output_config', $encoded); + } +} diff --git a/tests/Client/Anthropic/AnthropicEncoderMediaTest.php b/tests/Client/Anthropic/AnthropicEncoderMediaTest.php index 1808dbd..0ff900c 100644 --- a/tests/Client/Anthropic/AnthropicEncoderMediaTest.php +++ b/tests/Client/Anthropic/AnthropicEncoderMediaTest.php @@ -14,6 +14,8 @@ use Soukicz\Llm\Message\LLMMessageImage; use Soukicz\Llm\Message\LLMMessagePdf; use Soukicz\Llm\Message\LLMMessageText; +use Soukicz\Llm\Message\LLMMessageToolResult; +use Soukicz\Llm\Message\LLMMessageToolUse; class AnthropicEncoderMediaTest extends TestCase { public function testImageContent(): void { @@ -82,4 +84,31 @@ public function testPdfContent(): void { // Confirm no cache control since caching is not enabled $this->assertArrayNotHasKey('cache_control', $encoded['messages'][0]['content'][1]); } + + public function testPdfInsideToolResult(): void { + $conversation = new LLMConversation([ + LLMMessage::createFromUserString('Read the brand manual'), + LLMMessage::createFromAssistant(new LLMMessageContents([new LLMMessageToolUse('t1', 'uploads', ['fileId' => 'abc'])])), + LLMMessage::createFromUser(new LLMMessageContents([ + new LLMMessageToolResult('t1', new LLMMessageContents([ + new LLMMessageText('brand-manual.pdf'), + new LLMMessagePdf('base64', 'JVBERi0x'), + ])), + ])), + ]); + + $encoded = (new AnthropicEncoder())->encodeRequest(new LLMRequest( + model: new AnthropicClaude35Sonnet(AnthropicClaude35Sonnet::VERSION_20241022), + conversation: $conversation, + )); + + $toolResult = $encoded['messages'][2]['content'][0]; + $this->assertSame('tool_result', $toolResult['type']); + $this->assertSame(['type' => 'text', 'text' => 'brand-manual.pdf'], $toolResult['content'][0]); + $this->assertSame([ + 'type' => 'document', + 'source' => ['type' => 'base64', 'media_type' => 'application/pdf', 'data' => 'JVBERi0x'], + ], $toolResult['content'][1]); + $this->assertArrayNotHasKey('is_error', $toolResult); + } } diff --git a/tests/Client/Anthropic/AnthropicEncoderSamplingTest.php b/tests/Client/Anthropic/AnthropicEncoderSamplingTest.php new file mode 100644 index 0000000..be8245d --- /dev/null +++ b/tests/Client/Anthropic/AnthropicEncoderSamplingTest.php @@ -0,0 +1,70 @@ +encodeRequest(new LLMRequest( + model: $model, + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + temperature: $temperature, + )); + } + + /** + * @return array + */ + public static function modelsRejectingSampling(): array { + return [ + 'claude-opus-4-7' => [new AnthropicClaude47Opus()], + 'claude-opus-4-8' => [new AnthropicClaude48Opus()], + 'claude-opus-5' => [new AnthropicClaude5Opus()], + 'claude-opus-5-5' => [new AnthropicClaude55Opus()], + 'claude-sonnet-5' => [new AnthropicClaude5Sonnet()], + 'claude-fable-5' => [new AnthropicClaude5Fable()], + 'claude-fable-5-1' => [new AnthropicClaude51Fable()], + ]; + } + + /** + * @dataProvider modelsRejectingSampling + */ + public function testTemperatureIsNeverSentToModelsThatRejectIt(AnthropicModel $model): void { + $this->assertFalse($model->supportsSamplingParameters()); + $this->assertArrayNotHasKey('temperature', $this->encode($model, 0.7)); + $this->assertArrayNotHasKey('temperature', $this->encode($model, 0.0)); + } + + public function testNullTemperatureIsOmitted(): void { + $this->assertArrayNotHasKey('temperature', $this->encode(new AnthropicClaude35Sonnet(AnthropicClaude35Sonnet::VERSION_20241022), null)); + } + + public function testOlderModelsStillGetTheDefaultTemperature(): void { + $model = new AnthropicClaude45Sonnet(AnthropicClaude45Sonnet::VERSION_20250929); + + $this->assertTrue($model->supportsSamplingParameters()); + $encoded = (new AnthropicEncoder())->encodeRequest(new LLMRequest( + model: $model, + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + )); + $this->assertSame(0.0, $encoded['temperature']); + } +} diff --git a/tests/Client/Anthropic/AnthropicEncoderSystemMessagesTest.php b/tests/Client/Anthropic/AnthropicEncoderSystemMessagesTest.php new file mode 100644 index 0000000..2d30460 --- /dev/null +++ b/tests/Client/Anthropic/AnthropicEncoderSystemMessagesTest.php @@ -0,0 +1,140 @@ +encodeRequest(new LLMRequest( + model: $model, + conversation: new LLMConversation($messages), + conversationCacheConfig: $cache, + )); + } + + /** + * @return LLMMessage[] + */ + private static function turnWithStateNote(): array { + return [ + LLMMessage::createFromSystemString('You are the template agent.'), + LLMMessage::createFromUserString('Make a sale e-mail'), + LLMMessage::createFromAssistantString('Done.'), + LLMMessage::createFromUserString('Shorter please'), + LLMMessage::createFromSystemString('Editor state: 3 boxes.'), + ]; + } + + public function testCachedSystemPromptGetsABreakpoint(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), [ + LLMMessage::createFromSystemString('You are the template agent.', true), + LLMMessage::createFromUserString('Hi'), + ]); + + $this->assertSame([['type' => 'text', 'text' => 'You are the template agent.', 'cache_control' => ['type' => 'ephemeral']]], $encoded['system']); + } + + public function testUncachedSystemPromptStaysAString(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), [LLMMessage::createFromSystemString('Plain.'), LLMMessage::createFromUserString('Hi')]); + + $this->assertSame('Plain.', $encoded['system']); + } + + public function testConversationCacheTtlAppliesToTheSystemBreakpoint(): void { + $encoded = $this->encode( + new AnthropicClaude55Opus(), + [LLMMessage::createFromSystemString('Stable prompt.', true), LLMMessage::createFromUserString('Hi')], + new ConversationCacheConfig(CacheTtl::ONE_HOUR), + ); + + $this->assertSame(['type' => 'ephemeral', 'ttl' => '1h'], $encoded['system'][0]['cache_control']); + $this->assertSame(['type' => 'ephemeral', 'ttl' => '1h'], $encoded['messages'][0]['content'][0]['cache_control']); + } + + public function testLaterSystemMessageIsSentInPlaceWhereSupported(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), self::turnWithStateNote()); + + $this->assertSame('You are the template agent.', $encoded['system']); + $this->assertCount(4, $encoded['messages']); + $this->assertSame(['role' => 'system', 'content' => 'Editor state: 3 boxes.'], $encoded['messages'][3]); + } + + public function testLaterSystemMessageIsMergedIntoTheUserMessageElsewhere(): void { + $encoded = $this->encode(new AnthropicClaude5Sonnet(), self::turnWithStateNote()); + + $this->assertCount(3, $encoded['messages']); + $this->assertSame([ + ['type' => 'text', 'text' => 'Shorter please'], + ['type' => 'text', 'text' => 'Editor state: 3 boxes.'], + ], $encoded['messages'][2]['content']); + } + + public function testSystemMessageBetweenTwoUserMessagesIsMergedIntoThePreviousOne(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), [ + LLMMessage::createFromUserString('First'), + LLMMessage::createFromSystemString('Note.'), + LLMMessage::createFromUserString('Second'), + LLMMessage::createFromSystemString('Another note.'), + ]); + + $this->assertSame('user', $encoded['messages'][0]['role']); + $this->assertSame([['type' => 'text', 'text' => 'First'], ['type' => 'text', 'text' => 'Note.']], $encoded['messages'][0]['content']); + $this->assertSame(['role' => 'system', 'content' => 'Another note.'], $encoded['messages'][2]); + } + + /** + * Review focus 5: nothing may sit between an assistant tool_use and its tool_result + */ + public function testSystemNoteAfterToolUseGoesAfterTheToolResults(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), [ + // a leading system prompt, so the later note is not "the only system message" (which stays top-level) + LLMMessage::createFromSystemString('You are the shop agent.'), + LLMMessage::createFromUserString('Find shoes'), + LLMMessage::createFromAssistant(new LLMMessageContents([new LLMMessageToolUse('t1', 'search', ['q' => 'shoes'])])), + LLMMessage::createFromSystemString('Stock changed meanwhile.'), + LLMMessage::createFromUser(new LLMMessageContents([new LLMMessageToolResult('t1', LLMMessageContents::fromString('3 results'))])), + ]); + + $this->assertCount(3, $encoded['messages']); + $this->assertSame('assistant', $encoded['messages'][1]['role']); + $this->assertSame('tool_result', $encoded['messages'][2]['content'][0]['type']); + $this->assertSame(['type' => 'text', 'text' => 'Stock changed meanwhile.'], $encoded['messages'][2]['content'][1]); + } + + public function testMovingBreakpointsSkipSystemMessages(): void { + $encoded = $this->encode(new AnthropicClaude55Opus(), self::turnWithStateNote(), new ConversationCacheConfig()); + + $this->assertSame(['type' => 'ephemeral'], $encoded['messages'][2]['content'][0]['cache_control']); + $this->assertSame(['role' => 'system', 'content' => 'Editor state: 3 boxes.'], $encoded['messages'][3]); + } + + public function testSystemMessageWithoutANeighbouringUserMessageIsRejected(): void { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('A system message after the conversation start needs a user message next to it'); + + $this->encode(new AnthropicClaude55Opus(), [ + LLMMessage::createFromSystemString('Base prompt.'), + LLMMessage::createFromUserString('Hi'), + LLMMessage::createFromAssistantString('Hello'), + LLMMessage::createFromSystemString('Note.'), + ]); + } +} diff --git a/tests/Client/Anthropic/AnthropicEncoderThinkingDisplayTest.php b/tests/Client/Anthropic/AnthropicEncoderThinkingDisplayTest.php new file mode 100644 index 0000000..79567ca --- /dev/null +++ b/tests/Client/Anthropic/AnthropicEncoderThinkingDisplayTest.php @@ -0,0 +1,82 @@ +encodeRequest($this->request(new AnthropicClaude55Opus(), ThinkingDisplay::UPDATES, ReasoningEffort::HIGH)); + + $this->assertSame(['type' => 'adaptive', 'display' => 'updates'], $encoded['thinking']); + $this->assertSame('high', $encoded['output_config']['effort']); + } + + public function testDisplayAloneOnAModelThatThinksByDefault(): void { + $encoded = (new AnthropicEncoder())->encodeRequest($this->request(new AnthropicClaude55Opus(), ThinkingDisplay::SUMMARIZED)); + + $this->assertSame(['type' => 'adaptive', 'display' => 'summarized'], $encoded['thinking']); + } + + public function testDisplayIsIgnoredWhenNoThinkingRuns(): void { + $encoded = (new AnthropicEncoder())->encodeRequest($this->request(new AnthropicClaude45Sonnet(AnthropicClaude45Sonnet::VERSION_20250929), ThinkingDisplay::SUMMARIZED)); + + $this->assertArrayNotHasKey('thinking', $encoded); + } + + public function testUpdatesNeedTheirBetaHeader(): void { + $encoder = new AnthropicEncoder(); + + $this->assertSame(['thinking-display-updates-2026-08-18'], $encoder->getRequestBetas($this->request(new AnthropicClaude55Opus(), ThinkingDisplay::UPDATES))); + $this->assertSame([], $encoder->getRequestBetas($this->request(new AnthropicClaude55Opus(), ThinkingDisplay::SUMMARIZED))); + $this->assertSame([], $encoder->getRequestBetas($this->request(new AnthropicClaude55Opus(), null))); + } + + public function testUpdatesAreOnlySentToModelsThatSupportThem(): void { + $encoder = new AnthropicEncoder(); + foreach ([new AnthropicClaude5Fable(), new AnthropicClaude51Fable(), new AnthropicClaude55Opus(), new AnthropicClaude55Sonnet()] as $model) { + $request = $this->request($model, ThinkingDisplay::UPDATES); + $this->assertSame('updates', $encoder->encodeRequest($request)['thinking']['display'], $model::class); + $this->assertSame(['thinking-display-updates-2026-08-18'], $encoder->getRequestBetas($request), $model::class); + } + + foreach ([new AnthropicClaude5Opus(), new AnthropicClaude48Opus(), new AnthropicClaude5Sonnet(), new AnthropicClaude55Haiku()] as $model) { + $request = $this->request($model, ThinkingDisplay::UPDATES, ReasoningEffort::HIGH); + $this->assertArrayNotHasKey('display', $encoder->encodeRequest($request)['thinking'] ?? [], $model::class); + $this->assertSame([], $encoder->getRequestBetas($request), $model::class); + } + } + + public function testOtherDisplaysAreNotGatedByTheUpdatesFlag(): void { + $encoded = (new AnthropicEncoder())->encodeRequest($this->request(new AnthropicClaude5Opus(), ThinkingDisplay::SUMMARIZED, ReasoningEffort::HIGH)); + + $this->assertSame('summarized', $encoded['thinking']['display']); + } +} diff --git a/tests/Client/Anthropic/AnthropicEncoderToolsTest.php b/tests/Client/Anthropic/AnthropicEncoderToolsTest.php index 6effd23..dad9487 100644 --- a/tests/Client/Anthropic/AnthropicEncoderToolsTest.php +++ b/tests/Client/Anthropic/AnthropicEncoderToolsTest.php @@ -307,4 +307,30 @@ public function testReasoningEffortMax(): void { $this->assertEquals('adaptive', $encoded['thinking']['type']); $this->assertEquals('max', $encoded['output_config']['effort']); } + + public function testToolResultErrorFlagComesFromTheResultItself(): void { + $encoder = new AnthropicEncoder(); + + $conversation = new LLMConversation([ + LLMMessage::createFromUserString('What is 2+2 and 2/0?'), + LLMMessage::createFromAssistant(new LLMMessageContents([ + new LLMMessageToolUse('tool-ok', 'calculator', ['expression' => '2+2']), + new LLMMessageToolUse('tool-bad', 'calculator', ['expression' => '2/0']), + ])), + // the wrapping user message is NOT an error; only the second result is + LLMMessage::createFromUser(new LLMMessageContents([ + new LLMMessageToolResult('tool-ok', LLMMessageContents::fromArrayData(['result' => 4])), + new LLMMessageToolResult('tool-bad', LLMMessageContents::fromErrorString('ERROR: division by zero')), + ])), + ]); + + $encoded = $encoder->encodeRequest(new LLMRequest( + model: new AnthropicClaude35Sonnet(AnthropicClaude35Sonnet::VERSION_20241022), + conversation: $conversation, + )); + + $results = $encoded['messages'][2]['content']; + $this->assertArrayNotHasKey('is_error', $results[0]); + $this->assertTrue($results[1]['is_error']); + } } diff --git a/tests/Client/Anthropic/AnthropicModelCapabilitiesTest.php b/tests/Client/Anthropic/AnthropicModelCapabilitiesTest.php new file mode 100644 index 0000000..f86e8a5 --- /dev/null +++ b/tests/Client/Anthropic/AnthropicModelCapabilitiesTest.php @@ -0,0 +1,54 @@ + + */ + public static function models(): array { + return [ + // model, sampling, adaptive by default, mid-conversation system, 1h cache write $/MTok + 'claude-sonnet-4-5' => [new AnthropicClaude45Sonnet(AnthropicClaude45Sonnet::VERSION_20250929), true, false, false, 6.0], + 'claude-sonnet-4-6' => [new AnthropicClaude46Sonnet(), true, false, false, 6.0], + 'claude-opus-4-7' => [new AnthropicClaude47Opus(), false, false, false, 10.0], + 'claude-opus-4-8' => [new AnthropicClaude48Opus(), false, false, true, 10.0], + 'claude-opus-5' => [new AnthropicClaude5Opus(), false, true, true, 10.0], + 'claude-opus-5-5' => [new AnthropicClaude55Opus(), false, true, true, 8.0], + 'claude-sonnet-5' => [new AnthropicClaude5Sonnet(), false, true, false, 4.0], + 'claude-sonnet-5-5' => [new AnthropicClaude55Sonnet(), false, true, true, 4.0], + 'claude-haiku-5-5' => [new AnthropicClaude55Haiku(), false, true, true, 0.2], + 'claude-fable-5' => [new AnthropicClaude5Fable(), false, true, true, 20.0], + 'claude-fable-5-1' => [new AnthropicClaude51Fable(), false, true, true, 20.0], + ]; + } + + /** + * @dataProvider models + */ + public function testCapabilities(AnthropicModel $model, bool $sampling, bool $adaptiveByDefault, bool $midConversationSystem, float $cacheWrite1h): void { + $this->assertSame($sampling, $model->supportsSamplingParameters()); + $this->assertSame($adaptiveByDefault, $model->hasAdaptiveThinkingByDefault()); + $this->assertSame($midConversationSystem, $model->supportsMidConversationSystemMessages()); + $this->assertEqualsWithDelta($cacheWrite1h, $model->getCacheWrite1hPricePerMillionTokens(), 1e-9); + } +} diff --git a/tests/Client/Gemini/GeminiDecoderUsageTest.php b/tests/Client/Gemini/GeminiDecoderUsageTest.php new file mode 100644 index 0000000..5ed9d1c --- /dev/null +++ b/tests/Client/Gemini/GeminiDecoderUsageTest.php @@ -0,0 +1,32 @@ +decodeResponse($request, new ModelResponse([ + 'candidates' => [['content' => ['parts' => [['text' => 'Hello']]], 'finishReason' => 'STOP']], + 'usageMetadata' => ['promptTokenCount' => 200, 'candidatesTokenCount' => 40, 'thoughtsTokenCount' => 25], + ], 100)); + + $this->assertInstanceOf(LLMResponse::class, $response); + $this->assertEquals(new TokenUsage(input: 200, output: 40, reasoning: 25), $response->getStepUsage()); + } +} diff --git a/tests/Client/Gemini/GeminiEncoderEffortTest.php b/tests/Client/Gemini/GeminiEncoderEffortTest.php new file mode 100644 index 0000000..c14f2e2 --- /dev/null +++ b/tests/Client/Gemini/GeminiEncoderEffortTest.php @@ -0,0 +1,35 @@ + + */ + public static function topEfforts(): array { + return ['xhigh' => [ReasoningEffort::XHIGH], 'max' => [ReasoningEffort::MAX]]; + } + + /** + * @dataProvider topEfforts + */ + public function testTopEffortsMapToHigh(ReasoningEffort $effort): void { + $encoded = (new GeminiEncoder())->encodeRequest(new LLMRequest( + model: new Gemini38Flash(), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + reasoningConfig: $effort, + )); + + $this->assertSame(['thinkingLevel' => 'high'], $encoded['generationConfig']['thinkingConfig']); + } +} diff --git a/tests/Client/Gemini/GeminiEncoderSamplingTest.php b/tests/Client/Gemini/GeminiEncoderSamplingTest.php new file mode 100644 index 0000000..fafd74f --- /dev/null +++ b/tests/Client/Gemini/GeminiEncoderSamplingTest.php @@ -0,0 +1,25 @@ +encodeRequest(new LLMRequest( + model: new Gemini38Flash(), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + temperature: null, + )); + + $this->assertArrayNotHasKey('temperature', $encoded['generationConfig']); + $this->assertArrayHasKey('maxOutputTokens', $encoded['generationConfig']); + } +} diff --git a/tests/Client/LLMAgentClientTest.php b/tests/Client/LLMAgentClientTest.php index 0310d7e..52ddf21 100644 --- a/tests/Client/LLMAgentClientTest.php +++ b/tests/Client/LLMAgentClientTest.php @@ -509,6 +509,128 @@ private function createFinalResponse(LLMRequest $request, string $text): LLMResp ); } + /** + * A tool that throws must not abort the run: the model gets an error result and the loop goes on + */ + public function testThrowingToolBecomesErrorResultAndRunContinues(): void { + $failing = new CallbackToolDefinition( + 'lookup', + 'Lookup', + ['type' => 'object', 'properties' => ['q' => ['type' => 'string']], 'required' => ['q']], + function (array $input): LLMMessageContents { + throw new \DomainException('catalog offline'); + } + ); + + $request = new LLMRequest( + model: new GPT41(GPT41::VERSION_2025_04_14), + conversation: new LLMConversation([LLMMessage::createFromUserString('Find shoes')]), + tools: [$failing] + ); + + $sentRequests = []; + $queue = [$this->createToolUseResponse($request, 'tool-1', 'lookup', ['q' => 'shoes'])]; + $mockClient = $this->createMock(LLMClient::class); + $mockClient->method('sendRequestAsync') + ->willReturnCallback(function (LLMRequest $sent) use (&$queue, &$sentRequests) { + $sentRequests[] = $sent; + + return Create::promiseFor($queue !== [] ? array_shift($queue) : $this->createFinalResponse($sent, 'The catalog is offline.')); + }); + + $final = (new LLMAgentClient())->run($mockClient, $request); + + $this->assertSame('The catalog is offline.', $final->getLastText()); + $this->assertCount(2, $sentRequests); + $result = $sentRequests[1]->getLastMessage()->getContents()[0]; + $this->assertInstanceOf(LLMMessageToolResult::class, $result); + $this->assertTrue($result->getContent()->isError()); + $this->assertSame('ERROR: DomainException: catalog offline', $result->getContent()->getMessages()[0]->getText()); + } + + /** + * A rejected promise is an error result too, and the other tools of the same response still run + */ + public function testRejectedToolPromiseBecomesErrorResult(): void { + $schema = ['type' => 'object', 'properties' => ['q' => ['type' => 'string']], 'required' => ['q']]; + $slow = new CallbackToolDefinition('slow', 'Slow', $schema, fn(array $input): PromiseInterface => Create::rejectionFor(new RuntimeException('timeout'))); + $fast = new CallbackToolDefinition('fast', 'Fast', $schema, fn(array $input): LLMMessageContents => LLMMessageContents::fromString('ok')); + + $request = new LLMRequest( + model: new GPT41(GPT41::VERSION_2025_04_14), + conversation: new LLMConversation([LLMMessage::createFromUserString('Go')]), + tools: [$slow, $fast] + ); + $toolUse = new LLMResponse( + $request->withMessage(LLMMessage::createFromAssistant(new LLMMessageContents([ + new LLMMessageToolUse('t-slow', 'slow', ['q' => 'a']), + new LLMMessageToolUse('t-fast', 'fast', ['q' => 'b']), + ]))), + StopReason::TOOL_USE, + 100, 50, 4000, 0.001, 0.002, 500 + ); + $sentRequests = []; + $mockClient = $this->createMock(LLMClient::class); + $mockClient->method('sendRequestAsync') + ->willReturnCallback(function (LLMRequest $sent) use (&$sentRequests, $toolUse) { + $sentRequests[] = $sent; + + return Create::promiseFor(count($sentRequests) === 1 ? $toolUse : $this->createFinalResponse($sent, 'done')); + }); + + (new LLMAgentClient())->run($mockClient, $request); + + $results = $sentRequests[1]->getLastMessage()->getContents(); + $this->assertCount(2, $results); + $this->assertTrue($results[0]->getContent()->isError()); + $this->assertSame('ERROR: RuntimeException: timeout', $results[0]->getContent()->getMessages()[0]->getText()); + $this->assertFalse($results[1]->getContent()->isError()); + } + + /** + * Review focus 2: a promise resolving to something that is not LLMMessageContents is an error result, + * not a TypeError that kills the run + */ + public function testToolResolvingToTheWrongTypeBecomesAnErrorResult(): void { + $broken = new class implements \Soukicz\Llm\Tool\ToolDefinition { + public function getName(): string { + return 'broken'; + } + + public function getDescription(): string { + return 'Broken'; + } + + public function getInputSchema(): array { + return ['type' => 'object', 'properties' => new \stdClass()]; + } + + public function handle(array $input): PromiseInterface|LLMMessageContents { + return Create::promiseFor('plain string'); + } + }; + + $request = new LLMRequest( + model: new GPT41(GPT41::VERSION_2025_04_14), + conversation: new LLMConversation([LLMMessage::createFromUserString('Go')]), + tools: [$broken] + ); + $sentRequests = []; + $queue = [$this->createToolUseResponse($request, 'tool-1', 'broken', [])]; + $mockClient = $this->createMock(LLMClient::class); + $mockClient->method('sendRequestAsync') + ->willReturnCallback(function (LLMRequest $sent) use (&$queue, &$sentRequests) { + $sentRequests[] = $sent; + + return Create::promiseFor($queue !== [] ? array_shift($queue) : $this->createFinalResponse($sent, 'done')); + }); + + (new LLMAgentClient())->run($mockClient, $request); + + $result = $sentRequests[1]->getLastMessage()->getContents()[0]; + $this->assertTrue($result->getContent()->isError()); + $this->assertSame('ERROR: tool returned string, expected LLMMessageContents', $result->getContent()->getMessages()[0]->getText()); + } } diff --git a/tests/Client/ModelCatalogTest.php b/tests/Client/ModelCatalogTest.php index 498d8ab..edbadb2 100644 --- a/tests/Client/ModelCatalogTest.php +++ b/tests/Client/ModelCatalogTest.php @@ -23,12 +23,14 @@ use Soukicz\Llm\Client\Gemini\Model\Gemini38Flash; use Soukicz\Llm\Client\Gemini\Model\Gemini3FlashPreview; use Soukicz\Llm\Client\Gemini\Model\Gemini3ProImage; +use Soukicz\Llm\Client\Gemini\Model\Gemini4Argon; use Soukicz\Llm\Client\Gemini\Model\GeminiNanoBanana21; use Soukicz\Llm\Client\ModelInterface; use Soukicz\Llm\Client\OpenAI\Model\GPT55; use Soukicz\Llm\Client\OpenAI\Model\GPT56Luna; use Soukicz\Llm\Client\OpenAI\Model\GPT56Sol; use Soukicz\Llm\Client\OpenAI\Model\GPT56Terra; +use Soukicz\Llm\Client\OpenAI\Model\GPT61Sol; use Soukicz\Llm\Client\OpenAI\Model\GPT6Astra; use Soukicz\Llm\Client\OpenAI\Model\GPT6Luna; use Soukicz\Llm\Client\OpenAI\Model\GPT6Sol; @@ -46,9 +48,9 @@ public static function modelProvider(): array { 'claude-sonnet-5' => [new AnthropicClaude5Sonnet(), 'claude-sonnet-5', 2.0, 10.0, 2.5, 0.2], 'claude-fable-5' => [new AnthropicClaude5Fable(), 'claude-fable-5', 10.0, 50.0, 12.5, 1.0], 'claude-opus-5-5' => [new AnthropicClaude55Opus(), 'claude-opus-5-5', 4.0, 20.0, 5.0, 0.2], - 'claude-sonnet-5-5' => [new AnthropicClaude55Sonnet(), 'claude-sonnet-5-5', 2.0, 10.0, 2.5, 0.2], - 'claude-haiku-5-5' => [new AnthropicClaude55Haiku(), 'claude-haiku-5-5', 0.1, 0.5, 0.125, 0.01], 'claude-fable-5-1' => [new AnthropicClaude51Fable(), 'claude-fable-5-1', 10.0, 50.0, 12.5, 0.25], + 'claude-sonnet-5-5' => [new AnthropicClaude55Sonnet(), 'claude-sonnet-5-5', 2.0, 10.0, 2.5, 0.1], + 'claude-haiku-5-5' => [new AnthropicClaude55Haiku(), 'claude-haiku-5-5', 0.1, 0.5, 0.125, 0.01], 'gpt-5.5' => [new GPT55(GPT55::VERSION_2026_04_23), 'gpt-5.5-2026-04-23', 5.0, 30.0, 0.5, 0.0], 'gpt-5.6-sol' => [new GPT56Sol(), 'gpt-5.6-sol', 4.0, 20.0, 0.4, 0.0], 'gpt-5.6-terra' => [new GPT56Terra(), 'gpt-5.6-terra', 2.0, 12.0, 0.2, 0.0], @@ -56,7 +58,9 @@ public static function modelProvider(): array { 'gpt-6-astra' => [new GPT6Astra(), 'gpt-6-astra', 10.0, 50.0, 1.0, 0.0], 'gpt-6-sol' => [new GPT6Sol(), 'gpt-6-sol', 2.0, 10.0, 0.2, 0.0], 'gpt-6-luna' => [new GPT6Luna(), 'gpt-6-luna', 0.1, 0.5, 0.01, 0.0], + 'gpt-6.1-sol' => [new GPT61Sol(), 'gpt-6.1-sol', 2.0, 10.0, 0.1, 0.0], // Gemini reports cached prices equal to the uncached ones + 'gemini-4-argon' => [new Gemini4Argon(), 'gemini-4-argon', 2.0, 10.0, 2.0, 10.0], 'gemini-3-flash-preview' => [new Gemini3FlashPreview(), 'gemini-3-flash-preview', 0.5, 3.0, 0.5, 3.0], 'gemini-3.1-pro-preview' => [new Gemini31ProPreview(), 'gemini-3.1-pro-preview', 2.0, 12.0, 2.0, 12.0], 'gemini-3.1-flash-lite' => [new Gemini31FlashLite(), 'gemini-3.1-flash-lite', 0.25, 1.5, 0.25, 1.5], diff --git a/tests/Client/OpenAI/OpenAIDecoderPricingTest.php b/tests/Client/OpenAI/OpenAIDecoderPricingTest.php index d9410ef..2b7e4c4 100644 --- a/tests/Client/OpenAI/OpenAIDecoderPricingTest.php +++ b/tests/Client/OpenAI/OpenAIDecoderPricingTest.php @@ -13,6 +13,7 @@ use Soukicz\Llm\LLMRequest; use Soukicz\Llm\LLMResponse; use Soukicz\Llm\Message\LLMMessage; +use Soukicz\Llm\Usage\TokenUsage; class OpenAIDecoderPricingTest extends TestCase { private OpenAIEncoder $encoder; @@ -73,4 +74,16 @@ public function testCachedPromptTokensGetDiscountedRate(): void { // Token counts still report the full prompt size $this->assertSame(1_000_000, $response->getInputTokens()); } + + public function testStepUsageSeparatesCachedAndReasoningTokens(): void { + $response = $this->decode([ + 'prompt_tokens' => 1000, + 'completion_tokens' => 300, + 'prompt_tokens_details' => ['cached_tokens' => 400], + 'completion_tokens_details' => ['reasoning_tokens' => 120], + ]); + + $this->assertEquals(new TokenUsage(input: 600, output: 300, cacheRead: 400, reasoning: 120), $response->getStepUsage()); + $this->assertEquals($response->getStepUsage(), $response->getUsage()); + } } diff --git a/tests/Client/OpenAI/OpenAIEncoderEffortTest.php b/tests/Client/OpenAI/OpenAIEncoderEffortTest.php new file mode 100644 index 0000000..f3338df --- /dev/null +++ b/tests/Client/OpenAI/OpenAIEncoderEffortTest.php @@ -0,0 +1,35 @@ + + */ + public static function topEfforts(): array { + return ['xhigh' => [ReasoningEffort::XHIGH], 'max' => [ReasoningEffort::MAX], 'extra_high' => [ReasoningEffort::EXTRA_HIGH]]; + } + + /** + * @dataProvider topEfforts + */ + public function testTopEffortsMapToXhigh(ReasoningEffort $effort): void { + $encoded = (new OpenAIEncoder())->encodeRequest(new LLMRequest( + model: new GPT54(GPT54::VERSION_2026_03_05), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + reasoningConfig: $effort, + )); + + $this->assertSame('xhigh', $encoded['reasoning_effort']); + } +} diff --git a/tests/Client/OpenAI/OpenAIEncoderSamplingTest.php b/tests/Client/OpenAI/OpenAIEncoderSamplingTest.php new file mode 100644 index 0000000..013890b --- /dev/null +++ b/tests/Client/OpenAI/OpenAIEncoderSamplingTest.php @@ -0,0 +1,30 @@ +encodeRequest(new LLMRequest( + model: new GPT41(GPT41::VERSION_2025_04_14), + conversation: new LLMConversation([LLMMessage::createFromUserString('Hi')]), + temperature: $temperature, + )); + } + + public function testNullTemperatureIsOmitted(): void { + $this->assertArrayNotHasKey('temperature', $this->encode(null)); + } + + public function testExplicitTemperatureIsSent(): void { + $this->assertSame(0.5, $this->encode(0.5)['temperature']); + } +} diff --git a/tests/Client/ScriptedLLMClient.php b/tests/Client/ScriptedLLMClient.php new file mode 100644 index 0000000..d4bad30 --- /dev/null +++ b/tests/Client/ScriptedLLMClient.php @@ -0,0 +1,82 @@ + */ + public array $sent = []; + + /** @var list<\Closure(LLMRequest): LLMResponse> */ + private array $steps; + + public function __construct(\Closure ...$steps) { + $this->steps = array_values($steps); + } + + public function sendRequestAsync(LLMRequest $request): PromiseInterface { + $this->sent[] = $request; + $step = array_shift($this->steps); + if ($step === null) { + throw new \RuntimeException('ScriptedLLMClient: no scripted response left for request #' . count($this->sent)); + } + + return Create::promiseFor($step($request)); + } + + public static function reply(LLMMessageContents $contents, StopReason $stopReason, float $costUsd = 0.0, ?array $stopDetails = null): \Closure { + return static function (LLMRequest $request) use ($contents, $stopReason, $costUsd, $stopDetails): LLMResponse { + $step = new TokenUsage(input: 100, output: 50); + $request = $request + ->withCost(100, 50, $costUsd, 0.0) + ->withUsage($step) + ->withTime(10) + ->withMessage(LLMMessage::createFromAssistant($contents)); + + return new LLMResponse( + $request, + $stopReason, + $request->getPreviousInputTokens(), + $request->getPreviousOutputTokens(), + $request->getPreviousMaximumOutputTokens(), + $request->getPreviousInputCostUSD(), + $request->getPreviousOutputCostUSD(), + $request->getPreviousTimeMs(), + $request->getPreviousUsage(), + $step, + $stopDetails, + ); + }; + } + + public static function toolUse(string $id, string $name, array $input, float $costUsd = 0.0): \Closure { + return self::reply(new LLMMessageContents([new LLMMessageToolUse($id, $name, $input)]), StopReason::TOOL_USE, $costUsd); + } + + public static function text(string $text, float $costUsd = 0.0): \Closure { + return self::reply(LLMMessageContents::fromString($text), StopReason::FINISHED, $costUsd); + } + + /** + * @return list<\Closure(LLMRequest): LLMResponse> + */ + public static function repeat(int $times, \Closure $step): array { + return array_fill(0, $times, $step); + } +} diff --git a/tests/LLMConversationTest.php b/tests/LLMConversationTest.php index 4d6046d..9642c58 100644 --- a/tests/LLMConversationTest.php +++ b/tests/LLMConversationTest.php @@ -161,4 +161,24 @@ public function testFromJsonAcceptsLegacyContentFormat(): void { $this->assertSame('Hello', $contents[0]->getText()); $this->assertFalse($contents->isError()); } + + public function testPdfInsideToolResultSurvivesRoundTrip(): void { + $conversation = new LLMConversation([ + LLMMessage::createFromUser(new LLMMessageContents([ + new LLMMessageToolResult('t1', new LLMMessageContents([new LLMMessagePdf('base64', 'JVBERi0x')])), + ])), + ]); + + $restored = LLMConversation::fromJson(json_decode(json_encode($conversation, JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR)); + + $pdf = $restored->getMessages()[0]->getContents()[0]->getContent()->getMessages()[0]; + $this->assertInstanceOf(LLMMessagePdf::class, $pdf); + $this->assertSame('JVBERi0x', $pdf->getData()); + } + + public function testPdfFromJsonWithoutCachedFlag(): void { + $pdf = LLMMessagePdf::fromJson(['encoding' => 'base64', 'data' => 'JVBERi0x']); + + $this->assertFalse($pdf->isCached()); + } } diff --git a/tests/LLMResponseTest.php b/tests/LLMResponseTest.php new file mode 100644 index 0000000..fb269c8 --- /dev/null +++ b/tests/LLMResponseTest.php @@ -0,0 +1,46 @@ +request(), StopReason::FINISHED, 100, 50, 50, 0.25, 0.5, 10); + + $this->assertEquals(new TokenUsage(input: 100, output: 50), $response->getUsage()); + $this->assertEquals($response->getUsage(), $response->getStepUsage()); + $this->assertEqualsWithDelta(0.75, $response->getTotalCostUsd(), 1e-12); + } + + public function testMissingPricesCostNothing(): void { + $response = new LLMResponse($this->request(), StopReason::FINISHED, 1, 1, 1, null, null, 10); + + $this->assertSame(0.0, $response->getTotalCostUsd()); + } + + public function testExplicitUsageWins(): void { + $usage = new TokenUsage(input: 10, output: 5, cacheRead: 1000); + $step = new TokenUsage(input: 4, output: 2, cacheRead: 500); + $response = new LLMResponse($this->request(), StopReason::FINISHED, 10, 5, 5, 0.0, 0.0, 10, $usage, $step); + + $this->assertSame($usage, $response->getUsage()); + $this->assertSame($step, $response->getStepUsage()); + } +} diff --git a/tests/MarkdownFormatterTest.php b/tests/MarkdownFormatterTest.php index c97b3c9..2ed666c 100644 --- a/tests/MarkdownFormatterTest.php +++ b/tests/MarkdownFormatterTest.php @@ -13,6 +13,8 @@ use Soukicz\Llm\MarkdownFormatter; use Soukicz\Llm\Message\LLMMessage; use Soukicz\Llm\Message\LLMMessageContents; +use Soukicz\Llm\Message\LLMMessageOpaque; +use Soukicz\Llm\Message\LLMMessageRedactedReasoning; use Soukicz\Llm\Message\LLMMessageToolResult; use Soukicz\Llm\Message\LLMMessageToolUse; @@ -76,4 +78,22 @@ public function testResponseFormattingWithNullPrices(): void { $this->assertStringContainsString('price: $0.000', $markdown); } + + public function testRendersRedactedAndOpaqueBlocks(): void { + $request = new LLMRequest( + model: new AnthropicClaude45Haiku(AnthropicClaude45Haiku::VERSION_20251001), + conversation: new LLMConversation([ + LLMMessage::createFromUserString('Hi'), + LLMMessage::createFromAssistant(new LLMMessageContents([ + new LLMMessageRedactedReasoning('abc'), + new LLMMessageOpaque('anthropic', ['type' => 'server_tool_use']), + ])), + ]), + ); + + $markdown = $this->formatter->responseToMarkdown($request); + + $this->assertStringContainsString('**Redacted reasoning**', $markdown); + $this->assertStringContainsString('`server_tool_use`', $markdown); + } } diff --git a/tests/Stream/AnthropicStreamAccumulatorTest.php b/tests/Stream/AnthropicStreamAccumulatorTest.php index b8da66f..8ee5622 100644 --- a/tests/Stream/AnthropicStreamAccumulatorTest.php +++ b/tests/Stream/AnthropicStreamAccumulatorTest.php @@ -323,4 +323,25 @@ private function buildSse(array $events): string { return $sse; } + + public function testKeepsUnknownBlocksAndStopDetails(): void { + $sse = $this->buildSse([ + ['event' => 'message_start', 'data' => ['type' => 'message_start', 'message' => ['usage' => ['input_tokens' => 5]]]], + ['event' => 'content_block_start', 'data' => ['type' => 'content_block_start', 'index' => 0, 'content_block' => ['type' => 'redacted_thinking', 'data' => 'abc']]], + ['event' => 'content_block_stop', 'data' => ['type' => 'content_block_stop', 'index' => 0]], + ['event' => 'content_block_start', 'data' => ['type' => 'content_block_start', 'index' => 1, 'content_block' => ['type' => 'server_tool_use', 'id' => 'srv_1', 'name' => 'web_search', 'input' => []]]], + ['event' => 'content_block_delta', 'data' => ['type' => 'content_block_delta', 'index' => 1, 'delta' => ['type' => 'input_json_delta', 'partial_json' => '{"query":"shoes"}']]], + ['event' => 'content_block_stop', 'data' => ['type' => 'content_block_stop', 'index' => 1]], + ['event' => 'message_delta', 'data' => ['type' => 'message_delta', 'delta' => ['stop_reason' => 'refusal', 'stop_details' => ['type' => 'refusal', 'category' => 'bio', 'explanation' => null]], 'usage' => ['output_tokens' => 1]]], + ['event' => 'message_stop', 'data' => ['type' => 'message_stop']], + ]); + + $result = AnthropicStreamAccumulator::consume(Utils::streamFor($sse), new CallableStreamListener(static function (StreamEvent $event): void { + })); + + $this->assertSame(['type' => 'redacted_thinking', 'data' => 'abc'], $result['content'][0]); + $this->assertSame(['type' => 'server_tool_use', 'id' => 'srv_1', 'name' => 'web_search', 'input' => ['query' => 'shoes']], $result['content'][1]); + $this->assertSame('refusal', $result['stop_reason']); + $this->assertSame('bio', $result['stop_details']['category']); + } } diff --git a/tests/Usage/TokenUsageTest.php b/tests/Usage/TokenUsageTest.php new file mode 100644 index 0000000..8f1d236 --- /dev/null +++ b/tests/Usage/TokenUsageTest.php @@ -0,0 +1,30 @@ +add(new TokenUsage(10, 20, 30, 40, 50, 60)); + + $this->assertSame( + ['input' => 11, 'output' => 22, 'cacheWrite5m' => 33, 'cacheWrite1h' => 44, 'cacheRead' => 55, 'reasoning' => 66], + $sum->jsonSerialize() + ); + } + + public function testPromptTokensCountUncachedInputAndEveryCacheToken(): void { + $this->assertSame(1 + 3 + 4 + 5, (new TokenUsage(1, 2, 3, 4, 5, 6))->promptTokens()); + } + + public function testJsonUsesThePropertyNames(): void { + $this->assertSame( + '{"input":0,"output":0,"cacheWrite5m":0,"cacheWrite1h":0,"cacheRead":0,"reasoning":0}', + json_encode(new TokenUsage(), JSON_THROW_ON_ERROR) + ); + } +}