Sync upstream conductor-oss main (2026-10-02)

.github/actions/check-playback/README.md

@@ -0,0 +1,20 @@
+# Check LLM playback
+
+Validate SDK examples against a dedicated playback server:
+
+```yaml
+- uses: conductor-oss/conductor/.github/actions/check-playback@main
+  with:
+    server-url: http://localhost:8080/api
+```
+
+Completed workflows pass. Failed workflows pass only when persisted tasks show
+that a completed guardrail decision rejected the response and directly caused
+the failed termination. Failed LLM tasks, unrelated failures, and unfinished
+workflows fail. No SDK exception lists or example names are used.
+
+Requires `curl` and `jq`. Run locally with:
+
+```sh
+sh .github/actions/check-playback/check-playback.sh http://localhost:8080/api
+```

.github/actions/check-playback/action.yml

@@ -0,0 +1,12 @@
+name: Check LLM playback
+description: Validate completed workflows and guardrail rejections from persisted server state.
+inputs:
+  server-url:
+    description: Conductor API URL reachable from this runner, including /api.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Check playback
+      shell: sh
+      run: sh "$GITHUB_ACTION_PATH/check-playback.sh" "${{ inputs.server-url }}"

.github/actions/check-playback/check-playback.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+# Validate playback outcomes from persisted workflow state, for every SDK.
+set -eu
+if [ "$#" -ne 1 ]; then
+    printf '%s\n' "Usage: $0 <server-api-url>" >&2
+    exit 2
+fi
+server_url=${1%/}
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+start=0
+failures=0
+while :; do
+    result=$(curl --silent --show-error --fail-with-body --get \
+        --data-urlencode 'query=status IN (COMPLETED,RUNNING,PAUSED,FAILED,TERMINATED,TIMED_OUT)' \
+        --data-urlencode 'size=100' --data-urlencode "start=$start" \
+        "$server_url/workflow/search")
+    total=$(printf '%s' "$result" | jq -er '.totalHits')
+    rows=$(printf '%s' "$result" | jq '.results | length')
+    if [ "$rows" -eq 0 ]; then
+        if [ "$start" -lt "$total" ]; then
+            printf '%s\n' 'FAIL: workflow search returned an incomplete page'
+            exit 1
+        fi
+        break
+    fi
+    for id in $(printf '%s' "$result" | jq -r '.results[].workflowId | @uri'); do
+        workflow=$(curl --silent --show-error --fail-with-body "$server_url/workflow/$id?includeTasks=true")
+        if printf '%s' "$workflow" | jq -e '.status == "COMPLETED"' > /dev/null; then
+            continue
+        fi
+        if printf '%s' "$workflow" | jq -e -f "$script_dir/guardrail-rejection.jq" > /dev/null; then
+            printf 'PASS: %s rejected by its guardrail\n' "$id"
+        else
+            printf '%s' "$workflow" | jq -r '"\(.workflowType) \(.workflowId): \(.status) \(.reasonForIncompletion // "")"'
+            failures=$((failures + 1))
+        fi
+    done
+    start=$((start + rows))
+    [ "$start" -lt "$total" ] || break
+done
+if [ "$start" -eq 0 ]; then
+    printf '%s\n' 'FAIL: no workflows found; the SDK examples never ran'
+    exit 1
+fi
+if [ "$failures" -ne 0 ]; then
+    printf 'FAIL: %s unexpected workflow outcomes\n' "$failures"
+    exit 1
+fi
+printf '%s\n' 'PASS: every workflow completed or was rejected by its guardrail'

.github/actions/check-playback/guardrail-rejection.jq

@@ -0,0 +1,26 @@
+# GuardrailCompiler routes a rejected decision to a TERMINATE task whose reason
+# references that decision's message. Require that causal link, not just a
+# rejection somewhere in an otherwise broken workflow.
+. as $workflow
+| .status == "FAILED"
+  and any(.tasks[];
+    . as $guard
+    | (.outputData.result // .outputData) as $decision
+    | .status == "COMPLETED"
+      and ($decision | type) == "object"
+      and $decision.passed == false
+      and $decision.on_fail == "raise"
+      and ($decision.guardrail_name | type) == "string"
+      and ($decision.guardrail_name | length) > 0
+      and ($decision.message | type) == "string"
+      and $workflow.reasonForIncompletion == $decision.message
+      and any($workflow.tasks[];
+        .taskType == "TERMINATE"
+        and .status == "COMPLETED"
+        and .inputData.terminationStatus == "FAILED"
+        and .inputData.terminationReason == $decision.message
+        and (.workflowTask.inputParameters.terminationReason as $reason
+          | ($guard.workflowTask.taskReferenceName // $guard.referenceTaskName) as $ref
+          | $reason == ("${" + $ref + ".output.result.message}")
+            or $reason == ("${" + $ref + ".output.message}"))))
+  and all(.tasks[] | select(.taskType == "LLM_CHAT_COMPLETE"); .status == "COMPLETED")

.github/actions/check-playback/test_check_playback.py

@@ -0,0 +1,136 @@
+"""Exercise the shared verifier through its HTTP interface."""
+import copy
+import json
+import os
+from pathlib import Path
+import subprocess
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import parse_qs, urlparse
+
+SCRIPT = Path(__file__).with_name('check-playback.sh')
+
+
+def rejection():
+    return {
+        'workflowId': 'arbitrary-id', 'workflowType': 'any-agent', 'status': 'FAILED',
+        'reasonForIncompletion': 'policy rejected the response',
+        'tasks': [
+            {'taskType': 'LLM_CHAT_COMPLETE', 'status': 'COMPLETED'},
+            {'referenceTaskName': 'decision__1', 'status': 'COMPLETED',
+             'workflowTask': {'taskReferenceName': 'decision'},
+             'outputData': {'result': {'guardrail_name': 'any-policy', 'passed': False,
+                                       'on_fail': 'raise', 'message': 'policy rejected the response'}}},
+            {'taskType': 'TERMINATE', 'status': 'COMPLETED',
+             'inputData': {'terminationStatus': 'FAILED', 'terminationReason': 'policy rejected the response'},
+             'workflowTask': {'inputParameters': {'terminationReason': '${decision.output.result.message}'}}},
+        ],
+    }
+
+
+class PlaybackCheckTest(unittest.TestCase):
+    def verify(self, workflows, total=None):
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                url = urlparse(self.path)
+                if url.path == '/api/workflow/search':
+                    start = int(parse_qs(url.query)['start'][0])
+                    response = {'results': workflows[start:start + 100],
+                                'totalHits': len(workflows) if total is None else total}
+                else:
+                    workflow_id = url.path.rsplit('/', 1)[-1]
+                    response = next(w for w in workflows if w['workflowId'] == workflow_id)
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(json.dumps(response).encode())
+
+            def log_message(self, *_args):
+                pass
+
+        with HTTPServer(('127.0.0.1', 0), Handler) as server:
+            thread = threading.Thread(target=server.serve_forever)
+            thread.start()
+            try:
+                return subprocess.run(
+                    ['sh', str(SCRIPT), f'http://127.0.0.1:{server.server_port}/api'],
+                    env=os.environ, capture_output=True, text=True, timeout=30,
+                )
+            finally:
+                server.shutdown()
+                thread.join()
+
+    def assert_failed(self, workflow):
+        self.assertNotEqual(self.verify([workflow]).returncode, 0)
+
+    def test_completed_workflows_pass(self):
+        self.assertEqual(self.verify([]).returncode, 0)
+
+    def test_guardrail_rejection_passes_without_sdk_metadata(self):
+        result = self.verify([rejection()])
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_worker_guardrail_rejection_passes(self):
+        workflow = rejection()
+        guard = workflow['tasks'][1]
+        guard['outputData'] = guard['outputData']['result']
+        workflow['tasks'][2]['workflowTask']['inputParameters']['terminationReason'] = '${decision.output.message}'
+        self.assertEqual(self.verify([workflow]).returncode, 0)
+
+    def test_unrelated_failure_is_not_a_guardrail_rejection(self):
+        self.assert_failed({'workflowId': 'broken', 'status': 'FAILED', 'tasks': []})
+
+    def test_failed_llm_task_fails_even_with_guardrail_output(self):
+        workflow = rejection()
+        workflow['tasks'][0]['status'] = 'FAILED'
+        self.assert_failed(workflow)
+
+    def test_rejection_followed_by_an_unrelated_failure_fails(self):
+        workflow = rejection()
+        workflow['reasonForIncompletion'] = 'worker crashed'
+        self.assert_failed(workflow)
+
+    def test_termination_must_reference_the_rejecting_guardrail(self):
+        workflow = rejection()
+        workflow['tasks'][2]['workflowTask']['inputParameters']['terminationReason'] = '${another_task.output.message}'
+        self.assert_failed(workflow)
+
+    def test_missing_or_incomplete_termination_fails(self):
+        workflow = rejection()
+        workflow['tasks'].pop()
+        self.assert_failed(workflow)
+        workflow = rejection()
+        workflow['tasks'][2]['status'] = 'IN_PROGRESS'
+        self.assert_failed(workflow)
+
+    def test_non_rejecting_decisions_fail(self):
+        for field, value in [('passed', True), ('on_fail', 'retry'), ('guardrail_name', '')]:
+            with self.subTest(field=field):
+                workflow = rejection()
+                workflow['tasks'][1]['outputData']['result'][field] = value
+                self.assert_failed(workflow)
+
+    def test_unfinished_and_other_terminal_states_fail(self):
+        for status in ['RUNNING', 'PAUSED', 'TERMINATED', 'TIMED_OUT']:
+            with self.subTest(status=status):
+                workflow = rejection()
+                workflow['status'] = status
+                self.assert_failed(workflow)
+
+    def test_unrelated_failure_after_first_page_fails(self):
+        workflows = []
+        for index in range(100):
+            workflow = copy.deepcopy(rejection())
+            workflow['workflowId'] = str(index)
+            workflows.append(workflow)
+        workflows.append({'workflowId': 'broken', 'status': 'FAILED', 'tasks': []})
+        result = self.verify(workflows)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('broken', result.stdout)
+
+    def test_incomplete_search_page_fails(self):
+        self.assertNotEqual(self.verify([], total=1).returncode, 0)
+
+
+if __name__ == '__main__':
+    unittest.main()

.github/actions/start-playback-services/action.yml

@@ -0,0 +1,20 @@
+name: Start playback services
+description: Start and check the authenticated HTTP and MCP fixtures used by SDK playback.
+inputs:
+  work-directory:
+    description: Directory for fixture logs and PID files.
+    default: tmp/agent-playback
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install MCP test service
+      shell: bash
+      run: pip install mcp-testkit==1.0.4
+    - name: Start and check HTTP and MCP services
+      shell: bash
+      env:
+        CONDUCTOR_PLAYBACK_WORK_DIR: ${{ inputs.work-directory }}
+      run: bash "$GITHUB_ACTION_PATH/start-services.sh"

.github/actions/start-playback-services/http_fixture.py

@@ -0,0 +1,45 @@
+"""HTTP dependency for example 16e; uses the shared recording's response bytes.
+
+The agent, HTTP task, credential substitution, and model playback still run on
+Conductor. This fixture only replaces the external GitHub endpoint.
+"""
+import json
+import os
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+
+
+def github_response(recordings):
+    for path in (recordings / '16e_credentials_http_tool').glob('*.json'):
+        for message in json.loads(path.read_text())['request']['messages']:
+            for result in message['toolResults']:
+                if result['name'] == 'list_github_repos':
+                    return result['value']['response']
+    raise RuntimeError('Shared GitHub HTTP response is missing')
+
+
+class Handler(BaseHTTPRequestHandler):
+    response = None
+
+    def do_GET(self):
+        if self.path != '/users/Conductor/repos?per_page=5&sort=updated':
+            self.send_error(404)
+            return
+        if self.headers.get('Authorization') != 'Bearer playback-test-key':
+            self.send_error(401)
+            return
+        body = json.dumps(self.response['body'], separators=(',', ':')).encode()
+        self.send_response_only(self.response['statusCode'], self.response['reasonPhrase'])
+        for name, values in self.response['headers'].items():
+            if name.lower() == 'content-length':
+                continue
+            for value in values:
+                self.send_header(name, value)
+        self.send_header('Content-Length', str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+if __name__ == '__main__':
+    Handler.response = github_response(Path(os.environ['CONDUCTOR_RECORDINGS_DIR']))
+    HTTPServer(('127.0.0.1', 3002), Handler).serve_forever()

.github/actions/start-playback-services/start-services.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Start the authenticated HTTP/MCP fixtures; --check only checks existing services.
+set -euo pipefail
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+conductor_dir=$(cd "$script_dir/../../.." && pwd)
+export CONDUCTOR_RECORDINGS_DIR="$conductor_dir/llm-recordings"
+playback_dir=${CONDUCTOR_PLAYBACK_WORK_DIR:-"$PWD/tmp/agent-playback"}
+
+check_services() {
+  curl --fail --silent --show-error --max-time 3 \
+    -H 'Authorization: Bearer playback-test-key' -H 'Content-Type: application/json' \
+    --data '{"text":"hello world"}' http://localhost:3001/api/string/reverse > /dev/null &&
+  curl --fail --silent --show-error --max-time 3 \
+    -H 'Authorization: Bearer playback-test-key' -H 'Content-Type: application/json' \
+    -H 'Accept: application/json, text/event-stream' \
+    --data '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"sdk-playback","version":"1"}}}' \
+    http://localhost:3001/mcp > /dev/null &&
+  curl --fail --silent --show-error --max-time 3 \
+    -H 'Authorization: Bearer playback-test-key' \
+    'http://localhost:3002/users/Conductor/repos?per_page=5&sort=updated' > /dev/null
+}
+
+if [[ "${1:-}" == --check ]]; then
+  check_services
+  exit
+fi
+if [[ -n "${1:-}" ]]; then
+  echo "Unknown option: $1" >&2
+  exit 2
+fi
+command -v mcp-testkit > /dev/null
+# Never replace or stop an existing service.
+python3 - <<'PY'
+import socket
+for port in (3001, 3002):
+    with socket.socket() as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        sock.bind(('127.0.0.1', port))
+PY
+mkdir -p "$playback_dir"
+pids=()
+cleanup_on_error() {
+  result=$?
+  if ((result != 0)); then
+    for pid in "${pids[@]}"; do kill "$pid" 2>/dev/null || true; done
+  fi
+}
+trap cleanup_on_error EXIT
+nohup python3 "$script_dir/http_fixture.py" < /dev/null > "$playback_dir/http.log" 2>&1 &
+pids+=("$!")
+echo "$!" > "$playback_dir/http.pid"
+nohup mcp-testkit --transport http --host 127.0.0.1 --port 3001 --auth playback-test-key < /dev/null > "$playback_dir/mcp.log" 2>&1 &
+pids+=("$!")
+echo "$!" > "$playback_dir/mcp.pid"
+for attempt in $(seq 1 30); do
+  for pid in "${pids[@]}"; do
+    if ! kill -0 "$pid" 2>/dev/null; then
+      cat "$playback_dir/http.log" "$playback_dir/mcp.log"
+      exit 1
+    fi
+  done
+  if check_services > "$playback_dir/services-check.log" 2>&1; then
+    echo 'Authenticated HTTP and MCP fixtures are ready on ports 3001 and 3002.'
+    exit 0
+  fi
+  sleep 1
+done
+cat "$playback_dir/services-check.log"
+exit 1