From 5d8c01d171bd9b0771f5723bdf4fde3ea3c4d955 Mon Sep 17 00:00:00 2001
From: Luhaozhu
Date: Fri, 23 Oct 2026 20:48:42 -0700
Subject: [PATCH] Sync CE runtime and canvas preview improvements
---
.../references/api-envelope.md | 6 +-
.../references/architecture.md | 265 +-
.../references/error-codes.md | 2 +
.../scripts/scaffold_feature.sh | 145 +-
.../templates/extension.py | 38 +
.../hugagent-backend-dev/templates/model.py | 78 +-
.../templates/repository.py | 70 +-
.../hugagent-backend-dev/templates/route.py | 156 +-
.../hugagent-backend-dev/templates/service.py | 138 +-
.../hugagent-backend-dev/templates/test.py | 171 +-
.../assets/design-v2/icons-ui/attachment.svg | 3 +
.../assets/design-v2/icons-ui/automation.svg | 3 +
.../assets/design-v2/icons-ui/check.svg | 3 +
.../design-v2/icons-ui/chevron-right.svg | 3 +
.../assets/design-v2/icons-ui/close.svg | 3 +
.../assets/design-v2/icons-ui/knowledge.svg | 3 +
.../assets/design-v2/icons-ui/my-space.svg | 3 +
.../assets/design-v2/icons-ui/new-chat.svg | 3 +
.../assets/design-v2/icons-ui/search.svg | 3 +
.../assets/design-v2/icons-ui/send.svg | 3 +
.../assets/design-v2/icons-ui/settings.svg | 3 +
.../assets/design-v2/icons-ui/skill.svg | 3 +
.../assets/design-v2/logo/favicon.svg | 5 +
.../design-v2/logo/hugagentos-glyph-black.svg | 4 +
.../design-v2/logo/hugagentos-glyph-blue.svg | 5 +
.../design-v2/logo/hugagentos-glyph-white.svg | 4 +
.../logo/hugagentos-mark-mono-black.svg | 5 +
.../logo/hugagentos-mark-mono-white.svg | 5 +
.../logo/hugagentos-mark-reverse.svg | 6 +
.../design-v2/logo/hugagentos-mark-small.svg | 5 +
.../assets/design-v2/logo/hugagentos-mark.svg | 6 +
.../assets/design-v2/tokens/appTheme.v2.ts | 39 +
.../design-v2/tokens/hugagentos-tokens.v2.css | 123 +
.../tokens/hugagentos-tokens.v2.json | 223 +
.../design-v2/tokens/motionTokens.v2.ts | 32 +
.../references/css-variables.md | 277 +-
.../references/icon-assets.md | 46 +-
.../references/sse-events.md | 294 +-
.../references/ui-design-spec.md | 667 ++-
.../scripts/scaffold_component.sh | 5 +-
.../templates/component.tsx | 115 +-
.../templates/css-module.css | 114 +-
.../hugagent-frontend-dev/templates/hook.ts | 98 +-
.../hugagent-frontend-dev/templates/store.ts | 121 +-
.env.example | 6 +-
.gitignore | 10 +-
Makefile | 6 +-
README.md | 363 +-
README_CN.md | 290 +-
...uirements-desktop-linux-aarch64-py311.lock | 50 +-
...quirements-desktop-linux-x86_64-py311.lock | 58 +-
...uirements-desktop-macos-aarch64-py311.lock | 58 +-
...quirements-desktop-macos-x86_64-py311.lock | 58 +-
.../requirements-desktop-windows-py311.lock | 60 +-
desktop/requirements-desktop.txt | 11 +-
desktop/scripts/build-runtime.mjs | 3 +
desktop/scripts/desktop-dependencies.test.mjs | 33 +-
desktop/scripts/test-native-tools.py | 2 +-
docker-compose.yml | 17 +-
...Dockerfile.opensandbox-server.dockerignore | 16 -
docker/Dockerfile.personal-agent-test | 6 -
.../compose.application-hosting.yml | 4 +-
.../Dockerfile.object-storage | 0
.../compose.object-storage.yml | 4 +-
.../init-object-storage.py | 0
.../Dockerfile.opensandbox-server | 11 +-
...Dockerfile.opensandbox-server.dockerignore | 18 +
.../opensandbox-config.toml.tpl | 0
.../opensandbox_capacity.py | 31 +
.../opensandbox_expirations.py | 0
.../opensandbox_mount_recovery.py | 22 +-
.../opensandbox_mount_worker.py | 0
.../opensandbox_object_mounts.py | 0
.../patch_opensandbox_server.py | 11 +
.../render-opensandbox-config.py | 0
.../Dockerfile.personal-agent | 8 +-
.../compose.personal-agent.yml | 4 +-
.../desktop}/helpers.rc | 0
.../desktop}/mimeapps.list | 0
.../desktop}/pa-browser | 0
.../desktop}/pa-browser-helper.desktop | 0
.../desktop}/pa-browser.desktop | 0
.../desktop}/pa-stream.py | 0
.../hugagent-cli-entrypoint.sh | 0
.../personal-agent-entrypoint.sh | 4 +-
.../render-personal-agent-config.py | 0
docker/{ => sandbox}/Dockerfile.opensandbox | 27 +-
docker/{ => sandbox}/Dockerfile.script-runner | 8 +-
docker/{ => sandbox}/apt-skills.txt | 0
docker/{ => sandbox}/package-skills.json | 0
.../requirements-script-runner.txt | 0
docker/{ => sandbox}/requirements-skills.txt | 0
docker/sandbox/sandbox-init.c | 78 +
.../site-template/init-react-site.sh | 0
.../site-template/react-vite/.npmrc | 0
.../site-template/react-vite/index.html | 0
.../react-vite/package-lock.json | 0
.../site-template/react-vite/package.json | 0
.../site-template/react-vite/src/App.jsx | 0
.../site-template/react-vite/src/index.css | 0
.../react-vite/src/lib/siteApi.js | 0
.../site-template/react-vite/src/main.jsx | 0
.../react-vite/src/pages/Home.jsx | 0
.../site-template/react-vite/vite.config.mjs | 0
.../{ => sandbox}/verify-browser-runtime.py | 0
document/README.md | 173 +-
document/en/README.md | 129 +-
document/en/api/overview.md | 8 +-
document/en/architecture/backend.md | 18 +-
document/en/architecture/data-model.md | 6 +-
document/en/architecture/overview.md | 111 +-
document/en/deployment/README.md | 4 -
.../deployment/artifact-storage-key-repair.md | 55 -
document/en/deployment/desktop-local-mode.md | 2 +-
document/en/deployment/docker-compose.md | 16 +-
.../en/deployment/environment-variables.md | 20 +-
document/en/deployment/offline-production.md | 84 -
document/en/deployment/quick-install.md | 6 +-
document/en/deployment/space-files.md | 18 +-
document/en/deployment/uos1070-desktop.md | 101 -
document/en/development/backend.md | 16 +-
document/en/development/container-layout.md | 28 -
document/en/development/frontend.md | 9 +
.../en/development/workbench-migration.md | 29 +-
document/en/development/workbench-routing.md | 2 +-
document/en/editions/build-ce.md | 12 +-
document/en/getting-started/introduction.md | 170 +-
document/en/getting-started/quick-start.md | 119 +-
document/en/modules/admin-console.md | 16 +-
document/en/modules/agent-skills.md | 38 +-
document/en/modules/application-hosting.md | 32 +-
document/en/modules/auth.md | 8 +-
document/en/modules/automation.md | 50 +-
document/en/modules/autonomous-loop.md | 6 +-
document/en/modules/browser-automation.md | 2 +-
document/en/modules/canvas-artifacts.md | 14 +-
document/en/modules/catalog.md | 21 +-
document/en/modules/chat.md | 58 +-
document/en/modules/knowledge-base-wiki.md | 14 +-
document/en/modules/knowledge-base.md | 44 +-
document/en/modules/mcp-tools.md | 45 +-
document/en/modules/memory.md | 8 +-
document/en/modules/model-providers.md | 28 +-
document/en/modules/personal-agent.md | 10 +-
document/en/modules/projects-myspace.md | 31 +-
document/en/modules/prompts.md | 31 +-
document/en/modules/sandbox.md | 58 +-
document/en/modules/sites.md | 14 +-
document/en/modules/storage.md | 4 +-
document/en/modules/workflow-jobs.md | 8 -
document/zh-CN/README.md | 132 +-
document/zh-CN/api/overview.md | 8 +-
document/zh-CN/architecture/backend.md | 18 +-
document/zh-CN/architecture/data-model.md | 6 +-
document/zh-CN/architecture/overview.md | 99 +-
document/zh-CN/deployment/README.md | 4 -
.../deployment/artifact-storage-key-repair.md | 48 -
.../zh-CN/deployment/desktop-local-mode.md | 3 +-
document/zh-CN/deployment/docker-compose.md | 20 +-
.../zh-CN/deployment/environment-variables.md | 20 +-
.../zh-CN/deployment/offline-production.md | 84 -
document/zh-CN/deployment/quick-install.md | 6 +-
document/zh-CN/deployment/space-files.md | 28 +-
document/zh-CN/deployment/uos1070-desktop.md | 87 -
document/zh-CN/development/backend.md | 16 +-
.../zh-CN/development/container-layout.md | 24 -
document/zh-CN/development/frontend.md | 7 +
.../zh-CN/development/workbench-migration.md | 29 +-
.../zh-CN/development/workbench-routing.md | 2 +-
document/zh-CN/editions/build-ce.md | 12 +-
.../zh-CN/getting-started/introduction.md | 159 +-
document/zh-CN/getting-started/quick-start.md | 108 +-
document/zh-CN/modules/admin-console.md | 16 +-
document/zh-CN/modules/agent-skills.md | 38 +-
document/zh-CN/modules/application-hosting.md | 32 +-
document/zh-CN/modules/auth.md | 8 +-
document/zh-CN/modules/automation.md | 46 +-
document/zh-CN/modules/autonomous-loop.md | 6 +-
document/zh-CN/modules/browser-automation.md | 2 +-
document/zh-CN/modules/canvas-artifacts.md | 14 +-
document/zh-CN/modules/catalog.md | 21 +-
document/zh-CN/modules/chat.md | 58 +-
document/zh-CN/modules/knowledge-base-wiki.md | 14 +-
document/zh-CN/modules/knowledge-base.md | 44 +-
document/zh-CN/modules/mcp-tools.md | 45 +-
document/zh-CN/modules/memory.md | 8 +-
document/zh-CN/modules/model-providers.md | 28 +-
document/zh-CN/modules/personal-agent.md | 10 +-
document/zh-CN/modules/projects-myspace.md | 29 +-
document/zh-CN/modules/prompts.md | 31 +-
document/zh-CN/modules/sandbox.md | 49 +-
document/zh-CN/modules/sites.md | 14 +-
document/zh-CN/modules/storage.md | 4 +-
document/zh-CN/modules/workflow-jobs.md | 8 -
examples/customer-chat/.env.example | 6 -
examples/customer-chat/.gitignore | 8 -
examples/customer-chat/README.md | 37 -
examples/customer-chat/debug.html | 1 -
examples/customer-chat/index.html | 9 -
examples/customer-chat/package-lock.json | 1224 -----
examples/customer-chat/package.json | 41 -
examples/customer-chat/preview.html | 1 -
.../customer-chat/scripts/build-component.mjs | 14 -
.../customer-chat/scripts/developer-proxy.mjs | 43 -
.../customer-chat/scripts/test-package.mjs | 30 -
examples/customer-chat/scripts/test-proxy.mjs | 26 -
examples/customer-chat/session.html | 1 -
examples/customer-chat/src/controls.ts | 76 -
examples/customer-chat/src/css.d.ts | 1 -
examples/customer-chat/src/customization.tsx | 29 -
examples/customer-chat/src/debug.ts | 32 -
examples/customer-chat/src/developer.css | 3 -
examples/customer-chat/src/login.ts | 11 -
examples/customer-chat/src/preview.tsx | 5 -
examples/customer-chat/src/session.ts | 20 -
examples/customer-chat/src/standalone.css | 6 -
examples/customer-chat/src/standalone.ts | 142 -
examples/customer-chat/src/theme.css | 19 -
examples/customer-chat/tsconfig.json | 25 -
examples/customer-chat/vite.config.mjs | 30 -
install.sh | 2 +-
packages/cli/DESIGN.md | 164 +
pyproject.toml | 14 +-
requirements.txt | 17 +-
scripts/performance/benchmark_preview.py | 2 +-
scripts/performance/serve_fixture.py | 6 +-
scripts/refactor/__init__.py | 1 +
scripts/refactor/check_ce_paths.py | 79 +
scripts/refactor/check_line_budget.py | 37 +
scripts/refactor/check_test_baseline.py | 75 +
scripts/refactor/import_contract.py | 209 +
scripts/refactor/rewrite_imports.py | 357 ++
scripts/refactor/snapshot_routes.py | 35 +
scripts/refactor/snapshot_schema.py | 81 +
scripts/refactor/tests/test_edition_seams.py | 33 +
scripts/refactor/tests/test_guards.py | 127 +
.../refactor/tests/test_package_markers.py | 20 +
.../refactor/tests/test_private_imports.py | 41 +
.../refactor/tests/test_rewrite_imports.py | 142 +
.../refactor/tests/test_snapshot_schema.py | 33 +
.../refactor/tests/test_supplemental_moves.py | 47 +
scripts/refactor/tests/test_test_baseline.py | 22 +
scripts/refactor/tests/test_test_moves.py | 45 +
scripts/repair_artifact_storage_keys.py | 4 +-
scripts/sync-workbench-schema.mjs | 4 +-
scripts/test-workbench-build.mjs | 2 +-
src/backend/alembic/env.py | 4 +
.../alembic/versions/ce_0001_initial.py | 2 +-
.../versions/ce_0002_reconcile_schema.py | 2 +-
.../versions/ce_0003_mcp_marketplace.py | 2 +-
.../versions/ce_0009_desktop_delivery.py | 2 +-
.../versions/ce_0010_desktop_channel_relay.py | 2 +-
.../alembic/versions/ce_0012_model_steps.py | 3 +-
.../alembic/versions/ce_0013_tool_media.py | 2 +-
.../versions/ce_0014_site_access_password.py | 2 +-
.../versions/ce_0019_history_display.py | 2 +-
.../versions/ce_0020_plugin_resources.py | 2 +-
.../versions/ce_0021_retire_site_storage.py | 2 +-
.../versions/ce_0028_personal_agent.py | 4 +-
.../ce_0031_automation_lease_token.py | 19 +
.../versions/ce_0032_automation_queue.py | 25 +
.../versions/ce_0033_automation_targets.py | 17 +
src/backend/api/__init__.py | 19 +-
src/backend/api/app.py | 28 +-
src/backend/api/deps.py | 201 +-
src/backend/api/edition.py | 13 +
src/backend/api/health.py | 11 +-
src/backend/api/lifecycle.py | 18 +-
src/backend/api/middleware/agent_api_scope.py | 11 +-
src/backend/api/middleware/logging.py | 2 +-
src/backend/api/openapi.py | 2 +-
.../v1/agents.py => personal_agents.py} | 28 +-
src/backend/api/routes/__init__.py | 15 +-
src/backend/api/routes/file_archive.py | 26 +-
src/backend/api/routes/file_location.py | 2 +-
src/backend/api/routes/files.py | 56 +-
src/backend/api/routes/v1/__init__.py | 53 +-
src/backend/api/routes/v1/agent_api_keys.py | 30 +-
src/backend/api/routes/v1/agent_responses.py | 54 +-
src/backend/api/routes/v1/api_keys.py | 12 +-
src/backend/api/routes/v1/artifacts.py | 108 +-
src/backend/api/routes/v1/audio.py | 15 +-
src/backend/api/routes/v1/auth.py | 58 +-
src/backend/api/routes/v1/catalog.py | 37 +-
.../api/routes/v1/catalog_mutations.py | 12 +-
src/backend/api/routes/v1/chat_forks.py | 2 +-
src/backend/api/routes/v1/chat_modes.py | 2 +-
src/backend/api/routes/v1/chat_runs.py | 17 +-
src/backend/api/routes/v1/chats/confirms.py | 27 +-
src/backend/api/routes/v1/chats/feedback.py | 14 +-
src/backend/api/routes/v1/chats/history.py | 16 +-
.../api/routes/v1/chats/pending_access.py | 8 +-
src/backend/api/routes/v1/chats/questions.py | 28 +-
src/backend/api/routes/v1/chats/reruns.py | 111 +-
src/backend/api/routes/v1/chats/run_views.py | 40 +-
src/backend/api/routes/v1/chats/sessions.py | 20 +-
src/backend/api/routes/v1/chats/sidebar.py | 12 +-
src/backend/api/routes/v1/classify.py | 2 +-
src/backend/api/routes/v1/commands.py | 11 +-
src/backend/api/routes/v1/content.py | 11 +-
.../api/routes/v1/desktop_capability.py | 824 ---
src/backend/api/routes/v1/evolution.py | 29 +-
src/backend/api/routes/v1/file_parse.py | 3 +-
src/backend/api/routes/v1/file_text.py | 2 +-
src/backend/api/routes/v1/file_upload.py | 22 +-
src/backend/api/routes/v1/internal_jobs.py | 15 +-
.../api/routes/v1/job_callback_session.py | 2 +-
src/backend/api/routes/v1/jobs.py | 4 +-
src/backend/api/routes/v1/kb.py | 873 ----
src/backend/api/routes/v1/loops.py | 81 +-
src/backend/api/routes/v1/me_capabilities.py | 70 +-
src/backend/api/routes/v1/me_skill_files.py | 44 +-
src/backend/api/routes/v1/me_system.py | 12 +-
src/backend/api/routes/v1/memories.py | 81 +-
src/backend/api/routes/v1/mock_sso.py | 31 +-
src/backend/api/routes/v1/model_reasoning.py | 10 +-
src/backend/api/routes/v1/models.py | 176 +-
src/backend/api/routes/v1/myspace_folders.py | 4 +-
src/backend/api/routes/v1/ontologies.py | 20 +-
src/backend/api/routes/v1/plans.py | 199 +-
src/backend/api/routes/v1/plugin_resources.py | 98 +-
src/backend/api/routes/v1/plugins.py | 47 +-
src/backend/api/routes/v1/project_api_keys.py | 30 +-
.../api/routes/v1/project_environments.py | 8 +-
src/backend/api/routes/v1/projects.py | 36 +-
.../api/routes/v1/prompt_management.py | 95 +-
src/backend/api/routes/v1/public_chats.py | 23 +-
src/backend/api/routes/v1/space_files.py | 22 +-
src/backend/api/routes/v1/space_previews.py | 11 +-
src/backend/api/routes/v1/summary.py | 2 +-
src/backend/api/routes/v1/tool_approval.py | 10 +-
src/backend/api/routes/v1/users.py | 16 +-
src/backend/api/startup/bootstrap.py | 72 +-
src/backend/api/startup/commands.py | 8 +-
src/backend/api/startup/memory.py | 10 +-
src/backend/api/startup/plugins.py | 45 +-
src/backend/api/startup/runtime.py | 87 +-
src/backend/api/startup/workers.py | 63 +-
.../content => api/transports}/__init__.py | 0
src/backend/api/transports/run_sse.py | 85 +
src/backend/cli.py | 118 +-
src/backend/core/agent_runtime/README.md | 18 +
.../core/agent_runtime/README.zh-CN.md | 18 +
.../resources => agent_runtime}/__init__.py | 0
src/backend/core/agent_runtime/budget.py | 74 +
src/backend/core/agent_runtime/direct.py | 458 ++
.../core/agent_runtime/direct_state.py | 46 +
src/backend/core/agent_runtime/events.py | 71 +
src/backend/core/agent_runtime/host.py | 11 +
.../agent_runtime/integrations}/__init__.py | 0
.../integrations}/compaction_stream.py | 8 +-
.../agent_runtime/integrations/evolution.py | 160 +
.../core/agent_runtime/integrations/memory.py | 304 ++
.../integrations/memory_budget.py | 34 +
.../integrations/memory_identity.py | 74 +
.../integrations/memory_session.py | 178 +
.../integrations/memory_state.py | 23 +
.../integrations/ontology_evidence.py | 229 +
.../integrations/ontology_repair.py | 366 ++
src/backend/core/agent_runtime/main_state.py | 45 +
.../agent_runtime}/plugin_visibility.py | 14 +-
.../core/agent_runtime/prepare_direct.py | 126 +
.../core/agent_runtime/prepare_main.py | 289 +
src/backend/core/agent_runtime/runner.py | 444 ++
src/backend/core/agent_runtime/skills.py | 176 +
.../agent_runtime}/strategy.py | 0
.../core/agent_runtime/stream_direct.py | 298 ++
src/backend/core/agent_runtime/stream_main.py | 354 ++
src/backend/core/agent_runtime/targets.py | 208 +
.../core/agent_skills/backends/__init__.py | 14 -
.../core/agent_skills/cache_refresh.py | 51 -
src/backend/core/auth/__init__.py | 2 +-
.../agent_api.py} | 2 +-
src/backend/core/auth/agent_api_scope.py | 2 +-
.../core/{services => auth}/api_key_runs.py | 33 +-
.../api_key_service.py => auth/api_keys.py} | 12 +-
src/backend/core/auth/backend.py | 275 +-
src/backend/core/auth/bearer.py | 149 +
src/backend/core/auth/capabilities.py | 4 +-
src/backend/core/auth/edition.py | 5 +
src/backend/core/auth/local_user_bootstrap.py | 83 +
src/backend/core/auth/local_user_policy.py | 16 +
.../local_users.py} | 155 +-
.../core/auth/local_users_constants.py | 40 +
src/backend/core/auth/mock_ticket_store.py | 26 +-
src/backend/core/auth/model_selection.py | 52 +
src/backend/core/auth/oa_ticket_store.py | 23 +-
...t_policy.py => personal_account_policy.py} | 0
...count_view.py => personal_account_view.py} | 0
...es.py => personal_edition_capabilities.py} | 0
...iface.py => personal_permissions_iface.py} | 2 +-
src/backend/core/auth/project_api_scope.py | 2 +-
src/backend/core/auth/public_chat.py | 6 +-
src/backend/core/auth/public_identity.py | 36 +
src/backend/core/auth/session.py | 36 +-
.../user_service.py => auth/users.py} | 11 +-
src/backend/core/capabilities/__init__.py | 17 +-
.../core/capabilities/agents/__init__.py | 1 +
.../agents/available_resources.py | 273 +
src/backend/core/capabilities/agents/base.py | 435 ++
.../capabilities/agents/base_constants.py | 33 +
.../agents/builtin.py} | 4 +-
.../agents/edition.py} | 2 +-
.../agents/markdown.py} | 0
.../capabilities/agents/version_history.py | 180 +
src/backend/core/capabilities/archive.py | 18 +-
.../core/capabilities/catalog/__init__.py | 1 +
.../catalog/api.py} | 14 +-
.../catalog/common.py} | 4 +-
.../catalog/details.py} | 20 +-
.../catalog/loader.py} | 57 +-
.../catalog/migration.py} | 4 +-
.../catalog/overrides.py} | 0
.../catalog/resolver.py} | 78 +-
.../catalog/runtime.py} | 53 +-
.../core/capabilities/identity_rules.py | 36 +
.../manifest_protocol.py} | 3 +-
src/backend/core/capabilities/mcp/__init__.py | 1 +
.../mcp/builtin_specs.py} | 4 -
.../mcp/edition.py} | 0
.../mcp/management.py} | 64 +-
src/backend/core/capabilities/mcp/oauth.py | 385 ++
.../core/capabilities/mcp/oauth_storage.py | 124 +
.../mcp/service.py} | 60 +-
.../core/capabilities/mcp/tool_metadata.py | 13 +
.../core/capabilities/mcp/tool_schema.py | 20 +
.../capabilities/mcp/tool_schema_backfill.py | 120 +
.../core/capabilities/plugins/README.md | 23 +
.../{ => capabilities}/plugins/__init__.py | 0
.../{ => capabilities}/plugins/connections.py | 2 +-
.../plugins/management/__init__.py | 43 +
.../plugins/management/admin_config.py | 18 +-
.../plugins/management/bundle_upgrade.py} | 8 +-
.../plugins/management/cloud.py} | 52 +-
.../plugins/management/components.py | 32 +-
.../plugins/management/contract.py} | 0
.../plugins/management/defaults.py | 2 +-
.../plugins/management/details.py | 76 +-
.../plugins/management/errors.py} | 0
.../plugins/management/installation.py | 72 +-
.../plugins/management/lifecycle.py | 42 +-
.../plugins/management/package.py} | 10 +-
.../plugins/management/packages.py | 33 +-
.../plugins/management/projection.py | 28 +-
.../plugins/management/queries.py | 50 +-
.../plugins/packaging/__init__.py | 0
.../plugins/packaging/archive.py | 4 +-
.../plugins/packaging/definitions.py | 4 +-
.../plugins/packaging/importer.py | 24 +-
.../plugins/packaging/models.py | 0
.../plugins/packaging/presentation.py | 31 +
.../plugins/packaging/runtime_assets.py | 0
.../plugins/packaging/skill_transform.py | 2 +-
.../plugins/packaging/sources.py | 31 +-
.../plugins/resources/__init__.py | 1 +
.../plugins/resources/artifacts.py | 4 +-
.../plugins/resources/authentication.py | 4 +-
.../plugins/resources/configuration.py | 0
.../plugins/resources/confinement.py | 7 +-
.../plugins/resources/crypto.py | 0
.../plugins/resources/deadlines.py | 53 +
.../plugins/resources/installation.py | 41 +-
.../plugins/resources/lifecycle.py | 64 +-
.../capabilities/plugins/resources/owners.py | 66 +
.../capabilities/plugins/resources/prewarm.py | 177 +
.../plugins/resources/reservation.py | 84 +
.../capabilities/plugins/resources/service.py | 460 ++
.../plugins/resources/store.py | 2 +-
.../plugins/resources/stream.py | 58 +-
.../plugins/resources/transport.py | 0
.../plugins/runtime/__init__.py | 28 +-
.../plugins/runtime/activation_history.py | 2 +-
.../plugins/runtime/models.py | 0
.../plugins/runtime/resolution.py | 27 +-
.../plugins/runtime/sticky_selection.py | 35 +-
.../capabilities/{ref.py => reference.py} | 2 +-
.../core/capabilities/runtime/__init__.py | 37 -
src/backend/core/capabilities/skills.py | 656 ---
.../skills}/__init__.py | 1 -
.../capabilities/skills/backends/__init__.py | 14 +
.../skills}/backends/composite.py | 6 +-
.../skills}/backends/database.py | 2 +-
.../skills}/backends/filesystem.py | 2 +-
.../skills}/backends/protocol.py | 0
.../skills}/binary_files.py | 0
.../core/capabilities/skills/cache_refresh.py | 50 +
.../skills}/config.py | 56 +-
.../core/capabilities/skills/deps/__init__.py | 1 +
.../skills/deps/aggregator.py} | 20 +-
.../skills/deps/detector.py} | 0
.../skills/deps/prober.py} | 4 +-
.../skills/deps/requests.py} | 108 +-
.../skills/icons.py} | 0
.../core/capabilities/skills/installation.py | 117 +
.../skills}/loader.py | 165 +-
.../capabilities/skills/loader_factory.py | 21 +
.../skills/management.py} | 18 +-
.../skills}/publication.py | 41 +-
.../skills}/registry.py | 12 +-
.../capabilities/skills/runtime_loader.py | 115 +
.../capabilities/skills/script_discovery.py | 65 +
.../skills}/selection_record.py | 4 +-
.../skills}/skill_archive.py | 8 +-
.../core/capabilities/skills/zip_reader.py | 124 +
.../core/channels/adapters/dingtalk.py | 854 ---
src/backend/core/channels/adapters/lark.py | 683 ---
src/backend/core/channels/inbound.py | 1005 ----
.../{services => chat}/audio_transcription.py | 2 +-
src/backend/core/chat/compaction/__init__.py | 1 +
.../compaction/budget_checkpoint.py} | 4 +-
.../compaction/coordinator.py} | 24 +-
.../compaction/history.py} | 8 +-
.../compaction/progress.py} | 0
.../compaction/service.py} | 20 +-
.../compaction/summary.py} | 19 +-
.../compaction/usage_projection.py} | 4 +-
.../core/chat/compaction_checkpoints.py | 259 +
src/backend/core/chat/compaction_snapshots.py | 265 +
src/backend/core/chat/compaction_types.py | 36 +
src/backend/core/chat/context.py | 94 +-
.../chat_edition.py => chat/edition.py} | 0
.../{orchestration => core/chat}/followups.py | 57 +-
.../chat_fork_service.py => chat/fork.py} | 2 +-
src/backend/core/chat/history_operations.py | 49 +
src/backend/core/chat/limits.py | 88 +
.../chat}/message_parser.py | 5 +-
src/backend/core/chat/message_queries.py | 91 +
src/backend/core/chat/message_writes.py | 299 ++
src/backend/core/chat/mode_seed.py | 51 +
src/backend/core/chat/mode_types.py | 48 +
src/backend/core/chat/mode_validation.py | 107 +
.../chat_mode_service.py => chat/modes.py} | 295 +-
.../operations}/__init__.py | 0
src/backend/core/chat/operations/admission.py | 317 ++
.../operations}/admission_errors.py | 2 +-
.../operations}/agent_targets.py | 16 +-
.../operations}/commands.py | 18 +-
.../operations}/history.py | 10 +-
.../operations}/invocation.py | 31 +-
.../operations}/projects.py | 6 +-
.../operations}/request_context.py | 25 +-
.../operations}/runs.py | 14 +-
.../operations}/schemas.py | 0
.../operations}/session_context.py | 58 +-
.../operations}/sessions.py | 40 +-
.../plan_service.py => chat/plans.py} | 0
.../core/{services => chat}/project_init.py | 8 +-
src/backend/core/chat/projection/__init__.py | 0
src/backend/core/chat/projection/codec.py | 8 +
.../chat/projection/reducer.py} | 134 +-
.../chat/projection/storage.py} | 8 +-
.../chat/projection/subscription.py} | 32 +-
.../public_chat_service.py => chat/public.py} | 43 +-
.../references.py} | 6 +-
src/backend/core/chat/runtime/__init__.py | 0
src/backend/core/chat/runtime/admission.py | 98 +
src/backend/core/chat/runtime/chunks.py | 183 +
.../chat/runtime/compaction.py} | 23 +-
src/backend/core/chat/runtime/completion.py | 231 +
src/backend/core/chat/runtime/followups.py | 64 +
src/backend/core/chat/runtime/handoff.py | 272 +
.../chat/runtime}/job_wakeup.py | 224 +-
src/backend/core/chat/runtime/pauses.py | 58 +
src/backend/core/chat/runtime/recovery.py | 179 +
.../core/chat/runtime/recovery_snapshot.py | 204 +
src/backend/core/chat/runtime/snapshots.py | 201 +
src/backend/core/chat/runtime/steering.py | 199 +
src/backend/core/chat/runtime/turn_state.py | 44 +
src/backend/core/chat/runtime/worker.py | 405 ++
src/backend/core/chat/schemas.py | 31 +-
.../chat_sequencer.py => chat/sequencer.py} | 41 +-
src/backend/core/chat/service.py | 256 +
src/backend/core/chat/session_operations.py | 284 +
.../core/{services => chat}/sidebar_status.py | 0
.../chat_steer_service.py => chat/steer.py} | 7 +-
src/backend/core/chat/steer_hook.py | 70 +
.../core/{services => chat}/steer_queue.py | 0
.../subagent_routing.py} | 0
.../chat_titles.py => chat/titles.py} | 21 +-
src/backend/core/chat/tool_log.py | 2 +-
.../chat}/tool_payloads.py | 45 +-
src/backend/core/config/__init__.py | 11 +-
src/backend/core/config/display_names.py | 2 +-
.../{edition_display_names.py => edition.py} | 8 +
src/backend/core/config/environment.py | 3 -
src/backend/core/config/identity_settings.py | 11 +-
src/backend/core/config/mcp_config.py | 8 +-
src/backend/core/config/product_settings.py | 5 +-
src/backend/core/config/runtime_env.py | 49 +-
src/backend/core/config/runtime_settings.py | 33 +-
src/backend/core/config/sandbox_settings.py | 11 +-
src/backend/core/config/settings.py | 7 +-
src/backend/core/config/store/__init__.py | 1 +
src/backend/core/config/store/config_keys.py | 12 +
.../core/config/store/execution_options.py | 93 +
src/backend/core/config/store/runtime_env.py | 52 +
.../core/config/store/system_config.py | 120 +
.../config/store/system_config_constants.py | 302 ++
src/backend/core/config/user_intros.py | 2 +-
src/backend/core/content/content_blocks.py | 701 ---
src/backend/core/content/file_parser.py | 752 ---
src/backend/core/db/__init__.py | 28 +-
src/backend/core/db/artifact_scope.py | 13 +
src/backend/core/db/chat_search.py | 2 +-
src/backend/core/db/edition.py | 27 +
src/backend/core/db/engine.py | 45 +-
src/backend/core/db/history_projection.py | 2 +-
src/backend/core/db/models/__init__.py | 212 +-
src/backend/core/db/models/admin.py | 338 +-
src/backend/core/db/models/agent.py | 73 +-
src/backend/core/db/models/artifact.py | 17 +-
src/backend/core/db/models/channel_relay.py | 40 -
src/backend/core/db/models/chat.py | 350 +-
src/backend/core/db/models/chat_mode.py | 5 +-
src/backend/core/db/models/chat_resources.py | 116 +
src/backend/core/db/models/chat_runs.py | 199 +
src/backend/core/db/models/column_types.py | 9 +
src/backend/core/db/models/config.py | 12 +-
src/backend/core/db/models/evolution.py | 10 +-
src/backend/core/db/models/identity.py | 285 +-
src/backend/core/db/models/job.py | 14 +-
src/backend/core/db/models/kb_wiki.py | 224 -
src/backend/core/db/models/knowledge.py | 182 +-
src/backend/core/db/models/logs.py | 13 +-
src/backend/core/db/models/memory.py | 15 +-
src/backend/core/db/models/observability.py | 61 -
src/backend/core/db/models/ontology.py | 6 +-
src/backend/core/db/models/project.py | 14 +-
src/backend/core/db/personal_file_names.py | 13 +-
...extensions.py => personal_model_fields.py} | 0
.../{edition_tables.py => personal_schema.py} | 6 +-
src/backend/core/db/repository/__init__.py | 39 +-
src/backend/core/db/repository/artifact.py | 8 +-
src/backend/core/db/repository/catalog.py | 5 +-
src/backend/core/db/repository/chat.py | 15 +-
src/backend/core/db/repository/folder.py | 40 +
.../{agent.py => personal_agent.py} | 0
.../{audit.py => personal_audit.py} | 2 +-
src/backend/core/db/repository/user.py | 115 +-
.../core/db/site_storage_retirement.py | 8 +-
src/backend/core/db/system_configuration.py | 146 +
src/backend/core/db/threads.py | 99 +
src/backend/core/db/user_cleanup.py | 13 +
src/backend/core/evolution/__init__.py | 6 +-
src/backend/core/evolution/activation.py | 1502 ------
src/backend/core/evolution/agent_profile.py | 505 --
.../core/evolution/assessment/__init__.py | 1 +
.../{ => assessment}/applicability.py | 2 +-
.../evolution/{ => assessment}/benchmark.py | 6 +-
.../evolution/{ => assessment}/evaluator.py | 0
.../evolution/{ => assessment}/experiments.py | 2 +-
.../{ => assessment}/fault_injection.py | 15 +-
.../evolution/{ => assessment}/metrics.py | 0
.../evolution/{ => assessment}/similarity.py | 9 +-
.../evolution/{ => assessment}/sop_judge.py | 2 +-
src/backend/core/evolution/backfill.py | 9 +-
src/backend/core/evolution/contract.py | 4 +-
src/backend/core/evolution/cycle/__init__.py | 1 +
.../core/evolution/cycle/activation.py | 353 ++
.../evolution/cycle/activation_constants.py | 11 +
.../core/evolution/cycle/activation_policy.py | 196 +
.../core/evolution/cycle/activation_types.py | 9 +
.../core/evolution/cycle/candidate_records.py | 197 +
.../evolution/cycle/composition_candidates.py | 221 +
.../core/evolution/cycle/cycle_extras.py | 32 +
.../evolution/cycle/cycle_extras_constants.py | 20 +
.../core/evolution/cycle/episode_views.py | 178 +
.../core/evolution/cycle/evaluation.py | 153 +
.../core/evolution/cycle/extra_findings.py | 166 +
.../core/evolution/cycle/graph_candidates.py | 156 +
src/backend/core/evolution/cycle/loop.py | 292 ++
.../core/evolution/cycle/memory_candidates.py | 286 +
.../core/evolution/cycle/memory_findings.py | 109 +
src/backend/core/evolution/cycle/ordering.py | 103 +
src/backend/core/evolution/cycle/patterns.py | 113 +
.../core/evolution/cycle/profile_release.py | 281 +
src/backend/core/evolution/cycle/promotion.py | 317 ++
.../evolution/cycle/promotion_constants.py | 80 +
.../core/evolution/cycle/promotion_types.py | 94 +
.../core/evolution/cycle/prompt_candidates.py | 95 +
.../core/evolution/{ => cycle}/readiness.py | 25 +-
.../core/evolution/{ => cycle}/release.py | 0
.../core/evolution/cycle/retirement.py | 158 +
.../evolution/{ => cycle}/runtime_binding.py | 85 +-
.../core/evolution/cycle/skill_document.py | 298 ++
.../core/evolution/cycle/skill_release.py | 355 ++
.../core/evolution/{ => cycle}/timescales.py | 2 +-
.../core/evolution/{ => cycle}/trigger.py | 77 +-
.../core/evolution/cycle/usage_evidence.py | 283 +
src/backend/core/evolution/cycle_extras.py | 653 ---
.../core/evolution/evidence/__init__.py | 1 +
.../core/evolution/{ => evidence}/cassette.py | 19 +-
.../{ => evidence}/counterfactual.py | 2 +-
.../core/evolution/{ => evidence}/credit.py | 2 +-
.../core/evolution/{ => evidence}/events.py | 2 +-
.../{ => evidence}/evidence_contract.py | 0
.../{ => evidence}/evidence_resolver.py | 8 +-
.../core/evolution/{ => evidence}/exposure.py | 10 +-
.../core/evolution/{ => evidence}/lineage.py | 2 +-
.../core/evolution/{ => evidence}/outcomes.py | 0
.../core/evolution/{ => evidence}/replay.py | 0
.../{ => evidence}/trace_assembler.py | 6 +-
.../evolution/{ => evidence}/trace_store.py | 4 +-
.../core/evolution/generators/__init__.py | 1 +
.../evolution/{ => generators}/graph_scan.py | 0
.../{ => generators}/memory_apply.py | 8 +-
.../evolution/{ => generators}/memory_ops.py | 2 +-
.../evolution/{ => generators}/memory_scan.py | 2 +-
.../{ => generators}/ontology_bridge.py | 0
.../{ => generators}/orchestration_gen.py | 2 +-
.../evolution/{ => generators}/prompt_gen.py | 0
.../skill_candidate_engine.py | 14 +-
.../evolution/generators/skill_drafting.py | 185 +
.../evolution/generators/skill_evidence.py | 124 +
.../core/evolution/generators/skill_gen.py | 95 +
.../generators/skill_gen_constants.py | 109 +
src/backend/core/evolution/loop.py | 1576 ------
.../core/evolution/profiles/__init__.py | 1 +
.../core/evolution/profiles/agent_profile.py | 259 +
.../profiles/agent_profile_constants.py | 102 +
.../core/evolution/profiles/change_preview.py | 85 +
.../evolution/{ => profiles}/console_view.py | 2 -
.../evolution/{ => profiles}/federation.py | 2 +-
.../core/evolution/{ => profiles}/personal.py | 52 +-
.../personal_review.py} | 219 +-
.../core/evolution/{ => profiles}/policies.py | 22 +-
.../core/evolution/{ => profiles}/prefs.py | 0
.../evolution/profiles/profile_selection.py | 147 +
.../core/evolution/profiles/stall_hook.py | 103 +
.../core/evolution/profiles/user_settings.py | 70 +
.../profiles/user_settings_constants.py | 14 +
src/backend/core/evolution/promotion.py | 890 ----
.../core/evolution/settlement/__init__.py | 1 +
.../evolution/settlement/memory_service.py | 39 +
.../evolution/{ => settlement}/settlement.py | 0
.../{ => settlement}/settlement_runner.py | 43 +-
.../{ => settlement}/settlement_store.py | 2 +-
src/backend/core/evolution/skill_gen.py | 521 --
src/backend/core/extensions/__init__.py | 69 -
src/backend/core/extensions/registry.py | 107 -
src/backend/core/files/__init__.py | 1 +
src/backend/core/files/artifact_bytes.py | 75 +
.../core/{ => files}/artifacts/__init__.py | 1 -
.../artifacts/backfill.py} | 10 +-
src/backend/core/files/artifacts/generated.py | 210 +
.../artifacts/key_repair.py} | 0
.../{ => files}/artifacts/local_reference.py | 31 +-
.../artifacts/location.py} | 24 +-
.../artifacts/reader.py} | 19 +-
.../artifacts/refs.py} | 10 +-
.../artifacts/service.py} | 15 +-
.../core/{ => files}/artifacts/store.py | 26 +-
.../artifacts/summary.py} | 8 +-
.../artifacts/text.py} | 15 +-
src/backend/core/files/content/__init__.py | 1 +
.../{ => files}/content/archive_browser.py | 2 +-
.../{ => files}/content/archive_limits.py | 0
src/backend/core/files/content/branding.py | 299 ++
.../core/files/content/content_blocks.py | 183 +
.../files/content/content_blocks_constants.py | 181 +
.../core/files/content/file_formats.py | 33 +
src/backend/core/files/content/file_parser.py | 41 +
.../{ => files}/content/file_validation.py | 0
.../core/{ => files}/content/office.py | 0
.../content/office_preview_cache.py | 2 +-
.../core/files/content/parser_config.py | 47 +
src/backend/core/files/content/pdf_parser.py | 125 +
.../core/files/content/presentation_parser.py | 63 +
.../core/files/content/spreadsheet_parser.py | 332 ++
.../core/{ => files}/content/svg_fit.py | 5 +-
src/backend/core/files/content/text_parser.py | 21 +
src/backend/core/files/content/word_parser.py | 77 +
src/backend/core/files/edition.py | 98 +
.../core/{services => files}/folder_batch.py | 47 +-
.../personal_operations.py} | 10 +-
.../core/{ => files}/spaces/__init__.py | 0
src/backend/core/{ => files}/spaces/access.py | 4 +-
src/backend/core/{ => files}/spaces/files.py | 8 +-
src/backend/core/{ => files}/spaces/leases.py | 24 +-
src/backend/core/{ => files}/spaces/mounts.py | 32 +-
.../core/{ => files}/spaces/projects.py | 6 +-
.../core/{ => files}/spaces/recovery.py | 4 +-
.../core/{ => files}/spaces/sandbox.py | 26 +-
src/backend/core/{ => files}/spaces/search.py | 2 +-
src/backend/core/{ => files}/spaces/text.py | 4 +-
src/backend/core/{ => files}/spaces/tree.py | 0
.../core/{ => files}/storage/__init__.py | 17 +-
.../core/{ => files}/storage/entries.py | 0
.../core/{ => files}/storage/factory.py | 32 +-
src/backend/core/{ => files}/storage/local.py | 8 +-
.../{ => files}/storage/local_directory.py | 4 +-
.../core/{ => files}/storage/local_paths.py | 2 +-
.../core/{ => files}/storage/protocol.py | 0
src/backend/core/{ => files}/storage/s3.py | 4 +-
.../core/{ => files}/storage/s3_directory.py | 2 +-
.../core/{services => files}/upload_retry.py | 4 +-
.../capability_errors.py} | 0
src/backend/core/harness/context_items.py | 6 +
src/backend/core/harness/events.py | 67 +-
.../{llm => harness}/execution_manifest.py | 2 +-
src/backend/core/harness/hooks.py | 151 +-
.../harness_ledger.py => harness/ledger.py} | 163 +-
src/backend/core/harness/log_audit.py | 144 +
src/backend/core/harness/log_context.py | 161 +
src/backend/core/harness/log_records.py | 212 +
src/backend/core/harness/log_writer.py | 121 +
.../core/harness/log_writer_constants.py | 32 +
.../core/harness/observation_writer.py | 134 +
src/backend/core/harness/registry/__init__.py | 79 +
.../core/harness/registry/authentication.py | 39 +
.../core/harness/registry/build_governance.py | 51 +
.../harness/registry/cache_invalidation.py | 27 +
.../harness/registry/capability_management.py | 27 +
.../harness/registry/capability_revisions.py | 34 +
.../harness/registry/capability_runtime.py | 74 +
.../core/harness/registry/catalog_options.py | 26 +
.../core/harness/registry/chat_execution.py | 27 +
.../core/harness/registry/chat_history.py | 21 +
.../core/harness/registry/compaction.py | 62 +
.../core/harness/registry/device_catalog.py | 79 +
.../registry}/edition.py | 0
.../core/harness/registry/evaluation.py | 72 +
.../harness/registry/execution_account.py | 35 +
.../harness/registry/execution_catalog.py | 51 +
.../harness/registry/execution_identity.py | 78 +
.../core/harness/registry/hook_factories.py | 69 +
.../core/harness/registry/host_runtime.py | 13 +
.../core/harness/registry/job_operations.py | 21 +
.../core/harness/registry/local_files.py | 65 +
.../harness/registry/local_publication.py | 69 +
.../harness/registry/marketplace_catalog.py | 44 +
.../core/harness/registry/mcp_clients.py | 21 +
.../harness/registry/model_credentials.py | 70 +
.../core/harness/registry/model_services.py | 59 +
.../core/harness/registry/package_sources.py | 48 +
.../core/harness/registry/package_views.py | 66 +
src/backend/core/harness/registry/registry.py | 169 +
.../core/harness/registry/resource_owners.py | 48 +
.../harness/registry/retrieval_context.py | 37 +
.../core/harness/registry/run_cancellation.py | 37 +
.../core/harness/registry/run_capabilities.py | 114 +
.../core/harness/registry/schema_lifecycle.py | 36 +
.../core/harness/registry/skill_content.py | 24 +
.../harness/registry/system_configuration.py | 31 +
.../core/harness/registry/tool_operations.py | 59 +
.../registry/transfer_channels.py} | 64 +-
.../core/harness/registry/user_cleanup.py | 27 +
src/backend/core/harness/tool_call_context.py | 5 +
.../core/harness/tool_effect_context.py | 18 +
src/backend/core/harness/usage.py | 41 +-
src/backend/core/infra/background.py | 18 +-
src/backend/core/infra/byte_range.py | 71 +
src/backend/core/infra/crypto.py | 28 +-
.../junction.py => infra/directory_links.py} | 32 +-
src/backend/core/infra/dns_cache.py | 58 +-
src/backend/core/infra/failures.py | 13 +
.../lockfile.py => infra/file_lock.py} | 0
src/backend/core/{ => infra}/immutable.py | 0
src/backend/core/infra/json_output.py | 49 +
src/backend/core/infra/leader.py | 225 +-
.../core/infra/local_runtime/__init__.py | 1 +
.../infra/local_runtime/dynamic_servers.py | 64 +
.../infra/local_runtime/lifecycle.py} | 33 +-
.../local_runtime/runner_auth.py} | 0
.../infra/local_runtime/startup.py} | 95 +-
src/backend/core/infra/model_output.py | 95 +
src/backend/core/infra/probes/__init__.py | 1 +
.../probes/edition.py} | 0
.../probes}/service_probes.py | 25 +-
src/backend/core/infra/rate_limit.py | 21 -
src/backend/core/infra/redis.py | 27 +-
src/backend/core/infra/runtime_state.py | 24 +-
.../text_bounds.py} | 0
src/backend/core/infra/time.py | 14 +-
src/backend/core/infra/worker_count.py | 9 +-
.../{orchestration => core}/jobs/__init__.py | 0
.../{orchestration => core}/jobs/agent.py | 14 +-
.../{orchestration => core}/jobs/callback.py | 0
.../{orchestration => core}/jobs/files.py | 6 +-
.../jobs/notifications.py | 22 +-
.../{orchestration => core}/jobs/owner.py | 43 +-
.../{orchestration => core}/jobs/process.py | 13 +-
src/backend/core/jobs/reminder_hook.py | 80 +
.../{orchestration => core}/jobs/runner.py | 0
.../job_runtime.py => core/jobs/runtime.py} | 41 +-
.../{orchestration => core}/jobs/sdk.py | 0
.../{orchestration => core}/jobs/state.py | 2 +-
.../job_service.py => jobs/store.py} | 0
.../jobs/supervisor.py | 38 +-
src/backend/core/jobs/tool_operations.py | 410 ++
src/backend/core/kb/kb_assets.py | 567 --
src/backend/core/kb/kb_parser.py | 933 ----
src/backend/core/kb/kb_vector.py | 755 ---
src/backend/core/kb/wiki/local_provider.py | 523 --
src/backend/core/llm/__init__.py | 14 +-
src/backend/core/llm/_distill_shared.py | 156 -
src/backend/core/llm/api_runtime/__init__.py | 1 +
.../agent.py} | 6 +-
.../agent_skills.py} | 2 +-
.../agent_tools.py} | 27 +-
.../project.py} | 2 +-
src/backend/core/llm/capability_tools.py | 106 -
src/backend/core/llm/citations/__init__.py | 1 +
src/backend/core/llm/citations/allocator.py | 115 +
src/backend/core/llm/citations/anchor.py | 63 +
.../core/llm/citations/anchor_state.py | 101 +
src/backend/core/llm/citations/extraction.py | 208 +
src/backend/core/llm/citations/hook.py | 38 +
src/backend/core/llm/citations/types.py | 63 +
src/backend/core/llm/command_middleware.py | 39 -
src/backend/core/llm/context/__init__.py | 1 +
src/backend/core/llm/context/adapter.py | 362 ++
.../core/llm/context/attachment_hook.py | 81 +
.../context/budget.py} | 2 +-
.../llm/{ => context}/compacting_agent.py | 20 +-
.../core/llm/{ => context}/compaction.py | 11 +-
src/backend/core/llm/context/content.py | 212 +
src/backend/core/llm/context/ir.py | 327 ++
src/backend/core/llm/context/ir_constants.py | 86 +
src/backend/core/llm/context/items.py | 215 +
.../core/llm/context/message_defaults.py | 78 +
.../core/llm/{ => context}/offloader.py | 4 +-
.../core/llm/context/provider_conversion.py | 239 +
.../core/llm/context/reasoning_hooks.py | 49 +
.../core/llm/context/reference_conversion.py | 79 +
src/backend/core/llm/context/rendering.py | 114 +
.../{context_safety.py => context/safety.py} | 16 +-
.../core/llm/{ => context}/system_reminder.py | 4 +-
.../{context_usage.py => context/usage.py} | 2 +-
.../core/llm/context/workspace_hooks.py | 125 +
src/backend/core/llm/context_adapter.py | 867 ---
src/backend/core/llm/context_ir.py | 855 ---
src/backend/core/llm/factory/README.md | 6 +-
src/backend/core/llm/factory/__init__.py | 6 +-
src/backend/core/llm/factory/assembly.py | 18 +-
src/backend/core/llm/factory/build.py | 6 +-
src/backend/core/llm/factory/defaults.py | 16 +-
.../core/llm/factory/prompts/policy.py | 3 +-
.../core/llm/factory/prompts/system.py | 39 +-
src/backend/core/llm/factory/request.py | 4 +-
.../llm/factory/runtime/agent_construction.py | 21 +-
.../capability_bindings.py} | 11 +-
.../llm/factory/runtime/execution_budget.py | 10 +-
.../core/llm/factory/runtime/listeners.py | 33 +-
.../llm/factory/runtime/manifest_binding.py | 4 +-
.../llm/factory/runtime/middleware_stack.py | 130 +-
.../llm/factory/runtime/model_selection.py | 20 +-
.../llm/factory/selection/capabilities.py | 58 +-
.../llm/factory/selection/entry_policy.py | 20 +-
.../llm/factory/selection/mode_bindings.py | 7 +-
.../factory/selection/profile_selection.py | 12 +-
.../factory/selection/progressive_bindings.py | 24 +-
.../llm/factory/selection/server_selection.py | 10 +-
.../factory/selection/sticky_capabilities.py | 26 +-
.../factory/selection/subagent_bindings.py | 8 +-
.../registry.py => core/llm/factory/specs.py} | 2 -
.../core/llm/factory/tools/definitions.py | 35 +
.../core/llm/factory/tools/interactive.py | 2 +-
.../core/llm/factory/tools/mcp_config.py | 55 +-
.../core/llm/factory/tools/mcp_connections.py | 226 +-
src/backend/core/llm/factory/tools/native.py | 82 +-
.../llm/factory/tools/skill_registration.py | 26 +-
.../core/llm/factory/tools/tool_surface.py | 46 +-
src/backend/core/llm/finish_guard.py | 85 -
src/backend/core/llm/hooks.py | 588 ---
src/backend/core/llm/host_services.py | 31 +
src/backend/core/llm/management_tool.py | 84 -
src/backend/core/llm/mcp/__init__.py | 1 +
src/backend/core/llm/mcp/callbacks.py | 168 +
src/backend/core/llm/mcp/chat_sessions.py | 181 +
src/backend/core/llm/mcp/client.py | 149 +
src/backend/core/llm/mcp/context_tools.py | 69 +
src/backend/core/llm/mcp/deadline.py | 40 +
.../{mcp_invocation.py => mcp/invocation.py} | 10 +-
src/backend/core/llm/mcp/lifetime.py | 68 +
src/backend/core/llm/mcp/manager.py | 62 +
src/backend/core/llm/mcp/results.py | 67 +
src/backend/core/llm/mcp/session.py | 147 +
src/backend/core/llm/mcp/tools.py | 113 +
src/backend/core/llm/mcp_manager.py | 336 --
src/backend/core/llm/mcp_pool.py | 387 --
src/backend/core/llm/middleware/__init__.py | 1 +
.../migration_map.py} | 67 +-
src/backend/core/llm/middlewares.py | 1450 -----
src/backend/core/llm/persona_distiller.py | 226 -
src/backend/core/llm/ports.py | 79 +
.../llm/project_environment_middleware.py | 27 -
src/backend/core/llm/prompt_files.py | 138 +
src/backend/core/llm/prompt_kinds.py | 193 +
src/backend/core/llm/prompt_rendering.py | 137 +
src/backend/core/llm/prompt_storage.py | 99 +
src/backend/core/llm/prompt_versions.py | 296 ++
src/backend/core/llm/prompt_versions_state.py | 30 +
src/backend/core/llm/providers/__init__.py | 13 -
src/backend/core/llm/runtime/__init__.py | 1 +
.../{ => runtime}/agentscope_hook_adapter.py | 223 +-
.../core/llm/runtime/context_hook_adapter.py | 80 +
.../core/llm/runtime/delta_coalescer.py | 106 +
src/backend/core/llm/runtime/file_context.py | 277 +
src/backend/core/llm/runtime/finish_guard.py | 54 +
.../core/llm/runtime/hooks_constants.py | 17 +
.../llm/{ => runtime}/human_interaction.py | 4 +-
.../llm/{ => runtime}/interaction_store.py | 0
.../core/llm/{ => runtime}/manifest_agent.py | 17 +-
.../core/llm/{ => runtime}/message_compat.py | 16 +-
src/backend/core/llm/runtime/model_cache.py | 133 +
.../llm/runtime/model_selection_adapter.py | 28 +
.../core/llm/{ => runtime}/model_steps.py | 5 +-
.../llm/{ => runtime}/project_environment.py | 33 +-
.../llm/runtime/project_environment_hook.py | 19 +
.../core/llm/runtime/review_text_agent.py | 51 +
.../llm/{ => runtime}/session_capabilities.py | 2 +-
src/backend/core/llm/runtime/state.py | 75 +
src/backend/core/llm/runtime/stream_events.py | 261 +
src/backend/core/llm/runtime/stream_loop.py | 332 ++
src/backend/core/llm/runtime/stream_text.py | 63 +
src/backend/core/llm/runtime/streaming.py | 287 +
.../core/llm/runtime/streaming_constants.py | 20 +
.../llm/runtime}/tool_callbacks.py | 24 +-
.../llm/runtime/tool_execution_adapter.py | 210 +
.../core/llm/runtime/tool_hook_adapter.py | 81 +
.../llm/{ => runtime}/tool_media_store.py | 24 +-
.../core/llm/runtime/tool_result_adapter.py | 53 +
src/backend/core/llm/runtime/turn_state.py | 41 +
.../core/llm/{ => runtime}/workspace.py | 0
.../selector.py => llm/skill_selection.py} | 23 +-
src/backend/core/llm/steer_middleware.py | 96 -
src/backend/core/llm/subagents/__init__.py | 1 +
.../execution.py} | 48 +-
.../prompts.py} | 0
.../sessions.py} | 2 +-
.../stream.py} | 39 +-
.../{subagent_tool.py => subagents/tool.py} | 72 +-
src/backend/core/llm/tool_collector.py | 270 -
.../core/llm/tool_permissions/__init__.py | 95 +-
src/backend/core/llm/tool_permissions/hook.py | 71 +
.../core/llm/tool_permissions/middleware.py | 86 -
.../core/llm/tool_permissions/pending.py | 4 +-
.../core/llm/tool_permissions/service.py | 35 +-
src/backend/core/llm/tools/__init__.py | 9 +-
src/backend/core/llm/tools/_common.py | 23 +-
src/backend/core/llm/tools/_tool_helpers.py | 208 +-
.../core/llm/tools/ask_user_question_tool.py | 11 +-
.../core/llm/tools/chat_history_tool.py | 51 +-
.../cloud_gateway.py} | 119 +-
src/backend/core/llm/tools/cloud_manifest.py | 55 +
.../__init__.py | 0
.../gate.py | 3 +-
.../pending.py | 13 +-
.../shared.py | 2 +-
.../state.py | 104 +-
.../core/llm/tools/data_context_tool.py | 96 -
src/backend/core/llm/tools/edit_tool.py | 25 +-
src/backend/core/llm/tools/glob_tool.py | 7 +-
src/backend/core/llm/tools/grep_tool.py | 12 +-
src/backend/core/llm/tools/job_tool.py | 383 +-
src/backend/core/llm/tools/management.py | 45 +
.../tools/ontology_toolkit.py} | 30 +-
src/backend/core/llm/tools/pin_tool.py | 46 +-
.../plan_update.py} | 19 +-
.../tools/plugin_activation.py} | 122 +-
.../llm/tools/project_instructions_tool.py | 30 +-
.../core/llm/tools/project_source_access.py | 6 +-
.../core/llm/tools/read_artifact_tool.py | 42 +-
src/backend/core/llm/tools/read_image_tool.py | 34 +-
src/backend/core/llm/tools/read_tool.py | 27 +-
.../core/llm/tools/sandbox_artifact_tools.py | 34 +-
src/backend/core/llm/tools/sandbox_tool.py | 75 +-
src/backend/core/llm/tools/skill_tool.py | 51 +-
src/backend/core/llm/tools/user_questions.py | 67 +-
src/backend/core/llm/tools/write_tool.py | 30 +-
src/backend/core/memory/access.py | 48 +-
src/backend/core/memory/audit.py | 166 +-
src/backend/core/memory/backend.py | 96 +-
src/backend/core/memory/context.py | 2 +-
src/backend/core/memory/edition.py | 4 +
src/backend/core/memory/effect_lane.py | 75 +-
src/backend/core/memory/executor.py | 5 +-
src/backend/core/memory/extractors/_base.py | 49 +-
src/backend/core/memory/extractors/graph.py | 2 +-
src/backend/core/memory/extractors/writers.py | 10 +-
src/backend/core/memory/graph.py | 345 +-
src/backend/core/memory/graph_contract.py | 122 +
src/backend/core/memory/graph_state.py | 141 +
src/backend/core/memory/outbox/enqueue.py | 3 +-
src/backend/core/memory/outbox/jobs.py | 35 +-
src/backend/core/memory/outbox/leases.py | 6 +-
src/backend/core/memory/outbox/settlement.py | 18 +-
src/backend/core/memory/outbox/worker.py | 71 +-
src/backend/core/memory/pipeline.py | 31 +-
src/backend/core/memory/preview.py | 5 +-
src/backend/core/memory/procedures.py | 3 +-
src/backend/core/memory/queries.py | 3 +-
src/backend/core/memory/sanitizer.py | 39 +-
src/backend/core/memory/service.py | 9 +-
src/backend/core/memory/session_snapshot.py | 2 +-
.../settings.py} | 2 +-
src/backend/core/memory/settlement_port.py | 33 +
src/backend/core/memory/ttl_sweeper.py | 2 +-
src/backend/core/memory/weights.py | 23 +-
src/backend/core/models/__init__.py | 1 +
src/backend/core/models/admission.py | 21 +
.../core/models/api_protocol_backfill.py | 106 +
.../core/{llm => models}/chat_models.py | 47 +-
.../core/models/completions/__init__.py | 1 +
.../{llm => models/completions}/classifier.py | 25 +-
.../completions}/single_turn.py | 10 +-
.../{llm => models/completions}/summarizer.py | 25 +-
.../model_config.py => models/config.py} | 205 +-
src/backend/core/models/constants.py | 5 +
.../context_window.py} | 2 +-
src/backend/core/models/embedding.py | 123 +
src/backend/core/models/embedding_state.py | 20 +
src/backend/core/{llm => models}/failover.py | 36 +-
.../import_validation.py} | 4 +-
src/backend/core/models/providers/__init__.py | 3 +
.../providers/_image_tokens.py | 2 +-
.../providers/context_probe.py | 2 +-
.../providers/openai_compat.py | 20 +-
.../providers/protocol_probe.py | 0
.../providers/reasoning_probe.py | 6 +-
.../{llm => models}/providers/registry.py | 2 +-
.../{llm => models}/providers/stream_retry.py | 4 +-
.../providers/vendor_models.py | 12 +-
.../core/{llm => models}/reasoning_effort.py | 0
.../core/{llm => models}/reasoning_replay.py | 0
.../repository.py} | 4 +-
src/backend/core/models/reranking.py | 63 +
.../responses.py} | 97 +-
.../selection.py} | 46 +-
src/backend/core/models/selection_hook.py | 20 +
.../{llm => models}/tool_call_identity.py | 0
.../core/{llm => models}/tool_result_media.py | 63 +-
.../{llm/model_usage.py => models/usage.py} | 4 +-
src/backend/core/models/vision/__init__.py | 34 +
.../core/{ => models}/vision/attachments.py | 18 +-
.../core/{ => models}/vision/prompt.py | 2 +-
.../core/{ => models}/vision/provider.py | 12 +-
.../core/{ => models}/vision/render.py | 4 +-
.../core/{ => models}/vision/schema.py | 0
.../core/{ => models}/vision/service.py | 14 +-
src/backend/core/myspace/duplicate_cleanup.py | 63 -
src/backend/core/ontology/__init__.py | 3 +-
src/backend/core/ontology/audit_events.py | 71 +
src/backend/core/ontology/build_validator.py | 6 +-
.../evolution.py} | 115 +-
src/backend/core/ontology/host.py | 13 +
src/backend/core/ontology/manual_review.py | 85 +
.../core/ontology/output_review_state.py | 60 +
.../ontology_policy.py => ontology/policy.py} | 4 +-
.../core/ontology/review_operations.py | 102 +
src/backend/core/ontology/review_prompt.py | 99 +
src/backend/core/ontology/reviewer.py | 319 ++
src/backend/core/ontology/reviewer_state.py | 19 +
src/backend/core/ontology/runtime_gates.py | 157 +
src/backend/core/ontology/runtime_pack.py | 357 ++
src/backend/core/ontology/runtime_policy.py | 158 +
.../service.py} | 262 +-
src/backend/core/ontology/tool_hook.py | 183 +
.../core/ontology/validation_models.py | 33 +
src/backend/core/ontology/validator.py | 634 +--
src/backend/core/plugins/README.md | 28 -
.../core/plugins/management/__init__.py | 80 -
.../core/plugins/resources/application.py | 152 -
.../plugins/resources/application_cleanup.py | 48 -
.../core/plugins/resources/deadlines.py | 125 -
src/backend/core/plugins/resources/owners.py | 112 -
src/backend/core/plugins/resources/prewarm.py | 142 -
.../core/plugins/resources/reservation.py | 93 -
src/backend/core/plugins/resources/service.py | 347 --
src/backend/core/projects/__init__.py | 1 +
.../project_access.py => projects/access.py} | 2 +-
.../api.py} | 2 +-
.../project_service.py => projects/base.py} | 87 +-
src/backend/core/projects/edition.py | 8 +
.../environment}/__init__.py | 0
.../environment}/config.py | 0
.../environment}/dependencies.py | 2 +-
.../environment/service.py} | 12 +-
.../environment_resources.py} | 22 +-
.../files.py} | 14 +-
.../instructions.py} | 20 +-
.../local_workspace.py} | 2 +-
.../personal_scope.py} | 8 +-
src/backend/core/projects/personal_summary.py | 44 +
.../personal_transfer.py} | 0
src/backend/core/projects/project_names.py | 27 +
.../project_source.py => projects/source.py} | 20 +-
.../user_folders.py} | 23 +-
src/backend/core/runs/__init__.py | 0
.../runs/api_recovery.py} | 6 +-
.../run_binding.py => runs/binding.py} | 105 +-
src/backend/core/runs/clock.py | 55 +
src/backend/core/runs/events/__init__.py | 0
src/backend/core/runs/events/projection.py | 13 +
.../runs/events/store.py} | 88 +-
src/backend/core/runs/events/subscription.py | 81 +
src/backend/core/runs/events/writer.py | 197 +
src/backend/core/runs/host.py | 41 +
.../run_journal.py => runs/journal.py} | 32 +-
.../journal_domain.py} | 0
.../journal_lifecycle.py} | 20 +-
.../journal_recovery.py} | 9 +-
src/backend/core/runs/ownership.py | 276 +
src/backend/core/runs/runtime.py | 296 ++
src/backend/core/runs/settings.py | 98 +
src/backend/core/runs/tasks.py | 23 +
.../runs}/tool_effect_recovery.py | 99 +-
.../core/runs/tool_effects/__init__.py | 1 +
.../core/runs/tool_effects/arguments.py | 65 +
src/backend/core/runs/tool_effects/errors.py | 43 +
src/backend/core/runs/tool_effects/gateway.py | 256 +
src/backend/core/runs/tool_effects/hooks.py | 14 +
src/backend/core/runs/tool_effects/host.py | 28 +
.../runs/tool_effects/intent_transactions.py | 342 ++
.../runs/tool_effects/invocation_claims.py | 123 +
src/backend/core/runs/tool_effects/journal.py | 223 +
.../runs/tool_effects/journal_constants.py | 32 +
.../core/runs/tool_effects/recovery.py | 176 +
.../core/runs/tool_effects/recovery_claims.py | 166 +
.../core/runs/tool_effects/registry.py | 95 +
.../runs/tool_effects/result_transactions.py | 285 +
.../core/runs/tool_effects/retention.py | 154 +
src/backend/core/runs/tool_effects/types.py | 74 +
src/backend/core/runs/watchdog.py | 322 ++
src/backend/core/sandbox/commands/__init__.py | 1 +
.../core/sandbox/commands/completion_hook.py | 44 +
.../{services => sandbox}/commands/context.py | 10 +-
.../commands/delivery.py | 13 +-
.../sandbox}/commands/events.py | 15 +-
.../sandbox}/commands/notifications.py | 27 +-
src/backend/core/sandbox/commands/runtime.py | 449 ++
.../{services => sandbox}/commands/store.py | 13 +-
.../sandbox}/commands/subscription.py | 44 +-
.../core/sandbox/credential_dispatch.py | 30 +
src/backend/core/sandbox/factory.py | 33 +-
.../core/sandbox/oslayer/windows_runtime.py | 36 +-
.../permissions}/account.py | 4 +-
.../permissions/approval.py} | 6 +-
.../permissions/dispatch_models.py} | 4 +-
.../core/sandbox/permissions/os_gate.py | 2 +-
.../permissions/tool_specs.py} | 20 +-
.../core/sandbox/permissions/vocabulary.py | 26 +
src/backend/core/sandbox/processes/cloud.py | 28 +-
.../core/sandbox/processes/completion.py | 10 +-
.../core/sandbox/processes/dispatch.py | 2 +-
.../core/sandbox/processes/interactive.py | 3 +-
.../core/sandbox/providers/account_env.py | 27 +-
src/backend/core/sandbox/providers/edition.py | 2 +-
.../providers/opensandbox/account_volumes.py | 292 --
.../sandbox/providers/opensandbox/admin.py | 5 +-
.../sandbox/providers/opensandbox/adoption.py | 242 +-
.../sandbox/providers/opensandbox/binding.py | 2 +-
.../sandbox/providers/opensandbox/creation.py | 74 +-
.../providers/opensandbox/execution.py | 7 +-
.../providers/opensandbox/lifecycle.py | 4 +-
.../sandbox/providers/opensandbox/models.py | 9 +-
.../sandbox/providers/opensandbox/pool.py | 3 +-
.../sandbox/providers/opensandbox/provider.py | 59 +-
.../providers/opensandbox/reclamation.py | 4 +-
.../sandbox/providers/opensandbox/runtime.py | 25 +-
.../providers/opensandbox/snapshots.py | 44 +-
.../sandbox/providers/opensandbox/tracking.py | 31 +
.../providers/opensandbox/user_pool.py | 6 +-
.../sandbox/providers/opensandbox/volumes.py | 85 +-
.../providers/script_runner/provider.py | 44 +-
.../core/sandbox/runner/native_group.py | 28 +-
src/backend/core/sandbox/runner/server.py | 2 +-
src/backend/core/sandbox/sessions/registry.py | 21 +-
src/backend/core/sandbox/sessions/router.py | 15 +-
.../core/sandbox/workspace/container.py | 15 +-
.../core/sandbox/workspace/credentials.py | 141 -
src/backend/core/sandbox/workspace/desktop.py | 2 +-
.../core/sandbox/workspace/identity.py | 2 +-
.../workspace/resolution.py} | 14 +-
src/backend/core/services/__init__.py | 19 -
.../core/services/agent_market_categories.py | 77 -
.../core/services/agent_market_service.py | 940 ----
.../core/services/api_protocol_backfill.py | 94 -
src/backend/core/services/artifact_edition.py | 66 -
.../core/services/automation_tool_routing.py | 106 -
src/backend/core/services/chat_service.py | 1301 -----
.../core/services/commands/__init__.py | 1 -
.../core/services/desktop_cloud_bridge.py | 960 ----
.../core/services/edition_system_config.py | 6 -
.../core/services/edition_workspace.py | 52 -
src/backend/core/services/kb_edition.py | 12 -
src/backend/core/services/lark_service.py | 752 ---
src/backend/core/services/log_service.py | 649 ---
.../core/services/marketplace_service.py | 1400 -----
.../core/services/mcp_marketplace_service.py | 1772 -------
.../core/services/mcp_oauth_service.py | 529 --
.../core/services/mcp_tool_metadata.py | 19 -
.../core/services/mcp_tool_schema_backfill.py | 110 -
.../core/services/prompt_version_service.py | 889 ----
.../services/session_operations/admission.py | 262 -
.../core/services/skill_draft_service.py | 192 -
src/backend/core/services/system_config.py | 655 ---
.../core/services/tool_effect_ledger.py | 1808 -------
src/backend/core/services/user_agent_base.py | 854 ---
src/backend/core/spaces/edition_access.py | 17 -
src/backend/core/spaces/edition_revocation.py | 3 -
src/backend/core/vision/__init__.py | 45 -
src/backend/extensions/README.md | 2 +-
.../extensions/mcp_servers/_packaging.py | 6 +-
src/backend/extensions/mcp_servers/_serve.py | 28 +-
.../mcp_servers/agent_manager_mcp/impl.py | 48 +-
.../mcp_servers/agent_manager_mcp/server.py | 4 +-
.../mcp_servers/automation_task_mcp/impl.py | 18 +-
.../mcp_servers/automation_task_mcp/server.py | 16 +-
.../mcp_servers/browser_runtime_mcp/server.py | 8 +-
.../internet_search_mcp/_selftest.py | 2 +-
.../mcp_servers/internet_search_mcp/impl.py | 4 +-
.../mcp_servers/internet_search_mcp/server.py | 35 +-
.../mcp_servers/management_registration.py | 8 +-
.../mcp_servers/plugin_manager_mcp/impl.py | 11 +-
.../mcp_servers/plugin_manager_mcp/server.py | 4 +-
.../retrieve_dataset_content_mcp/impl.py | 18 +-
.../local_impl.py | 25 +-
.../retrieve_dataset_content_mcp/server.py | 47 +-
.../retrieve_dataset_content_mcp/wiki_impl.py | 12 +-
.../mcp_servers/site_publish_mcp/server.py | 6 +-
.../mcp_servers/skill_manager_mcp/impl.py | 21 +-
.../mcp_servers/skill_manager_mcp/server.py | 4 +-
.../space_files_mcp/permissions.py | 8 +-
.../mcp_servers/space_files_mcp/server.py | 10 +-
.../mcp_servers/web_fetch_mcp/server.py | 4 +-
.../marketplace/agent-manager/mcp.json | 2 +-
.../plugins/marketplace/automation/mcp.json | 2 +-
.../marketplace/plugin-manager/mcp.json | 2 +-
.../plugin-creator/scripts/validate_plugin.py | 2 +-
.../plugins/marketplace/sites/mcp.json | 2 +-
.../marketplace/skill-manager/mcp.json | 2 +-
.../skill-creator/references/skill-anatomy.md | 2 +-
.../skill-creator/scripts/quick_validate.py | 2 +-
.../skills/marketplace/data-mask/config.yaml | 166 +-
src/backend/features/__init__.py | 157 +
src/backend/features/automation/README.md | 15 +
src/backend/features/automation/__init__.py | 1 +
src/backend/features/automation/admission.py | 80 +
.../features/automation/api/__init__.py | 1 +
.../automation/api}/automations.py | 27 +-
.../automation}/delivery_targets.py | 19 +-
.../automation/execution.py} | 4 +-
src/backend/features/automation/extension.py | 28 +
.../automation/loops.py} | 5 +-
.../automation/models.py} | 59 +-
.../automation/notifications.py} | 0
.../automation/remote_effect.py} | 3 +-
src/backend/features/automation/runs.py | 66 +
src/backend/features/automation/scheduling.py | 23 +
src/backend/features/automation/schema.py | 59 +
.../automation/service.py} | 155 +-
.../features/automation/tool_routing.py | 116 +
src/backend/features/desktop/README.md | 11 +
src/backend/features/desktop/__init__.py | 1 +
src/backend/features/desktop/api/__init__.py | 1 +
.../features/desktop/api/capability_auth.py | 48 +
.../features/desktop/api/capability_bridge.py | 92 +
.../desktop/api/capability_dependencies.py | 107 +
.../desktop/api/capability_manifests.py | 145 +
.../desktop/api/capability_mcp_gateway.py | 79 +
.../desktop/api/capability_model_gateway.py | 89 +
.../features/desktop/api/capability_tools.py | 192 +
.../desktop/api/capability_workcopies.py | 38 +
.../v1 => features/desktop/api}/desktop.py | 2 +-
.../desktop/api}/desktop_capabilities.py | 104 +-
.../desktop/api/desktop_capability.py | 12 +
.../desktop/api/desktop_capability_state.py | 52 +
.../desktop/api}/desktop_login.py | 2 +-
.../desktop/api}/desktop_observability.py | 8 +-
.../desktop/api}/device_link.py | 6 +-
.../v1 => features/desktop/api}/local.py | 20 +-
.../desktop/api}/local_file_actions.py | 4 +-
src/backend/features/desktop/auth/__init__.py | 1 +
.../desktop/auth/bridge.py} | 13 +-
.../desktop/auth/login_store.py} | 0
.../desktop/auth/ticket_store.py} | 0
.../features/desktop/bridge/__init__.py | 1 +
.../desktop/bridge/bundles.py} | 42 +-
src/backend/features/desktop/bridge/client.py | 214 +
.../features/desktop/bridge/client_state.py | 48 +
.../features/desktop/bridge/gateway.py | 225 +
.../features/desktop/bridge/identity.py | 254 +
.../desktop/bridge/managed_connectors.py | 144 +
.../desktop/bridge/model_credentials.py} | 25 +-
.../desktop/bridge/release_events.py} | 0
.../desktop/bridge/resource_plugins.py} | 19 +-
.../desktop/bridge/skills.py} | 47 +-
src/backend/features/desktop/bridge/status.py | 113 +
.../features/desktop/capabilities/__init__.py | 16 +
.../desktop}/capabilities/agents.py | 17 +-
.../desktop}/capabilities/change_merge.py | 10 +-
.../desktop}/capabilities/change_signal.py | 0
.../desktop}/capabilities/change_sync.py | 60 +-
.../desktop}/capabilities/connectors.py | 12 +-
.../desktop}/capabilities/credentials.py | 0
.../desktop}/capabilities/dependency.py | 28 +-
.../capabilities/dependency_declarations.py | 0
.../desktop}/capabilities/device_catalog.py | 90 +-
.../capabilities/device_plugin_catalog.py | 69 +-
.../desktop}/capabilities/discovery.py | 6 +-
.../desktop}/capabilities/invocation.py | 34 +-
.../capabilities/local_plugin_runtime.py | 21 +-
.../desktop}/capabilities/local_projection.py | 27 +-
.../capabilities/management_context.py | 17 +-
.../desktop}/capabilities/manifest_order.py | 2 +-
.../desktop}/capabilities/mcp_json.py | 8 +-
.../desktop}/capabilities/migration.py | 25 +-
.../desktop}/capabilities/paths.py | 31 +-
.../desktop}/capabilities/plugins.py | 47 +-
.../desktop}/capabilities/preparation.py | 28 +-
.../desktop}/capabilities/readiness.py | 24 +-
.../desktop}/capabilities/registry.py | 115 +-
.../desktop/capabilities/registry_types.py | 79 +
.../desktop}/capabilities/resolver.py | 4 +-
.../desktop/capabilities/runtime/__init__.py | 37 +
.../capabilities/runtime/connectors.py | 23 +-
.../capabilities/runtime/dependencies.py | 18 +-
.../capabilities/runtime/preparation.py | 25 +-
.../desktop}/capabilities/runtime/state.py | 52 +-
.../capabilities/session_availability.py | 0
.../desktop/capabilities/skill_account.py | 54 +
.../desktop/capabilities/skill_candidates.py | 140 +
.../desktop/capabilities/skill_hashes.py | 101 +
.../desktop/capabilities/skill_paths.py | 29 +
.../desktop/capabilities/skill_publication.py | 88 +
.../desktop/capabilities/skill_views.py | 62 +
.../features/desktop/capabilities/skills.py | 161 +
.../desktop/capabilities/skills_state.py | 35 +
.../desktop}/capabilities/store.py | 17 +-
.../desktop}/capabilities/view.py | 4 +-
.../desktop}/capabilities/warmup.py | 17 +-
.../features/desktop/capability/__init__.py | 1 +
.../desktop/capability/configs.py} | 6 +-
.../desktop/capability/credentials.py} | 2 +-
.../desktop/capability/entities.py} | 24 +-
.../desktop/capability/mcp.py} | 40 +-
.../desktop/capability/models.py} | 16 +-
.../desktop/capability/revision_provider.py | 22 +
.../desktop/capability/security.py} | 9 +-
.../desktop/capability/service.py} | 43 +-
.../desktop/capability/skills.py} | 18 +-
.../desktop/capability/sync_check.py} | 29 +-
.../desktop/capability/workcopies.py} | 45 +-
src/backend/features/desktop/edition.py | 23 +
src/backend/features/desktop/extension.py | 4 +
.../features/desktop/gateway/__init__.py | 1 +
.../desktop/gateway/observability.py} | 4 +-
.../features/desktop/local/__init__.py | 1 +
.../desktop/local/file_actions.py} | 2 +-
.../desktop/local/grants.py} | 0
.../desktop/local/management_migration.py} | 16 +-
.../desktop/local/manager_execution.py | 51 +
.../desktop/local/market_install.py} | 14 +-
.../desktop/local}/native_app_icons.py | 0
.../desktop/local/plugins}/__init__.py | 0
.../desktop/local/plugins}/assets.py | 10 +-
.../local/plugins}/legacy_migration.py | 27 +-
.../desktop/local/plugins}/service.py | 66 +-
.../desktop/local/plugins/sidecars.py | 33 +
.../desktop/local/publication_provider.py | 101 +
.../features/desktop/local/skills/__init__.py | 1 +
.../desktop/local/skills/editor.py} | 28 +-
.../desktop/local/skills/service.py} | 36 +-
.../desktop/local/skills/upload.py} | 34 +-
.../desktop/local/snapshots.py} | 0
.../desktop/models.py} | 69 +-
.../desktop/observability/__init__.py | 1 +
.../desktop/observability/context.py} | 8 +-
.../desktop/observability/delivery.py} | 11 +-
.../desktop/observability/fields.py} | 2 +-
.../desktop/observability/projection.py} | 8 +-
.../desktop/observability/sync.py} | 13 +-
.../desktop/plugin_resolution.py} | 39 +-
.../desktop/run_capabilities/__init__.py | 1 +
.../desktop/run_capabilities/preparation.py} | 22 +-
.../desktop/run_capabilities/provider.py | 118 +
.../desktop/skill_backend.py} | 20 +-
src/backend/features/evaluation/README.md | 7 +
src/backend/features/evaluation/__init__.py | 1 +
.../evaluation/admission.py} | 2 +-
.../features/evaluation/api/__init__.py | 1 +
.../evaluation/api}/evaluation_sandboxes.py | 2 +-
src/backend/features/evaluation/extension.py | 23 +
src/backend/features/evaluation/models.py | 1 +
src/backend/features/evaluation/policy.py | 27 +
.../features/evaluation/runtime/__init__.py | 1 +
.../evaluation/runtime/paths.py} | 0
.../evaluation/runtime/process_tools.py} | 7 +-
.../evaluation/runtime/runtime.py} | 0
.../evaluation/runtime/tools.py} | 59 +-
.../evaluation/sandbox_provider}/__init__.py | 0
.../evaluation/sandbox_provider}/binding.py | 0
.../sandbox_provider}/process_guard.py | 0
.../evaluation/sandbox_provider}/provider.py | 4 +-
.../evaluation/sandboxes.py} | 24 +-
src/backend/features/hosted_apps/README.md | 9 +
src/backend/features/hosted_apps/__init__.py | 1 +
.../features/hosted_apps/api/__init__.py | 1 +
.../api}/application_transports.py | 88 +-
.../hosted_apps/api}/applications.py | 65 +-
.../hosted_apps/collections.py} | 8 +-
.../hosted_apps/config.py} | 2 +-
.../hosted_apps/data.py} | 20 +-
.../hosted_apps/deployments.py} | 13 +-
.../hosted_apps/export.py} | 6 +-
src/backend/features/hosted_apps/extension.py | 17 +
.../hosted_apps/history.py} | 8 +-
.../hosted_apps/hosting_setup.py} | 4 +-
.../hosted_apps/industry.py} | 9 +-
.../hosted_apps/lifecycle.py} | 14 +-
.../hosted_apps/mcp.py} | 34 +-
src/backend/features/hosted_apps/models.py | 5 +
.../hosted_apps/operations.py} | 21 +-
.../hosted_apps/projects.py} | 10 +-
.../hosted_apps/publication.py} | 19 +-
.../hosted_apps/relational.py} | 2 +-
.../hosted_apps/resources/__init__.py | 1 +
.../hosted_apps/resources/admission.py | 159 +
.../features/hosted_apps/resources/cleanup.py | 71 +
.../hosted_apps/schema.py} | 0
.../hosted_apps/sql_policy.py} | 2 +-
.../hosted_apps/store.py} | 10 +-
src/backend/features/installed.py | 3 +
src/backend/features/knowledge/README.md | 11 +
.../kb => features/knowledge}/__init__.py | 2 +-
.../features/knowledge/api/__init__.py | 1 +
src/backend/features/knowledge/api/kb.py | 7 +
.../features/knowledge/api/kb_assets.py | 147 +
.../features/knowledge/api/kb_chunks.py | 175 +
.../features/knowledge/api/kb_documents.py | 254 +
.../features/knowledge/api/kb_helpers.py | 105 +
.../features/knowledge/api/kb_mutations.py | 117 +
.../features/knowledge/api/kb_spaces.py | 116 +
.../features/knowledge/api/kb_state.py | 8 +
.../v1 => features/knowledge/api}/kb_wiki.py | 24 +-
.../features/knowledge/asset_captions.py | 181 +
.../features/knowledge/asset_indexing.py | 88 +
.../features/knowledge/catalog_options.py | 7 +
.../knowledge/disabled_provider.py} | 4 +-
.../knowledge/disabled_retrieval.py} | 0
.../knowledge}/document_status.py | 0
src/backend/features/knowledge/edition.py | 33 +
src/backend/features/knowledge/enrichment.py | 126 +
src/backend/features/knowledge/extension.py | 19 +
.../kb => features/knowledge}/index_queue.py | 4 +-
.../kb => features/knowledge}/index_worker.py | 8 +-
src/backend/features/knowledge/kb_assets.py | 231 +
.../features/knowledge/kb_assets_state.py | 31 +
src/backend/features/knowledge/kb_parser.py | 116 +
.../features/knowledge/kb_parser_state.py | 34 +
src/backend/features/knowledge/kb_vector.py | 149 +
.../features/knowledge/kb_vector_state.py | 27 +
src/backend/features/knowledge/models.py | 377 ++
.../features/knowledge/parser_chunks.py | 278 +
.../features/knowledge/parser_extract.py | 302 ++
.../features/knowledge/parser_semantic.py | 130 +
.../features/knowledge/parser_tokens.py | 26 +
.../features/knowledge/parser_types.py | 43 +
.../knowledge/personal_permissions.py} | 2 +-
.../knowledge/personal_request_models.py} | 0
.../knowledge/processing.py} | 193 +-
.../knowledge/repository.py} | 4 +-
.../knowledge/service.py} | 34 +-
.../features/knowledge/vector_client.py | 166 +
.../features/knowledge/vector_fields.py | 16 +
.../features/knowledge/vector_indexing.py | 204 +
.../knowledge}/wiki/__init__.py | 2 +-
.../kb => features/knowledge}/wiki/cite.py | 10 +-
.../kb => features/knowledge}/wiki/config.py | 2 +-
.../kb => features/knowledge}/wiki/dedup.py | 16 +-
.../kb => features/knowledge}/wiki/extract.py | 12 +-
.../knowledge}/wiki/finalize.py | 17 +-
.../kb => features/knowledge}/wiki/jobs.py | 4 +-
.../kb => features/knowledge}/wiki/linkify.py | 0
.../kb => features/knowledge}/wiki/llm.py | 10 +-
.../features/knowledge/wiki/local_graph.py | 156 +
.../knowledge/wiki/local_projection.py | 51 +
.../features/knowledge/wiki/local_provider.py | 271 +
.../knowledge/wiki/local_provider_state.py | 15 +
.../knowledge}/wiki/pipeline.py | 26 +-
.../kb => features/knowledge}/wiki/prompts.py | 0
.../kb => features/knowledge}/wiki/reduce.py | 17 +-
.../kb => features/knowledge}/wiki/slug.py | 0
.../kb => features/knowledge}/wiki/store.py | 8 +-
.../knowledge}/wiki/taxonomy.py | 14 +-
.../kb => features/knowledge}/wiki/worker.py | 12 +-
.../kb => features/knowledge}/wiki_router.py | 16 +-
src/backend/features/marketplace/README.md | 15 +
src/backend/features/marketplace/__init__.py | 1 +
.../features/marketplace/agent_catalog.py | 119 +
.../features/marketplace/agent_categories.py | 38 +
.../marketplace/agent_installation.py | 240 +
.../features/marketplace/agent_metadata.py | 101 +
.../features/marketplace/agent_publishing.py | 165 +
.../marketplace/agent_submission_data.py | 155 +
.../features/marketplace/agent_submissions.py | 160 +
.../marketplace/agent_versions.py} | 31 +-
.../features/marketplace/agents_state.py | 14 +
.../features/marketplace/api/__init__.py | 1 +
.../marketplace/api}/agent_marketplace.py | 28 +-
.../marketplace/api}/catalog_ownership.py | 9 +-
.../marketplace/api}/marketplace.py | 27 +-
.../marketplace/api}/mcp_marketplace.py | 59 +-
.../features/marketplace/core_catalog.py | 108 +
src/backend/features/marketplace/edition.py | 10 +
src/backend/features/marketplace/extension.py | 4 +
.../marketplace/mcp_administration.py | 291 ++
.../marketplace/mcp_catalog.py} | 0
.../features/marketplace/mcp_credentials.py | 130 +
.../features/marketplace/mcp_curated.py | 127 +
.../features/marketplace/mcp_installation.py | 267 +
.../features/marketplace/mcp_listing.py | 142 +
.../marketplace/mcp_monitor.py} | 41 +-
.../features/marketplace/mcp_publishing.py | 193 +
.../features/marketplace/mcp_revalidation.py | 146 +
src/backend/features/marketplace/mcp_state.py | 17 +
.../features/marketplace/mcp_submissions.py | 319 ++
.../features/marketplace/mcp_validation.py | 166 +
.../features/marketplace/mcp_versions.py | 221 +
src/backend/features/marketplace/models.py | 388 ++
.../features/marketplace/package_source.py | 35 +
.../marketplace/personal_listing.py} | 2 +-
.../marketplace/personal_visibility.py} | 0
.../marketplace/plugin_market.py} | 63 +-
.../features/marketplace/skill_archives.py | 49 +
.../features/marketplace/skill_catalog.py | 184 +
.../marketplace/skill_installation.py | 199 +
.../features/marketplace/skill_metadata.py | 174 +
.../features/marketplace/skill_publishing.py | 231 +
.../marketplace/skill_submission_data.py | 80 +
.../features/marketplace/skill_submissions.py | 190 +
.../marketplace/skill_versions.py} | 70 +-
.../features/marketplace/skills_state.py | 56 +
.../features/marketplace/versions/__init__.py | 1 +
.../marketplace/versions/archive.py} | 0
.../marketplace/versions/config.py} | 4 +-
.../marketplace/versions/sources.py} | 50 +-
.../marketplace/versions/store.py} | 37 +-
src/backend/features/sites/README.md | 11 +
src/backend/features/sites/__init__.py | 1 +
src/backend/features/sites/api/__init__.py | 1 +
.../sites/api}/internal_site_auth.py | 0
.../sites/api}/internal_sites.py | 39 +-
.../sites/api}/local_site_sources.py | 3 +-
.../sites/api}/site_gate.py | 0
.../routes/v1 => features/sites/api}/sites.py | 11 +-
.../sites/api}/sites_serve.py | 71 +-
.../sites/builtin_upgrade.py} | 29 +-
.../sites/desktop_publish.py} | 64 +-
src/backend/features/sites/edition.py | 10 +
src/backend/features/sites/extension.py | 34 +
.../sites/listing.py} | 8 +-
.../sites/local_sources.py} | 28 +-
.../sites/management.py} | 14 +-
.../site.py => features/sites/models.py} | 2 +-
.../sites/packaging.py} | 17 +-
.../sites/password.py} | 0
.../sites/personal_access.py} | 2 +-
.../sites/personal_scope.py} | 0
.../sites/rate_limit.py} | 0
.../site.py => features/sites/repository.py} | 2 +-
.../sites/service.py} | 17 +-
src/backend/features/sites/tools/__init__.py | 1 +
.../sites/tools/design_picker.py} | 46 +-
src/backend/features/workbench/README.md | 9 +
.../workbench}/__init__.py | 0
.../workbench}/activity.py | 6 +-
.../features/workbench/api/__init__.py | 1 +
.../workbench/api}/plugin_ui.py | 16 +-
.../workbench/api}/workbench.py | 67 +-
.../workbench/api}/workbench_activity.py | 38 +-
.../api}/workbench_application_config.py | 14 +-
.../workbench/api}/workbench_auth.py | 21 +-
.../workbench/api}/workbench_capabilities.py | 104 +-
.../workbench/api}/workbench_diagnostics.py | 6 +-
.../workbench/api}/workbench_events.py | 9 +-
.../workbench/api}/workbench_preferences.py | 2 +-
.../workbench/api}/workbench_resources.py | 47 +-
.../api}/workbench_revision_leases.py | 2 +-
.../workbench/api}/workbench_state.py | 2 +-
.../workbench/api}/workbench_upgrades.py | 2 +-
.../workbench/api}/workbench_workspace.py | 12 +-
.../workbench}/application_config.py | 14 +-
.../workbench}/application_config_bindings.py | 26 +-
.../workbench}/background.py | 124 +-
.../workbench}/capabilities.py | 14 +-
.../workbench}/catalog.py | 6 +-
.../workbench}/coordination.py | 0
.../workbench}/diagnostic_probe.py | 2 +-
.../workbench}/diagnostic_resources.py | 0
.../workbench}/diagnostics.py | 2 +-
.../workbench}/directory_revision.py | 2 +-
src/backend/features/workbench/edition.py | 10 +
src/backend/features/workbench/extension.py | 4 +
.../workbench}/external.py | 12 +-
.../workbench}/external_auth.py | 4 +-
.../workbench}/external_boundary.py | 2 +-
.../workbench}/external_embed.py | 2 +-
.../workbench}/installation_binding.py | 24 +-
.../workbench}/local_plugin_upgrades.py | 32 +-
.../workbench}/migration.py | 14 +-
.../workbench}/migration_compat.py | 25 +-
.../workbench}/migration_plan.py | 12 +-
.../workbench}/migration_snapshot.py | 10 +-
src/backend/features/workbench/models.py | 1 +
.../workbench}/origin.py | 0
.../features/workbench/package_source.py | 19 +
.../workbench}/plugin_candidate_storage.py | 33 +-
.../workbench}/plugin_catalog.py | 10 +-
.../workbench}/plugin_revisions.py | 36 +-
.../workbench}/plugin_upgrades.py | 26 +-
.../workbench}/policy.py | 6 +-
.../workbench}/preferences.py | 4 +-
.../workbench}/protocol.py | 0
.../features/workbench/resource_deadlines.py | 122 +
.../features/workbench/resource_owner.py | 65 +
.../workbench}/revocation.py | 54 +-
.../workbench}/revocation_sweep.py | 30 +-
.../schemas/application.schema.json | 0
.../workbench}/schemas/host.json | 0
.../workbench}/schemas/module.schema.json | 0
.../workbench}/state.py | 8 +-
.../workbench}/state_recovery.py | 0
.../workbench}/ui/__init__.py | 0
.../workbench}/ui/contract.py | 2 +-
.../workbench}/ui/contributions.py | 20 +-
.../workbench}/ui/data_proxy.py | 12 +-
.../workbench}/ui/declarations.py | 2 +-
.../workbench}/ui/primitives.py | 2 +-
.../workbench}/ui/workspace_enabled.py | 0
.../workbench}/workspace_page.py | 4 +-
src/backend/hugagent_cli/schema.py | 20 +
src/backend/integrations/__init__.py | 8 +
src/backend/integrations/api/__init__.py | 1 +
.../v1 => integrations/api}/integrations.py | 49 +-
src/backend/integrations/channels/README.md | 9 +
.../channels/__init__.py | 9 +-
.../channels/adapters/__init__.py | 0
.../channels/adapters/dingtalk.py | 325 ++
.../channels/adapters/dingtalk_delivery.py | 213 +
.../channels/adapters/dingtalk_history.py | 361 ++
.../channels/adapters/dingtalk_settings.py | 3 +
.../integrations/channels/adapters/lark.py | 261 +
.../channels/adapters/lark_delivery.py | 152 +
.../channels/adapters/lark_messages.py | 317 ++
.../channels/adapters/lark_settings.py | 3 +
.../channels/adapters/wecom.py | 2 +-
.../channels/adapters/weixin.py | 2 +-
.../integrations/channels/api/__init__.py | 1 +
.../channels/api}/channel_desktop.py | 39 +-
.../channels/api}/channels.py | 46 +-
.../channels/desktop_worker.py} | 110 +-
.../integrations/channels/extension.py | 7 +
src/backend/integrations/channels/inbound.py | 368 ++
.../channels/inbound_attachments.py | 82 +
.../integrations/channels/inbound_delivery.py | 118 +
.../integrations/channels/inbound_history.py | 157 +
.../channels/inbound_observation.py | 168 +
.../integrations/channels/inbound_sessions.py | 131 +
.../integrations/channels/inbound_state.py | 46 +
.../channels/manager.py | 14 +-
.../channels/markdown.py | 0
src/backend/integrations/channels/models.py | 133 +
.../channels/outbound.py | 72 +-
.../channels/protocol.py | 0
.../channels/registry.py | 10 +-
.../channels/relay.py} | 4 +-
.../channels/relay_dispatch.py} | 98 +-
.../channels/relay_rpc.py} | 113 +-
.../channels/repository.py} | 2 +-
.../channels/service.py} | 242 +-
.../integrations/channels/tools/__init__.py | 1 +
.../channels/tools/attachment.py} | 163 +-
src/backend/integrations/dingtalk/README.md | 5 +
src/backend/integrations/dingtalk/__init__.py | 1 +
.../integrations/dingtalk/cube_credentials.py | 129 +
.../integrations/dingtalk/environment.py | 29 +
.../integrations/dingtalk/extension.py | 10 +
src/backend/integrations/dingtalk/models.py | 61 +
src/backend/integrations/dingtalk/mounts.py | 77 +
src/backend/integrations/dingtalk/parsing.py | 105 +
.../integrations/dingtalk/repository.py | 39 +
src/backend/integrations/dingtalk/sandbox.py | 15 +
.../dingtalk/service.py} | 164 +-
src/backend/integrations/dingtalk/storage.py | 34 +
src/backend/integrations/email/README.md | 5 +
src/backend/integrations/email/__init__.py | 1 +
.../integrations/email/cube_credentials.py | 53 +
src/backend/integrations/email/extension.py | 10 +
src/backend/integrations/email/models.py | 65 +
src/backend/integrations/email/mounts.py | 72 +
src/backend/integrations/email/repository.py | 37 +
src/backend/integrations/email/sandbox.py | 14 +
.../email/service.py} | 65 +-
src/backend/integrations/email/storage.py | 36 +
src/backend/integrations/lark/README.md | 5 +
src/backend/integrations/lark/__init__.py | 1 +
.../integrations/lark/configuration.py | 200 +
.../integrations/lark/cube_credentials.py | 90 +
src/backend/integrations/lark/extension.py | 10 +
src/backend/integrations/lark/login.py | 114 +
src/backend/integrations/lark/models.py | 67 +
src/backend/integrations/lark/mounts.py | 75 +
src/backend/integrations/lark/parsing.py | 141 +
src/backend/integrations/lark/process.py | 59 +
src/backend/integrations/lark/repository.py | 37 +
src/backend/integrations/lark/sandbox.py | 14 +
src/backend/integrations/lark/service.py | 189 +
src/backend/integrations/lark/state.py | 49 +
src/backend/integrations/lark/storage.py | 51 +
src/backend/integrations/yida/README.md | 5 +
src/backend/integrations/yida/__init__.py | 1 +
.../integrations/yida/cube_credentials.py | 152 +
.../yida/execution.py} | 4 +-
src/backend/integrations/yida/extension.py | 13 +
src/backend/integrations/yida/models.py | 1 +
src/backend/integrations/yida/mounts.py | 68 +
src/backend/integrations/yida/sandbox.py | 18 +
.../yida/service.py} | 6 +-
.../integrations/yida/skill_content.py | 12 +
src/backend/integrations/yida/storage.py | 52 +
src/backend/orchestration/__init__.py | 5 +-
.../orchestration/automations/__init__.py | 0
.../automation_artifacts.py | 3 +-
.../automations/automation_conversation.py | 396 ++
.../automations/automation_delivery.py | 62 +
.../automations/automation_dispatch.py | 306 ++
.../automations/automation_lease.py | 113 +
.../automations/automation_recovery.py | 39 +
.../automations/automation_scheduler.py | 136 +
.../orchestration/chat_run_executor.py | 4651 -----------------
src/backend/orchestration/citation_anchor.py | 551 --
.../orchestration/commands/__init__.py | 1 -
src/backend/orchestration/commands/runtime.py | 262 -
src/backend/orchestration/configuration.py | 6 +
.../orchestration/local_plugin_sidecars.py | 75 -
src/backend/orchestration/loops/__init__.py | 0
src/backend/orchestration/loops/admission.py | 122 +
.../{autonomous_loop.py => loops/driver.py} | 26 +-
.../{loop_planner.py => loops/planner.py} | 14 +-
.../orchestration/loops/presentation.py | 71 +
src/backend/orchestration/loops/recovery.py | 93 +
.../requirements.py} | 59 +-
.../loop_reviewer.py => loops/reviewer.py} | 161 +-
src/backend/orchestration/loops/run.py | 509 ++
.../orchestration/memory_integration.py | 576 --
src/backend/orchestration/pending.py | 36 +
src/backend/orchestration/plans/__init__.py | 0
src/backend/orchestration/plans/admission.py | 171 +
.../orchestration/plans/execute_run.py | 338 ++
.../orchestration/plans/generate_run.py | 278 +
.../{subagents/plan_mode.py => plans/mode.py} | 272 +-
.../schedulers/automation_conversation.py | 307 --
.../schedulers/automation_dispatch.py | 279 -
.../schedulers/automation_lease.py | 58 -
.../schedulers/automation_recovery.py | 45 -
.../schedulers/automation_scheduler.py | 330 --
src/backend/orchestration/streaming.py | 999 ----
.../orchestration/subagents/__init__.py | 2 -
.../subagents/ontology_reviewer.py | 656 ---
.../{schedulers => workers}/__init__.py | 0
.../{schedulers => workers}/_worker_base.py | 12 +-
.../workers}/edition_startup.py | 0
.../evolution_scheduler.py | 4 +-
.../memory_ttl_scheduler.py | 6 +-
.../persona_distill_worker.py | 6 +-
.../orchestration/workers/run_recovery.py | 37 +
src/backend/orchestration/workflow.py | 3463 ------------
src/backend/personal_agent/README.md | 2 +-
src/backend/personal_agent/access.py | 2 +-
.../personal_agent/api/conversation.py | 2 +-
src/backend/personal_agent/api/sessions.py | 11 +-
src/backend/personal_agent/background.py | 8 +-
.../personal_agent/channels/channel_bridge.py | 64 +-
.../personal_agent/computer/computer.py | 5 +-
.../personal_agent/computer/devices.py | 45 +-
.../personal_agent/computer/driver_install.py | 2 +-
.../personal_agent/computer/live_stream.py | 27 +-
.../personal_agent/computer/sandbox.py | 42 +-
.../computer/screenshot_pruning.py | 72 +-
.../conversation/conversations.py | 3 +-
.../personal_agent/conversation/front.py | 41 +-
.../personal_agent/delivery/delivery.py | 4 +-
.../personal_agent/device_link/README.md | 3 +
.../device_link/__init__.py | 2 +-
.../device_link/driver.py | 0
.../device_link/local_api.py | 0
.../device_link/runner.py | 6 +-
.../device_link/store.py | 0
src/backend/personal_agent/extension.py | 26 +-
.../personal_agent/identity/capabilities.py | 8 +-
src/backend/personal_agent/identity/config.py | 2 +-
.../personal_agent/identity/service.py | 2 +-
.../personal_agent/scheduling/schedules.py | 2 +-
.../personal_agent/sessions/session_auth.py | 43 +-
.../personal_agent/sessions/session_cloud.py | 89 +-
.../sessions/session_manager.py | 151 +-
.../personal_agent/sessions/session_tool.py | 37 +-
.../personal_agent/sessions/session_watch.py | 2 +-
.../personal_agent/tasks/attachments.py | 10 +-
src/backend/personal_agent/tasks/runtime.py | 90 +-
src/backend/personal_agent/tasks/service.py | 2 +-
src/backend/personal_agent/tasks/steering.py | 6 +-
.../personal_agent/tasks/task_threads.py | 6 +-
src/backend/personal_agent/tasks/worker.py | 182 +-
src/backend/personal_agent/tools/email.py | 5 +-
.../personal_agent/tools/front_tools.py | 52 +-
src/backend/personal_agent/tools/responses.py | 4 +-
src/backend/personal_agent/tools/tools.py | 49 +-
.../personal_agent/workspace/dream_plan.py | 4 +-
.../personal_agent/workspace/dreaming.py | 8 +-
.../personal_agent/workspace/prompting.py | 7 +-
src/backend/prompts/desktop_templates.py | 5 +-
src/backend/prompts/desktop_workspace.py | 6 +-
src/backend/prompts/kb_lite_section.py | 4 +-
src/backend/prompts/project_section.py | 7 +-
src/backend/prompts/prompt_runtime.py | 20 +-
src/backend/prompts/system_builder.py | 13 +-
src/backend/scripts/_loop_convergence_unit.py | 147 -
src/backend/scripts/_loop_db_resume_unit.py | 92 -
.../scripts/_loop_evaluator_route_unit.py | 88 -
src/backend/scripts/_loop_integration_test.py | 88 -
src/backend/scripts/_loop_llmeval_smoke.py | 48 -
src/backend/scripts/_loop_preflight.py | 73 -
src/backend/scripts/_loop_selfverify_smoke.py | 66 -
src/backend/scripts/_loop_smoke.py | 90 -
src/backend/scripts/_loop_test_edgebench.py | 99 -
src/backend/scripts/backend_entrypoint.sh | 4 +-
src/backend/scripts/desktop_cloud_wrapper.py | 76 -
src/backend/scripts/desktop_fixture_mcp.py | 50 -
src/backend/scripts/dual_mode_e2e.py | 947 ----
src/backend/scripts/edgebench/ann_judge.py | 136 -
.../scripts/edgebench/perm_opt_judge.py | 88 -
src/backend/scripts/edit_chat_message.py | 198 -
src/backend/scripts/evaluation_sandbox_e2e.py | 89 -
src/backend/scripts/export_content.py | 317 --
.../scripts/harness_fault_injection.py | 637 ---
src/backend/scripts/import_cherry_agents.py | 372 --
src/backend/scripts/import_content.py | 379 --
.../scripts/run_evolution_benchmark.py | 308 --
src/backend/scripts/run_evolution_llm_ab.py | 621 ---
src/backend/scripts/workbench_migration.py | 66 -
src/backend/tests/_capability_index.py | 11 +-
.../tests/api/api_routes_v1_selftest.py | 2 +-
src/backend/tests/api/catalog_api_selftest.py | 9 +-
src/backend/tests/api/chat_fork_fixtures.py | 6 +-
src/backend/tests/api/test_agent_api_keys.py | 9 +-
.../api/test_agent_response_admission.py | 100 +-
.../tests/api/test_agent_response_scope.py | 24 +-
src/backend/tests/api/test_agent_responses.py | 14 +-
.../test_api_key_run_lifecycle_postgres.py | 6 +-
src/backend/tests/api/test_archive_files.py | 6 +-
src/backend/tests/api/test_archive_safety.py | 4 +-
.../tests/api/test_artifacts_backfill_once.py | 4 +-
.../tests/api/test_cancel_continue_e2e.py | 34 +-
.../tests/api/test_catalog_event_loop.py | 27 +-
.../api/test_chat_connector_selection.py | 4 +-
.../api/test_chat_event_loop_performance.py | 8 +-
.../tests/api/test_chat_fork_derived_data.py | 5 +-
src/backend/tests/api/test_chat_forks.py | 13 +-
.../tests/api/test_chat_reference_api.py | 4 +-
.../tests/api/test_chat_rerun_options.py | 22 +-
.../api/test_chat_rerun_start_failure.py | 10 +-
.../tests/api/test_chat_run_steer_api.py | 9 +-
.../tests/api/test_chat_stream_sequencer.py | 61 +-
.../tests/api/test_chat_thinking_mode.py | 4 +-
.../tests/api/test_cross_turn_file_access.py | 10 +-
.../tests/api/test_deep_folder_repair.py | 2 +-
.../tests/api/test_evaluation_sandboxes.py | 2 +-
src/backend/tests/api/test_file_location.py | 2 +-
.../tests/api/test_file_open_browser_e2e.py | 4 +-
.../tests/api/test_loop_cancel_scope.py | 41 +
src/backend/tests/api/test_me_system.py | 2 +-
.../api/test_model_reasoning_settings.py | 2 +
.../tests/api/test_page_config_backfill.py | 6 +-
.../tests/api/test_plugin_delete_isolation.py | 4 +-
.../tests/api/test_project_environments.py | 7 +-
src/backend/tests/api/test_public_chats.py | 42 +-
.../auth/test_auth_dependencies_off_loop.py | 176 +
.../tests/auth/test_bearer_identity.py | 180 +
.../tests/browser_automation_server.py | 81 +-
.../test_credential_continuity.py | 189 -
.../ce_release/test_ce_api_key_lifecycle.py | 8 +-
.../ce_release/test_ce_history_display.py | 2 +-
.../ce_release/test_ce_release_regressions.py | 159 +-
.../tests/ce_release/test_ce_tool_media.py | 8 +-
.../test_desktop_capability_wiring.py | 6 +-
.../ce_release/test_personal_agent_ce.py | 12 +-
.../test_workbench_migration_snapshot.py | 6 +-
src/backend/tests/chat/test_display_bounds.py | 2 +-
.../tests/chat/test_quoted_follow_up.py | 93 +
src/backend/tests/chat/test_sidebar_order.py | 14 +-
.../chat/test_snapshot_transaction_cleanup.py | 23 +
.../tests/chat/test_user_questions_api.py | 35 +-
src/backend/tests/conftest.py | 48 +-
.../tests/content/artifacts_selftest.py | 4 +-
src/backend/tests/content/catalog_selftest.py | 5 +-
.../tests/content/content_blocks_selftest.py | 2 +-
.../tests/content/security_selftest.py | 28 +-
.../content/test_artifact_default_path.py | 24 +
.../content/test_artifact_record_store.py | 4 +-
.../content/test_artifact_storage_keys.py | 10 +-
src/backend/tests/content/test_extra_files.py | 42 +-
.../tests/content/test_file_preview.py | 72 +-
.../tests/content/test_local_file_location.py | 6 +-
.../content/test_office_preview_cache.py | 6 +-
.../test_storage_existence_is_strict.py | 6 +-
.../runs/test_automation_journal_guard.py | 89 +
src/backend/tests/db_support.py | 43 +
.../tests/e2e/_e2e_regression_smoke.py | 2 +-
src/backend/tests/e2e/agentscope_poc.py | 2 +-
src/backend/tests/e2e/core_external_server.py | 4 +-
src/backend/tests/e2e/core_refactor_server.py | 2 +-
src/backend/tests/e2e/model_tests.py | 4 +-
src/backend/tests/evolution/conftest.py | 2 +-
.../test_benchmark_and_activation.py | 24 +-
.../evolution/test_candidate_reasoning.py | 32 +-
.../evolution/test_cassette_and_replay.py | 4 +-
.../tests/evolution/test_compiler_v2.py | 73 +-
.../evolution/test_console_and_metrics.py | 8 +-
.../tests/evolution/test_credit_assigner.py | 2 +-
.../evolution/test_engines_end_to_end.py | 71 +-
.../tests/evolution/test_episode_assembler.py | 32 +-
.../test_experiments_and_federation.py | 4 +-
.../tests/evolution/test_exposure_and_ramp.py | 9 +-
.../evolution/test_fault_injection_gate.py | 4 +-
.../tests/evolution/test_mechanisms.py | 6 +-
.../tests/evolution/test_ontology_bridge.py | 2 +-
.../evolution/test_outcomes_and_backfill.py | 6 +-
.../tests/evolution/test_personal_approval.py | 61 +-
.../test_personal_cycle_and_prefs.py | 20 +-
.../evolution/test_policies_and_memory_ops.py | 67 +-
.../tests/evolution/test_promotion_chain.py | 98 +-
.../evolution/test_react_intervention.py | 29 +-
.../tests/evolution/test_readiness_gate.py | 18 +-
.../tests/evolution/test_realistic_corpus.py | 46 +-
.../evolution/test_release_state_machine.py | 2 +-
.../tests/evolution/test_runtime_binding.py | 8 +-
.../tests/evolution/test_settlement.py | 2 +-
.../tests/evolution/test_settlement_runner.py | 4 +-
.../evolution/test_skill_selection_record.py | 6 +-
src/backend/tests/evolution/test_sop_judge.py | 2 +-
.../tests/evolution/test_three_engines.py | 59 +-
src/backend/tests/evolution/test_trigger.py | 9 +-
.../tests/evolution/test_turn_card_edits.py | 4 +-
.../tests/evolution/test_user_contribution.py | 72 +-
src/backend/tests/features/__init__.py | 1 +
.../tests/features/automation/__init__.py | 1 +
.../tests/features/automation/run_fixtures.py | 15 +
.../test_automation_delivery_deadline.py | 179 +
.../test_automation_execution_location.py | 94 +-
.../automation/test_automation_lease_close.py | 199 +
.../test_automation_manual_trigger.py | 81 +
.../test_automation_notifications.py | 2 +-
.../test_automation_oneshot_finalize.py | 49 +
.../test_automation_ownership_postgres.py | 173 +
.../test_automation_remote_effect.py | 64 +-
.../test_automation_run_ownership.py | 256 +
.../test_delivery_target_migration.py | 31 +
.../automation/test_durable_admission.py | 96 +
.../automation}/test_loop_service_harness.py | 2 +-
.../automation/test_queue_postgres.py | 94 +
.../tests/features/desktop/__init__.py | 1 +
.../features/desktop/capabilities/__init__.py | 1 +
.../desktop}/capabilities/_cloud_identity.py | 0
.../desktop}/capabilities/conftest.py | 36 +-
.../capabilities/management_browser_server.py | 17 +-
.../capabilities/runtime_recovery_support.py | 19 +-
.../test_account_authorization.py | 30 +-
.../capabilities/test_agent_rerun_identity.py | 83 +-
.../capabilities/test_agent_runtime_grants.py | 22 +-
.../capabilities/test_agents_plugins.py | 42 +-
.../test_autonomous_loop_scopes.py | 95 +-
.../capabilities/test_available_runtime.py | 65 +-
.../test_capability_change_sync.py | 56 +-
.../test_catalog_cloud_only_in_hybrid.py | 41 +-
.../test_catalog_plugin_components_hidden.py | 8 +-
.../capabilities/test_change_merge.py | 4 +-
.../capabilities/test_change_signal.py | 2 +-
.../capabilities/test_cloud_identity.py | 2 +-
.../capabilities/test_cloud_management.py | 17 +-
.../test_cloud_parallel_preparation.py | 33 +-
.../test_cloud_plugin_binding_runtime.py | 32 +-
.../capabilities/test_cloud_sticky_plugins.py | 52 +-
.../capabilities/test_cloud_workcopies.py | 25 +-
.../desktop}/capabilities/test_core_layers.py | 13 +-
.../test_credential_continuity.py | 193 +
.../capabilities/test_dependency_closure.py | 53 +-
.../test_desktop_capabilities_api.py | 52 +-
.../test_desktop_content_safety.py | 67 +-
.../test_desktop_declaration_export.py | 21 +-
.../test_desktop_model_reasoning_safety.py | 21 +-
.../test_desktop_resource_plugins.py | 28 +-
.../test_desktop_token_security.py | 36 +-
.../capabilities/test_desktop_warmup.py | 28 +-
.../capabilities/test_device_api_contract.py | 58 +-
.../test_device_catalog_projection.py | 34 +-
.../test_dynamic_model_epoch_cache.py | 28 +-
.../test_explicit_desktop_invocation.py | 27 +-
.../test_gateway_invocation_proof.py | 16 +-
.../test_gateway_unknown_outcome.py | 52 +-
.../capabilities/test_hybrid_owned_plugins.py | 22 +-
.../test_initial_sync_readiness.py | 30 +-
.../test_junction_windows_paths.py | 10 +-
.../capabilities/test_loader_store_mode.py | 34 +-
.../test_local_folder_discovery.py | 4 +-
.../test_local_management_migration.py | 22 +-
.../test_local_skill_projection.py | 17 +-
.../capabilities/test_management_readiness.py | 26 +-
.../capabilities/test_management_tools.py | 61 +-
.../capabilities/test_manager_lifecycle.py | 8 +-
.../capabilities/test_manager_regressions.py | 29 +-
.../capabilities/test_manifest_order.py | 32 +-
.../capabilities/test_manifest_tool_schema.py | 27 +-
.../desktop}/capabilities/test_mcp_json.py | 2 +-
.../capabilities/test_mcp_json_consistency.py | 2 +-
.../capabilities/test_memory_epoch_cache.py | 31 +-
.../desktop}/capabilities/test_migration.py | 9 +-
.../test_migration_preservation.py | 5 +-
.../test_model_credential_reference.py | 13 +-
.../test_model_error_diagnostics.py | 6 +-
.../test_owned_connector_assembly.py | 38 +-
.../test_partial_session_availability.py | 33 +-
.../capabilities/test_plugin_connections.py | 12 +-
.../capabilities/test_plugin_device_assets.py | 10 +-
.../test_prepare_lock_not_held_over_io.py | 2 +-
.../capabilities/test_progressive_plugins.py | 166 +-
.../test_runtime_frozen_bindings.py | 142 +-
.../test_runtime_publication_recovery.py | 120 +-
.../capabilities/test_runtime_recovery.py | 72 +-
.../test_runtime_scope_consumers.py | 40 +-
.../capabilities/test_runtime_scopes.py | 104 +-
.../test_runtime_selection_recovery.py | 99 +-
.../test_runtime_session_recovery.py | 103 +-
.../test_selection_dependency_sources.py | 36 +-
.../capabilities/test_selection_readiness.py | 14 +-
.../capabilities/test_store_file_integrity.py | 19 +-
.../test_toolkit_runtime_alias.py | 40 +-
.../capabilities/test_user_facing_errors.py | 11 +-
.../capabilities/test_view_os_errors.py | 7 +-
.../test_workbench_local_upgrades.py | 16 +-
.../desktop}/desktop_login_browser_server.py | 6 +-
.../tests/features/desktop/index_fixtures.py | 13 +
.../features/desktop/observability_cases_1.py | 312 ++
.../features/desktop/observability_cases_2.py | 270 +
.../features/desktop/observability_support.py | 32 +
.../desktop}/run_desktop_browser_e2e.py | 2 +-
.../desktop}/test_desktop_auth.py | 4 +-
.../desktop}/test_desktop_bridge.py | 26 +-
.../desktop}/test_desktop_capability.py | 101 +-
.../test_desktop_capability_models.py | 27 +-
.../desktop}/test_desktop_cloud_skills.py | 70 +-
.../desktop}/test_desktop_device_login.py | 9 +-
.../test_desktop_memory_persistence.py | 8 +-
.../desktop}/test_desktop_model_gateway.py | 45 +-
.../desktop/test_desktop_observability.py | 20 +
...op_observability_authenticated_delivery.py | 54 +-
.../test_desktop_observability_delivery.py | 64 +-
.../desktop}/test_desktop_release_events.py | 4 +-
.../desktop}/test_desktop_sync_locking.py | 34 +-
.../desktop}/test_desktop_updates.py | 4 +-
.../desktop}/test_local_file_actions.py | 20 +-
.../desktop}/test_local_grant_service.py | 2 +-
.../desktop}/test_local_snapshot_service.py | 2 +-
.../tests/features/evaluation/__init__.py | 1 +
.../tests/features/evaluation/conftest.py | 9 +
.../evaluation}/test_evaluation_admission.py | 4 +-
.../evaluation}/test_evaluation_binding.py | 2 +-
.../evaluation}/test_evaluation_factory.py | 15 +-
.../test_evaluation_process_guard.py | 2 +-
.../evaluation}/test_evaluation_provider.py | 10 +-
.../evaluation}/test_evaluation_router.py | 38 +-
.../evaluation}/test_evaluation_runtime.py | 36 +-
.../evaluation}/test_evaluation_service.py | 7 +-
.../tests/features/hosted_apps/__init__.py | 1 +
.../application_hosting_support.py | 21 +-
.../test_application_agent_contract.py | 14 +-
.../test_application_collections.py | 18 +-
.../hosted_apps}/test_application_desktop.py | 69 +-
.../hosted_apps}/test_application_export.py | 2 +-
.../hosted_apps}/test_application_hosting.py | 42 +-
.../test_application_personal_mcp.py | 32 +-
.../test_application_project_tools.py | 16 +-
.../hosted_apps}/test_application_projects.py | 9 +-
.../test_application_retirement.py | 10 +-
.../hosted_apps}/test_hosting_remediation.py | 28 +-
.../hosted_apps/test_schema_lifecycle.py | 60 +
.../tests/features/knowledge/__init__.py | 1 +
.../knowledge/kb_multimodal_assets_cases_1.py | 353 ++
.../knowledge/kb_multimodal_assets_cases_2.py | 120 +
.../knowledge/kb_multimodal_assets_support.py | 235 +
.../knowledge/kb_wiki_end_to_end_cases_1.py | 349 ++
.../knowledge/kb_wiki_end_to_end_cases_2.py | 175 +
.../knowledge/kb_wiki_end_to_end_support.py | 197 +
.../knowledge/test_host_contributions.py | 68 +
.../knowledge}/test_kb_backend_routing.py | 59 +-
.../knowledge}/test_kb_document_status.py | 7 +-
.../knowledge}/test_kb_index_queue.py | 5 +-
.../knowledge}/test_kb_index_worker.py | 12 +-
.../knowledge/test_kb_multimodal_assets.py | 37 +
.../knowledge}/test_kb_vector_safety.py | 51 +-
.../knowledge}/test_kb_vector_timeouts.py | 27 +-
.../knowledge/test_kb_wiki_end_to_end.py | 31 +
.../knowledge}/test_kb_wiki_pipeline.py | 24 +-
.../tests/features/marketplace/__init__.py | 1 +
.../marketplace/mcp_marketplace_cases_1.py | 432 ++
.../marketplace/mcp_marketplace_cases_2.py | 460 ++
.../marketplace/mcp_marketplace_cases_3.py | 457 ++
.../marketplace/mcp_marketplace_cases_4.py | 418 ++
.../marketplace/mcp_marketplace_support.py | 275 +
.../marketplace}/test_agent_marketplace.py | 49 +-
.../marketplace}/test_marketplace_service.py | 83 +-
.../test_marketplace_submission.py | 90 +-
.../marketplace/test_mcp_marketplace.py | 69 +
.../marketplace/test_oauth_sdk2_contract.py | 114 +
src/backend/tests/features/sites/__init__.py | 1 +
src/backend/tests/features/sites/conftest.py | 2 +
.../sites}/test_cloud_site_publish.py | 38 +-
.../sites}/test_design_pick.py | 2 +-
.../sites}/test_desktop_site_paths.py | 20 +-
.../sites}/test_local_project_site_publish.py | 18 +-
.../sites}/test_local_site_sources.py | 56 +-
.../sites}/test_site_access_password.py | 21 +-
.../sites}/test_site_contention_postgres.py | 17 +-
.../sites}/test_site_listing.py | 19 +-
.../sites}/test_site_plugin_upgrade.py | 40 +-
.../sites}/test_site_rate_limit.py | 2 +-
.../test_execution_identity_contract.py | 44 +
.../tests/features/test_host_runtime.py | 18 +
.../tests/features/test_installed_modules.py | 63 +
.../features/test_run_capability_host.py | 15 +
.../tests/features/workbench/__init__.py | 1 +
.../workbench}/test_plugin_ui_contract.py | 10 +-
.../workbench}/test_plugin_ui_routes.py | 8 +-
.../test_workbench_activity_routes.py | 13 +-
.../test_workbench_actual_process_observer.py | 4 +-
.../test_workbench_application_config_http.py | 12 +-
.../test_workbench_approval_lease_http.py | 16 +-
.../workbench}/test_workbench_auth_routes.py | 12 +-
.../test_workbench_background_subpage_http.py | 8 +-
.../test_workbench_candidate_storage.py | 12 +-
.../test_workbench_capability_routes.py | 11 +-
.../test_workbench_closing_isolation_http.py | 14 +-
.../test_workbench_cookie_boundaries.py | 2 +-
.../workbench}/test_workbench_coordination.py | 2 +-
.../test_workbench_diagnostic_failure_http.py | 6 +-
.../workbench}/test_workbench_diagnostics.py | 9 +-
.../workbench}/test_workbench_embed_routes.py | 2 +-
.../test_workbench_external_routes.py | 5 +-
.../workbench}/test_workbench_invalidation.py | 14 +-
.../workbench}/test_workbench_migration.py | 8 +-
.../test_workbench_migration_auth_gate.py | 6 +-
.../test_workbench_observer_paths.py | 2 +-
...st_workbench_official_capabilities_http.py | 4 +-
.../workbench}/test_workbench_origins.py | 10 +-
.../test_workbench_plugin_routes.py | 9 +-
.../test_workbench_preferences_routes.py | 9 +-
.../workbench}/test_workbench_protocol.py | 8 +-
.../test_workbench_provider_scope_http.py | 4 +-
...est_workbench_resource_owner_migrations.py | 2 +-
.../workbench}/test_workbench_resources.py | 58 +-
.../test_workbench_revocation_failure_http.py | 30 +-
.../test_workbench_routing_routes.py | 4 +-
.../test_workbench_state_binding.py | 2 +-
.../test_workbench_state_concurrency.py | 6 +-
.../workbench}/test_workbench_state_routes.py | 6 +-
.../test_workbench_upgrade_routes.py | 4 +-
.../workbench_capabilities_http_fixture.py | 6 +-
.../workbench_diagnostic_http_fixture.py | 2 +-
.../workbench_embed_http_fixture.py | 2 +-
.../workbench}/workbench_http_fixture.py | 2 +-
.../workbench_observed_runner_fixture.py | 4 +-
.../workbench}/workbench_observer_paths.py | 2 +-
.../workbench_overflow_http_fixture.py | 2 +-
.../workbench_plugin_http_fixture.py | 2 +-
.../workbench}/workbench_plugin_support.py | 0
.../workbench}/workbench_resource_observer.py | 6 +-
.../workbench_routing_http_fixture.py | 2 +-
src/backend/tests/infra/test_byte_range.py | 56 +
src/backend/tests/infra/test_core_cleanup.py | 23 +-
.../infra/test_db_off_loop_call_sites.py | 115 +
.../tests/infra/test_db_thread_budget.py | 120 +
.../infra/test_no_session_on_event_loop.py | 100 +
.../integration/background_command_server.py | 13 +-
.../integration/job_agent_local_smoke.py | 15 +-
.../tests/integration/job_local_smoke.py | 18 +-
.../tests/integration/job_managed_smoke.py | 12 +-
src/backend/tests/integrations/__init__.py | 1 +
.../tests/integrations/channels/__init__.py | 1 +
.../integrations/channels/bot_cases_1.py | 383 ++
.../integrations/channels/bot_cases_2.py | 362 ++
.../integrations/channels/bot_cases_3.py | 374 ++
.../integrations/channels/bot_cases_4.py | 383 ++
.../integrations/channels/bot_cases_5.py | 388 ++
.../integrations/channels/bot_cases_6.py | 145 +
.../integrations/channels/bot_support.py | 187 +
.../channels/test_adapter_method_binding.py | 16 +
.../channels/test_channel_bots.py | 115 +
.../channels}/test_channel_bots_api.py | 8 +-
.../channels}/test_channel_desktop_e2e.py | 54 +-
.../channels}/test_channel_desktop_http.py | 48 +-
.../test_channel_desktop_migration.py | 2 +-
.../channels}/test_channel_desktop_relay.py | 13 +-
.../test_channel_desktop_relay_api.py | 36 +-
.../channels}/test_channel_desktop_worker.py | 50 +-
.../test_channel_receiver_lifecycle.py | 10 +-
.../test_weixin_connection_health.py | 11 +-
.../channels}/test_weixin_qr_protocol.py | 7 +-
.../dingtalk}/test_dingtalk_integration.py | 51 +-
.../email}/test_email_integration.py | 70 +-
.../integrations/lark/test_app_lifecycle.py | 48 +
.../lark}/test_lark_auth_status.py | 39 +-
.../yida}/test_yida_integration.py | 29 +-
src/backend/tests/llm/command_test_support.py | 15 +-
.../tests/llm/e2e_multiturn_toolcall.py | 4 +-
...le_context_hook_user_id_gating_selftest.py | 8 +-
.../tests/llm/local_project_test_support.py | 29 +-
src/backend/tests/llm/mcp_env_debug.py | 6 +-
.../tests/llm/test_agent_api_execution.py | 15 +-
.../tests/llm/test_agent_api_runtime.py | 51 +-
.../tests/llm/test_api_protocol_backfill.py | 8 +-
src/backend/tests/llm/test_approval_modes.py | 24 +-
src/backend/tests/llm/test_artifact_refs.py | 2 +-
.../tests/llm/test_ask_user_question_tool.py | 14 +-
.../tests/llm/test_bash_process_tools.py | 23 +-
.../tests/llm/test_builtin_subagents.py | 19 +-
.../tests/llm/test_cancelled_replay.py | 2 +-
.../tests/llm/test_chat_history_tool.py | 18 +-
.../tests/llm/test_chat_model_stream_retry.py | 8 +-
src/backend/tests/llm/test_compaction.py | 2 +-
src/backend/tests/llm/test_compaction_e2e.py | 14 +-
.../tests/llm/test_compaction_integration.py | 22 +-
.../tests/llm/test_compaction_mid_turn.py | 35 +-
.../tests/llm/test_compaction_observation.py | 16 +-
.../tests/llm/test_compaction_progress.py | 56 +-
.../llm/test_compaction_summary_requests.py | 6 +-
.../tests/llm/test_compaction_watermark.py | 38 +-
.../llm/test_configured_reasoning_effort.py | 10 +-
src/backend/tests/llm/test_context_adapter.py | 26 +-
src/backend/tests/llm/test_context_ir.py | 8 +-
.../llm/test_context_order_user_input.py | 16 +-
src/backend/tests/llm/test_context_probe.py | 4 +-
src/backend/tests/llm/test_context_safety.py | 28 +-
.../tests/llm/test_context_step_units.py | 25 +-
.../llm/test_context_usage_adversarial.py | 10 +-
.../tests/llm/test_context_usage_snapshot.py | 13 +-
src/backend/tests/llm/test_delta_coalescer.py | 145 +
.../tests/llm/test_execution_manifest.py | 10 +-
.../llm/test_execution_surface_limits.py | 37 +-
.../llm/test_execution_surface_snapshot.py | 24 +-
.../tests/llm/test_failover_context_safety.py | 8 +-
...st_file_context_offloads_blocking_parse.py | 25 +-
.../tests/llm/test_file_context_preview.py | 18 +-
.../llm/test_live_approval_integration.py | 8 +-
.../tests/llm/test_live_approval_repair.py | 30 +-
.../tests/llm/test_local_bash_permission.py | 52 +-
.../llm/test_local_confinement_contract.py | 11 +-
.../tests/llm/test_local_file_permission.py | 46 +-
.../tests/llm/test_local_myspace_block.py | 2 +-
.../tests/llm/test_local_project_cwd.py | 59 +-
.../tests/llm/test_local_project_delivery.py | 73 +-
.../tests/llm/test_local_project_pin.py | 80 +-
.../tests/llm/test_local_session_reference.py | 32 +-
.../tests/llm/test_manifest_mcp_client.py | 27 +-
src/backend/tests/llm/test_mcp_factory_e2e.py | 88 +
.../tests/llm/test_mcp_human_wait_timeout.py | 271 +
.../tests/llm/test_mcp_native_adversarial.py | 219 +
.../tests/llm/test_mcp_native_capabilities.py | 169 +
.../tests/llm/test_mcp_parallel_execution.py | 36 +-
src/backend/tests/llm/test_mcp_pool_cache.py | 203 -
.../tests/llm/test_mcp_runtime_pool.py | 272 +
.../tests/llm/test_mcp_session_ownership.py | 128 +
src/backend/tests/llm/test_mcp_timeouts.py | 32 +-
.../llm/test_migration_vendor_datasource.py | 114 +
src/backend/tests/llm/test_model_failover.py | 27 +-
src/backend/tests/llm/test_model_steps.py | 26 +-
.../tests/llm/test_model_usage_extraction.py | 2 +-
.../llm/test_model_usage_scope_teardown.py | 2 +-
src/backend/tests/llm/test_offloader.py | 40 +-
src/backend/tests/llm/test_openai_provider.py | 14 +-
.../llm/test_permission_ticket_isolation.py | 21 +-
.../llm/test_plan_update_tool_description.py | 10 +-
.../llm/test_post_turn_measured_compaction.py | 14 +-
.../llm/test_project_environment_factory.py | 32 +-
.../llm/test_project_environment_policy.py | 14 +-
src/backend/tests/llm/test_protocol_probe.py | 8 +-
.../tests/llm/test_read_image_native.py | 26 +-
.../tests/llm/test_reasoning_budget.py | 6 +-
.../llm/test_reasoning_echo_formatter.py | 4 +-
src/backend/tests/llm/test_reasoning_probe.py | 2 +-
.../tests/llm/test_reasoning_replay_policy.py | 10 +-
.../llm/test_reasoning_retained_in_history.py | 8 +-
.../llm/test_responses_reasoning_replay.py | 7 +-
.../llm/test_responses_thinking_stream.py | 94 +-
.../tests/llm/test_responses_tool_call_id.py | 20 +-
.../llm/test_sandbox_session_resolution.py | 8 +-
.../llm/test_selected_model_context_window.py | 27 +-
.../llm/test_shared_human_interactions.py | 8 +-
.../tests/llm/test_single_turn_protocol.py | 2 +-
.../tests/llm/test_stream_wrapper_cleanup.py | 30 +
.../llm/test_structured_reasoning_flag.py | 4 +-
.../tests/llm/test_subagent_cancellation.py | 25 +-
.../tests/llm/test_subagent_model_config.py | 17 +-
src/backend/tests/llm/test_subagent_resume.py | 10 +-
.../tests/llm/test_subagent_tool_streaming.py | 17 +-
src/backend/tests/llm/test_system_reminder.py | 7 +-
.../test_thinking_modes_across_protocols.py | 10 +-
.../tests/llm/test_tool_call_identity.py | 2 +-
.../tests/llm/test_tool_media_no_disk.py | 10 +-
.../llm/test_tool_permission_hardening.py | 17 +-
.../tests/llm/test_tool_permissions.py | 43 +-
.../tests/llm/test_tool_result_images.py | 46 +-
src/backend/tests/llm/test_vendor_timeouts.py | 4 +-
src/backend/tests/llm/test_vision_bridge.py | 20 +-
.../tests/llm/test_vision_bridge_e2e.py | 6 +-
.../tests/llm/test_vision_injection.py | 76 +-
.../tests/llm/test_vision_progress_events.py | 44 +-
.../tests/llm/test_vision_role_capability.py | 7 +-
src/backend/tests/llm/test_workspace.py | 13 +-
.../tests/llm/test_workspace_profiles.py | 99 +-
src/backend/tests/llm/tool_call_debug.py | 4 +
.../tests/llm/vision_fullstack_check.py | 4 +-
src/backend/tests/llm/vision_live_check.py | 12 +-
.../tests/llm/vision_role_live_check.py | 10 +-
src/backend/tests/mcp/test_internet_search.py | 12 +-
.../tests/mcp/test_wiki_tool_gating.py | 13 +-
.../tests/mcp_servers/test_search_batch.py | 28 +-
.../tests/mcp_servers/test_web_fetch.py | 3 +-
.../tests/memory/outbox_test_support.py | 3 +-
.../test_ce_overlay_memory_contracts.py | 41 +-
src/backend/tests/memory/test_graph_memory.py | 22 +-
.../memory/test_identity_frozen_block.py | 6 +-
.../tests/memory/test_mem0_real_lifecycle.py | 1 +
.../tests/memory/test_memory_outbox.py | 4 +-
.../memory/test_memory_outbox_receipts.py | 8 +-
.../memory/test_memory_outbox_recovery.py | 16 +-
.../memory/test_memory_retrieval_lifecycle.py | 36 +-
.../memory/test_session_frozen_snapshot.py | 34 +-
.../misc/chat_metadata_merge_selftest.py | 2 +-
src/backend/tests/misc/conftest.py | 15 +-
src/backend/tests/misc/test_identity_dedup.py | 2 +-
.../tests/ontology/test_ontology_reviewer.py | 40 +-
.../ontology/test_ontology_revision_accept.py | 10 +-
.../test_ontology_workflow_evidence.py | 17 +-
.../agent_mention_parse_selftest.py | 2 +-
.../orchestration/automation_test_support.py | 17 +
.../orchestration/command_test_support.py | 2 +-
.../tests/orchestration/hook_support.py | 53 +
.../orchestration/recovery_test_support.py | 17 +-
.../orchestration/router_strategy_selftest.py | 2 +-
.../orchestration/routing_import_selftest.py | 2 +-
.../orchestration/stream_protocol_selftest.py | 7 +-
.../streaming_tool_call_dedupe_selftest.py | 15 +-
.../orchestration/test_agent_api_recovery.py | 40 +-
.../test_agent_runtime_resources.py | 20 +
.../test_agentscope_hook_adapter.py | 31 +-
.../test_automation_admission.py | 50 +-
.../orchestration/test_automation_handoff.py | 50 +-
.../test_automation_live_conversations.py | 125 +-
.../orchestration/test_automation_reaper.py | 4 +-
.../test_automation_unbounded.py | 65 +-
.../test_autonomous_loop_driver.py | 12 +-
.../orchestration/test_background_commands.py | 87 +-
.../test_chat_resume_checkpoint.py | 18 +-
.../test_chat_run_cancel_persistence.py | 57 +-
.../test_chat_run_follow_recovery.py | 104 +-
.../orchestration/test_chat_run_human_wait.py | 24 +-
.../orchestration/test_chat_run_recovery.py | 37 +-
.../test_chat_run_restart_recovery.py | 89 +-
.../test_chat_run_worker_recovery.py | 56 +-
.../tests/orchestration/test_chat_steer.py | 104 +-
.../orchestration/test_command_events.py | 61 +-
.../orchestration/test_command_recovery.py | 158 +-
.../test_command_worker_lifecycle.py | 18 +-
.../orchestration/test_context_budget_stop.py | 41 +-
.../orchestration/test_context_provenance.py | 11 +-
.../test_context_stats_finalization_race.py | 6 +-
.../test_direct_subagent_routing.py | 79 +-
.../test_event_offset_fencing.py | 11 +-
.../test_execution_manifest_binding.py | 128 +-
.../tests/orchestration/test_finish_guard.py | 79 +-
.../test_harness_e2e_fault_matrix.py | 18 +-
.../test_harness_ledger_postgres.py | 4 +-
.../test_harness_process_restart.py | 52 -
.../test_harness_usage_ledger.py | 57 +-
.../tests/orchestration/test_hook_bus.py | 141 +
.../orchestration/test_inactivity_watchdog.py | 2 +-
.../orchestration/test_inflight_message.py | 40 +-
.../test_job_callback_transport.py | 7 +-
.../orchestration/test_job_conflict_policy.py | 13 +-
.../orchestration/test_job_derived_agent.py | 56 +-
.../tests/orchestration/test_job_failfast.py | 35 +-
.../test_job_inherited_grants.py | 18 +-
.../orchestration/test_job_local_callback.py | 5 +-
.../orchestration/test_job_managed_runtime.py | 50 +-
.../orchestration/test_job_orphan_reaper.py | 25 +-
.../orchestration/test_job_resilience.py | 4 +-
.../orchestration/test_job_sdk_ledger.py | 2 +-
.../orchestration/test_job_ui_progress.py | 12 +-
.../test_job_wake_delivery_format.py | 2 +-
.../test_job_wake_message_visibility.py | 34 +-
.../tests/orchestration/test_job_wake_mode.py | 36 +-
.../test_job_windows_attempt_paths.py | 5 +-
.../test_loop_reviewer_registry_tool.py | 4 +-
.../test_missing_tool_call_synthesis.py | 4 +-
.../test_model_progress_signal.py | 60 +-
.../test_model_steps_pipeline.py | 25 +-
.../test_model_steps_plan_projection.py | 66 +-
.../orchestration/test_model_steps_worker.py | 47 +-
.../orchestration/test_observation_writer.py | 146 +
.../orchestration/test_pending_workflows.py | 68 +
.../orchestration/test_plan_early_cancel.py | 84 +
.../test_plan_progress_closeout.py | 9 +-
.../orchestration/test_plan_stale_reminder.py | 57 +-
.../test_project_api_recovery.py | 25 +-
.../test_project_init_stream_input.py | 5 +-
.../orchestration/test_run_event_stream.py | 16 +-
.../tests/orchestration/test_run_journal.py | 9 +-
.../test_run_journal_postgres.py | 15 +-
.../test_runtime_phase_lifecycle.py | 203 +
.../orchestration/test_site_mode_prompt.py | 11 +-
.../tests/orchestration/test_stale_reaper.py | 123 +-
.../tests/orchestration/test_steer_queue.py | 13 +-
.../test_structured_reasoning_protocol.py | 2 +-
.../test_thinking_block_persistence.py | 22 +-
.../test_thinking_suppression.py | 2 +-
.../orchestration/test_tool_effect_ledger.py | 1405 +----
.../test_tool_effect_ledger_2.py | 429 ++
.../test_tool_effect_ledger_3.py | 475 ++
.../test_tool_effect_ledger_4.py | 459 ++
.../test_tool_effect_ledger_5.py | 133 +
.../test_user_question_passthrough.py | 42 +-
.../tests/orchestration/test_worker_base.py | 3 +-
.../test_workflow_hint_consistency.py | 2 +-
.../test_workflow_model_runtime.py | 10 +-
.../orchestration/tool_effect_support.py | 81 +
.../{ => personal_agent}/test_device_link.py | 8 +-
.../prompts/test_desktop_delivery_tools.py | 2 +-
.../tests/prompts/test_desktop_environment.py | 13 +-
.../prompts/test_desktop_prompt_versions.py | 57 +-
...est_local_mode_drops_container_sections.py | 16 +-
.../prompts/test_local_project_section.py | 16 +-
.../tests/prompts/test_project_isolation.py | 2 +-
.../prompts/test_prompt_cache_full_hash.py | 4 +-
.../test_prompt_parts_from_filesystem.py | 56 +-
.../prompts/test_prompt_version_freshness.py | 44 +-
.../tests/run_browser_opensandbox_e2e.py | 8 +-
.../tests/run_command_opensandbox_e2e.py | 12 +-
.../tests/run_personal_agent_cli_grant_e2e.py | 3 +-
src/backend/tests/run_personal_agent_e2e.py | 6 +-
.../tests/run_personal_agent_key_e2e.py | 2 +-
...run_personal_agent_project_sessions_e2e.py | 5 +-
.../tests/sandbox/e2e_myspace_bulk_live.py | 6 +-
.../tests/sandbox/fixtures/egress_gateway.py | 14 +-
.../tests/sandbox/provider_test_support.py | 5 +-
.../test_adopt_respects_session_binding.py | 3 -
.../tests/sandbox/test_artifact_streaming.py | 8 +-
.../sandbox/test_cloud_process_sessions.py | 12 +-
.../tests/sandbox/test_controller_capacity.py | 37 +
.../sandbox/test_creation_adoption_race.py | 9 +-
.../test_credential_purge_preserves_mounts.py | 37 +-
.../sandbox/test_cube_skill_push_ownership.py | 7 +-
.../sandbox/test_idle_reclaim_snapshot.py | 3 +-
.../sandbox/test_multiworker_lifecycle.py | 2 +-
.../sandbox/test_multiworker_transactions.py | 2 +-
.../test_opensandbox_controller_patch.py | 2 +-
.../sandbox/test_opensandbox_expirations.py | 4 +-
.../sandbox/test_opensandbox_provider.py | 30 +
src/backend/tests/sandbox/test_os_sandbox.py | 4 +-
.../sandbox/test_pool_retirement_guards.py | 18 +-
.../sandbox/test_product_session_scope.py | 2 +-
.../tests/sandbox/test_runner_package.py | 2 +-
.../tests/sandbox/test_runtime_ownership.py | 61 +
.../tests/sandbox/test_sandbox_accounting.py | 14 +-
.../tests/sandbox/test_sandbox_init.py | 70 +
.../tests/sandbox/test_sandbox_packaging.py | 18 +-
.../sandbox/test_service_loop_affinity.py | 4 +-
.../tests/sandbox/test_session_affinity.py | 2 +-
.../sandbox/test_session_lock_cross_loop.py | 2 +-
.../tests/sandbox/test_user_bound_sandbox.py | 21 -
.../services/project_instructions_support.py | 5 +-
.../tests/services/test_artifact_reader.py | 41 +-
.../tests/services/test_artifact_summary.py | 20 +-
src/backend/tests/services/test_audio_live.py | 2 +-
.../tests/services/test_audio_models.py | 16 +-
.../tests/services/test_audio_routes.py | 6 +-
.../services/test_audio_transcription.py | 4 +-
.../tests/services/test_chat_mode_service.py | 3 +-
.../services/test_chat_reference_service.py | 11 +-
.../tests/services/test_chat_sequencer.py | 74 +-
.../tests/services/test_chat_titles.py | 35 +-
.../services/test_kb_multimodal_assets.py | 657 ---
.../tests/services/test_kb_wiki_end_to_end.py | 663 ---
.../services/test_memory_settings_service.py | 16 +-
.../services/test_model_config_consistency.py | 50 +-
.../services/test_model_config_export.py | 8 +-
.../services/test_plan_execution_snapshot.py | 2 +-
.../test_project_context_no_inventory.py | 8 +-
.../services/test_project_environment.py | 17 +-
.../services/test_project_instructions.py | 39 +-
.../test_project_instructions_dispatch.py | 26 +-
.../tests/services/test_retire_chart_mcp.py | 2 +-
.../services/test_turbo_capability_config.py | 14 +-
.../services/test_user_model_selection.py | 13 +-
.../services/user_folder_service_selftest.py | 6 +-
.../tests/skills/agent_skills_selftest.py | 4 +-
src/backend/tests/skills/backends_selftest.py | 6 +-
src/backend/tests/skills/loader_selftest.py | 7 +-
.../tests/skills/skill_loading_selftest.py | 2 +-
.../tests/skills/skill_selection_selftest.py | 8 +-
.../skills/test_database_backend_lazy.py | 10 +-
.../tests/skills/test_deps_detector.py | 2 +-
.../skills/test_read_artifact_pptx_xlsx.py | 54 +-
src/backend/tests/skills/test_registry.py | 6 +-
.../tests/skills/test_sandbox_skill_purge.py | 17 +-
.../skills/test_skill_deps_aggregator.py | 20 +-
.../tests/skills/test_skill_publication.py | 67 +-
.../skills/test_skill_publication_docker.py | 2 +-
.../skills/test_skill_selector_context_ir.py | 6 +-
.../tests/skills/test_skill_zip_export.py | 20 +-
.../tests/test_add_agent_permission.py | 4 +-
.../tests/test_agent_manager_plugin.py | 13 +-
src/backend/tests/test_agent_markdown.py | 7 +-
.../tests/test_agent_version_history.py | 15 +
.../tests/test_automation_manual_trigger.py | 77 -
.../tests/test_automation_oneshot_finalize.py | 109 -
src/backend/tests/test_browser_prewarm.py | 75 +-
src/backend/tests/test_browser_remediation.py | 6 +-
.../tests/test_browser_resource_assets.py | 6 +-
.../tests/test_browser_resource_security.py | 39 +-
.../tests/test_browser_resource_transport.py | 14 +-
src/backend/tests/test_ce_agent_visibility.py | 6 +-
.../tests/test_ce_runtime_contracts.py | 47 +-
src/backend/tests/test_channel_bots.py | 2109 --------
src/backend/tests/test_citation_anchor.py | 16 +-
src/backend/tests/test_cli_cold_bootstrap.py | 48 +
.../tests/test_content_block_id_migration.py | 2 +-
src/backend/tests/test_credential_registry.py | 44 +
src/backend/tests/test_db_import_order.py | 103 +
.../tests/test_default_plugin_bootstrap.py | 10 +-
.../tests/test_desktop_observability.py | 633 ---
.../test_explicit_capability_invocation.py | 21 +-
src/backend/tests/test_extensions_layout.py | 28 +-
src/backend/tests/test_hook_contributions.py | 47 +
.../tests/test_iter_budget_middleware.py | 69 +-
src/backend/tests/test_leader_election.py | 48 +
.../tests/test_local_mcp_http_readiness.py | 33 +
.../tests/test_local_memory_defaults.py | 3 +-
src/backend/tests/test_local_onboarding.py | 95 +
.../tests/test_local_plugin_sidecars.py | 75 +-
src/backend/tests/test_local_profile.py | 158 +-
src/backend/tests/test_local_subprocess.py | 95 +-
src/backend/tests/test_management_failures.py | 8 +-
src/backend/tests/test_mcp_canonical_url.py | 30 +
src/backend/tests/test_mcp_marketplace.py | 1758 -------
src/backend/tests/test_package_boundaries.py | 16 +-
src/backend/tests/test_package_sources.py | 23 +
src/backend/tests/test_personal_agent.py | 16 +-
.../tests/test_personal_agent_access.py | 10 +-
.../tests/test_personal_agent_capabilities.py | 14 +-
.../test_personal_agent_channel_files.py | 8 +-
.../test_personal_agent_channel_sessions.py | 11 +-
.../tests/test_personal_agent_cold_start.py | 10 +-
.../test_personal_agent_computer_wait.py | 6 +-
.../tests/test_personal_agent_delivery.py | 11 +-
.../tests/test_personal_agent_dreaming.py | 4 +-
.../tests/test_personal_agent_fixes.py | 35 +-
.../tests/test_personal_agent_monitor.py | 3 +-
.../tests/test_personal_agent_persona.py | 15 +-
.../tests/test_personal_agent_prompt_cache.py | 22 +
.../tests/test_personal_agent_routing.py | 28 +-
.../tests/test_personal_agent_session_tool.py | 5 +-
.../tests/test_personal_agent_sessions.py | 12 +-
.../tests/test_personal_agent_steer.py | 21 +-
.../test_personal_agent_tool_regressions.py | 30 +-
.../tests/test_personal_canonical_history.py | 4 +-
.../test_personal_task_transcript_order.py | 18 +-
src/backend/tests/test_plugin_components.py | 73 +-
.../tests/test_plugin_manager_plugin.py | 17 +-
.../tests/test_plugin_progressive_loading.py | 89 +-
.../tests/test_plugin_resource_contract.py | 2 +-
src/backend/tests/test_plugin_system.py | 17 +-
src/backend/tests/test_redis_stream_pool.py | 6 +-
.../test_retired_batch_chat_admission_http.py | 32 +-
src/backend/tests/test_route_import_order.py | 74 +
.../test_script_runner_process_limits.py | 10 +-
.../tests/test_session_capabilities.py | 4 +-
.../tests/test_session_database_isolation.py | 23 +
src/backend/tests/test_sidebar_task_status.py | 13 +-
.../tests/test_skill_manager_plugin.py | 23 +-
src/backend/tests/test_tool_contributions.py | 90 +
.../tests/test_user_agent_resources.py | 16 +-
src/backend/tests/test_user_cleanup.py | 34 +
src/backend/tests/test_visualize_plugin.py | 18 +-
src/backend/tests/test_yida_local_login.py | 12 +-
src/frontend/Dockerfile | 3 +-
src/frontend/Dockerfile.dockerignore | 8 +-
src/frontend/package-lock.json | 263 +
src/frontend/package.json | 3 +
src/frontend/scripts/archive-e2e-fixtures.py | 4 +-
.../scripts/chat-customization-environment.py | 2 +-
.../scripts/component-e2e-environment.py | 2 +-
.../scripts/customer-devkit-environment.py | 4 +-
.../scripts/desktop-lint-baseline.json | 1 -
.../scripts/grapheme-e2e-environment.py | 2 +-
src/frontend/scripts/test-attachment-send.tsx | 4 +-
.../scripts/test-canvas-quote-browser.mjs | 131 +
.../scripts/test-canvas-viewers-browser.mjs | 213 +
.../scripts/test-canvas-zoom-browser.mjs | 254 +
src/frontend/scripts/test-chat-fork.tsx | 2 +-
.../scripts/test-desktop-login-browser.mjs | 2 +-
.../scripts/test-queued-invocation.ts | 12 +
.../scripts/test-reference-motion-browser.mjs | 34 +-
.../test-sidebar-task-status-browser.mjs | 16 +-
.../scripts/test-subagent-canvas-browser.mjs | 3 +-
.../scripts/test-turn-status-alignment.mjs | 2 +-
.../test-workbench-capabilities-browser.mjs | 2 +-
.../test-workbench-diagnostics-browser.mjs | 2 +-
.../scripts/test-workbench-embed-browser.mjs | 2 +-
.../scripts/test-workbench-plugin-browser.mjs | 4 +-
.../scripts/test-workspace-http-browser.mjs | 4 +-
.../automation/AutomationTaskSections.tsx | 6 +-
.../automation/RunTimelinePanel.tsx | 64 +-
.../components/automation/automationUtils.ts | 9 +-
.../components/canvas/CanvasFileHeader.css | 6 +
.../components/canvas/CanvasFileHeader.tsx | 2 +
.../components/canvas/CanvasFileRenderers.tsx | 338 --
.../src/components/canvas/CanvasPanel.tsx | 45 +-
.../components/canvas/CanvasZoomControl.tsx | 81 +
.../src/components/canvas/FileTextPanel.tsx | 6 +-
.../components/canvas/UniverSpreadsheet.tsx | 31 +
.../src/components/canvas/canvasDocument.ts | 121 +
.../components/canvas/canvasFileCategory.ts | 22 -
.../src/components/canvas/canvasZoom.ts | 85 +
.../src/components/canvas/find/CanvasFind.css | 45 +
.../src/components/canvas/find/CanvasFind.tsx | 196 +
.../components/canvas/find/domFindSource.ts | 17 +
.../src/components/canvas/find/index.ts | 1 +
.../src/components/canvas/find/textSearch.ts | 76 +
.../canvas/outline/CanvasOutline.css | 143 +
.../canvas/outline/CanvasOutline.tsx | 225 +
.../components/canvas/outline/domOutline.ts | 54 +
.../src/components/canvas/outline/index.ts | 1 +
.../canvas/renderers/DocxRenderer.tsx | 100 +
.../canvas/renderers/FallbackRenderers.tsx | 40 +
.../canvas/renderers/HtmlRenderer.tsx | 62 +
.../canvas/renderers/ImageRenderer.tsx | 102 +
.../canvas/renderers/PdfRenderer.tsx | 60 +
.../components/canvas/renderers/PdfViewer.css | 104 +
.../components/canvas/renderers/PdfViewer.tsx | 399 ++
.../canvas/renderers/TextRenderers.css | 66 +
.../canvas/renderers/TextRenderers.tsx | 138 +
.../src/components/canvas/renderers/index.ts | 6 +
.../src/components/canvas/renderers/pdfjs.ts | 68 +
.../canvas/renderers/previewError.ts | 11 +
.../selection/SpreadsheetQuoteButton.css | 56 +
.../selection/SpreadsheetQuoteButton.tsx | 23 +
.../components/canvas/selection/canvasChat.ts | 48 +
.../canvas/selection/canvasLocator.ts | 37 +
.../canvas/selection/useCanvasSelection.tsx | 33 +
.../src/components/chat/AgentCursor.tsx | 72 +
.../src/components/chat/InputArea.tsx | 22 +-
.../src/components/chat/MessageBody.tsx | 111 +-
.../src/components/chat/MessageBubble.tsx | 5 +-
.../components/chat/OntologyRevisionPanel.tsx | 7 -
.../components/chat/StreamWaitIndicator.tsx | 4 +-
.../components/chat/TurnStatusIndicator.tsx | 56 +-
.../components/chat/useMessageSelection.tsx | 298 +-
.../src/components/common/AgentMark.tsx | 48 +
.../src/components/common/ElapsedTimer.tsx | 8 +-
.../src/components/common/RollingText.tsx | 105 +
.../common/selection/SelectionToolbar.css | 79 +
.../common/selection/useSelectionToolbar.tsx | 226 +
.../src/components/file/FilePreviewPane.tsx | 80 +-
.../components/file/FilePreviewViewers.tsx | 30 +-
.../src/components/file/MediaPlayer.css | 84 +
.../src/components/file/MediaPlayer.tsx | 66 +
.../src/components/file/filePreviewUrls.ts | 5 -
.../shared/PersonalAgentNavStatus.tsx | 9 +-
.../personal-agent/shared/notifications.ts | 15 +-
.../src/components/tool/CompactionStepRow.tsx | 10 +-
.../src/components/tool/PendingStepRow.tsx | 34 -
.../src/components/tool/ThinkingStepRow.tsx | 7 +-
.../src/components/tool/ToolCallRow.tsx | 72 +-
.../components/tool/ToolCapabilityIcon.tsx | 8 +-
.../src/components/tool/ToolKindStack.tsx | 69 +
.../src/components/tool/ToolRunShell.tsx | 151 +-
.../src/components/tool/toolIcons.tsx | 68 +
src/frontend/src/hooks/chatHistoryMessage.ts | 10 +-
src/frontend/src/hooks/chatStream/consume.ts | 1 -
.../src/hooks/chatStream/lifecycleAndTools.ts | 46 +-
.../src/hooks/chatStream/messageStore.ts | 6 +-
src/frontend/src/hooks/chatStream/snapshot.ts | 1 -
src/frontend/src/hooks/chatStream/state.ts | 1 -
.../hooks/chatStream/toolResultsAndMeta.ts | 1 -
src/frontend/src/hooks/streamQueue.ts | 6 +-
src/frontend/src/hooks/streamSend.ts | 17 +-
src/frontend/src/hooks/useActivityAge.ts | 47 +
src/frontend/src/hooks/useStallDetector.ts | 6 +-
src/frontend/src/hooks/useToolCapability.ts | 18 +-
src/frontend/src/i18n/en/canvas.ts | 39 +
src/frontend/src/i18n/en/chat.ts | 5 +-
src/frontend/src/i18n/en/lab.ts | 4 +-
src/frontend/src/i18n/en/myspace.ts | 2 +
src/frontend/src/i18n/en/panels.ts | 3 +-
src/frontend/src/i18n/en/tool.ts | 27 +-
src/frontend/src/plugin-ui/README.md | 4 +-
.../src/stores/automationChatStore.ts | 35 +-
src/frontend/src/stores/canvasStore.ts | 39 +-
src/frontend/src/stores/composerStore.ts | 4 +-
src/frontend/src/styles/canvas.css | 290 +-
src/frontend/src/styles/chat.css | 1 +
.../src/styles/chat/agent-activity.css | 226 +
src/frontend/src/styles/chat/attachments.css | 18 +-
.../src/styles/chat/import-browser.css | 54 -
.../src/styles/chat/message-actions.css | 68 -
src/frontend/src/styles/chat/messages.css | 12 +-
src/frontend/src/styles/chat/turn-status.css | 28 +-
src/frontend/src/styles/tool.css | 144 +-
src/frontend/src/styles/variables.css | 15 +
src/frontend/src/types/automations.ts | 2 +-
src/frontend/src/types/conversations.ts | 23 +-
src/frontend/src/types/messages.ts | 1 -
src/frontend/src/utils/chatInvocation.ts | 10 +
src/frontend/src/utils/elapsed.ts | 5 +
src/frontend/src/utils/filePreviewKind.ts | 92 +
src/frontend/src/utils/filePreviewSafety.ts | 24 +-
src/frontend/src/utils/index.ts | 2 +-
src/frontend/src/utils/quotedFollowUp.ts | 64 +
src/frontend/src/utils/toolActionLabel.ts | 84 +-
src/frontend/src/utils/toolIconKind.ts | 12 +
src/frontend/src/utils/turnActivity.ts | 69 +
2834 files changed, 102814 insertions(+), 90978 deletions(-)
create mode 100644 .claude/skills/hugagent-backend-dev/templates/extension.py
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/attachment.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/automation.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/check.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/chevron-right.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/close.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/knowledge.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/my-space.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/new-chat.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/search.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/send.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/settings.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/skill.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/favicon.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-black.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-blue.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-white.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-black.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-white.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-reverse.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-small.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark.svg
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/appTheme.v2.ts
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.css
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.json
create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/motionTokens.v2.ts
delete mode 100644 docker/Dockerfile.opensandbox-server.dockerignore
delete mode 100644 docker/Dockerfile.personal-agent-test
rename docker-compose.application-hosting.yml => docker/application-hosting/compose.application-hosting.yml (90%)
rename docker/{ => object-storage}/Dockerfile.object-storage (100%)
rename docker/{ => object-storage}/compose.object-storage.yml (91%)
rename docker/{ => object-storage}/init-object-storage.py (100%)
rename docker/{ => opensandbox-server}/Dockerfile.opensandbox-server (52%)
create mode 100644 docker/opensandbox-server/Dockerfile.opensandbox-server.dockerignore
rename docker/{ => opensandbox-server}/opensandbox-config.toml.tpl (100%)
create mode 100644 docker/opensandbox-server/opensandbox_capacity.py
rename docker/{ => opensandbox-server}/opensandbox_expirations.py (100%)
rename docker/{ => opensandbox-server}/opensandbox_mount_recovery.py (91%)
rename docker/{ => opensandbox-server}/opensandbox_mount_worker.py (100%)
rename docker/{ => opensandbox-server}/opensandbox_object_mounts.py (100%)
rename docker/{ => opensandbox-server}/patch_opensandbox_server.py (96%)
rename docker/{ => opensandbox-server}/render-opensandbox-config.py (100%)
rename docker/{ => personal-agent}/Dockerfile.personal-agent (86%)
rename docker/{ => personal-agent}/compose.personal-agent.yml (91%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/helpers.rc (100%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/mimeapps.list (100%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser (100%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser-helper.desktop (100%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser.desktop (100%)
rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-stream.py (100%)
rename docker/{ => personal-agent}/hugagent-cli-entrypoint.sh (100%)
rename docker/{ => personal-agent}/personal-agent-entrypoint.sh (94%)
rename docker/{ => personal-agent}/render-personal-agent-config.py (100%)
rename docker/{ => sandbox}/Dockerfile.opensandbox (92%)
rename docker/{ => sandbox}/Dockerfile.script-runner (96%)
rename docker/{ => sandbox}/apt-skills.txt (100%)
rename docker/{ => sandbox}/package-skills.json (100%)
rename docker/{ => sandbox}/requirements-script-runner.txt (100%)
rename docker/{ => sandbox}/requirements-skills.txt (100%)
create mode 100644 docker/sandbox/sandbox-init.c
rename docker/{ => sandbox}/site-template/init-react-site.sh (100%)
rename docker/{ => sandbox}/site-template/react-vite/.npmrc (100%)
rename docker/{ => sandbox}/site-template/react-vite/index.html (100%)
rename docker/{ => sandbox}/site-template/react-vite/package-lock.json (100%)
rename docker/{ => sandbox}/site-template/react-vite/package.json (100%)
rename docker/{ => sandbox}/site-template/react-vite/src/App.jsx (100%)
rename docker/{ => sandbox}/site-template/react-vite/src/index.css (100%)
rename docker/{ => sandbox}/site-template/react-vite/src/lib/siteApi.js (100%)
rename docker/{ => sandbox}/site-template/react-vite/src/main.jsx (100%)
rename docker/{ => sandbox}/site-template/react-vite/src/pages/Home.jsx (100%)
rename docker/{ => sandbox}/site-template/react-vite/vite.config.mjs (100%)
rename docker/{ => sandbox}/verify-browser-runtime.py (100%)
delete mode 100644 document/en/deployment/artifact-storage-key-repair.md
delete mode 100644 document/en/deployment/offline-production.md
delete mode 100644 document/en/deployment/uos1070-desktop.md
delete mode 100644 document/en/development/container-layout.md
delete mode 100644 document/zh-CN/deployment/artifact-storage-key-repair.md
delete mode 100644 document/zh-CN/deployment/offline-production.md
delete mode 100644 document/zh-CN/deployment/uos1070-desktop.md
delete mode 100644 document/zh-CN/development/container-layout.md
delete mode 100644 examples/customer-chat/.env.example
delete mode 100644 examples/customer-chat/.gitignore
delete mode 100644 examples/customer-chat/README.md
delete mode 100644 examples/customer-chat/debug.html
delete mode 100644 examples/customer-chat/index.html
delete mode 100644 examples/customer-chat/package-lock.json
delete mode 100644 examples/customer-chat/package.json
delete mode 100644 examples/customer-chat/preview.html
delete mode 100644 examples/customer-chat/scripts/build-component.mjs
delete mode 100644 examples/customer-chat/scripts/developer-proxy.mjs
delete mode 100644 examples/customer-chat/scripts/test-package.mjs
delete mode 100644 examples/customer-chat/scripts/test-proxy.mjs
delete mode 100644 examples/customer-chat/session.html
delete mode 100644 examples/customer-chat/src/controls.ts
delete mode 100644 examples/customer-chat/src/css.d.ts
delete mode 100644 examples/customer-chat/src/customization.tsx
delete mode 100644 examples/customer-chat/src/debug.ts
delete mode 100644 examples/customer-chat/src/developer.css
delete mode 100644 examples/customer-chat/src/login.ts
delete mode 100644 examples/customer-chat/src/preview.tsx
delete mode 100644 examples/customer-chat/src/session.ts
delete mode 100644 examples/customer-chat/src/standalone.css
delete mode 100644 examples/customer-chat/src/standalone.ts
delete mode 100644 examples/customer-chat/src/theme.css
delete mode 100644 examples/customer-chat/tsconfig.json
delete mode 100644 examples/customer-chat/vite.config.mjs
create mode 100644 packages/cli/DESIGN.md
create mode 100644 scripts/refactor/__init__.py
create mode 100644 scripts/refactor/check_ce_paths.py
create mode 100644 scripts/refactor/check_line_budget.py
create mode 100644 scripts/refactor/check_test_baseline.py
create mode 100644 scripts/refactor/import_contract.py
create mode 100644 scripts/refactor/rewrite_imports.py
create mode 100644 scripts/refactor/snapshot_routes.py
create mode 100644 scripts/refactor/snapshot_schema.py
create mode 100644 scripts/refactor/tests/test_edition_seams.py
create mode 100644 scripts/refactor/tests/test_guards.py
create mode 100644 scripts/refactor/tests/test_package_markers.py
create mode 100644 scripts/refactor/tests/test_private_imports.py
create mode 100644 scripts/refactor/tests/test_rewrite_imports.py
create mode 100644 scripts/refactor/tests/test_snapshot_schema.py
create mode 100644 scripts/refactor/tests/test_supplemental_moves.py
create mode 100644 scripts/refactor/tests/test_test_baseline.py
create mode 100644 scripts/refactor/tests/test_test_moves.py
create mode 100644 src/backend/alembic/versions/ce_0031_automation_lease_token.py
create mode 100644 src/backend/alembic/versions/ce_0032_automation_queue.py
create mode 100644 src/backend/alembic/versions/ce_0033_automation_targets.py
create mode 100644 src/backend/api/edition.py
rename src/backend/api/{routes/v1/agents.py => personal_agents.py} (94%)
delete mode 100644 src/backend/api/routes/v1/desktop_capability.py
delete mode 100644 src/backend/api/routes/v1/kb.py
rename src/backend/{core/content => api/transports}/__init__.py (100%)
create mode 100644 src/backend/api/transports/run_sse.py
create mode 100644 src/backend/core/agent_runtime/README.md
create mode 100644 src/backend/core/agent_runtime/README.zh-CN.md
rename src/backend/core/{plugins/resources => agent_runtime}/__init__.py (100%)
create mode 100644 src/backend/core/agent_runtime/budget.py
create mode 100644 src/backend/core/agent_runtime/direct.py
create mode 100644 src/backend/core/agent_runtime/direct_state.py
create mode 100644 src/backend/core/agent_runtime/events.py
create mode 100644 src/backend/core/agent_runtime/host.py
rename src/backend/{tests/capabilities => core/agent_runtime/integrations}/__init__.py (100%)
rename src/backend/{orchestration => core/agent_runtime/integrations}/compaction_stream.py (95%)
create mode 100644 src/backend/core/agent_runtime/integrations/evolution.py
create mode 100644 src/backend/core/agent_runtime/integrations/memory.py
create mode 100644 src/backend/core/agent_runtime/integrations/memory_budget.py
create mode 100644 src/backend/core/agent_runtime/integrations/memory_identity.py
create mode 100644 src/backend/core/agent_runtime/integrations/memory_session.py
create mode 100644 src/backend/core/agent_runtime/integrations/memory_state.py
create mode 100644 src/backend/core/agent_runtime/integrations/ontology_evidence.py
create mode 100644 src/backend/core/agent_runtime/integrations/ontology_repair.py
create mode 100644 src/backend/core/agent_runtime/main_state.py
rename src/backend/{orchestration/subagents => core/agent_runtime}/plugin_visibility.py (91%)
create mode 100644 src/backend/core/agent_runtime/prepare_direct.py
create mode 100644 src/backend/core/agent_runtime/prepare_main.py
create mode 100644 src/backend/core/agent_runtime/runner.py
create mode 100644 src/backend/core/agent_runtime/skills.py
rename src/backend/{orchestration => core/agent_runtime}/strategy.py (100%)
create mode 100644 src/backend/core/agent_runtime/stream_direct.py
create mode 100644 src/backend/core/agent_runtime/stream_main.py
create mode 100644 src/backend/core/agent_runtime/targets.py
delete mode 100644 src/backend/core/agent_skills/backends/__init__.py
delete mode 100644 src/backend/core/agent_skills/cache_refresh.py
rename src/backend/core/{services/agent_api_service.py => auth/agent_api.py} (99%)
rename src/backend/core/{services => auth}/api_key_runs.py (85%)
rename src/backend/core/{services/api_key_service.py => auth/api_keys.py} (95%)
create mode 100644 src/backend/core/auth/bearer.py
create mode 100644 src/backend/core/auth/edition.py
create mode 100644 src/backend/core/auth/local_user_bootstrap.py
create mode 100644 src/backend/core/auth/local_user_policy.py
rename src/backend/core/{services/local_user_service.py => auth/local_users.py} (79%)
create mode 100644 src/backend/core/auth/local_users_constants.py
create mode 100644 src/backend/core/auth/model_selection.py
rename src/backend/core/auth/{account_policy.py => personal_account_policy.py} (100%)
rename src/backend/core/auth/{account_view.py => personal_account_view.py} (100%)
rename src/backend/core/auth/{edition_capabilities.py => personal_edition_capabilities.py} (100%)
rename src/backend/core/auth/{permissions_iface.py => personal_permissions_iface.py} (99%)
create mode 100644 src/backend/core/auth/public_identity.py
rename src/backend/core/{services/user_service.py => auth/users.py} (92%)
create mode 100644 src/backend/core/capabilities/agents/__init__.py
create mode 100644 src/backend/core/capabilities/agents/available_resources.py
create mode 100644 src/backend/core/capabilities/agents/base.py
create mode 100644 src/backend/core/capabilities/agents/base_constants.py
rename src/backend/core/{llm/builtin_subagents.py => capabilities/agents/builtin.py} (98%)
rename src/backend/core/{services/user_agent_service.py => capabilities/agents/edition.py} (64%)
rename src/backend/core/{services/agent_markdown.py => capabilities/agents/markdown.py} (100%)
create mode 100644 src/backend/core/capabilities/agents/version_history.py
create mode 100644 src/backend/core/capabilities/catalog/__init__.py
rename src/backend/core/{config/catalog.py => capabilities/catalog/api.py} (92%)
rename src/backend/core/{config/catalog_common.py => capabilities/catalog/common.py} (90%)
rename src/backend/core/{config/catalog_details.py => capabilities/catalog/details.py} (96%)
rename src/backend/core/{config/catalog_loader.py => capabilities/catalog/loader.py} (85%)
rename src/backend/core/{config/catalog_migration.py => capabilities/catalog/migration.py} (97%)
rename src/backend/core/{services/catalog_service.py => capabilities/catalog/overrides.py} (100%)
rename src/backend/core/{config/catalog_resolver.py => capabilities/catalog/resolver.py} (87%)
rename src/backend/core/{config/catalog_runtime.py => capabilities/catalog/runtime.py} (87%)
create mode 100644 src/backend/core/capabilities/identity_rules.py
rename src/backend/core/{services/desktop_capability_protocol.py => capabilities/manifest_protocol.py} (99%)
create mode 100644 src/backend/core/capabilities/mcp/__init__.py
rename src/backend/core/{services/mcp_builtin_specs.py => capabilities/mcp/builtin_specs.py} (94%)
rename src/backend/core/{services/edition_mcp.py => capabilities/mcp/edition.py} (100%)
rename src/backend/core/{services/mcp_management_service.py => capabilities/mcp/management.py} (89%)
create mode 100644 src/backend/core/capabilities/mcp/oauth.py
create mode 100644 src/backend/core/capabilities/mcp/oauth_storage.py
rename src/backend/core/{services/mcp_service.py => capabilities/mcp/service.py} (91%)
create mode 100644 src/backend/core/capabilities/mcp/tool_metadata.py
create mode 100644 src/backend/core/capabilities/mcp/tool_schema.py
create mode 100644 src/backend/core/capabilities/mcp/tool_schema_backfill.py
create mode 100644 src/backend/core/capabilities/plugins/README.md
rename src/backend/core/{ => capabilities}/plugins/__init__.py (100%)
rename src/backend/core/{ => capabilities}/plugins/connections.py (83%)
create mode 100644 src/backend/core/capabilities/plugins/management/__init__.py
rename src/backend/core/{ => capabilities}/plugins/management/admin_config.py (88%)
rename src/backend/core/{services/manager_bundle_upgrade.py => capabilities/plugins/management/bundle_upgrade.py} (92%)
rename src/backend/core/{services/cloud_management.py => capabilities/plugins/management/cloud.py} (78%)
rename src/backend/core/{ => capabilities}/plugins/management/components.py (88%)
rename src/backend/core/{services/management_contract.py => capabilities/plugins/management/contract.py} (100%)
rename src/backend/core/{ => capabilities}/plugins/management/defaults.py (93%)
rename src/backend/core/{ => capabilities}/plugins/management/details.py (58%)
rename src/backend/core/{services/management_errors.py => capabilities/plugins/management/errors.py} (100%)
rename src/backend/core/{ => capabilities}/plugins/management/installation.py (81%)
rename src/backend/core/{ => capabilities}/plugins/management/lifecycle.py (87%)
rename src/backend/core/{services/management_package.py => capabilities/plugins/management/package.py} (91%)
rename src/backend/core/{ => capabilities}/plugins/management/packages.py (70%)
rename src/backend/core/{ => capabilities}/plugins/management/projection.py (63%)
rename src/backend/core/{ => capabilities}/plugins/management/queries.py (86%)
rename src/backend/core/{ => capabilities}/plugins/packaging/__init__.py (100%)
rename src/backend/core/{ => capabilities}/plugins/packaging/archive.py (96%)
rename src/backend/core/{ => capabilities}/plugins/packaging/definitions.py (94%)
rename src/backend/core/{ => capabilities}/plugins/packaging/importer.py (96%)
rename src/backend/core/{ => capabilities}/plugins/packaging/models.py (100%)
create mode 100644 src/backend/core/capabilities/plugins/packaging/presentation.py
rename src/backend/core/{ => capabilities}/plugins/packaging/runtime_assets.py (100%)
rename src/backend/core/{ => capabilities}/plugins/packaging/skill_transform.py (96%)
rename src/backend/core/{ => capabilities}/plugins/packaging/sources.py (83%)
create mode 100644 src/backend/core/capabilities/plugins/resources/__init__.py
rename src/backend/core/{ => capabilities}/plugins/resources/artifacts.py (93%)
rename src/backend/core/{ => capabilities}/plugins/resources/authentication.py (93%)
rename src/backend/core/{ => capabilities}/plugins/resources/configuration.py (100%)
rename src/backend/core/{ => capabilities}/plugins/resources/confinement.py (83%)
rename src/backend/core/{ => capabilities}/plugins/resources/crypto.py (100%)
create mode 100644 src/backend/core/capabilities/plugins/resources/deadlines.py
rename src/backend/core/{ => capabilities}/plugins/resources/installation.py (56%)
rename src/backend/core/{ => capabilities}/plugins/resources/lifecycle.py (59%)
create mode 100644 src/backend/core/capabilities/plugins/resources/owners.py
create mode 100644 src/backend/core/capabilities/plugins/resources/prewarm.py
create mode 100644 src/backend/core/capabilities/plugins/resources/reservation.py
create mode 100644 src/backend/core/capabilities/plugins/resources/service.py
rename src/backend/core/{ => capabilities}/plugins/resources/store.py (97%)
rename src/backend/core/{ => capabilities}/plugins/resources/stream.py (72%)
rename src/backend/core/{ => capabilities}/plugins/resources/transport.py (100%)
rename src/backend/core/{ => capabilities}/plugins/runtime/__init__.py (76%)
rename src/backend/core/{ => capabilities}/plugins/runtime/activation_history.py (97%)
rename src/backend/core/{ => capabilities}/plugins/runtime/models.py (100%)
rename src/backend/core/{ => capabilities}/plugins/runtime/resolution.py (91%)
rename src/backend/core/{ => capabilities}/plugins/runtime/sticky_selection.py (85%)
rename src/backend/core/capabilities/{ref.py => reference.py} (98%)
delete mode 100644 src/backend/core/capabilities/runtime/__init__.py
delete mode 100644 src/backend/core/capabilities/skills.py
rename src/backend/core/{agent_skills => capabilities/skills}/__init__.py (97%)
create mode 100644 src/backend/core/capabilities/skills/backends/__init__.py
rename src/backend/core/{agent_skills => capabilities/skills}/backends/composite.py (97%)
rename src/backend/core/{agent_skills => capabilities/skills}/backends/database.py (99%)
rename src/backend/core/{agent_skills => capabilities/skills}/backends/filesystem.py (98%)
rename src/backend/core/{agent_skills => capabilities/skills}/backends/protocol.py (100%)
rename src/backend/core/{agent_skills => capabilities/skills}/binary_files.py (100%)
create mode 100644 src/backend/core/capabilities/skills/cache_refresh.py
rename src/backend/core/{agent_skills => capabilities/skills}/config.py (89%)
create mode 100644 src/backend/core/capabilities/skills/deps/__init__.py
rename src/backend/core/{services/skill_deps_aggregator.py => capabilities/skills/deps/aggregator.py} (92%)
rename src/backend/core/{agent_skills/deps_detector.py => capabilities/skills/deps/detector.py} (100%)
rename src/backend/core/{services/skill_deps_prober.py => capabilities/skills/deps/prober.py} (98%)
rename src/backend/core/{services/skill_deps_request_service.py => capabilities/skills/deps/requests.py} (81%)
rename src/backend/core/{services/skill_icon_service.py => capabilities/skills/icons.py} (100%)
create mode 100644 src/backend/core/capabilities/skills/installation.py
rename src/backend/core/{agent_skills => capabilities/skills}/loader.py (77%)
create mode 100644 src/backend/core/capabilities/skills/loader_factory.py
rename src/backend/core/{services/skill_management_service.py => capabilities/skills/management.py} (93%)
rename src/backend/core/{agent_skills => capabilities/skills}/publication.py (89%)
rename src/backend/core/{agent_skills => capabilities/skills}/registry.py (97%)
create mode 100644 src/backend/core/capabilities/skills/runtime_loader.py
create mode 100644 src/backend/core/capabilities/skills/script_discovery.py
rename src/backend/core/{agent_skills => capabilities/skills}/selection_record.py (98%)
rename src/backend/core/{agent_skills => capabilities/skills}/skill_archive.py (88%)
create mode 100644 src/backend/core/capabilities/skills/zip_reader.py
delete mode 100644 src/backend/core/channels/adapters/dingtalk.py
delete mode 100644 src/backend/core/channels/adapters/lark.py
delete mode 100644 src/backend/core/channels/inbound.py
rename src/backend/core/{services => chat}/audio_transcription.py (98%)
create mode 100644 src/backend/core/chat/compaction/__init__.py
rename src/backend/core/{services/context_budget_checkpoint.py => chat/compaction/budget_checkpoint.py} (97%)
rename src/backend/core/{services/compaction_coordinator.py => chat/compaction/coordinator.py} (94%)
rename src/backend/core/{services/compaction_history.py => chat/compaction/history.py} (93%)
rename src/backend/core/{services/compaction_progress.py => chat/compaction/progress.py} (100%)
rename src/backend/core/{services/compaction_service.py => chat/compaction/service.py} (96%)
rename src/backend/core/{services/compaction_summary.py => chat/compaction/summary.py} (93%)
rename src/backend/core/{services/context_usage_projection.py => chat/compaction/usage_projection.py} (97%)
create mode 100644 src/backend/core/chat/compaction_checkpoints.py
create mode 100644 src/backend/core/chat/compaction_snapshots.py
create mode 100644 src/backend/core/chat/compaction_types.py
rename src/backend/core/{services/chat_edition.py => chat/edition.py} (100%)
rename src/backend/{orchestration => core/chat}/followups.py (83%)
rename src/backend/core/{services/chat_fork_service.py => chat/fork.py} (99%)
create mode 100644 src/backend/core/chat/history_operations.py
create mode 100644 src/backend/core/chat/limits.py
rename src/backend/{orchestration => core/chat}/message_parser.py (97%)
create mode 100644 src/backend/core/chat/message_queries.py
create mode 100644 src/backend/core/chat/message_writes.py
create mode 100644 src/backend/core/chat/mode_seed.py
create mode 100644 src/backend/core/chat/mode_types.py
create mode 100644 src/backend/core/chat/mode_validation.py
rename src/backend/core/{services/chat_mode_service.py => chat/modes.py} (62%)
rename src/backend/core/{services/session_operations => chat/operations}/__init__.py (100%)
create mode 100644 src/backend/core/chat/operations/admission.py
rename src/backend/core/{services/session_operations => chat/operations}/admission_errors.py (92%)
rename src/backend/core/{services/session_operations => chat/operations}/agent_targets.py (94%)
rename src/backend/core/{services/session_operations => chat/operations}/commands.py (82%)
rename src/backend/core/{services/session_operations => chat/operations}/history.py (92%)
rename src/backend/core/{services/session_operations => chat/operations}/invocation.py (85%)
rename src/backend/core/{services/session_operations => chat/operations}/projects.py (90%)
rename src/backend/core/{services/session_operations => chat/operations}/request_context.py (91%)
rename src/backend/core/{services/session_operations => chat/operations}/runs.py (90%)
rename src/backend/core/{services/session_operations => chat/operations}/schemas.py (100%)
rename src/backend/core/{services/session_operations => chat/operations}/session_context.py (69%)
rename src/backend/core/{services/session_operations => chat/operations}/sessions.py (86%)
rename src/backend/core/{services/plan_service.py => chat/plans.py} (100%)
rename src/backend/core/{services => chat}/project_init.py (91%)
create mode 100644 src/backend/core/chat/projection/__init__.py
create mode 100644 src/backend/core/chat/projection/codec.py
rename src/backend/{orchestration/run_projection.py => core/chat/projection/reducer.py} (62%)
rename src/backend/{orchestration/run_projection_storage.py => core/chat/projection/storage.py} (89%)
rename src/backend/{orchestration/run_subscription.py => core/chat/projection/subscription.py} (80%)
rename src/backend/core/{services/public_chat_service.py => chat/public.py} (78%)
rename src/backend/core/{services/chat_reference_service.py => chat/references.py} (98%)
create mode 100644 src/backend/core/chat/runtime/__init__.py
create mode 100644 src/backend/core/chat/runtime/admission.py
create mode 100644 src/backend/core/chat/runtime/chunks.py
rename src/backend/{orchestration/post_turn_compaction.py => core/chat/runtime/compaction.py} (84%)
create mode 100644 src/backend/core/chat/runtime/completion.py
create mode 100644 src/backend/core/chat/runtime/followups.py
create mode 100644 src/backend/core/chat/runtime/handoff.py
rename src/backend/{orchestration => core/chat/runtime}/job_wakeup.py (64%)
create mode 100644 src/backend/core/chat/runtime/pauses.py
create mode 100644 src/backend/core/chat/runtime/recovery.py
create mode 100644 src/backend/core/chat/runtime/recovery_snapshot.py
create mode 100644 src/backend/core/chat/runtime/snapshots.py
create mode 100644 src/backend/core/chat/runtime/steering.py
create mode 100644 src/backend/core/chat/runtime/turn_state.py
create mode 100644 src/backend/core/chat/runtime/worker.py
rename src/backend/core/{services/chat_sequencer.py => chat/sequencer.py} (92%)
create mode 100644 src/backend/core/chat/service.py
create mode 100644 src/backend/core/chat/session_operations.py
rename src/backend/core/{services => chat}/sidebar_status.py (100%)
rename src/backend/core/{services/chat_steer_service.py => chat/steer.py} (95%)
create mode 100644 src/backend/core/chat/steer_hook.py
rename src/backend/core/{services => chat}/steer_queue.py (100%)
rename src/backend/core/{services/subagent_routing_service.py => chat/subagent_routing.py} (100%)
rename src/backend/core/{services/chat_titles.py => chat/titles.py} (92%)
rename src/backend/{orchestration => core/chat}/tool_payloads.py (92%)
rename src/backend/core/config/{edition_display_names.py => edition.py} (78%)
create mode 100644 src/backend/core/config/store/__init__.py
create mode 100644 src/backend/core/config/store/config_keys.py
create mode 100644 src/backend/core/config/store/execution_options.py
create mode 100644 src/backend/core/config/store/runtime_env.py
create mode 100644 src/backend/core/config/store/system_config.py
create mode 100644 src/backend/core/config/store/system_config_constants.py
delete mode 100644 src/backend/core/content/content_blocks.py
delete mode 100644 src/backend/core/content/file_parser.py
create mode 100644 src/backend/core/db/artifact_scope.py
create mode 100644 src/backend/core/db/edition.py
delete mode 100644 src/backend/core/db/models/channel_relay.py
create mode 100644 src/backend/core/db/models/chat_resources.py
create mode 100644 src/backend/core/db/models/chat_runs.py
create mode 100644 src/backend/core/db/models/column_types.py
delete mode 100644 src/backend/core/db/models/kb_wiki.py
delete mode 100644 src/backend/core/db/models/observability.py
rename src/backend/core/db/{model_extensions.py => personal_model_fields.py} (100%)
rename src/backend/core/db/{edition_tables.py => personal_schema.py} (97%)
create mode 100644 src/backend/core/db/repository/folder.py
rename src/backend/core/db/repository/{agent.py => personal_agent.py} (100%)
rename src/backend/core/db/repository/{audit.py => personal_audit.py} (96%)
create mode 100644 src/backend/core/db/system_configuration.py
create mode 100644 src/backend/core/db/threads.py
create mode 100644 src/backend/core/db/user_cleanup.py
delete mode 100644 src/backend/core/evolution/activation.py
delete mode 100644 src/backend/core/evolution/agent_profile.py
create mode 100644 src/backend/core/evolution/assessment/__init__.py
rename src/backend/core/evolution/{ => assessment}/applicability.py (98%)
rename src/backend/core/evolution/{ => assessment}/benchmark.py (98%)
rename src/backend/core/evolution/{ => assessment}/evaluator.py (100%)
rename src/backend/core/evolution/{ => assessment}/experiments.py (99%)
rename src/backend/core/evolution/{ => assessment}/fault_injection.py (98%)
rename src/backend/core/evolution/{ => assessment}/metrics.py (100%)
rename src/backend/core/evolution/{ => assessment}/similarity.py (98%)
rename src/backend/core/evolution/{ => assessment}/sop_judge.py (98%)
create mode 100644 src/backend/core/evolution/cycle/__init__.py
create mode 100644 src/backend/core/evolution/cycle/activation.py
create mode 100644 src/backend/core/evolution/cycle/activation_constants.py
create mode 100644 src/backend/core/evolution/cycle/activation_policy.py
create mode 100644 src/backend/core/evolution/cycle/activation_types.py
create mode 100644 src/backend/core/evolution/cycle/candidate_records.py
create mode 100644 src/backend/core/evolution/cycle/composition_candidates.py
create mode 100644 src/backend/core/evolution/cycle/cycle_extras.py
create mode 100644 src/backend/core/evolution/cycle/cycle_extras_constants.py
create mode 100644 src/backend/core/evolution/cycle/episode_views.py
create mode 100644 src/backend/core/evolution/cycle/evaluation.py
create mode 100644 src/backend/core/evolution/cycle/extra_findings.py
create mode 100644 src/backend/core/evolution/cycle/graph_candidates.py
create mode 100644 src/backend/core/evolution/cycle/loop.py
create mode 100644 src/backend/core/evolution/cycle/memory_candidates.py
create mode 100644 src/backend/core/evolution/cycle/memory_findings.py
create mode 100644 src/backend/core/evolution/cycle/ordering.py
create mode 100644 src/backend/core/evolution/cycle/patterns.py
create mode 100644 src/backend/core/evolution/cycle/profile_release.py
create mode 100644 src/backend/core/evolution/cycle/promotion.py
create mode 100644 src/backend/core/evolution/cycle/promotion_constants.py
create mode 100644 src/backend/core/evolution/cycle/promotion_types.py
create mode 100644 src/backend/core/evolution/cycle/prompt_candidates.py
rename src/backend/core/evolution/{ => cycle}/readiness.py (93%)
rename src/backend/core/evolution/{ => cycle}/release.py (100%)
create mode 100644 src/backend/core/evolution/cycle/retirement.py
rename src/backend/core/evolution/{ => cycle}/runtime_binding.py (87%)
create mode 100644 src/backend/core/evolution/cycle/skill_document.py
create mode 100644 src/backend/core/evolution/cycle/skill_release.py
rename src/backend/core/evolution/{ => cycle}/timescales.py (99%)
rename src/backend/core/evolution/{ => cycle}/trigger.py (81%)
create mode 100644 src/backend/core/evolution/cycle/usage_evidence.py
delete mode 100644 src/backend/core/evolution/cycle_extras.py
create mode 100644 src/backend/core/evolution/evidence/__init__.py
rename src/backend/core/evolution/{ => evidence}/cassette.py (94%)
rename src/backend/core/evolution/{ => evidence}/counterfactual.py (99%)
rename src/backend/core/evolution/{ => evidence}/credit.py (99%)
rename src/backend/core/evolution/{ => evidence}/events.py (99%)
rename src/backend/core/evolution/{ => evidence}/evidence_contract.py (100%)
rename src/backend/core/evolution/{ => evidence}/evidence_resolver.py (98%)
rename src/backend/core/evolution/{ => evidence}/exposure.py (95%)
rename src/backend/core/evolution/{ => evidence}/lineage.py (99%)
rename src/backend/core/evolution/{ => evidence}/outcomes.py (100%)
rename src/backend/core/evolution/{ => evidence}/replay.py (100%)
rename src/backend/core/evolution/{ => evidence}/trace_assembler.py (98%)
rename src/backend/core/evolution/{ => evidence}/trace_store.py (97%)
create mode 100644 src/backend/core/evolution/generators/__init__.py
rename src/backend/core/evolution/{ => generators}/graph_scan.py (100%)
rename src/backend/core/evolution/{ => generators}/memory_apply.py (98%)
rename src/backend/core/evolution/{ => generators}/memory_ops.py (99%)
rename src/backend/core/evolution/{ => generators}/memory_scan.py (99%)
rename src/backend/core/evolution/{ => generators}/ontology_bridge.py (100%)
rename src/backend/core/evolution/{ => generators}/orchestration_gen.py (99%)
rename src/backend/core/evolution/{ => generators}/prompt_gen.py (100%)
rename src/backend/core/evolution/{ => generators}/skill_candidate_engine.py (96%)
create mode 100644 src/backend/core/evolution/generators/skill_drafting.py
create mode 100644 src/backend/core/evolution/generators/skill_evidence.py
create mode 100644 src/backend/core/evolution/generators/skill_gen.py
create mode 100644 src/backend/core/evolution/generators/skill_gen_constants.py
delete mode 100644 src/backend/core/evolution/loop.py
create mode 100644 src/backend/core/evolution/profiles/__init__.py
create mode 100644 src/backend/core/evolution/profiles/agent_profile.py
create mode 100644 src/backend/core/evolution/profiles/agent_profile_constants.py
create mode 100644 src/backend/core/evolution/profiles/change_preview.py
rename src/backend/core/evolution/{ => profiles}/console_view.py (99%)
rename src/backend/core/evolution/{ => profiles}/federation.py (99%)
rename src/backend/core/evolution/{ => profiles}/personal.py (88%)
rename src/backend/core/evolution/{user_settings.py => profiles/personal_review.py} (60%)
rename src/backend/core/evolution/{ => profiles}/policies.py (88%)
rename src/backend/core/evolution/{ => profiles}/prefs.py (100%)
create mode 100644 src/backend/core/evolution/profiles/profile_selection.py
create mode 100644 src/backend/core/evolution/profiles/stall_hook.py
create mode 100644 src/backend/core/evolution/profiles/user_settings.py
create mode 100644 src/backend/core/evolution/profiles/user_settings_constants.py
delete mode 100644 src/backend/core/evolution/promotion.py
create mode 100644 src/backend/core/evolution/settlement/__init__.py
create mode 100644 src/backend/core/evolution/settlement/memory_service.py
rename src/backend/core/evolution/{ => settlement}/settlement.py (100%)
rename src/backend/core/evolution/{ => settlement}/settlement_runner.py (88%)
rename src/backend/core/evolution/{ => settlement}/settlement_store.py (98%)
delete mode 100644 src/backend/core/evolution/skill_gen.py
delete mode 100644 src/backend/core/extensions/__init__.py
delete mode 100644 src/backend/core/extensions/registry.py
create mode 100644 src/backend/core/files/__init__.py
create mode 100644 src/backend/core/files/artifact_bytes.py
rename src/backend/core/{ => files}/artifacts/__init__.py (96%)
rename src/backend/core/{services/artifact_backfill.py => files/artifacts/backfill.py} (95%)
create mode 100644 src/backend/core/files/artifacts/generated.py
rename src/backend/core/{content/artifact_key_repair.py => files/artifacts/key_repair.py} (100%)
rename src/backend/core/{ => files}/artifacts/local_reference.py (89%)
rename src/backend/core/{services/artifact_location.py => files/artifacts/location.py} (52%)
rename src/backend/core/{content/artifact_reader.py => files/artifacts/reader.py} (91%)
rename src/backend/core/{content/artifact_refs.py => files/artifacts/refs.py} (93%)
rename src/backend/core/{services/artifact_service.py => files/artifacts/service.py} (93%)
rename src/backend/core/{ => files}/artifacts/store.py (96%)
rename src/backend/core/{content/artifact_summary.py => files/artifacts/summary.py} (97%)
rename src/backend/core/{services/artifact_text_service.py => files/artifacts/text.py} (92%)
create mode 100644 src/backend/core/files/content/__init__.py
rename src/backend/core/{ => files}/content/archive_browser.py (99%)
rename src/backend/core/{ => files}/content/archive_limits.py (100%)
create mode 100644 src/backend/core/files/content/branding.py
create mode 100644 src/backend/core/files/content/content_blocks.py
create mode 100644 src/backend/core/files/content/content_blocks_constants.py
create mode 100644 src/backend/core/files/content/file_formats.py
create mode 100644 src/backend/core/files/content/file_parser.py
rename src/backend/core/{ => files}/content/file_validation.py (100%)
rename src/backend/core/{ => files}/content/office.py (100%)
rename src/backend/core/{ => files}/content/office_preview_cache.py (98%)
create mode 100644 src/backend/core/files/content/parser_config.py
create mode 100644 src/backend/core/files/content/pdf_parser.py
create mode 100644 src/backend/core/files/content/presentation_parser.py
create mode 100644 src/backend/core/files/content/spreadsheet_parser.py
rename src/backend/core/{ => files}/content/svg_fit.py (97%)
create mode 100644 src/backend/core/files/content/text_parser.py
create mode 100644 src/backend/core/files/content/word_parser.py
create mode 100644 src/backend/core/files/edition.py
rename src/backend/core/{services => files}/folder_batch.py (69%)
rename src/backend/core/{services/personal_file_operations.py => files/personal_operations.py} (86%)
rename src/backend/core/{ => files}/spaces/__init__.py (100%)
rename src/backend/core/{ => files}/spaces/access.py (95%)
rename src/backend/core/{ => files}/spaces/files.py (96%)
rename src/backend/core/{ => files}/spaces/leases.py (86%)
rename src/backend/core/{ => files}/spaces/mounts.py (78%)
rename src/backend/core/{ => files}/spaces/projects.py (94%)
rename src/backend/core/{ => files}/spaces/recovery.py (94%)
rename src/backend/core/{ => files}/spaces/sandbox.py (73%)
rename src/backend/core/{ => files}/spaces/search.py (97%)
rename src/backend/core/{ => files}/spaces/text.py (92%)
rename src/backend/core/{ => files}/spaces/tree.py (100%)
rename src/backend/core/{ => files}/storage/__init__.py (56%)
rename src/backend/core/{ => files}/storage/entries.py (100%)
rename src/backend/core/{ => files}/storage/factory.py (81%)
rename src/backend/core/{ => files}/storage/local.py (96%)
rename src/backend/core/{ => files}/storage/local_directory.py (94%)
rename src/backend/core/{ => files}/storage/local_paths.py (98%)
rename src/backend/core/{ => files}/storage/protocol.py (100%)
rename src/backend/core/{ => files}/storage/s3.py (98%)
rename src/backend/core/{ => files}/storage/s3_directory.py (96%)
rename src/backend/core/{services => files}/upload_retry.py (92%)
rename src/backend/core/{capabilities/errors.py => harness/capability_errors.py} (100%)
create mode 100644 src/backend/core/harness/context_items.py
rename src/backend/core/{llm => harness}/execution_manifest.py (99%)
rename src/backend/core/{services/harness_ledger.py => harness/ledger.py} (53%)
create mode 100644 src/backend/core/harness/log_audit.py
create mode 100644 src/backend/core/harness/log_context.py
create mode 100644 src/backend/core/harness/log_records.py
create mode 100644 src/backend/core/harness/log_writer.py
create mode 100644 src/backend/core/harness/log_writer_constants.py
create mode 100644 src/backend/core/harness/observation_writer.py
create mode 100644 src/backend/core/harness/registry/__init__.py
create mode 100644 src/backend/core/harness/registry/authentication.py
create mode 100644 src/backend/core/harness/registry/build_governance.py
create mode 100644 src/backend/core/harness/registry/cache_invalidation.py
create mode 100644 src/backend/core/harness/registry/capability_management.py
create mode 100644 src/backend/core/harness/registry/capability_revisions.py
create mode 100644 src/backend/core/harness/registry/capability_runtime.py
create mode 100644 src/backend/core/harness/registry/catalog_options.py
create mode 100644 src/backend/core/harness/registry/chat_execution.py
create mode 100644 src/backend/core/harness/registry/chat_history.py
create mode 100644 src/backend/core/harness/registry/compaction.py
create mode 100644 src/backend/core/harness/registry/device_catalog.py
rename src/backend/core/{extensions => harness/registry}/edition.py (100%)
create mode 100644 src/backend/core/harness/registry/evaluation.py
create mode 100644 src/backend/core/harness/registry/execution_account.py
create mode 100644 src/backend/core/harness/registry/execution_catalog.py
create mode 100644 src/backend/core/harness/registry/execution_identity.py
create mode 100644 src/backend/core/harness/registry/hook_factories.py
create mode 100644 src/backend/core/harness/registry/host_runtime.py
create mode 100644 src/backend/core/harness/registry/job_operations.py
create mode 100644 src/backend/core/harness/registry/local_files.py
create mode 100644 src/backend/core/harness/registry/local_publication.py
create mode 100644 src/backend/core/harness/registry/marketplace_catalog.py
create mode 100644 src/backend/core/harness/registry/mcp_clients.py
create mode 100644 src/backend/core/harness/registry/model_credentials.py
create mode 100644 src/backend/core/harness/registry/model_services.py
create mode 100644 src/backend/core/harness/registry/package_sources.py
create mode 100644 src/backend/core/harness/registry/package_views.py
create mode 100644 src/backend/core/harness/registry/registry.py
create mode 100644 src/backend/core/harness/registry/resource_owners.py
create mode 100644 src/backend/core/harness/registry/retrieval_context.py
create mode 100644 src/backend/core/harness/registry/run_cancellation.py
create mode 100644 src/backend/core/harness/registry/run_capabilities.py
create mode 100644 src/backend/core/harness/registry/schema_lifecycle.py
create mode 100644 src/backend/core/harness/registry/skill_content.py
create mode 100644 src/backend/core/harness/registry/system_configuration.py
create mode 100644 src/backend/core/harness/registry/tool_operations.py
rename src/backend/core/{services/desktop_gateway_uploads.py => harness/registry/transfer_channels.py} (53%)
create mode 100644 src/backend/core/harness/registry/user_cleanup.py
create mode 100644 src/backend/core/harness/tool_call_context.py
create mode 100644 src/backend/core/harness/tool_effect_context.py
create mode 100644 src/backend/core/infra/byte_range.py
rename src/backend/core/{capabilities/junction.py => infra/directory_links.py} (92%)
create mode 100644 src/backend/core/infra/failures.py
rename src/backend/core/{capabilities/lockfile.py => infra/file_lock.py} (100%)
rename src/backend/core/{ => infra}/immutable.py (100%)
create mode 100644 src/backend/core/infra/json_output.py
create mode 100644 src/backend/core/infra/local_runtime/__init__.py
create mode 100644 src/backend/core/infra/local_runtime/dynamic_servers.py
rename src/backend/{orchestration/local_sidecar_lifecycle.py => core/infra/local_runtime/lifecycle.py} (89%)
rename src/backend/core/{sandbox/providers/script_runner/auth.py => infra/local_runtime/runner_auth.py} (100%)
rename src/backend/{orchestration/local_subprocess.py => core/infra/local_runtime/startup.py} (87%)
create mode 100644 src/backend/core/infra/model_output.py
create mode 100644 src/backend/core/infra/probes/__init__.py
rename src/backend/core/{services/edition_service_probe.py => infra/probes/edition.py} (100%)
rename src/backend/core/{services => infra/probes}/service_probes.py (89%)
delete mode 100644 src/backend/core/infra/rate_limit.py
rename src/backend/core/{chat/display_bounds.py => infra/text_bounds.py} (100%)
rename src/backend/{orchestration => core}/jobs/__init__.py (100%)
rename src/backend/{orchestration => core}/jobs/agent.py (94%)
rename src/backend/{orchestration => core}/jobs/callback.py (100%)
rename src/backend/{orchestration => core}/jobs/files.py (95%)
rename src/backend/{orchestration => core}/jobs/notifications.py (81%)
rename src/backend/{orchestration => core}/jobs/owner.py (62%)
rename src/backend/{orchestration => core}/jobs/process.py (95%)
create mode 100644 src/backend/core/jobs/reminder_hook.py
rename src/backend/{orchestration => core}/jobs/runner.py (100%)
rename src/backend/{orchestration/job_runtime.py => core/jobs/runtime.py} (85%)
rename src/backend/{orchestration => core}/jobs/sdk.py (100%)
rename src/backend/{orchestration => core}/jobs/state.py (98%)
rename src/backend/core/{services/job_service.py => jobs/store.py} (100%)
rename src/backend/{orchestration => core}/jobs/supervisor.py (88%)
create mode 100644 src/backend/core/jobs/tool_operations.py
delete mode 100644 src/backend/core/kb/kb_assets.py
delete mode 100644 src/backend/core/kb/kb_parser.py
delete mode 100644 src/backend/core/kb/kb_vector.py
delete mode 100644 src/backend/core/kb/wiki/local_provider.py
delete mode 100644 src/backend/core/llm/_distill_shared.py
create mode 100644 src/backend/core/llm/api_runtime/__init__.py
rename src/backend/core/llm/{agent_api_runtime.py => api_runtime/agent.py} (97%)
rename src/backend/core/llm/{agent_api_skills.py => api_runtime/agent_skills.py} (96%)
rename src/backend/core/llm/{agent_api_tools.py => api_runtime/agent_tools.py} (90%)
rename src/backend/core/llm/{project_api_runtime.py => api_runtime/project.py} (94%)
delete mode 100644 src/backend/core/llm/capability_tools.py
create mode 100644 src/backend/core/llm/citations/__init__.py
create mode 100644 src/backend/core/llm/citations/allocator.py
create mode 100644 src/backend/core/llm/citations/anchor.py
create mode 100644 src/backend/core/llm/citations/anchor_state.py
create mode 100644 src/backend/core/llm/citations/extraction.py
create mode 100644 src/backend/core/llm/citations/hook.py
create mode 100644 src/backend/core/llm/citations/types.py
delete mode 100644 src/backend/core/llm/command_middleware.py
create mode 100644 src/backend/core/llm/context/__init__.py
create mode 100644 src/backend/core/llm/context/adapter.py
create mode 100644 src/backend/core/llm/context/attachment_hook.py
rename src/backend/core/{services/compaction_budget.py => llm/context/budget.py} (98%)
rename src/backend/core/llm/{ => context}/compacting_agent.py (96%)
rename src/backend/core/llm/{ => context}/compaction.py (98%)
create mode 100644 src/backend/core/llm/context/content.py
create mode 100644 src/backend/core/llm/context/ir.py
create mode 100644 src/backend/core/llm/context/ir_constants.py
create mode 100644 src/backend/core/llm/context/items.py
create mode 100644 src/backend/core/llm/context/message_defaults.py
rename src/backend/core/llm/{ => context}/offloader.py (96%)
create mode 100644 src/backend/core/llm/context/provider_conversion.py
create mode 100644 src/backend/core/llm/context/reasoning_hooks.py
create mode 100644 src/backend/core/llm/context/reference_conversion.py
create mode 100644 src/backend/core/llm/context/rendering.py
rename src/backend/core/llm/{context_safety.py => context/safety.py} (87%)
rename src/backend/core/llm/{ => context}/system_reminder.py (94%)
rename src/backend/core/llm/{context_usage.py => context/usage.py} (99%)
create mode 100644 src/backend/core/llm/context/workspace_hooks.py
delete mode 100644 src/backend/core/llm/context_adapter.py
delete mode 100644 src/backend/core/llm/context_ir.py
rename src/backend/core/llm/factory/{selection/device_bindings.py => runtime/capability_bindings.py} (93%)
rename src/backend/{orchestration/registry.py => core/llm/factory/specs.py} (99%)
create mode 100644 src/backend/core/llm/factory/tools/definitions.py
delete mode 100644 src/backend/core/llm/finish_guard.py
delete mode 100644 src/backend/core/llm/hooks.py
create mode 100644 src/backend/core/llm/host_services.py
delete mode 100644 src/backend/core/llm/management_tool.py
create mode 100644 src/backend/core/llm/mcp/__init__.py
create mode 100644 src/backend/core/llm/mcp/callbacks.py
create mode 100644 src/backend/core/llm/mcp/chat_sessions.py
create mode 100644 src/backend/core/llm/mcp/client.py
create mode 100644 src/backend/core/llm/mcp/context_tools.py
create mode 100644 src/backend/core/llm/mcp/deadline.py
rename src/backend/core/llm/{mcp_invocation.py => mcp/invocation.py} (87%)
create mode 100644 src/backend/core/llm/mcp/lifetime.py
create mode 100644 src/backend/core/llm/mcp/manager.py
create mode 100644 src/backend/core/llm/mcp/results.py
create mode 100644 src/backend/core/llm/mcp/session.py
create mode 100644 src/backend/core/llm/mcp/tools.py
delete mode 100644 src/backend/core/llm/mcp_manager.py
delete mode 100644 src/backend/core/llm/mcp_pool.py
create mode 100644 src/backend/core/llm/middleware/__init__.py
rename src/backend/core/llm/{middleware_migration.py => middleware/migration_map.py} (51%)
delete mode 100644 src/backend/core/llm/middlewares.py
delete mode 100644 src/backend/core/llm/persona_distiller.py
create mode 100644 src/backend/core/llm/ports.py
delete mode 100644 src/backend/core/llm/project_environment_middleware.py
create mode 100644 src/backend/core/llm/prompt_files.py
create mode 100644 src/backend/core/llm/prompt_kinds.py
create mode 100644 src/backend/core/llm/prompt_rendering.py
create mode 100644 src/backend/core/llm/prompt_storage.py
create mode 100644 src/backend/core/llm/prompt_versions.py
create mode 100644 src/backend/core/llm/prompt_versions_state.py
delete mode 100644 src/backend/core/llm/providers/__init__.py
create mode 100644 src/backend/core/llm/runtime/__init__.py
rename src/backend/core/llm/{ => runtime}/agentscope_hook_adapter.py (63%)
create mode 100644 src/backend/core/llm/runtime/context_hook_adapter.py
create mode 100644 src/backend/core/llm/runtime/delta_coalescer.py
create mode 100644 src/backend/core/llm/runtime/file_context.py
create mode 100644 src/backend/core/llm/runtime/finish_guard.py
create mode 100644 src/backend/core/llm/runtime/hooks_constants.py
rename src/backend/core/llm/{ => runtime}/human_interaction.py (86%)
rename src/backend/core/llm/{ => runtime}/interaction_store.py (100%)
rename src/backend/core/llm/{ => runtime}/manifest_agent.py (93%)
rename src/backend/core/llm/{ => runtime}/message_compat.py (93%)
create mode 100644 src/backend/core/llm/runtime/model_cache.py
create mode 100644 src/backend/core/llm/runtime/model_selection_adapter.py
rename src/backend/core/llm/{ => runtime}/model_steps.py (98%)
rename src/backend/core/llm/{ => runtime}/project_environment.py (88%)
create mode 100644 src/backend/core/llm/runtime/project_environment_hook.py
create mode 100644 src/backend/core/llm/runtime/review_text_agent.py
rename src/backend/core/llm/{ => runtime}/session_capabilities.py (97%)
create mode 100644 src/backend/core/llm/runtime/state.py
create mode 100644 src/backend/core/llm/runtime/stream_events.py
create mode 100644 src/backend/core/llm/runtime/stream_loop.py
create mode 100644 src/backend/core/llm/runtime/stream_text.py
create mode 100644 src/backend/core/llm/runtime/streaming.py
create mode 100644 src/backend/core/llm/runtime/streaming_constants.py
rename src/backend/{orchestration => core/llm/runtime}/tool_callbacks.py (70%)
create mode 100644 src/backend/core/llm/runtime/tool_execution_adapter.py
create mode 100644 src/backend/core/llm/runtime/tool_hook_adapter.py
rename src/backend/core/llm/{ => runtime}/tool_media_store.py (95%)
create mode 100644 src/backend/core/llm/runtime/tool_result_adapter.py
create mode 100644 src/backend/core/llm/runtime/turn_state.py
rename src/backend/core/llm/{ => runtime}/workspace.py (100%)
rename src/backend/core/{agent_skills/selector.py => llm/skill_selection.py} (91%)
delete mode 100644 src/backend/core/llm/steer_middleware.py
create mode 100644 src/backend/core/llm/subagents/__init__.py
rename src/backend/core/llm/{_subagent_execution.py => subagents/execution.py} (90%)
rename src/backend/core/llm/{_subagent_prompts.py => subagents/prompts.py} (100%)
rename src/backend/core/llm/{subagent_sessions.py => subagents/sessions.py} (98%)
rename src/backend/core/llm/{_subagent_stream.py => subagents/stream.py} (80%)
rename src/backend/core/llm/{subagent_tool.py => subagents/tool.py} (89%)
delete mode 100644 src/backend/core/llm/tool_collector.py
create mode 100644 src/backend/core/llm/tool_permissions/hook.py
delete mode 100644 src/backend/core/llm/tool_permissions/middleware.py
rename src/backend/core/llm/{gateway_mcp_tool.py => tools/cloud_gateway.py} (71%)
create mode 100644 src/backend/core/llm/tools/cloud_manifest.py
rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/__init__.py (100%)
rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/gate.py (99%)
rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/pending.py (96%)
rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/shared.py (98%)
rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/state.py (89%)
delete mode 100644 src/backend/core/llm/tools/data_context_tool.py
create mode 100644 src/backend/core/llm/tools/management.py
rename src/backend/core/{ontology/toolkit.py => llm/tools/ontology_toolkit.py} (85%)
rename src/backend/core/llm/{plan_update_tool.py => tools/plan_update.py} (92%)
rename src/backend/core/{plugins/runtime/activation.py => llm/tools/plugin_activation.py} (78%)
create mode 100644 src/backend/core/memory/edition.py
create mode 100644 src/backend/core/memory/graph_contract.py
create mode 100644 src/backend/core/memory/graph_state.py
rename src/backend/core/{services/memory_settings_service.py => memory/settings.py} (98%)
create mode 100644 src/backend/core/memory/settlement_port.py
create mode 100644 src/backend/core/models/__init__.py
create mode 100644 src/backend/core/models/admission.py
create mode 100644 src/backend/core/models/api_protocol_backfill.py
rename src/backend/core/{llm => models}/chat_models.py (92%)
create mode 100644 src/backend/core/models/completions/__init__.py
rename src/backend/core/{llm => models/completions}/classifier.py (84%)
rename src/backend/core/{llm => models/completions}/single_turn.py (96%)
rename src/backend/core/{llm => models/completions}/summarizer.py (86%)
rename src/backend/core/{services/model_config.py => models/config.py} (59%)
create mode 100644 src/backend/core/models/constants.py
rename src/backend/core/{llm/context_manager.py => models/context_window.py} (97%)
create mode 100644 src/backend/core/models/embedding.py
create mode 100644 src/backend/core/models/embedding_state.py
rename src/backend/core/{llm => models}/failover.py (94%)
rename src/backend/core/{db/model_import_validation.py => models/import_validation.py} (93%)
create mode 100644 src/backend/core/models/providers/__init__.py
rename src/backend/core/{llm => models}/providers/_image_tokens.py (96%)
rename src/backend/core/{llm => models}/providers/context_probe.py (99%)
rename src/backend/core/{llm => models}/providers/openai_compat.py (96%)
rename src/backend/core/{llm => models}/providers/protocol_probe.py (100%)
rename src/backend/core/{llm => models}/providers/reasoning_probe.py (97%)
rename src/backend/core/{llm => models}/providers/registry.py (99%)
rename src/backend/core/{llm => models}/providers/stream_retry.py (96%)
rename src/backend/core/{llm => models}/providers/vendor_models.py (97%)
rename src/backend/core/{llm => models}/reasoning_effort.py (100%)
rename src/backend/core/{llm => models}/reasoning_replay.py (100%)
rename src/backend/core/{db/model_repository.py => models/repository.py} (99%)
create mode 100644 src/backend/core/models/reranking.py
rename src/backend/core/{llm/responses_models.py => models/responses.py} (80%)
rename src/backend/core/{services/user_model_selection.py => models/selection.py} (63%)
create mode 100644 src/backend/core/models/selection_hook.py
rename src/backend/core/{llm => models}/tool_call_identity.py (100%)
rename src/backend/core/{llm => models}/tool_result_media.py (69%)
rename src/backend/core/{llm/model_usage.py => models/usage.py} (98%)
create mode 100644 src/backend/core/models/vision/__init__.py
rename src/backend/core/{ => models}/vision/attachments.py (83%)
rename src/backend/core/{ => models}/vision/prompt.py (98%)
rename src/backend/core/{ => models}/vision/provider.py (97%)
rename src/backend/core/{ => models}/vision/render.py (98%)
rename src/backend/core/{ => models}/vision/schema.py (100%)
rename src/backend/core/{ => models}/vision/service.py (96%)
delete mode 100644 src/backend/core/myspace/duplicate_cleanup.py
create mode 100644 src/backend/core/ontology/audit_events.py
rename src/backend/core/{services/ontology_evolution_service.py => ontology/evolution.py} (87%)
create mode 100644 src/backend/core/ontology/host.py
create mode 100644 src/backend/core/ontology/manual_review.py
create mode 100644 src/backend/core/ontology/output_review_state.py
rename src/backend/core/{services/ontology_policy.py => ontology/policy.py} (97%)
create mode 100644 src/backend/core/ontology/review_operations.py
create mode 100644 src/backend/core/ontology/review_prompt.py
create mode 100644 src/backend/core/ontology/reviewer.py
create mode 100644 src/backend/core/ontology/reviewer_state.py
create mode 100644 src/backend/core/ontology/runtime_gates.py
create mode 100644 src/backend/core/ontology/runtime_pack.py
create mode 100644 src/backend/core/ontology/runtime_policy.py
rename src/backend/core/{services/ontology_service.py => ontology/service.py} (68%)
create mode 100644 src/backend/core/ontology/tool_hook.py
create mode 100644 src/backend/core/ontology/validation_models.py
delete mode 100644 src/backend/core/plugins/README.md
delete mode 100644 src/backend/core/plugins/management/__init__.py
delete mode 100644 src/backend/core/plugins/resources/application.py
delete mode 100644 src/backend/core/plugins/resources/application_cleanup.py
delete mode 100644 src/backend/core/plugins/resources/deadlines.py
delete mode 100644 src/backend/core/plugins/resources/owners.py
delete mode 100644 src/backend/core/plugins/resources/prewarm.py
delete mode 100644 src/backend/core/plugins/resources/reservation.py
delete mode 100644 src/backend/core/plugins/resources/service.py
create mode 100644 src/backend/core/projects/__init__.py
rename src/backend/core/{services/project_access.py => projects/access.py} (90%)
rename src/backend/core/{services/project_api_service.py => projects/api.py} (97%)
rename src/backend/core/{services/project_service.py => projects/base.py} (85%)
create mode 100644 src/backend/core/projects/edition.py
rename src/backend/core/{project_environment => projects/environment}/__init__.py (100%)
rename src/backend/core/{project_environment => projects/environment}/config.py (100%)
rename src/backend/core/{project_environment => projects/environment}/dependencies.py (96%)
rename src/backend/core/{services/project_environment.py => projects/environment/service.py} (86%)
rename src/backend/core/{services/project_environment_resources.py => projects/environment_resources.py} (90%)
rename src/backend/core/{services/project_file_service.py => projects/files.py} (78%)
rename src/backend/core/{services/project_instructions.py => projects/instructions.py} (95%)
rename src/backend/core/{services/local_project_workspace.py => projects/local_workspace.py} (97%)
rename src/backend/core/{services/project_scope.py => projects/personal_scope.py} (95%)
create mode 100644 src/backend/core/projects/personal_summary.py
rename src/backend/core/{services/project_transfer.py => projects/personal_transfer.py} (100%)
create mode 100644 src/backend/core/projects/project_names.py
rename src/backend/core/{services/project_source.py => projects/source.py} (91%)
rename src/backend/core/{services/user_folder_service.py => projects/user_folders.py} (95%)
create mode 100644 src/backend/core/runs/__init__.py
rename src/backend/{orchestration/agent_api_recovery.py => core/runs/api_recovery.py} (94%)
rename src/backend/core/{services/run_binding.py => runs/binding.py} (73%)
create mode 100644 src/backend/core/runs/clock.py
create mode 100644 src/backend/core/runs/events/__init__.py
create mode 100644 src/backend/core/runs/events/projection.py
rename src/backend/{orchestration/run_event_stream.py => core/runs/events/store.py} (84%)
create mode 100644 src/backend/core/runs/events/subscription.py
create mode 100644 src/backend/core/runs/events/writer.py
create mode 100644 src/backend/core/runs/host.py
rename src/backend/core/{services/run_journal.py => runs/journal.py} (94%)
rename src/backend/core/{services/run_journal_domain.py => runs/journal_domain.py} (100%)
rename src/backend/core/{services/run_journal_lifecycle.py => runs/journal_lifecycle.py} (93%)
rename src/backend/core/{services/run_journal_recovery.py => runs/journal_recovery.py} (97%)
create mode 100644 src/backend/core/runs/ownership.py
create mode 100644 src/backend/core/runs/runtime.py
create mode 100644 src/backend/core/runs/settings.py
create mode 100644 src/backend/core/runs/tasks.py
rename src/backend/{orchestration => core/runs}/tool_effect_recovery.py (68%)
create mode 100644 src/backend/core/runs/tool_effects/__init__.py
create mode 100644 src/backend/core/runs/tool_effects/arguments.py
create mode 100644 src/backend/core/runs/tool_effects/errors.py
create mode 100644 src/backend/core/runs/tool_effects/gateway.py
create mode 100644 src/backend/core/runs/tool_effects/hooks.py
create mode 100644 src/backend/core/runs/tool_effects/host.py
create mode 100644 src/backend/core/runs/tool_effects/intent_transactions.py
create mode 100644 src/backend/core/runs/tool_effects/invocation_claims.py
create mode 100644 src/backend/core/runs/tool_effects/journal.py
create mode 100644 src/backend/core/runs/tool_effects/journal_constants.py
create mode 100644 src/backend/core/runs/tool_effects/recovery.py
create mode 100644 src/backend/core/runs/tool_effects/recovery_claims.py
create mode 100644 src/backend/core/runs/tool_effects/registry.py
create mode 100644 src/backend/core/runs/tool_effects/result_transactions.py
create mode 100644 src/backend/core/runs/tool_effects/retention.py
create mode 100644 src/backend/core/runs/tool_effects/types.py
create mode 100644 src/backend/core/runs/watchdog.py
create mode 100644 src/backend/core/sandbox/commands/__init__.py
create mode 100644 src/backend/core/sandbox/commands/completion_hook.py
rename src/backend/core/{services => sandbox}/commands/context.py (85%)
rename src/backend/core/{services => sandbox}/commands/delivery.py (92%)
rename src/backend/{orchestration => core/sandbox}/commands/events.py (93%)
rename src/backend/{orchestration => core/sandbox}/commands/notifications.py (77%)
create mode 100644 src/backend/core/sandbox/commands/runtime.py
rename src/backend/core/{services => sandbox}/commands/store.py (95%)
rename src/backend/{orchestration => core/sandbox}/commands/subscription.py (66%)
create mode 100644 src/backend/core/sandbox/credential_dispatch.py
rename src/backend/core/{llm/tool_permissions => sandbox/permissions}/account.py (90%)
rename src/backend/core/{llm/tool_permissions/preferences.py => sandbox/permissions/approval.py} (94%)
rename src/backend/core/{llm/tool_permissions/models.py => sandbox/permissions/dispatch_models.py} (98%)
rename src/backend/core/{llm/tool_permissions/specs.py => sandbox/permissions/tool_specs.py} (95%)
create mode 100644 src/backend/core/sandbox/permissions/vocabulary.py
delete mode 100644 src/backend/core/sandbox/providers/opensandbox/account_volumes.py
rename src/backend/core/{llm/tools/_paths.py => sandbox/workspace/resolution.py} (88%)
delete mode 100644 src/backend/core/services/__init__.py
delete mode 100644 src/backend/core/services/agent_market_categories.py
delete mode 100644 src/backend/core/services/agent_market_service.py
delete mode 100644 src/backend/core/services/api_protocol_backfill.py
delete mode 100644 src/backend/core/services/artifact_edition.py
delete mode 100644 src/backend/core/services/automation_tool_routing.py
delete mode 100644 src/backend/core/services/chat_service.py
delete mode 100644 src/backend/core/services/commands/__init__.py
delete mode 100644 src/backend/core/services/desktop_cloud_bridge.py
delete mode 100644 src/backend/core/services/edition_system_config.py
delete mode 100644 src/backend/core/services/edition_workspace.py
delete mode 100644 src/backend/core/services/kb_edition.py
delete mode 100644 src/backend/core/services/lark_service.py
delete mode 100644 src/backend/core/services/log_service.py
delete mode 100644 src/backend/core/services/marketplace_service.py
delete mode 100644 src/backend/core/services/mcp_marketplace_service.py
delete mode 100644 src/backend/core/services/mcp_oauth_service.py
delete mode 100644 src/backend/core/services/mcp_tool_metadata.py
delete mode 100644 src/backend/core/services/mcp_tool_schema_backfill.py
delete mode 100644 src/backend/core/services/prompt_version_service.py
delete mode 100644 src/backend/core/services/session_operations/admission.py
delete mode 100644 src/backend/core/services/skill_draft_service.py
delete mode 100644 src/backend/core/services/system_config.py
delete mode 100644 src/backend/core/services/tool_effect_ledger.py
delete mode 100644 src/backend/core/services/user_agent_base.py
delete mode 100644 src/backend/core/spaces/edition_access.py
delete mode 100644 src/backend/core/spaces/edition_revocation.py
delete mode 100644 src/backend/core/vision/__init__.py
create mode 100644 src/backend/features/__init__.py
create mode 100644 src/backend/features/automation/README.md
create mode 100644 src/backend/features/automation/__init__.py
create mode 100644 src/backend/features/automation/admission.py
create mode 100644 src/backend/features/automation/api/__init__.py
rename src/backend/{api/routes/v1 => features/automation/api}/automations.py (94%)
rename src/backend/{core/services => features/automation}/delivery_targets.py (75%)
rename src/backend/{core/services/automation_execution.py => features/automation/execution.py} (96%)
create mode 100644 src/backend/features/automation/extension.py
rename src/backend/{core/services/loop_service.py => features/automation/loops.py} (99%)
rename src/backend/{core/db/models/automation.py => features/automation/models.py} (64%)
rename src/backend/{core/services/automation_notifications.py => features/automation/notifications.py} (100%)
rename src/backend/{core/services/automation_remote_effect.py => features/automation/remote_effect.py} (97%)
create mode 100644 src/backend/features/automation/runs.py
create mode 100644 src/backend/features/automation/scheduling.py
create mode 100644 src/backend/features/automation/schema.py
rename src/backend/{core/services/automation_service.py => features/automation/service.py} (73%)
create mode 100644 src/backend/features/automation/tool_routing.py
create mode 100644 src/backend/features/desktop/README.md
create mode 100644 src/backend/features/desktop/__init__.py
create mode 100644 src/backend/features/desktop/api/__init__.py
create mode 100644 src/backend/features/desktop/api/capability_auth.py
create mode 100644 src/backend/features/desktop/api/capability_bridge.py
create mode 100644 src/backend/features/desktop/api/capability_dependencies.py
create mode 100644 src/backend/features/desktop/api/capability_manifests.py
create mode 100644 src/backend/features/desktop/api/capability_mcp_gateway.py
create mode 100644 src/backend/features/desktop/api/capability_model_gateway.py
create mode 100644 src/backend/features/desktop/api/capability_tools.py
create mode 100644 src/backend/features/desktop/api/capability_workcopies.py
rename src/backend/{api/routes/v1 => features/desktop/api}/desktop.py (99%)
rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_capabilities.py (80%)
create mode 100644 src/backend/features/desktop/api/desktop_capability.py
create mode 100644 src/backend/features/desktop/api/desktop_capability_state.py
rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_login.py (99%)
rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_observability.py (90%)
rename src/backend/{api/routes/v1 => features/desktop/api}/device_link.py (95%)
rename src/backend/{api/routes/v1 => features/desktop/api}/local.py (82%)
rename src/backend/{api/routes => features/desktop/api}/local_file_actions.py (93%)
create mode 100644 src/backend/features/desktop/auth/__init__.py
rename src/backend/{core/auth/desktop_bridge.py => features/desktop/auth/bridge.py} (92%)
rename src/backend/{core/auth/desktop_login_store.py => features/desktop/auth/login_store.py} (100%)
rename src/backend/{core/auth/desktop_ticket_store.py => features/desktop/auth/ticket_store.py} (100%)
create mode 100644 src/backend/features/desktop/bridge/__init__.py
rename src/backend/{core/services/desktop_cloud_bundles.py => features/desktop/bridge/bundles.py} (88%)
create mode 100644 src/backend/features/desktop/bridge/client.py
create mode 100644 src/backend/features/desktop/bridge/client_state.py
create mode 100644 src/backend/features/desktop/bridge/gateway.py
create mode 100644 src/backend/features/desktop/bridge/identity.py
create mode 100644 src/backend/features/desktop/bridge/managed_connectors.py
rename src/backend/{core/services/desktop_model_credentials.py => features/desktop/bridge/model_credentials.py} (86%)
rename src/backend/{core/services/desktop_release_events.py => features/desktop/bridge/release_events.py} (100%)
rename src/backend/{core/services/desktop_resource_plugins.py => features/desktop/bridge/resource_plugins.py} (86%)
rename src/backend/{core/services/desktop_cloud_skills.py => features/desktop/bridge/skills.py} (86%)
create mode 100644 src/backend/features/desktop/bridge/status.py
create mode 100644 src/backend/features/desktop/capabilities/__init__.py
rename src/backend/{core => features/desktop}/capabilities/agents.py (93%)
rename src/backend/{core => features/desktop}/capabilities/change_merge.py (94%)
rename src/backend/{core => features/desktop}/capabilities/change_signal.py (100%)
rename src/backend/{core => features/desktop}/capabilities/change_sync.py (87%)
rename src/backend/{core => features/desktop}/capabilities/connectors.py (90%)
rename src/backend/{core => features/desktop}/capabilities/credentials.py (100%)
rename src/backend/{core => features/desktop}/capabilities/dependency.py (94%)
rename src/backend/{core => features/desktop}/capabilities/dependency_declarations.py (100%)
rename src/backend/{core => features/desktop}/capabilities/device_catalog.py (81%)
rename src/backend/{core => features/desktop}/capabilities/device_plugin_catalog.py (69%)
rename src/backend/{core => features/desktop}/capabilities/discovery.py (85%)
rename src/backend/{core => features/desktop}/capabilities/invocation.py (74%)
rename src/backend/{core => features/desktop}/capabilities/local_plugin_runtime.py (72%)
rename src/backend/{core => features/desktop}/capabilities/local_projection.py (79%)
rename src/backend/{core => features/desktop}/capabilities/management_context.py (82%)
rename src/backend/{core => features/desktop}/capabilities/manifest_order.py (96%)
rename src/backend/{core => features/desktop}/capabilities/mcp_json.py (98%)
rename src/backend/{core => features/desktop}/capabilities/migration.py (92%)
rename src/backend/{core => features/desktop}/capabilities/paths.py (80%)
rename src/backend/{core => features/desktop}/capabilities/plugins.py (87%)
rename src/backend/{core => features/desktop}/capabilities/preparation.py (89%)
rename src/backend/{core => features/desktop}/capabilities/readiness.py (92%)
rename src/backend/{core => features/desktop}/capabilities/registry.py (84%)
create mode 100644 src/backend/features/desktop/capabilities/registry_types.py
rename src/backend/{core => features/desktop}/capabilities/resolver.py (98%)
create mode 100644 src/backend/features/desktop/capabilities/runtime/__init__.py
rename src/backend/{core => features/desktop}/capabilities/runtime/connectors.py (92%)
rename src/backend/{core => features/desktop}/capabilities/runtime/dependencies.py (93%)
rename src/backend/{core => features/desktop}/capabilities/runtime/preparation.py (86%)
rename src/backend/{core => features/desktop}/capabilities/runtime/state.py (89%)
rename src/backend/{core => features/desktop}/capabilities/session_availability.py (100%)
create mode 100644 src/backend/features/desktop/capabilities/skill_account.py
create mode 100644 src/backend/features/desktop/capabilities/skill_candidates.py
create mode 100644 src/backend/features/desktop/capabilities/skill_hashes.py
create mode 100644 src/backend/features/desktop/capabilities/skill_paths.py
create mode 100644 src/backend/features/desktop/capabilities/skill_publication.py
create mode 100644 src/backend/features/desktop/capabilities/skill_views.py
create mode 100644 src/backend/features/desktop/capabilities/skills.py
create mode 100644 src/backend/features/desktop/capabilities/skills_state.py
rename src/backend/{core => features/desktop}/capabilities/store.py (94%)
rename src/backend/{core => features/desktop}/capabilities/view.py (97%)
rename src/backend/{core => features/desktop}/capabilities/warmup.py (72%)
create mode 100644 src/backend/features/desktop/capability/__init__.py
rename src/backend/{core/services/desktop_capability_configs.py => features/desktop/capability/configs.py} (91%)
rename src/backend/{core/services/desktop_capability_credentials.py => features/desktop/capability/credentials.py} (98%)
rename src/backend/{core/services/desktop_capability_entities.py => features/desktop/capability/entities.py} (91%)
rename src/backend/{core/services/desktop_capability_mcp.py => features/desktop/capability/mcp.py} (85%)
rename src/backend/{core/services/desktop_capability_models.py => features/desktop/capability/models.py} (93%)
create mode 100644 src/backend/features/desktop/capability/revision_provider.py
rename src/backend/{core/services/desktop_capability_security.py => features/desktop/capability/security.py} (96%)
rename src/backend/{core/services/desktop_capability.py => features/desktop/capability/service.py} (90%)
rename src/backend/{core/services/desktop_capability_skills.py => features/desktop/capability/skills.py} (86%)
rename src/backend/{core/services/desktop_capability_sync_check.py => features/desktop/capability/sync_check.py} (79%)
rename src/backend/{core/services/capability_workcopies.py => features/desktop/capability/workcopies.py} (89%)
create mode 100644 src/backend/features/desktop/edition.py
create mode 100644 src/backend/features/desktop/extension.py
create mode 100644 src/backend/features/desktop/gateway/__init__.py
rename src/backend/{core/services/desktop_gateway_observability.py => features/desktop/gateway/observability.py} (98%)
create mode 100644 src/backend/features/desktop/local/__init__.py
rename src/backend/{core/services/local_file_actions.py => features/desktop/local/file_actions.py} (99%)
rename src/backend/{core/services/local_grant_service.py => features/desktop/local/grants.py} (100%)
rename src/backend/{core/services/local_management_migration.py => features/desktop/local/management_migration.py} (85%)
create mode 100644 src/backend/features/desktop/local/manager_execution.py
rename src/backend/{core/services/local_market_install.py => features/desktop/local/market_install.py} (61%)
rename src/backend/{core/services => features/desktop/local}/native_app_icons.py (100%)
rename src/backend/{core/plugins/local => features/desktop/local/plugins}/__init__.py (100%)
rename src/backend/{core/plugins/local => features/desktop/local/plugins}/assets.py (92%)
rename src/backend/{core/plugins/local => features/desktop/local/plugins}/legacy_migration.py (87%)
rename src/backend/{core/plugins/local => features/desktop/local/plugins}/service.py (85%)
create mode 100644 src/backend/features/desktop/local/plugins/sidecars.py
create mode 100644 src/backend/features/desktop/local/publication_provider.py
create mode 100644 src/backend/features/desktop/local/skills/__init__.py
rename src/backend/{core/services/local_skill_editor.py => features/desktop/local/skills/editor.py} (86%)
rename src/backend/{core/services/local_skill_service.py => features/desktop/local/skills/service.py} (81%)
rename src/backend/{core/services/local_skill_upload.py => features/desktop/local/skills/upload.py} (74%)
rename src/backend/{core/services/local_snapshot_service.py => features/desktop/local/snapshots.py} (100%)
rename src/backend/{core/db/models/capability.py => features/desktop/models.py} (60%)
create mode 100644 src/backend/features/desktop/observability/__init__.py
rename src/backend/{core/services/desktop_observation_context.py => features/desktop/observability/context.py} (78%)
rename src/backend/{core/services/desktop_observability_delivery.py => features/desktop/observability/delivery.py} (86%)
rename src/backend/{core/services/observability_projection.py => features/desktop/observability/fields.py} (96%)
rename src/backend/{core/services/desktop_observability.py => features/desktop/observability/projection.py} (98%)
rename src/backend/{core/services/desktop_observability_sync.py => features/desktop/observability/sync.py} (96%)
rename src/backend/{core/plugins/runtime/desktop_resolution.py => features/desktop/plugin_resolution.py} (85%)
create mode 100644 src/backend/features/desktop/run_capabilities/__init__.py
rename src/backend/{core/llm/factory/selection/device_preparation.py => features/desktop/run_capabilities/preparation.py} (83%)
create mode 100644 src/backend/features/desktop/run_capabilities/provider.py
rename src/backend/{core/agent_skills/backends/capability_store.py => features/desktop/skill_backend.py} (86%)
create mode 100644 src/backend/features/evaluation/README.md
create mode 100644 src/backend/features/evaluation/__init__.py
rename src/backend/{core/services/evaluation_admission.py => features/evaluation/admission.py} (93%)
create mode 100644 src/backend/features/evaluation/api/__init__.py
rename src/backend/{api/routes/v1 => features/evaluation/api}/evaluation_sandboxes.py (98%)
create mode 100644 src/backend/features/evaluation/extension.py
create mode 100644 src/backend/features/evaluation/models.py
create mode 100644 src/backend/features/evaluation/policy.py
create mode 100644 src/backend/features/evaluation/runtime/__init__.py
rename src/backend/{core/llm/evaluation_paths.py => features/evaluation/runtime/paths.py} (100%)
rename src/backend/{core/llm/evaluation_process_tools.py => features/evaluation/runtime/process_tools.py} (87%)
rename src/backend/{core/llm/evaluation_runtime.py => features/evaluation/runtime/runtime.py} (100%)
rename src/backend/{core/llm/evaluation_tools.py => features/evaluation/runtime/tools.py} (57%)
rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/__init__.py (100%)
rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/binding.py (100%)
rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/process_guard.py (100%)
rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/provider.py (98%)
rename src/backend/{core/services/evaluation_sandbox_service.py => features/evaluation/sandboxes.py} (90%)
create mode 100644 src/backend/features/hosted_apps/README.md
create mode 100644 src/backend/features/hosted_apps/__init__.py
create mode 100644 src/backend/features/hosted_apps/api/__init__.py
rename src/backend/{api/routes/v1 => features/hosted_apps/api}/application_transports.py (68%)
rename src/backend/{api/routes/v1 => features/hosted_apps/api}/applications.py (76%)
rename src/backend/{core/services/application_collections.py => features/hosted_apps/collections.py} (92%)
rename src/backend/{core/config/application_hosting.py => features/hosted_apps/config.py} (87%)
rename src/backend/{core/services/application_data.py => features/hosted_apps/data.py} (95%)
rename src/backend/{core/services/application_deployments.py => features/hosted_apps/deployments.py} (92%)
rename src/backend/{core/services/application_export.py => features/hosted_apps/export.py} (89%)
create mode 100644 src/backend/features/hosted_apps/extension.py
rename src/backend/{core/services/application_history.py => features/hosted_apps/history.py} (91%)
rename src/backend/{core/services/application_hosting_setup.py => features/hosted_apps/hosting_setup.py} (96%)
rename src/backend/{core/services/application_industry.py => features/hosted_apps/industry.py} (95%)
rename src/backend/{core/services/application_lifecycle.py => features/hosted_apps/lifecycle.py} (88%)
rename src/backend/{core/services/application_mcp.py => features/hosted_apps/mcp.py} (82%)
create mode 100644 src/backend/features/hosted_apps/models.py
rename src/backend/{core/services/application_operations.py => features/hosted_apps/operations.py} (84%)
rename src/backend/{core/services/application_projects.py => features/hosted_apps/projects.py} (95%)
rename src/backend/{core/services/application_publication.py => features/hosted_apps/publication.py} (92%)
rename src/backend/{core/services/application_relational.py => features/hosted_apps/relational.py} (98%)
create mode 100644 src/backend/features/hosted_apps/resources/__init__.py
create mode 100644 src/backend/features/hosted_apps/resources/admission.py
create mode 100644 src/backend/features/hosted_apps/resources/cleanup.py
rename src/backend/{core/services/application_schema.py => features/hosted_apps/schema.py} (100%)
rename src/backend/{core/services/application_sql_policy.py => features/hosted_apps/sql_policy.py} (96%)
rename src/backend/{core/services/application_store.py => features/hosted_apps/store.py} (94%)
create mode 100644 src/backend/features/installed.py
create mode 100644 src/backend/features/knowledge/README.md
rename src/backend/{core/kb => features/knowledge}/__init__.py (64%)
create mode 100644 src/backend/features/knowledge/api/__init__.py
create mode 100644 src/backend/features/knowledge/api/kb.py
create mode 100644 src/backend/features/knowledge/api/kb_assets.py
create mode 100644 src/backend/features/knowledge/api/kb_chunks.py
create mode 100644 src/backend/features/knowledge/api/kb_documents.py
create mode 100644 src/backend/features/knowledge/api/kb_helpers.py
create mode 100644 src/backend/features/knowledge/api/kb_mutations.py
create mode 100644 src/backend/features/knowledge/api/kb_spaces.py
create mode 100644 src/backend/features/knowledge/api/kb_state.py
rename src/backend/{api/routes/v1 => features/knowledge/api}/kb_wiki.py (93%)
create mode 100644 src/backend/features/knowledge/asset_captions.py
create mode 100644 src/backend/features/knowledge/asset_indexing.py
create mode 100644 src/backend/features/knowledge/catalog_options.py
rename src/backend/{core/kb/external_provider.py => features/knowledge/disabled_provider.py} (85%)
rename src/backend/{core/kb/external_retrieval.py => features/knowledge/disabled_retrieval.py} (100%)
rename src/backend/{core/kb => features/knowledge}/document_status.py (100%)
create mode 100644 src/backend/features/knowledge/edition.py
create mode 100644 src/backend/features/knowledge/enrichment.py
create mode 100644 src/backend/features/knowledge/extension.py
rename src/backend/{core/kb => features/knowledge}/index_queue.py (99%)
rename src/backend/{core/kb => features/knowledge}/index_worker.py (97%)
create mode 100644 src/backend/features/knowledge/kb_assets.py
create mode 100644 src/backend/features/knowledge/kb_assets_state.py
create mode 100644 src/backend/features/knowledge/kb_parser.py
create mode 100644 src/backend/features/knowledge/kb_parser_state.py
create mode 100644 src/backend/features/knowledge/kb_vector.py
create mode 100644 src/backend/features/knowledge/kb_vector_state.py
create mode 100644 src/backend/features/knowledge/models.py
create mode 100644 src/backend/features/knowledge/parser_chunks.py
create mode 100644 src/backend/features/knowledge/parser_extract.py
create mode 100644 src/backend/features/knowledge/parser_semantic.py
create mode 100644 src/backend/features/knowledge/parser_tokens.py
create mode 100644 src/backend/features/knowledge/parser_types.py
rename src/backend/{core/auth/kb_permissions.py => features/knowledge/personal_permissions.py} (97%)
rename src/backend/{api/routes/v1/kb_models.py => features/knowledge/personal_request_models.py} (100%)
rename src/backend/{core/content/kb_processing.py => features/knowledge/processing.py} (67%)
rename src/backend/{core/db/repository/kb.py => features/knowledge/repository.py} (97%)
rename src/backend/{core/services/kb_service.py => features/knowledge/service.py} (95%)
create mode 100644 src/backend/features/knowledge/vector_client.py
create mode 100644 src/backend/features/knowledge/vector_fields.py
create mode 100644 src/backend/features/knowledge/vector_indexing.py
rename src/backend/{core/kb => features/knowledge}/wiki/__init__.py (92%)
rename src/backend/{core/kb => features/knowledge}/wiki/cite.py (95%)
rename src/backend/{core/kb => features/knowledge}/wiki/config.py (99%)
rename src/backend/{core/kb => features/knowledge}/wiki/dedup.py (94%)
rename src/backend/{core/kb => features/knowledge}/wiki/extract.py (93%)
rename src/backend/{core/kb => features/knowledge}/wiki/finalize.py (92%)
rename src/backend/{core/kb => features/knowledge}/wiki/jobs.py (99%)
rename src/backend/{core/kb => features/knowledge}/wiki/linkify.py (100%)
rename src/backend/{core/kb => features/knowledge}/wiki/llm.py (96%)
create mode 100644 src/backend/features/knowledge/wiki/local_graph.py
create mode 100644 src/backend/features/knowledge/wiki/local_projection.py
create mode 100644 src/backend/features/knowledge/wiki/local_provider.py
create mode 100644 src/backend/features/knowledge/wiki/local_provider_state.py
rename src/backend/{core/kb => features/knowledge}/wiki/pipeline.py (95%)
rename src/backend/{core/kb => features/knowledge}/wiki/prompts.py (100%)
rename src/backend/{core/kb => features/knowledge}/wiki/reduce.py (93%)
rename src/backend/{core/kb => features/knowledge}/wiki/slug.py (100%)
rename src/backend/{core/kb => features/knowledge}/wiki/store.py (98%)
rename src/backend/{core/kb => features/knowledge}/wiki/taxonomy.py (89%)
rename src/backend/{core/kb => features/knowledge}/wiki/worker.py (94%)
rename src/backend/{core/kb => features/knowledge}/wiki_router.py (88%)
create mode 100644 src/backend/features/marketplace/README.md
create mode 100644 src/backend/features/marketplace/__init__.py
create mode 100644 src/backend/features/marketplace/agent_catalog.py
create mode 100644 src/backend/features/marketplace/agent_categories.py
create mode 100644 src/backend/features/marketplace/agent_installation.py
create mode 100644 src/backend/features/marketplace/agent_metadata.py
create mode 100644 src/backend/features/marketplace/agent_publishing.py
create mode 100644 src/backend/features/marketplace/agent_submission_data.py
create mode 100644 src/backend/features/marketplace/agent_submissions.py
rename src/backend/{core/services/marketplace_agent_versions.py => features/marketplace/agent_versions.py} (88%)
create mode 100644 src/backend/features/marketplace/agents_state.py
create mode 100644 src/backend/features/marketplace/api/__init__.py
rename src/backend/{api/routes/v1 => features/marketplace/api}/agent_marketplace.py (82%)
rename src/backend/{api/routes/v1 => features/marketplace/api}/catalog_ownership.py (94%)
rename src/backend/{api/routes/v1 => features/marketplace/api}/marketplace.py (84%)
rename src/backend/{api/routes/v1 => features/marketplace/api}/mcp_marketplace.py (79%)
create mode 100644 src/backend/features/marketplace/core_catalog.py
create mode 100644 src/backend/features/marketplace/edition.py
create mode 100644 src/backend/features/marketplace/extension.py
create mode 100644 src/backend/features/marketplace/mcp_administration.py
rename src/backend/{core/config/mcp_marketplace_catalog.py => features/marketplace/mcp_catalog.py} (100%)
create mode 100644 src/backend/features/marketplace/mcp_credentials.py
create mode 100644 src/backend/features/marketplace/mcp_curated.py
create mode 100644 src/backend/features/marketplace/mcp_installation.py
create mode 100644 src/backend/features/marketplace/mcp_listing.py
rename src/backend/{core/services/mcp_marketplace_monitor.py => features/marketplace/mcp_monitor.py} (55%)
create mode 100644 src/backend/features/marketplace/mcp_publishing.py
create mode 100644 src/backend/features/marketplace/mcp_revalidation.py
create mode 100644 src/backend/features/marketplace/mcp_state.py
create mode 100644 src/backend/features/marketplace/mcp_submissions.py
create mode 100644 src/backend/features/marketplace/mcp_validation.py
create mode 100644 src/backend/features/marketplace/mcp_versions.py
create mode 100644 src/backend/features/marketplace/models.py
create mode 100644 src/backend/features/marketplace/package_source.py
rename src/backend/{core/services/marketplace_listing.py => features/marketplace/personal_listing.py} (97%)
rename src/backend/{core/auth/marketplace_visibility.py => features/marketplace/personal_visibility.py} (100%)
rename src/backend/{core/plugins/management/market.py => features/marketplace/plugin_market.py} (73%)
create mode 100644 src/backend/features/marketplace/skill_archives.py
create mode 100644 src/backend/features/marketplace/skill_catalog.py
create mode 100644 src/backend/features/marketplace/skill_installation.py
create mode 100644 src/backend/features/marketplace/skill_metadata.py
create mode 100644 src/backend/features/marketplace/skill_publishing.py
create mode 100644 src/backend/features/marketplace/skill_submission_data.py
create mode 100644 src/backend/features/marketplace/skill_submissions.py
rename src/backend/{core/services/marketplace_skill_versions.py => features/marketplace/skill_versions.py} (72%)
create mode 100644 src/backend/features/marketplace/skills_state.py
create mode 100644 src/backend/features/marketplace/versions/__init__.py
rename src/backend/{core/services/marketplace_version_archive.py => features/marketplace/versions/archive.py} (100%)
rename src/backend/{core/services/marketplace_version_config.py => features/marketplace/versions/config.py} (95%)
rename src/backend/{core/services/marketplace_version_sources.py => features/marketplace/versions/sources.py} (84%)
rename src/backend/{core/services/marketplace_version_store.py => features/marketplace/versions/store.py} (90%)
create mode 100644 src/backend/features/sites/README.md
create mode 100644 src/backend/features/sites/__init__.py
create mode 100644 src/backend/features/sites/api/__init__.py
rename src/backend/{api/routes/v1 => features/sites/api}/internal_site_auth.py (100%)
rename src/backend/{api/routes/v1 => features/sites/api}/internal_sites.py (94%)
rename src/backend/{api/routes/v1 => features/sites/api}/local_site_sources.py (94%)
rename src/backend/{api/routes => features/sites/api}/site_gate.py (100%)
rename src/backend/{api/routes/v1 => features/sites/api}/sites.py (95%)
rename src/backend/{api/routes => features/sites/api}/sites_serve.py (82%)
rename src/backend/{core/plugins/local/site_upgrade.py => features/sites/builtin_upgrade.py} (83%)
rename src/backend/{core/services/desktop_site_publish.py => features/sites/desktop_publish.py} (83%)
create mode 100644 src/backend/features/sites/edition.py
create mode 100644 src/backend/features/sites/extension.py
rename src/backend/{core/services/site_listing.py => features/sites/listing.py} (94%)
rename src/backend/{core/services/local_site_sources.py => features/sites/local_sources.py} (93%)
rename src/backend/{core/services/site_management.py => features/sites/management.py} (95%)
rename src/backend/{core/db/models/site.py => features/sites/models.py} (97%)
rename src/backend/{core/services/site_packaging.py => features/sites/packaging.py} (92%)
rename src/backend/{core/services/site_password.py => features/sites/password.py} (100%)
rename src/backend/{core/services/site_access_policy.py => features/sites/personal_access.py} (97%)
rename src/backend/{core/db/models/site_scope.py => features/sites/personal_scope.py} (100%)
rename src/backend/{core/services/site_rate_limit.py => features/sites/rate_limit.py} (100%)
rename src/backend/{core/db/repository/site.py => features/sites/repository.py} (98%)
rename src/backend/{core/services/site_service.py => features/sites/service.py} (97%)
create mode 100644 src/backend/features/sites/tools/__init__.py
rename src/backend/{core/llm/tools/design_picker_tool.py => features/sites/tools/design_picker.py} (86%)
create mode 100644 src/backend/features/workbench/README.md
rename src/backend/{core/applications => features/workbench}/__init__.py (100%)
rename src/backend/{core/applications => features/workbench}/activity.py (98%)
create mode 100644 src/backend/features/workbench/api/__init__.py
rename src/backend/{api/routes/v1 => features/workbench/api}/plugin_ui.py (91%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench.py (82%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_activity.py (82%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_application_config.py (88%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_auth.py (92%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_capabilities.py (81%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_diagnostics.py (97%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_events.py (85%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_preferences.py (97%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_resources.py (69%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_revision_leases.py (97%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_state.py (98%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_upgrades.py (98%)
rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_workspace.py (83%)
rename src/backend/{core/applications => features/workbench}/application_config.py (93%)
rename src/backend/{core/applications => features/workbench}/application_config_bindings.py (82%)
rename src/backend/{core/applications => features/workbench}/background.py (63%)
rename src/backend/{core/applications => features/workbench}/capabilities.py (95%)
rename src/backend/{core/applications => features/workbench}/catalog.py (96%)
rename src/backend/{core/applications => features/workbench}/coordination.py (100%)
rename src/backend/{core/applications => features/workbench}/diagnostic_probe.py (98%)
rename src/backend/{core/applications => features/workbench}/diagnostic_resources.py (100%)
rename src/backend/{core/applications => features/workbench}/diagnostics.py (99%)
rename src/backend/{core/applications => features/workbench}/directory_revision.py (96%)
create mode 100644 src/backend/features/workbench/edition.py
create mode 100644 src/backend/features/workbench/extension.py
rename src/backend/{core/applications => features/workbench}/external.py (94%)
rename src/backend/{core/applications => features/workbench}/external_auth.py (99%)
rename src/backend/{core/applications => features/workbench}/external_boundary.py (97%)
rename src/backend/{core/applications => features/workbench}/external_embed.py (97%)
rename src/backend/{core/applications => features/workbench}/installation_binding.py (77%)
rename src/backend/{core/applications => features/workbench}/local_plugin_upgrades.py (86%)
rename src/backend/{core/applications => features/workbench}/migration.py (97%)
rename src/backend/{core/applications => features/workbench}/migration_compat.py (88%)
rename src/backend/{core/applications => features/workbench}/migration_plan.py (96%)
rename src/backend/{core/applications => features/workbench}/migration_snapshot.py (91%)
create mode 100644 src/backend/features/workbench/models.py
rename src/backend/{core/applications => features/workbench}/origin.py (100%)
create mode 100644 src/backend/features/workbench/package_source.py
rename src/backend/{core/applications => features/workbench}/plugin_candidate_storage.py (88%)
rename src/backend/{core/applications => features/workbench}/plugin_catalog.py (95%)
rename src/backend/{core/applications => features/workbench}/plugin_revisions.py (89%)
rename src/backend/{core/applications => features/workbench}/plugin_upgrades.py (90%)
rename src/backend/{core/applications => features/workbench}/policy.py (94%)
rename src/backend/{core/applications => features/workbench}/preferences.py (96%)
rename src/backend/{core/applications => features/workbench}/protocol.py (100%)
create mode 100644 src/backend/features/workbench/resource_deadlines.py
create mode 100644 src/backend/features/workbench/resource_owner.py
rename src/backend/{core/applications => features/workbench}/revocation.py (87%)
rename src/backend/{core/applications => features/workbench}/revocation_sweep.py (78%)
rename src/backend/{core/applications => features/workbench}/schemas/application.schema.json (100%)
rename src/backend/{core/applications => features/workbench}/schemas/host.json (100%)
rename src/backend/{core/applications => features/workbench}/schemas/module.schema.json (100%)
rename src/backend/{core/applications => features/workbench}/state.py (96%)
rename src/backend/{core/applications => features/workbench}/state_recovery.py (100%)
rename src/backend/{core/plugins => features/workbench}/ui/__init__.py (100%)
rename src/backend/{core/plugins => features/workbench}/ui/contract.py (99%)
rename src/backend/{core/plugins => features/workbench}/ui/contributions.py (87%)
rename src/backend/{core/plugins => features/workbench}/ui/data_proxy.py (97%)
rename src/backend/{core/plugins => features/workbench}/ui/declarations.py (99%)
rename src/backend/{core/plugins => features/workbench}/ui/primitives.py (99%)
rename src/backend/{core/plugins => features/workbench}/ui/workspace_enabled.py (100%)
rename src/backend/{core/applications => features/workbench}/workspace_page.py (98%)
create mode 100644 src/backend/hugagent_cli/schema.py
create mode 100644 src/backend/integrations/__init__.py
create mode 100644 src/backend/integrations/api/__init__.py
rename src/backend/{api/routes/v1 => integrations/api}/integrations.py (86%)
create mode 100644 src/backend/integrations/channels/README.md
rename src/backend/{core => integrations}/channels/__init__.py (75%)
rename src/backend/{core => integrations}/channels/adapters/__init__.py (100%)
create mode 100644 src/backend/integrations/channels/adapters/dingtalk.py
create mode 100644 src/backend/integrations/channels/adapters/dingtalk_delivery.py
create mode 100644 src/backend/integrations/channels/adapters/dingtalk_history.py
create mode 100644 src/backend/integrations/channels/adapters/dingtalk_settings.py
create mode 100644 src/backend/integrations/channels/adapters/lark.py
create mode 100644 src/backend/integrations/channels/adapters/lark_delivery.py
create mode 100644 src/backend/integrations/channels/adapters/lark_messages.py
create mode 100644 src/backend/integrations/channels/adapters/lark_settings.py
rename src/backend/{core => integrations}/channels/adapters/wecom.py (99%)
rename src/backend/{core => integrations}/channels/adapters/weixin.py (99%)
create mode 100644 src/backend/integrations/channels/api/__init__.py
rename src/backend/{api/routes/v1 => integrations/channels/api}/channel_desktop.py (75%)
rename src/backend/{api/routes/v1 => integrations/channels/api}/channels.py (89%)
rename src/backend/{core/services/channel_desktop_worker.py => integrations/channels/desktop_worker.py} (81%)
create mode 100644 src/backend/integrations/channels/extension.py
create mode 100644 src/backend/integrations/channels/inbound.py
create mode 100644 src/backend/integrations/channels/inbound_attachments.py
create mode 100644 src/backend/integrations/channels/inbound_delivery.py
create mode 100644 src/backend/integrations/channels/inbound_history.py
create mode 100644 src/backend/integrations/channels/inbound_observation.py
create mode 100644 src/backend/integrations/channels/inbound_sessions.py
create mode 100644 src/backend/integrations/channels/inbound_state.py
rename src/backend/{core => integrations}/channels/manager.py (96%)
rename src/backend/{core => integrations}/channels/markdown.py (100%)
create mode 100644 src/backend/integrations/channels/models.py
rename src/backend/{core => integrations}/channels/outbound.py (70%)
rename src/backend/{core => integrations}/channels/protocol.py (100%)
rename src/backend/{core => integrations}/channels/registry.py (88%)
rename src/backend/{core/services/channel_relay.py => integrations/channels/relay.py} (98%)
rename src/backend/{core/services/channel_relay_dispatch.py => integrations/channels/relay_dispatch.py} (54%)
rename src/backend/{core/services/channel_relay_rpc.py => integrations/channels/relay_rpc.py} (63%)
rename src/backend/{core/db/repository/channel.py => integrations/channels/repository.py} (98%)
rename src/backend/{core/services/channel_service.py => integrations/channels/service.py} (70%)
create mode 100644 src/backend/integrations/channels/tools/__init__.py
rename src/backend/{core/llm/tools/channel_attachment_tool.py => integrations/channels/tools/attachment.py} (52%)
create mode 100644 src/backend/integrations/dingtalk/README.md
create mode 100644 src/backend/integrations/dingtalk/__init__.py
create mode 100644 src/backend/integrations/dingtalk/cube_credentials.py
create mode 100644 src/backend/integrations/dingtalk/environment.py
create mode 100644 src/backend/integrations/dingtalk/extension.py
create mode 100644 src/backend/integrations/dingtalk/models.py
create mode 100644 src/backend/integrations/dingtalk/mounts.py
create mode 100644 src/backend/integrations/dingtalk/parsing.py
create mode 100644 src/backend/integrations/dingtalk/repository.py
create mode 100644 src/backend/integrations/dingtalk/sandbox.py
rename src/backend/{core/services/dingtalk_service.py => integrations/dingtalk/service.py} (75%)
create mode 100644 src/backend/integrations/dingtalk/storage.py
create mode 100644 src/backend/integrations/email/README.md
create mode 100644 src/backend/integrations/email/__init__.py
create mode 100644 src/backend/integrations/email/cube_credentials.py
create mode 100644 src/backend/integrations/email/extension.py
create mode 100644 src/backend/integrations/email/models.py
create mode 100644 src/backend/integrations/email/mounts.py
create mode 100644 src/backend/integrations/email/repository.py
create mode 100644 src/backend/integrations/email/sandbox.py
rename src/backend/{core/services/email_service.py => integrations/email/service.py} (91%)
create mode 100644 src/backend/integrations/email/storage.py
create mode 100644 src/backend/integrations/lark/README.md
create mode 100644 src/backend/integrations/lark/__init__.py
create mode 100644 src/backend/integrations/lark/configuration.py
create mode 100644 src/backend/integrations/lark/cube_credentials.py
create mode 100644 src/backend/integrations/lark/extension.py
create mode 100644 src/backend/integrations/lark/login.py
create mode 100644 src/backend/integrations/lark/models.py
create mode 100644 src/backend/integrations/lark/mounts.py
create mode 100644 src/backend/integrations/lark/parsing.py
create mode 100644 src/backend/integrations/lark/process.py
create mode 100644 src/backend/integrations/lark/repository.py
create mode 100644 src/backend/integrations/lark/sandbox.py
create mode 100644 src/backend/integrations/lark/service.py
create mode 100644 src/backend/integrations/lark/state.py
create mode 100644 src/backend/integrations/lark/storage.py
create mode 100644 src/backend/integrations/yida/README.md
create mode 100644 src/backend/integrations/yida/__init__.py
create mode 100644 src/backend/integrations/yida/cube_credentials.py
rename src/backend/{core/services/yida_execution.py => integrations/yida/execution.py} (97%)
create mode 100644 src/backend/integrations/yida/extension.py
create mode 100644 src/backend/integrations/yida/models.py
create mode 100644 src/backend/integrations/yida/mounts.py
create mode 100644 src/backend/integrations/yida/sandbox.py
rename src/backend/{core/services/yida_service.py => integrations/yida/service.py} (99%)
create mode 100644 src/backend/integrations/yida/skill_content.py
create mode 100644 src/backend/integrations/yida/storage.py
create mode 100644 src/backend/orchestration/automations/__init__.py
rename src/backend/orchestration/{schedulers => automations}/automation_artifacts.py (97%)
create mode 100644 src/backend/orchestration/automations/automation_conversation.py
create mode 100644 src/backend/orchestration/automations/automation_delivery.py
create mode 100644 src/backend/orchestration/automations/automation_dispatch.py
create mode 100644 src/backend/orchestration/automations/automation_lease.py
create mode 100644 src/backend/orchestration/automations/automation_recovery.py
create mode 100644 src/backend/orchestration/automations/automation_scheduler.py
delete mode 100644 src/backend/orchestration/chat_run_executor.py
delete mode 100644 src/backend/orchestration/citation_anchor.py
delete mode 100644 src/backend/orchestration/commands/__init__.py
delete mode 100644 src/backend/orchestration/commands/runtime.py
create mode 100644 src/backend/orchestration/configuration.py
delete mode 100644 src/backend/orchestration/local_plugin_sidecars.py
create mode 100644 src/backend/orchestration/loops/__init__.py
create mode 100644 src/backend/orchestration/loops/admission.py
rename src/backend/orchestration/{autonomous_loop.py => loops/driver.py} (98%)
rename src/backend/orchestration/{loop_planner.py => loops/planner.py} (97%)
create mode 100644 src/backend/orchestration/loops/presentation.py
create mode 100644 src/backend/orchestration/loops/recovery.py
rename src/backend/orchestration/{loop_evaluator.py => loops/requirements.py} (83%)
rename src/backend/orchestration/{subagents/loop_reviewer.py => loops/reviewer.py} (78%)
create mode 100644 src/backend/orchestration/loops/run.py
delete mode 100644 src/backend/orchestration/memory_integration.py
create mode 100644 src/backend/orchestration/pending.py
create mode 100644 src/backend/orchestration/plans/__init__.py
create mode 100644 src/backend/orchestration/plans/admission.py
create mode 100644 src/backend/orchestration/plans/execute_run.py
create mode 100644 src/backend/orchestration/plans/generate_run.py
rename src/backend/orchestration/{subagents/plan_mode.py => plans/mode.py} (89%)
delete mode 100644 src/backend/orchestration/schedulers/automation_conversation.py
delete mode 100644 src/backend/orchestration/schedulers/automation_dispatch.py
delete mode 100644 src/backend/orchestration/schedulers/automation_lease.py
delete mode 100644 src/backend/orchestration/schedulers/automation_recovery.py
delete mode 100644 src/backend/orchestration/schedulers/automation_scheduler.py
delete mode 100644 src/backend/orchestration/streaming.py
delete mode 100644 src/backend/orchestration/subagents/__init__.py
delete mode 100644 src/backend/orchestration/subagents/ontology_reviewer.py
rename src/backend/orchestration/{schedulers => workers}/__init__.py (100%)
rename src/backend/orchestration/{schedulers => workers}/_worker_base.py (96%)
rename src/backend/{core/services => orchestration/workers}/edition_startup.py (100%)
rename src/backend/orchestration/{schedulers => workers}/evolution_scheduler.py (98%)
rename src/backend/orchestration/{schedulers => workers}/memory_ttl_scheduler.py (95%)
rename src/backend/orchestration/{schedulers => workers}/persona_distill_worker.py (94%)
create mode 100644 src/backend/orchestration/workers/run_recovery.py
delete mode 100644 src/backend/orchestration/workflow.py
create mode 100644 src/backend/personal_agent/device_link/README.md
rename src/backend/{core => personal_agent}/device_link/__init__.py (84%)
rename src/backend/{core => personal_agent}/device_link/driver.py (100%)
rename src/backend/{core => personal_agent}/device_link/local_api.py (100%)
rename src/backend/{core => personal_agent}/device_link/runner.py (98%)
rename src/backend/{core => personal_agent}/device_link/store.py (100%)
delete mode 100644 src/backend/scripts/_loop_convergence_unit.py
delete mode 100644 src/backend/scripts/_loop_db_resume_unit.py
delete mode 100644 src/backend/scripts/_loop_evaluator_route_unit.py
delete mode 100644 src/backend/scripts/_loop_integration_test.py
delete mode 100644 src/backend/scripts/_loop_llmeval_smoke.py
delete mode 100644 src/backend/scripts/_loop_preflight.py
delete mode 100644 src/backend/scripts/_loop_selfverify_smoke.py
delete mode 100644 src/backend/scripts/_loop_smoke.py
delete mode 100644 src/backend/scripts/_loop_test_edgebench.py
delete mode 100644 src/backend/scripts/desktop_cloud_wrapper.py
delete mode 100644 src/backend/scripts/desktop_fixture_mcp.py
delete mode 100644 src/backend/scripts/dual_mode_e2e.py
delete mode 100644 src/backend/scripts/edgebench/ann_judge.py
delete mode 100644 src/backend/scripts/edgebench/perm_opt_judge.py
delete mode 100644 src/backend/scripts/edit_chat_message.py
delete mode 100644 src/backend/scripts/evaluation_sandbox_e2e.py
delete mode 100644 src/backend/scripts/export_content.py
delete mode 100755 src/backend/scripts/harness_fault_injection.py
delete mode 100644 src/backend/scripts/import_cherry_agents.py
delete mode 100644 src/backend/scripts/import_content.py
delete mode 100644 src/backend/scripts/run_evolution_benchmark.py
delete mode 100644 src/backend/scripts/run_evolution_llm_ab.py
delete mode 100644 src/backend/scripts/workbench_migration.py
create mode 100644 src/backend/tests/api/test_loop_cancel_scope.py
create mode 100644 src/backend/tests/auth/test_auth_dependencies_off_loop.py
create mode 100644 src/backend/tests/auth/test_bearer_identity.py
delete mode 100644 src/backend/tests/capabilities/test_credential_continuity.py
create mode 100644 src/backend/tests/chat/test_quoted_follow_up.py
create mode 100644 src/backend/tests/chat/test_snapshot_transaction_cleanup.py
create mode 100644 src/backend/tests/content/test_artifact_default_path.py
create mode 100644 src/backend/tests/core/runs/test_automation_journal_guard.py
create mode 100644 src/backend/tests/db_support.py
create mode 100644 src/backend/tests/features/__init__.py
create mode 100644 src/backend/tests/features/automation/__init__.py
create mode 100644 src/backend/tests/features/automation/run_fixtures.py
create mode 100644 src/backend/tests/features/automation/test_automation_delivery_deadline.py
rename src/backend/tests/{ => features/automation}/test_automation_execution_location.py (80%)
create mode 100644 src/backend/tests/features/automation/test_automation_lease_close.py
create mode 100644 src/backend/tests/features/automation/test_automation_manual_trigger.py
rename src/backend/tests/{services => features/automation}/test_automation_notifications.py (98%)
create mode 100644 src/backend/tests/features/automation/test_automation_oneshot_finalize.py
create mode 100644 src/backend/tests/features/automation/test_automation_ownership_postgres.py
rename src/backend/tests/{ => features/automation}/test_automation_remote_effect.py (74%)
create mode 100644 src/backend/tests/features/automation/test_automation_run_ownership.py
create mode 100644 src/backend/tests/features/automation/test_delivery_target_migration.py
create mode 100644 src/backend/tests/features/automation/test_durable_admission.py
rename src/backend/tests/{services => features/automation}/test_loop_service_harness.py (98%)
create mode 100644 src/backend/tests/features/automation/test_queue_postgres.py
create mode 100644 src/backend/tests/features/desktop/__init__.py
create mode 100644 src/backend/tests/features/desktop/capabilities/__init__.py
rename src/backend/tests/{ => features/desktop}/capabilities/_cloud_identity.py (100%)
rename src/backend/tests/{ => features/desktop}/capabilities/conftest.py (65%)
rename src/backend/tests/{ => features/desktop}/capabilities/management_browser_server.py (88%)
rename src/backend/tests/{ => features/desktop}/capabilities/runtime_recovery_support.py (75%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_account_authorization.py (52%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_agent_rerun_identity.py (75%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_agent_runtime_grants.py (62%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_agents_plugins.py (90%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_autonomous_loop_scopes.py (73%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_available_runtime.py (82%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_capability_change_sync.py (81%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_catalog_cloud_only_in_hybrid.py (72%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_catalog_plugin_components_hidden.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_change_merge.py (90%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_change_signal.py (97%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_identity.py (96%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_management.py (88%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_parallel_preparation.py (68%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_plugin_binding_runtime.py (76%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_sticky_plugins.py (83%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_workcopies.py (91%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_core_layers.py (96%)
create mode 100644 src/backend/tests/features/desktop/capabilities/test_credential_continuity.py
rename src/backend/tests/{ => features/desktop}/capabilities/test_dependency_closure.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_capabilities_api.py (85%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_content_safety.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_declaration_export.py (89%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_model_reasoning_safety.py (88%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_resource_plugins.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_token_security.py (90%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_warmup.py (61%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_device_api_contract.py (74%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_device_catalog_projection.py (91%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_dynamic_model_epoch_cache.py (80%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_explicit_desktop_invocation.py (79%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_gateway_invocation_proof.py (78%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_gateway_unknown_outcome.py (61%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_hybrid_owned_plugins.py (50%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_initial_sync_readiness.py (51%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_junction_windows_paths.py (86%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_loader_store_mode.py (74%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_local_folder_discovery.py (90%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_local_management_migration.py (83%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_local_skill_projection.py (83%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_management_readiness.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_management_tools.py (56%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_manager_lifecycle.py (91%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_manager_regressions.py (85%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_manifest_order.py (85%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_manifest_tool_schema.py (76%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_mcp_json.py (98%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_mcp_json_consistency.py (99%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_memory_epoch_cache.py (79%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_migration.py (89%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_migration_preservation.py (96%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_model_credential_reference.py (86%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_model_error_diagnostics.py (83%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_owned_connector_assembly.py (55%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_partial_session_availability.py (73%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_plugin_connections.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_plugin_device_assets.py (89%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_prepare_lock_not_held_over_io.py (98%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_progressive_plugins.py (72%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_frozen_bindings.py (63%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_publication_recovery.py (70%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_recovery.py (58%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_scope_consumers.py (87%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_scopes.py (81%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_selection_recovery.py (66%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_session_recovery.py (68%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_selection_dependency_sources.py (79%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_selection_readiness.py (80%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_store_file_integrity.py (88%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_toolkit_runtime_alias.py (83%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_user_facing_errors.py (81%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_view_os_errors.py (95%)
rename src/backend/tests/{ => features/desktop}/capabilities/test_workbench_local_upgrades.py (84%)
rename src/backend/tests/{ => features/desktop}/desktop_login_browser_server.py (91%)
create mode 100644 src/backend/tests/features/desktop/index_fixtures.py
create mode 100644 src/backend/tests/features/desktop/observability_cases_1.py
create mode 100644 src/backend/tests/features/desktop/observability_cases_2.py
create mode 100644 src/backend/tests/features/desktop/observability_support.py
rename src/backend/tests/{ => features/desktop}/run_desktop_browser_e2e.py (99%)
rename src/backend/tests/{ => features/desktop}/test_desktop_auth.py (96%)
rename src/backend/tests/{misc => features/desktop}/test_desktop_bridge.py (90%)
rename src/backend/tests/{misc => features/desktop}/test_desktop_capability.py (81%)
rename src/backend/tests/{misc => features/desktop}/test_desktop_capability_models.py (91%)
rename src/backend/tests/{misc => features/desktop}/test_desktop_cloud_skills.py (84%)
rename src/backend/tests/{ => features/desktop}/test_desktop_device_login.py (97%)
rename src/backend/tests/{ => features/desktop}/test_desktop_memory_persistence.py (96%)
rename src/backend/tests/{api => features/desktop}/test_desktop_model_gateway.py (88%)
create mode 100644 src/backend/tests/features/desktop/test_desktop_observability.py
rename src/backend/tests/{ => features/desktop}/test_desktop_observability_authenticated_delivery.py (77%)
rename src/backend/tests/{ => features/desktop}/test_desktop_observability_delivery.py (75%)
rename src/backend/tests/{api => features/desktop}/test_desktop_release_events.py (96%)
rename src/backend/tests/{ => features/desktop}/test_desktop_sync_locking.py (74%)
rename src/backend/tests/{api => features/desktop}/test_desktop_updates.py (98%)
rename src/backend/tests/{api => features/desktop}/test_local_file_actions.py (89%)
rename src/backend/tests/{services => features/desktop}/test_local_grant_service.py (98%)
rename src/backend/tests/{services => features/desktop}/test_local_snapshot_service.py (96%)
create mode 100644 src/backend/tests/features/evaluation/__init__.py
create mode 100644 src/backend/tests/features/evaluation/conftest.py
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_admission.py (97%)
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_binding.py (98%)
rename src/backend/tests/{llm => features/evaluation}/test_evaluation_factory.py (87%)
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_process_guard.py (97%)
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_provider.py (96%)
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_router.py (78%)
rename src/backend/tests/{llm => features/evaluation}/test_evaluation_runtime.py (88%)
rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_service.py (93%)
create mode 100644 src/backend/tests/features/hosted_apps/__init__.py
rename src/backend/tests/{api => features/hosted_apps}/application_hosting_support.py (75%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_agent_contract.py (76%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_collections.py (87%)
rename src/backend/tests/{services => features/hosted_apps}/test_application_desktop.py (80%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_export.py (91%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_hosting.py (92%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_personal_mcp.py (91%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_project_tools.py (87%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_projects.py (94%)
rename src/backend/tests/{api => features/hosted_apps}/test_application_retirement.py (94%)
rename src/backend/tests/{api => features/hosted_apps}/test_hosting_remediation.py (82%)
create mode 100644 src/backend/tests/features/hosted_apps/test_schema_lifecycle.py
create mode 100644 src/backend/tests/features/knowledge/__init__.py
create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_cases_1.py
create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_cases_2.py
create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_support.py
create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_1.py
create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_2.py
create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_support.py
create mode 100644 src/backend/tests/features/knowledge/test_host_contributions.py
rename src/backend/tests/{mcp => features/knowledge}/test_kb_backend_routing.py (74%)
rename src/backend/tests/{ => features/knowledge}/test_kb_document_status.py (89%)
rename src/backend/tests/{services => features/knowledge}/test_kb_index_queue.py (98%)
rename src/backend/tests/{services => features/knowledge}/test_kb_index_worker.py (88%)
create mode 100644 src/backend/tests/features/knowledge/test_kb_multimodal_assets.py
rename src/backend/tests/{mcp => features/knowledge}/test_kb_vector_safety.py (57%)
rename src/backend/tests/{mcp => features/knowledge}/test_kb_vector_timeouts.py (68%)
create mode 100644 src/backend/tests/features/knowledge/test_kb_wiki_end_to_end.py
rename src/backend/tests/{services => features/knowledge}/test_kb_wiki_pipeline.py (96%)
create mode 100644 src/backend/tests/features/marketplace/__init__.py
create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_1.py
create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_2.py
create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_3.py
create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_4.py
create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_support.py
rename src/backend/tests/{ => features/marketplace}/test_agent_marketplace.py (69%)
rename src/backend/tests/{skills => features/marketplace}/test_marketplace_service.py (73%)
rename src/backend/tests/{skills => features/marketplace}/test_marketplace_submission.py (54%)
create mode 100644 src/backend/tests/features/marketplace/test_mcp_marketplace.py
create mode 100644 src/backend/tests/features/marketplace/test_oauth_sdk2_contract.py
create mode 100644 src/backend/tests/features/sites/__init__.py
create mode 100644 src/backend/tests/features/sites/conftest.py
rename src/backend/tests/{capabilities => features/sites}/test_cloud_site_publish.py (82%)
rename src/backend/tests/{llm => features/sites}/test_design_pick.py (99%)
rename src/backend/tests/{llm => features/sites}/test_desktop_site_paths.py (88%)
rename src/backend/tests/{capabilities => features/sites}/test_local_project_site_publish.py (93%)
rename src/backend/tests/{capabilities => features/sites}/test_local_site_sources.py (88%)
rename src/backend/tests/{api => features/sites}/test_site_access_password.py (93%)
rename src/backend/tests/{api => features/sites}/test_site_contention_postgres.py (92%)
rename src/backend/tests/{capabilities => features/sites}/test_site_listing.py (90%)
rename src/backend/tests/{capabilities => features/sites}/test_site_plugin_upgrade.py (85%)
rename src/backend/tests/{services => features/sites}/test_site_rate_limit.py (97%)
create mode 100644 src/backend/tests/features/test_execution_identity_contract.py
create mode 100644 src/backend/tests/features/test_host_runtime.py
create mode 100644 src/backend/tests/features/test_installed_modules.py
create mode 100644 src/backend/tests/features/test_run_capability_host.py
create mode 100644 src/backend/tests/features/workbench/__init__.py
rename src/backend/tests/{ => features/workbench}/test_plugin_ui_contract.py (97%)
rename src/backend/tests/{ => features/workbench}/test_plugin_ui_routes.py (96%)
rename src/backend/tests/{ => features/workbench}/test_workbench_activity_routes.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_actual_process_observer.py (93%)
rename src/backend/tests/{ => features/workbench}/test_workbench_application_config_http.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_approval_lease_http.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_auth_routes.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_background_subpage_http.py (96%)
rename src/backend/tests/{ => features/workbench}/test_workbench_candidate_storage.py (91%)
rename src/backend/tests/{ => features/workbench}/test_workbench_capability_routes.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_closing_isolation_http.py (94%)
rename src/backend/tests/{ => features/workbench}/test_workbench_cookie_boundaries.py (94%)
rename src/backend/tests/{ => features/workbench}/test_workbench_coordination.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_diagnostic_failure_http.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_diagnostics.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_embed_routes.py (96%)
rename src/backend/tests/{ => features/workbench}/test_workbench_external_routes.py (98%)
rename src/backend/tests/{ => features/workbench}/test_workbench_invalidation.py (94%)
rename src/backend/tests/{ => features/workbench}/test_workbench_migration.py (98%)
rename src/backend/tests/{ => features/workbench}/test_workbench_migration_auth_gate.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_observer_paths.py (93%)
rename src/backend/tests/{ => features/workbench}/test_workbench_official_capabilities_http.py (93%)
rename src/backend/tests/{ => features/workbench}/test_workbench_origins.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_plugin_routes.py (97%)
rename src/backend/tests/{ => features/workbench}/test_workbench_preferences_routes.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_protocol.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_provider_scope_http.py (93%)
rename src/backend/tests/{ => features/workbench}/test_workbench_resource_owner_migrations.py (99%)
rename src/backend/tests/{ => features/workbench}/test_workbench_resources.py (85%)
rename src/backend/tests/{ => features/workbench}/test_workbench_revocation_failure_http.py (90%)
rename src/backend/tests/{ => features/workbench}/test_workbench_routing_routes.py (94%)
rename src/backend/tests/{ => features/workbench}/test_workbench_state_binding.py (94%)
rename src/backend/tests/{ => features/workbench}/test_workbench_state_concurrency.py (87%)
rename src/backend/tests/{ => features/workbench}/test_workbench_state_routes.py (95%)
rename src/backend/tests/{ => features/workbench}/test_workbench_upgrade_routes.py (97%)
rename src/backend/tests/{ => features/workbench}/workbench_capabilities_http_fixture.py (96%)
rename src/backend/tests/{ => features/workbench}/workbench_diagnostic_http_fixture.py (95%)
rename src/backend/tests/{ => features/workbench}/workbench_embed_http_fixture.py (95%)
rename src/backend/tests/{ => features/workbench}/workbench_http_fixture.py (95%)
rename src/backend/tests/{ => features/workbench}/workbench_observed_runner_fixture.py (90%)
rename src/backend/tests/{ => features/workbench}/workbench_observer_paths.py (98%)
rename src/backend/tests/{ => features/workbench}/workbench_overflow_http_fixture.py (95%)
rename src/backend/tests/{ => features/workbench}/workbench_plugin_http_fixture.py (96%)
rename src/backend/tests/{ => features/workbench}/workbench_plugin_support.py (100%)
rename src/backend/tests/{ => features/workbench}/workbench_resource_observer.py (92%)
rename src/backend/tests/{ => features/workbench}/workbench_routing_http_fixture.py (88%)
create mode 100644 src/backend/tests/infra/test_byte_range.py
create mode 100644 src/backend/tests/infra/test_db_off_loop_call_sites.py
create mode 100644 src/backend/tests/infra/test_db_thread_budget.py
create mode 100644 src/backend/tests/infra/test_no_session_on_event_loop.py
create mode 100644 src/backend/tests/integrations/__init__.py
create mode 100644 src/backend/tests/integrations/channels/__init__.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_1.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_2.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_3.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_4.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_5.py
create mode 100644 src/backend/tests/integrations/channels/bot_cases_6.py
create mode 100644 src/backend/tests/integrations/channels/bot_support.py
create mode 100644 src/backend/tests/integrations/channels/test_adapter_method_binding.py
create mode 100644 src/backend/tests/integrations/channels/test_channel_bots.py
rename src/backend/tests/{ => integrations/channels}/test_channel_bots_api.py (93%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_e2e.py (84%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_http.py (84%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_migration.py (97%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_relay.py (88%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_relay_api.py (83%)
rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_worker.py (75%)
rename src/backend/tests/{ => integrations/channels}/test_channel_receiver_lifecycle.py (94%)
rename src/backend/tests/{ => integrations/channels}/test_weixin_connection_health.py (95%)
rename src/backend/tests/{ => integrations/channels}/test_weixin_qr_protocol.py (95%)
rename src/backend/tests/{ => integrations/dingtalk}/test_dingtalk_integration.py (91%)
rename src/backend/tests/{ => integrations/email}/test_email_integration.py (87%)
create mode 100644 src/backend/tests/integrations/lark/test_app_lifecycle.py
rename src/backend/tests/{ => integrations/lark}/test_lark_auth_status.py (79%)
rename src/backend/tests/{ => integrations/yida}/test_yida_integration.py (94%)
create mode 100644 src/backend/tests/llm/test_delta_coalescer.py
create mode 100644 src/backend/tests/llm/test_mcp_factory_e2e.py
create mode 100644 src/backend/tests/llm/test_mcp_human_wait_timeout.py
create mode 100644 src/backend/tests/llm/test_mcp_native_adversarial.py
create mode 100644 src/backend/tests/llm/test_mcp_native_capabilities.py
delete mode 100644 src/backend/tests/llm/test_mcp_pool_cache.py
create mode 100644 src/backend/tests/llm/test_mcp_runtime_pool.py
create mode 100644 src/backend/tests/llm/test_mcp_session_ownership.py
create mode 100644 src/backend/tests/llm/test_migration_vendor_datasource.py
create mode 100644 src/backend/tests/llm/test_stream_wrapper_cleanup.py
create mode 100644 src/backend/tests/orchestration/automation_test_support.py
create mode 100644 src/backend/tests/orchestration/hook_support.py
create mode 100644 src/backend/tests/orchestration/test_agent_runtime_resources.py
delete mode 100644 src/backend/tests/orchestration/test_harness_process_restart.py
create mode 100644 src/backend/tests/orchestration/test_observation_writer.py
create mode 100644 src/backend/tests/orchestration/test_pending_workflows.py
create mode 100644 src/backend/tests/orchestration/test_plan_early_cancel.py
create mode 100644 src/backend/tests/orchestration/test_runtime_phase_lifecycle.py
create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_2.py
create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_3.py
create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_4.py
create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_5.py
create mode 100644 src/backend/tests/orchestration/tool_effect_support.py
rename src/backend/tests/{ => personal_agent}/test_device_link.py (97%)
create mode 100644 src/backend/tests/sandbox/test_controller_capacity.py
create mode 100644 src/backend/tests/sandbox/test_runtime_ownership.py
create mode 100644 src/backend/tests/sandbox/test_sandbox_init.py
delete mode 100644 src/backend/tests/services/test_kb_multimodal_assets.py
delete mode 100644 src/backend/tests/services/test_kb_wiki_end_to_end.py
create mode 100644 src/backend/tests/test_agent_version_history.py
delete mode 100644 src/backend/tests/test_automation_manual_trigger.py
delete mode 100644 src/backend/tests/test_automation_oneshot_finalize.py
delete mode 100644 src/backend/tests/test_channel_bots.py
create mode 100644 src/backend/tests/test_cli_cold_bootstrap.py
create mode 100644 src/backend/tests/test_credential_registry.py
create mode 100644 src/backend/tests/test_db_import_order.py
delete mode 100644 src/backend/tests/test_desktop_observability.py
create mode 100644 src/backend/tests/test_hook_contributions.py
create mode 100644 src/backend/tests/test_local_mcp_http_readiness.py
create mode 100644 src/backend/tests/test_local_onboarding.py
create mode 100644 src/backend/tests/test_mcp_canonical_url.py
delete mode 100644 src/backend/tests/test_mcp_marketplace.py
create mode 100644 src/backend/tests/test_package_sources.py
create mode 100644 src/backend/tests/test_personal_agent_prompt_cache.py
create mode 100644 src/backend/tests/test_route_import_order.py
create mode 100644 src/backend/tests/test_session_database_isolation.py
create mode 100644 src/backend/tests/test_tool_contributions.py
create mode 100644 src/backend/tests/test_user_cleanup.py
create mode 100644 src/frontend/scripts/test-canvas-quote-browser.mjs
create mode 100644 src/frontend/scripts/test-canvas-viewers-browser.mjs
create mode 100644 src/frontend/scripts/test-canvas-zoom-browser.mjs
delete mode 100644 src/frontend/src/components/canvas/CanvasFileRenderers.tsx
create mode 100644 src/frontend/src/components/canvas/CanvasZoomControl.tsx
create mode 100644 src/frontend/src/components/canvas/canvasDocument.ts
delete mode 100644 src/frontend/src/components/canvas/canvasFileCategory.ts
create mode 100644 src/frontend/src/components/canvas/canvasZoom.ts
create mode 100644 src/frontend/src/components/canvas/find/CanvasFind.css
create mode 100644 src/frontend/src/components/canvas/find/CanvasFind.tsx
create mode 100644 src/frontend/src/components/canvas/find/domFindSource.ts
create mode 100644 src/frontend/src/components/canvas/find/index.ts
create mode 100644 src/frontend/src/components/canvas/find/textSearch.ts
create mode 100644 src/frontend/src/components/canvas/outline/CanvasOutline.css
create mode 100644 src/frontend/src/components/canvas/outline/CanvasOutline.tsx
create mode 100644 src/frontend/src/components/canvas/outline/domOutline.ts
create mode 100644 src/frontend/src/components/canvas/outline/index.ts
create mode 100644 src/frontend/src/components/canvas/renderers/DocxRenderer.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/FallbackRenderers.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/HtmlRenderer.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/ImageRenderer.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/PdfRenderer.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/PdfViewer.css
create mode 100644 src/frontend/src/components/canvas/renderers/PdfViewer.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/TextRenderers.css
create mode 100644 src/frontend/src/components/canvas/renderers/TextRenderers.tsx
create mode 100644 src/frontend/src/components/canvas/renderers/index.ts
create mode 100644 src/frontend/src/components/canvas/renderers/pdfjs.ts
create mode 100644 src/frontend/src/components/canvas/renderers/previewError.ts
create mode 100644 src/frontend/src/components/canvas/selection/SpreadsheetQuoteButton.css
create mode 100644 src/frontend/src/components/canvas/selection/SpreadsheetQuoteButton.tsx
create mode 100644 src/frontend/src/components/canvas/selection/canvasChat.ts
create mode 100644 src/frontend/src/components/canvas/selection/canvasLocator.ts
create mode 100644 src/frontend/src/components/canvas/selection/useCanvasSelection.tsx
create mode 100644 src/frontend/src/components/chat/AgentCursor.tsx
create mode 100644 src/frontend/src/components/common/AgentMark.tsx
create mode 100644 src/frontend/src/components/common/RollingText.tsx
create mode 100644 src/frontend/src/components/common/selection/SelectionToolbar.css
create mode 100644 src/frontend/src/components/common/selection/useSelectionToolbar.tsx
create mode 100644 src/frontend/src/components/file/MediaPlayer.css
create mode 100644 src/frontend/src/components/file/MediaPlayer.tsx
delete mode 100644 src/frontend/src/components/tool/PendingStepRow.tsx
create mode 100644 src/frontend/src/components/tool/ToolKindStack.tsx
create mode 100644 src/frontend/src/components/tool/toolIcons.tsx
create mode 100644 src/frontend/src/hooks/useActivityAge.ts
create mode 100644 src/frontend/src/styles/chat/agent-activity.css
create mode 100644 src/frontend/src/utils/elapsed.ts
create mode 100644 src/frontend/src/utils/filePreviewKind.ts
create mode 100644 src/frontend/src/utils/quotedFollowUp.ts
create mode 100644 src/frontend/src/utils/toolIconKind.ts
create mode 100644 src/frontend/src/utils/turnActivity.ts
diff --git a/.claude/skills/hugagent-backend-dev/references/api-envelope.md b/.claude/skills/hugagent-backend-dev/references/api-envelope.md
index 841012ce..a0d9525c 100644
--- a/.claude/skills/hugagent-backend-dev/references/api-envelope.md
+++ b/.claude/skills/hugagent-backend-dev/references/api-envelope.md
@@ -78,4 +78,8 @@ return error_response(code=20001, message="Bad request", status_code=400)
1. **路由中只用 `success_response` / `created_response` / `paginated_response`**
2. **错误通过抛 `AppException` 子类,由全局 handler 转 `error_response`**
3. **trace_id 默认自动生成,无需手动传**
-4. **SSE 端点(流式聊天)不使用信封,直接发 SSE 事件**
+4. **SSE 端点不使用信封**:用 `sse_response(generator, extra_headers=...)` 返回(自带 `Cache-Control: no-cache`、
+ `X-Accel-Buffering: no` 等 `SSE_HEADERS`);聊天回合的 SSE 输出统一走 `api/transports/run_sse.py`,不要自写一套
+5. **License 402** 只来自 `edition_ee/licensing/features.py` 的 `FeatureNotLicensed`(40201)/ `SeatLimitExceeded`(40202),
+ 路由与服务层不要手搓 402
+6. 时间字段序列化用 `.isoformat()`(模型存的是带时区的 UTC),不要输出无时区字符串
diff --git a/.claude/skills/hugagent-backend-dev/references/architecture.md b/.claude/skills/hugagent-backend-dev/references/architecture.md
index 34c6a766..306f4283 100644
--- a/.claude/skills/hugagent-backend-dev/references/architecture.md
+++ b/.claude/skills/hugagent-backend-dev/references/architecture.md
@@ -1,129 +1,164 @@
# 后端架构参考
-## 请求流转
+## 1. 顶层布局与依赖方向
```
-Browser → Nginx (:3000 /api/ proxy) → FastAPI (api/app.py)
- → api/middleware/ (CORS → Logging → Error Handler → license_gate)
- → api/routes/v1/*.py (路由层;CE 注册表 api/routes/v1/__init__.py,EE 注册表 edition_ee/routes/registry.py)
- → core/services/*.py (服务层 — 业务逻辑)
- → core/db/repository/ (仓库层 — 数据访问,按领域分文件)
- → core/db/models/ (ORM 包 → PostgreSQL/SQLite)
+src/backend/
+├── api/ HTTP 协议层:app.py(组装)、deps.py、middleware/、routes/v1/(CE_ROUTERS)、transports/run_sse.py、startup/
+├── core/ 内核:领域逻辑与可复用运行时(不导入 features / integrations / personal_agent / orchestration)
+├── orchestration/ 模式编排:plans/、loops/、automations/、workers/(周期任务与恢复)
+├── features/ 可安装的产品功能模块(各自 README + extension.py + models.py + api/)
+├── integrations/ 外部系统集成:channels(IM 机器人框架)、dingtalk、lark、email、yida、api
+├── personal_agent/ Personal Agent 模块(同样经 extension.py 接入)
+├── edition_ee/ 商业版专属:路由注册表、License、Team/RBAC、EE 表、KB provider、EE 服务(CE 树物理删除)
+├── extensions/ 随包资源:skills/、agents/、plugins/(default + marketplace)、mcp_servers/;根路径唯一来源 paths.py
+├── prompts/ 提示词装配运行时 + prompt_text/ 文件兜底
+├── hugagent_cli/ 命令行客户端
+├── alembic/ 迁移(EE 主链;CE 用独立 ce_0001 基线)
+└── tests/ pytest(按领域子目录:api/ chat/ core/ features// infra/ llm/ orchestration/ …)
```
-## 分层职责
+```
+api ──► orchestration ──► core
+ │ │ ▲
+ └──► features / integrations / personal_agent ──┘ (通过 core.harness.registry 注册钩子接入)
+edition_ee ──► core / features (CE 派生树中不存在)
+```
+
+- **core 永远不导入功能模块的实现**。功能模块在 `extension.setup()` 里调用 `core.harness.registry.register_*` 把能力挂进来;
+ 关闭时什么也不注册,core 的行为等同于未安装。
+- **不保留转发垫片**:模块搬家时直接改所有调用方的 import,不留旧路径 re-export(ADR 20261021)。
+- 单个模块 ≤ 500 行(存量超限文件已登记为例外,不要新增)。
+
+## 2. core/ 子包职责
+
+| 子包 | 职责 |
+|---|---|
+| `agent_runtime/` | 一次普通智能体回合:`runner.astream_chat_workflow` 选主智能体 / 直连路径;`prepare_*`(建 agent、组装输入)、`stream_*`(模型与工具事件 → 既有事件协议)、`*_state`(单轮有类型状态)、`integrations/`(记忆、压缩、本体证据/修复、演化) |
+| `runs/` | 持久化执行所有权:租约 / 心跳、取消、事件日志与续播(`events/`)、看门狗、工具副作用账本(`tool_effects/`)、恢复 |
+| `chat/` | 会话与消息读写、模式(modes)、分叉、引用、追问、压缩;`runtime/`(聊天 worker:消费回合事件、在 runs 租约下持久化输出、排队 / 插话 / 暂停 / 恢复);`operations/`、`projection/` |
+| `llm/` | AgentScope 适配与智能体装配:`factory/`(build / assembly / tools / prompts)、`runtime/`(流式循环、钩子适配、工具执行、人机交互)、`citations/`(锚点发号)、`mcp/`、`subagents/`、`tools/`、`tool_permissions/`、提示词版本池 |
+| `models/` | 模型供应商与选择:chat_models、providers、failover、reasoning effort、embedding / reranking、vision、用量 |
+| `capabilities/` | 能力生命周期:`catalog/`、`skills/`、`plugins/`(含插件资源)、`mcp/`、`agents/`、manifest 协议 |
+| `harness/` | 宿主接缝:`registry/`(功能模块注册钩子的唯一入口)、执行清单、观测事件、日志写入、用量 |
+| `auth/` | `backend.py`(`get_current_user` / `UserContext`)、`bearer.py`(mock/remote Bearer 异步校验 + 共享缓存)、session、本地账号、API Key、能力位、权限接口 |
+| `db/` | `engine.py`(Base / SessionLocal / `get_db` / 线程预算)、`threads.py`(`run_db` / `run_in_session` / `off_loop`)、`models/`(核心 ORM 包)、`repository/`、`utc_datetime.py`、各类升级 / 修复脚本 |
+| `config/` | `settings.py`(`settings` 冻结 dataclass:deploy / auth / sso / db / llm / memory / storage / kb / redis / server / sandbox / edition / branding …)、`catalog.json`、`mcp_config.py`、`display_names.py`、`runtime_env.py` |
+| `infra/` | `responses.py`、`exceptions.py`、`logging.py`、`time.py`(`utc_now`)、`background.py`(`spawn`)、redis、metrics、限流 / 熔断、leader 选举、加密、脱敏 |
+| `files/` | 产物(artifacts)、内容解析(content)、空间文件(spaces)、存储协议(storage:local / s3 / oss) |
+| `projects/` | 项目、项目环境、指令、本地工作区、个人项目作用域 |
+| `memory/` | 分层记忆(L1 画像 / L2 向量 / L3 图谱)、抽取流水线、outbox、脱敏、TTL |
+| `ontology/` | 本体治理:schema、校验、运行时闸门、输出评审、修订 |
+| `evolution/` | 协同进化共享契约与证据(控制面在 EE) |
+| `jobs/` | 后台作业运行时(作业 SDK、监督、通知、文件) |
+| `sandbox/` | 沙箱协议 / 工厂 / providers(script_runner、opensandbox、cube[EE])、runner 进程服务、进程 / 会话 / 工作区 / 权限 / OS 隔离 |
+
+## 3. 功能模块(features / integrations / personal_agent)
+
+`features/installed.py::INSTALLED_FEATURES` 是全量安装清单(CE 由 `ce/overlay/src/backend/features/installed.py` 覆盖)。
+
+| 模块 | 内容 |
+|---|---|
+| `features/automation` | 定时任务:调度、执行、投递目标、通知、远程副作用 |
+| `features/knowledge` | 自建知识库:解析、分块、向量、资产索引、Wiki;外部 provider 接缝 |
+| `features/marketplace` | 技能 / 智能体 / MCP 市场:目录、安装、发布、版本、导出 |
+| `features/workbench` | 可扩展工作台:应用目录、外部应用 / 嵌入 / 外部认证、插件 UI、资源实例、偏好、诊断、迁移 |
+| `features/hosted_apps` | 托管业务应用(SQL / MCP):部署、数据、发布、资源 |
+| `features/sites` | 站点托管与发布(静态服务、访问密码、打包、本机发布) |
+| `features/desktop` | 桌面双端:能力桥、网关、本机能力、登录桥接、可观测性 |
+| `features/evaluation` | 评测沙箱(严格 OpenSandbox) |
+| `features/distillation` · `security_console` · `datasources` · `sandbox_ops` | EE:技能 / 人格蒸馏、安全台、数据源、沙箱运维 |
+| `integrations/channels` + `dingtalk` / `lark` / `email` / `yida` | 入站渠道机器人框架与各平台适配 |
+| `personal_agent` | Personal Agent:会话、渠道、调度、投递、设备连接、工作区人格 |
+
+模块约定:
```
-┌─────────────────────────────────────────┐
-│ Routes (api/routes/v1/) │ ← HTTP 入口、参数校验、响应包装
-│ - 依赖注入: auth, db │
-│ - Pydantic 请求模型 │
-│ - _to_dict() 转换函数 │
-│ - 调用 Service │
-├─────────────────────────────────────────┤
-│ Services (core/services/) │ ← 业务逻辑、权限校验、审计
-│ - 构造函数接收 db: Session │
-│ - 内部创建 Repository │
-│ - 幂等操作 (ensure_xxx) │
-│ - 抛 AppException │
-├─────────────────────────────────────────┤
-│ Repository (core/db/repository/) │ ← 数据访问抽象(包,10 个领域文件)
-│ - CRUD 操作 │
-│ - 分页查询 │
-│ - 软删除过滤 │
-│ - 不含业务逻辑 │
-├─────────────────────────────────────────┤
-│ Models (core/db/models/) │ ← ORM 定义(包,17 个领域文件)
-│ - SQLAlchemy declarative │
-│ - 索引、约束、关系 │
-│ - EE 专属表登记 edition_ee/db/edition_tables.py │
-│ - Alembic 迁移 │
-└─────────────────────────────────────────┘
+features//
+├── README.md 负责什么、公开入口、依赖方向、版本差异
+├── extension.py NAME + setup() -> bool;可选 register_tools / register_gateway_contributions /
+│ register_catalog_options / register_retrieval_context
+├── models.py ORM(按安装清单加载,与运行开关无关)
+├── edition.py (可选)CE/EE 差异唯一入口,CE overlay 只覆盖它
+├── service.py / repository.py / <领域>.py
+└── api/ 路由文件(多数进 CE_ROUTERS / EE_ROUTERS,可关闭的由 setup() 内 register_router 挂载)
```
-## 聊天流式请求流转
+## 4. 启动与路由装配(api/app.py)
+
+1. `orchestration.configuration.register_configuration()`、`features.register_run_capabilities()`、
+ `register_model_credentials()`、`integrations.register_credentials()`
+2. 挂 health、`api/routes` 文件下载路由
+3. `iter_edition_routers(CE_ROUTERS)` 全部 include;`iter_edition_routers(EE_ROUTERS)` include 并附加
+ `edition_router_dependencies(policy_key)`(License 能力位,未授权 402)
+4. `register_product_tools()` / `register_catalog_options()` / `register_package_sources()` / `register_resource_owners()`
+5. `load_extensions(INSTALLED_FEATURES)`:先 `load_models`,再逐个 `.extension.setup()`;然后挂
+ `core.harness.registry.routers()` 里登记的模块路由
+6. `sites_serve_router`(静态站点兜底)→ 登录路由 → mock SSO(按登录模式)→ 本地模式前端静态托管(最后)
+
+注册表规则:
+
+- `CE_ROUTERS`(`api/routes/v1/__init__.py`):二元组 `(module, attr)`;短名解析为 `api.routes.v1.`,
+ 含点号的写绝对路径(`"features.sites.api.sites"`)。**顺序即注册顺序**,同前缀族的先后有语义(例:`plugin_ui` 必须在
+ `plugins` 之前,否则 `/v1/plugins/{slug}` 会吞掉 `/v1/plugins/ui-contributions`)
+- `EE_ROUTERS`(`edition_ee/routes/registry.py`):三元组 `(module, attr, policy_key)`;`policy_key` 取
+ `edition_ee/licensing/features.py::Feature` 的值(`content_admin` / `audit` / `multi_tenancy` / `system_config` /
+ `billing` / `model_gateway` / `evolution_control` …);`None` 仅用于 License 失效时也必须可达的入口(登录、换证)
+- `api.edition.EE_ROUTERS` 是懒加载接缝;CE overlay 把它换成空表
+- **不要**在 `app.py` 手工 `include_router()`
+
+## 5. 一次聊天回合的流转
```
-POST /v1/agents/responses
- → api/routes/v1/chats/__init__.py
- → orchestration/chat_run_executor.py # ChatRun + Redis Stream(后台 run,SSE 跟随,断线续播)
- → orchestration/workflow.py # 流式编排主入口
- → core/services/chat_mode_service.py # 解析对话模式 ChatModeSpec(standard/turbo/市场模式;装配契约)
- → orchestration/memory_integration.py # 检索分层记忆并注入(user-role 冻结块,600ms 预算)
- → core/llm/factory/build.py # 构建 AgentScope 2.0 ReActAgent(按 ChatModeSpec 收窄工具面)
- → core/llm/mcp_manager.py # MCP 客户端池(streamable-http → mcp 容器)
- → core/config/mcp_config.py # MCP server 定义(端口真源 mcp_servers/_ports.py)
- → core/llm/middlewares.py # AS2 中间件(动态模型、文件上下文、CitationAnchorMiddleware、OntologyGateMiddleware)
- → prompts/prompt_runtime.py # 装配系统提示词(DB 版本池优先,prompt_text/ 兜底)
- → orchestration/streaming.py # StreamingAgent:把 AS2 事件流映射为内部块事件
- → orchestration/citation_anchor.py # 引用锚点唯一发号(AnchorAllocator,e1/e2/…,经中间件回注工具结果)
- → core/memory/service.py # 流结束后后台保存记忆(L1/L2/L3)
- → orchestration/followups.py # 流结束后后台生成追问建议(写入消息 extra_data)
+POST /v1/agents/responses api/routes/v1/agent_responses.py(+ chats/ 子路由)
+ → 准入与回合登记 core/chat/runtime/admission.py · core/runs(租约、事件日志)
+ → 聊天 worker core/chat/runtime/worker.py(消费事件、按顺序持久化输出)
+ → core/agent_runtime/runner.astream_chat_workflow
+ → prepare_main / prepare_direct 建 agent、组装输入(模式 core/chat/modes.py、记忆 integrations/memory.py)
+ → core/llm/factory/build.py AgentScope agent:模型选择、工具集、MCP(core/llm/mcp)、技能、提示词
+ → stream_main / stream_direct 模型与工具事件 → 事件协议(core/llm/runtime/streaming.py、stream_events.py)
+ → core/llm/citations/ 引用锚点发号(e1/e2…)
+ → 结束后:本体评审、记忆 / 演化结算、追问(core/chat/runtime/followups.py)
+ → SSE 输出 api/transports/run_sse.py(跟随事件日志;续播 GET /v1/chats/stream/{run_id}/subscription)
```
-> `orchestration/strategy.py` 仍在但已退化为占位符(两个分支都返回 MainOnlyStrategy),不再是主链路环节。
-
-**平级编排入口**(不走普通 chat run 的独立驱动):
-
-| 入口 | 文件 | 说明 |
-|------|------|------|
-| 自主循环 | `orchestration/autonomous_loop.py` + `loop_planner.py` / `loop_evaluator.py` + `subagents/loop_reviewer.py` | driver 持有需求台账 feature_list.json,逐条注入 + 只读 reviewer 验收;路由 `v1/loops.py` |
-| 批量执行 | `orchestration/batch_orchestrator.py` | batch_confirm 确认后逐行执行 |
-| 计划模式 | `orchestration/subagents/plan_mode.py` | plan_generate / plan_execute run |
-| 渠道入站 | `core/channels/`(protocol/registry/manager/inbound/outbound) | IM 消息归一为 InboundMsg 后复用聊天编排,回复经 markdown.py 降级回发 |
-| 定时任务 | `orchestration/schedulers/`(automation / distillation_cron / evolution / memory_ttl) | croniter 调度 |
-
-SSE 事件(前端解析真源 `src/frontend/src/hooks/chatStream.ts`):
-
-- 生命周期:`run_started`(首帧,携带续播 run_id)、`meta`、`error`、`end` / `data: [DONE]`(终止)、心跳为 SSE 注释行
-- 正文/思考:`content`(delta 增量;别名 ai_message/text/delta)、`content_replace`、`thinking`、`compaction_notice`、`steer_applied`
-- 工具:`tool_call_start`、`tool_call_delta`(参数流式)、`tool_call`、`tool_result`、`tool_pending`、`model_progress`(活性信号)
-- 交互确认:`batch_confirm`、`file_confirm`、`design_pick`
-- 计划/子智能体:`plan_update`、`subagent_event`
-- 本体治理:`ontology_activation` / `ontology_gate` / `ontology_review` / `ontology_repair` / `ontology_revision` / `ontology_revision_thinking`
-
-断线续播:`GET /v1/chats/stream/{run_id}`。追问建议(follow_up_questions)流结束后后台生成,也可能以 `follow_up` 事件流内直送。
-
-## 模块索引
-
-| 模块 | 路径 | 职责 |
-|------|------|------|
-| App 入口 | `api/app.py` | FastAPI 实例、中间件;路由按注册表自动注册 |
-| CE 路由注册表 | `api/routes/v1/__init__.py` | **CE_ROUTERS 单一真源**(二元组);re-export EE_ROUTERS |
-| EE 路由注册表 | `edition_ee/routes/registry.py` | **EE_ROUTERS 单一真源**(三元组,带 license 能力位) |
-| 依赖注入 | `api/deps.py` | require_admin(ADMIN_TOKEN)、require_config(CONFIG_TOKEN)、require_admin_or_config、require_system_settings、require_super_admin |
-| 用户认证 | `core/auth/backend.py` | get_current_user / UserContext(桌面桥接 → session → API-Key → mock/remote) |
-| 健康检查 | `api/health.py` | /health, /ready, /live |
-| Schema | `api/schemas.py` | 请求/响应 Pydantic 模型 |
-| 路由 | `api/routes/v1/` | 66 个路由文件 |
-| 技能引擎 | `core/agent_skills/` | SKILL.md 解析、多源加载、{dir} 沙箱路径注入 |
-| 生成物 | `core/artifacts/` | 注册与下载(store.py,local/oss 双模) |
-| 认证 | `core/auth/` | backend.py, session.py, sso.py, permissions_iface.py(CE/EE 接缝) |
-| 渠道 | `core/channels/` | 入站机器人框架:钉钉/飞书/企微/微信 adapter;加渠道 = 写 adapter + registry.py 注册 |
-| 聊天 | `core/chat/` | context.py, tool_log.py |
-| 配置 | `core/config/` | settings.py + catalog 五件套(catalog.json/catalog.py/loader/resolver/migration)+ mcp_config.py |
-| 内容 | `core/content/` | 内容块、file_parser.py |
-| 数据库 | `core/db/` | engine.py, models/(17 领域文件), repository/(10 领域文件), model_extensions.py, edition_tables.py(垫片) |
-| 进化 | `core/evolution/` | GCE 共享契约(Episode/候选/结算);控制面在 edition_ee/evolution |
-| 基础设施 | `core/infra/` | exceptions, responses, logging, metrics, 限流, Redis, crypto |
-| 知识库 | `core/kb/` | 分块、向量化、混合检索 + wiki/(LLM-Wiki 管线)+ wiki_router.py + external_provider.py(外接接缝) |
-| License | `core/licensing/` | features.py(能力位+402)、manager.py(状态机) |
-| LLM | `core/llm/` | agent_factory(含 turbo 参数), chat_models, middlewares, mcp_manager/mcp_pool, offloader, subagent_tool, tools/ |
-| 记忆 | `core/memory/` | service.py(mem0)、pipeline.py、profile.py(L1)、sanitizer.py、audit.py |
-| 本体 | `core/ontology/` | Domain Pack 校验、运行时选取、工具闸(schemas/validator/revision/toolkit) |
-| 沙箱 | `core/sandbox/` | protocol.py + script_runner / opensandbox / cube 三实现 |
-| 服务 | `core/services/` | 70+ 业务服务(user/chat/chat_mode/channel/loop/kb/plugin/prompt_version/marketplace…) |
-| 存储 | `core/storage/` | protocol, local, s3, oss |
-| 编排 | `orchestration/` | workflow, chat_run_executor, streaming, citation_anchor, autonomous_loop, followups, batch_orchestrator, schedulers/, subagents/ |
-| 提示词 | `prompts/` | prompt_runtime + provider + prompt_text/{default,code_exec,distillation,plan_mode,subagents,turbo}/ |
-| MCP 服务器 | `mcp_servers/` | 12 个 server 目录、11 个在役(端口 9100–9114,真源 _ports.py;9105/9109-9111 为退役保留位) |
-| 技能资产 | `skill_bundles/` | default/(5 内置)+ marketplace/(40+ 可安装) |
-| EE 树 | `edition_ee/` | EE 专属路由/模型/KB provider(dify/fastgpt/weknora)/evolution 控制面;CE 派生树物理删除 |
-
-## 新路由注册步骤
-
-1. 创建 `api/routes/v1/my_feature.py`(EE 专属则放 `edition_ee/routes/`),文件内定义 `router = APIRouter(prefix="/v1/xxx", tags=["Xxx"])`
-2. 注册进对应注册表:
- - 通用功能 → `api/routes/v1/__init__.py` 的 `CE_ROUTERS` 加 `("my_feature", "router")`
- - 企业版功能 → `edition_ee/routes/registry.py` 的 `EE_ROUTERS` 加 `("edition_ee.routes.my_feature", "router", "")`(能力位 `None` 表示显式豁免 feature 守卫)
-3. **不要**在 `api/app.py` 手工 `include_router()` —— app 启动时经 `iter_edition_routers()` 按表自动注册,模块缺失(CE 派生树物理删除了 EE 文件)会被静默跳过
+平级编排入口(不走普通回合):
+
+| 入口 | 位置 |
+|---|---|
+| 计划模式 | `orchestration/plans/`(generate_run / execute_run / mode) |
+| 自主循环 | `orchestration/loops/`(driver / planner / reviewer / requirements / recovery) |
+| 定时任务派发 | `orchestration/automations/` + `features/automation` |
+| 周期任务 / 恢复 | `orchestration/workers/`(`_worker_base.py`、run_recovery、memory_ttl、evolution、蒸馏 cron) |
+| 渠道入站 | `integrations/channels/`(归一为入站消息后复用聊天回合) |
+
+## 6. 数据访问与事件循环
+
+- 同步 SQLAlchemy Session 的每次查询都是阻塞 I/O。
+- **只用 Session 不 await 的路由写 `def`**(FastAPI 线程池执行);`def` 路由里没有事件循环,不能 `create_task` / `get_running_loop`。
+- **要 await 网络的 async 代码不持有 Session**:每段 DB 工作写成同步函数,用 `await run_in_session(fn, ...)`(新开并关闭会话)
+ 或 `await run_db(fn, ...)`(已有会话)在线程里跑;需要 async 接口的纯 DB 函数用 `@off_loop` 包装。
+- 线程池与连接池按 `core.db.engine.thread_budget()` 同一预算配置(`apply_thread_budget()`)。
+- 跨请求存活的工作:落库交给 worker(`orchestration/workers/_worker_base.py`);事件循环上的临时任务用 `core.infra.background.spawn(coro, name=...)`。
+- 守卫测试:`tests/test_no_blocking_async_routes.py`、`tests/test_no_event_loop_in_sync_routes.py`、
+ `tests/infra/test_no_session_on_event_loop.py`(全仓零例外:任何 async 函数都不收 `db` / `Session` / `Depends(get_db)`、
+ 不调 `SessionLocal()`、不 `with … as db`;测试里覆盖 `get_db` 时同时用 `tests/db_support.py` 让 `run_in_session` 指向同一个库)。
+
+## 7. 模块索引(常用入口)
+
+| 主题 | 路径 |
+|---|---|
+| 应用组装 | `api/app.py`、`api/lifecycle.py`、`api/startup/` |
+| 路由注册表 | `api/routes/v1/__init__.py`(CE)、`edition_ee/routes/registry.py`(EE)、`core/harness/registry`(模块路由) |
+| 依赖注入 | `api/deps.py`(require_admin / require_config / require_admin_or_config / require_system_settings / require_ontology_governance / require_super_admin / console_actor) |
+| 认证 | `core/auth/backend.py`(get_current_user / UserContext)、`core/auth/bearer.py` |
+| DB | `core/db/engine.py`(get_db)、`core/db/threads.py`、`core/db/models/`、`core/db/repository/` |
+| 信封 / 异常 | `core/infra/responses.py`、`core/infra/exceptions.py`、`api/middleware/error_handler.py` |
+| 配置 | `core/config/settings.py`、`core/config/catalog.json` |
+| 智能体回合 | `core/agent_runtime/`、`core/llm/factory/`、`core/llm/runtime/` |
+| 运行与续播 | `core/runs/`、`core/chat/runtime/`、`api/transports/run_sse.py` |
+| MCP 服务 | `extensions/mcp_servers/`(`registry.py`、`_ports.py` 端口稳定契约、`_launcher.py`)、`core/config/mcp_config.py` |
+| 技能 / 插件 / 智能体资源 | `extensions/{skills,plugins,agents}/{default,marketplace}`、`core/capabilities/` |
+| 提示词 | `prompts/prompt_runtime.py`(DB 版本池优先,`prompt_text/` 兜底)、`core/llm/prompt_*.py` |
+| License | `edition_ee/licensing/`(features / deps / gate / manager)、`api/middleware/edition.py` |
+| CE 派生 | `ce/manifest.yaml`(排除)、`ce/overlay/`(覆盖接缝文件) |
diff --git a/.claude/skills/hugagent-backend-dev/references/error-codes.md b/.claude/skills/hugagent-backend-dev/references/error-codes.md
index 18149924..73d26851 100644
--- a/.claude/skills/hugagent-backend-dev/references/error-codes.md
+++ b/.claude/skills/hugagent-backend-dev/references/error-codes.md
@@ -35,6 +35,8 @@
|------|------|----------------|------|
| 40001 | 404 | `ResourceNotFoundError` | 资源未找到 |
| 40002 | 404 | `EndpointNotFoundError` | API 端点不存在 |
+| 40201 | 402 | `FeatureNotLicensed`(`edition_ee/licensing/features.py`) | License 未授权该能力(EE 路由能力位守卫) |
+| 40202 | 402 | `SeatLimitExceeded`(同上) | 席位不足 / License 无效,无法新增用户 |
| 41001 | 409 | `ResourceAlreadyExistsError` | 资源已存在 |
| 41002 | 409 | `ConcurrentModificationError` | 并发修改冲突 |
| 42001 | 429 | `RateLimitExceededError` | 速率限制 |
diff --git a/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh b/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh
index 313654df..ff2ca623 100755
--- a/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh
+++ b/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh
@@ -1,83 +1,108 @@
#!/usr/bin/env bash
#
-# 后端新功能脚手架生成器
+# 后端功能模块脚手架:生成 features// 完整骨架
#
# 用法:
-# bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh
+# bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh
#
# 示例:
# bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh bookmark Bookmark bookmarks
#
-# 生成文件:
-# - src/backend/api/routes/v1/s.py (路由)
-# - src/backend/core/services/_service.py (服务)
-# - src/backend/tests/test_.py (测试)
+# 生成:
+# src/backend/features//__init__.py
+# src/backend/features//README.md 职责、公开入口、依赖方向
+# src/backend/features//extension.py NAME + setup()
+# src/backend/features//models.py ORM(UTCDateTime + utc_now + JSONType)
+# src/backend/features//repository.py
+# src/backend/features//service.py
+# src/backend/features//api/__init__.py
+# src/backend/features//api/s.py 路由(同步 def + run_in_session 示例)
+# src/backend/tests/features//__init__.py
+# src/backend/tests/features//test_.py
#
-# 注意: model 和 repository 需手动合并到对应领域文件中
-# - core/db/models/<领域>.py ← 添加 ORM 类(并在包 __init__.py re-export)
-# - core/db/repository/<领域>.py ← 添加 Repository 类
-# - api/routes/v1/__init__.py ← 注册进 CE_ROUTERS / EE_ROUTERS 注册表
-# - 运行: alembic revision --autogenerate -m "add ${table_name} table"
+# 只建新文件,已存在的文件跳过;不改任何注册表(手动步骤见脚本末尾输出)。
+# 不是所有新功能都该进 features/:先按 SKILL.md「放哪里」判断。
set -euo pipefail
-FEATURE="${1:?Usage: $0 }"
-FEATURE_CAP="${2:?Usage: $0 }"
-TABLE="${3:?Usage: $0 }"
+FEATURE="${1:?Usage: $0 }"
+FEATURE_CAP="${2:?Usage: $0 }"
+TABLE="${3:?Usage: $0 }"
+FEATURE_UPPER="$(echo "$FEATURE" | tr '[:lower:]' '[:upper:]')"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
TEMPLATE_DIR="$SCRIPT_DIR/../templates"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../../" && pwd)"
BACKEND="$PROJECT_ROOT/src/backend"
+PKG="$BACKEND/features/$FEATURE"
+TESTS="$BACKEND/tests/features/$FEATURE"
-echo "=== HugAgentOS Backend Scaffold ==="
-echo "Feature: $FEATURE / $FEATURE_CAP"
-echo "Table: $TABLE"
-echo ""
-
-# Route file
-ROUTE_FILE="$BACKEND/api/routes/v1/${FEATURE}s.py"
-if [ ! -f "$ROUTE_FILE" ]; then
+render() { # render
+ local src="$1" dest="$2"
+ if [ -f "$dest" ]; then
+ echo "⊘ Skipped ${dest#$PROJECT_ROOT/} (exists)"
+ return
+ fi
sed -e "s/\${feature}/$FEATURE/g" \
-e "s/\${Feature}/$FEATURE_CAP/g" \
- -e "s/\${FEATURE_NAME}/$FEATURE_CAP/g" \
- "$TEMPLATE_DIR/route.py" > "$ROUTE_FILE"
- echo "✓ Created $ROUTE_FILE"
-else
- echo "⊘ Skipped $ROUTE_FILE (exists)"
-fi
+ -e "s/\${FEATURE}/$FEATURE_UPPER/g" \
+ -e "s/\${table_name}/$TABLE/g" \
+ "$src" > "$dest"
+ echo "✓ Created ${dest#$PROJECT_ROOT/}"
+}
-# Service file
-SERVICE_FILE="$BACKEND/core/services/${FEATURE}_service.py"
-if [ ! -f "$SERVICE_FILE" ]; then
- sed -e "s/\${feature}/$FEATURE/g" \
- -e "s/\${Feature}/$FEATURE_CAP/g" \
- "$TEMPLATE_DIR/service.py" > "$SERVICE_FILE"
- echo "✓ Created $SERVICE_FILE"
-else
- echo "⊘ Skipped $SERVICE_FILE (exists)"
-fi
-
-# Test file
-TEST_FILE="$BACKEND/tests/test_${FEATURE}.py"
-if [ ! -f "$TEST_FILE" ]; then
- sed -e "s/\${feature}/$FEATURE/g" \
- -e "s/\${Feature}/$FEATURE_CAP/g" \
- "$TEMPLATE_DIR/test.py" > "$TEST_FILE"
- echo "✓ Created $TEST_FILE"
-else
- echo "⊘ Skipped $TEST_FILE (exists)"
-fi
+touch_file() { # touch_file
+ if [ -f "$1" ]; then
+ echo "⊘ Skipped ${1#$PROJECT_ROOT/} (exists)"
+ else
+ printf '%s\n' "$2" > "$1"
+ echo "✓ Created ${1#$PROJECT_ROOT/}"
+ fi
+}
+echo "=== HugAgentOS backend feature scaffold ==="
+echo "Feature: $FEATURE / $FEATURE_CAP Table: $TABLE"
echo ""
-echo "=== 手动步骤 ==="
-echo "1. 将 model 模板合并到 core/db/models/<领域>.py(替换 \${Feature}=$FEATURE_CAP, \${table_name}=$TABLE),并在包 __init__.py re-export"
-echo "2. 将 repository 模板合并到 core/db/repository/<领域>.py(替换 \${Feature}=$FEATURE_CAP)"
-echo "3. 在 api/routes/v1/__init__.py 的注册表中添加:"
-echo " CE_ROUTERS += ((\"${FEATURE}s\", \"router\"),) # 或 EE_ROUTERS(带 license 能力位)"
-echo "4. 创建迁移:"
-echo " alembic revision --autogenerate -m \"add $TABLE table\""
-echo "5. 应用迁移:"
-echo " alembic upgrade head"
-echo ""
-echo "Done!"
+
+mkdir -p "$PKG/api" "$TESTS"
+
+touch_file "$PKG/__init__.py" "\"\"\"${FEATURE_CAP} feature module.\"\"\""
+touch_file "$PKG/api/__init__.py" ""
+touch_file "$TESTS/__init__.py" ""
+touch_file "$PKG/README.md" "# ${FEATURE_CAP}
+
+负责:(一句话写清本模块拥有的业务能力与数据)。
+
+公开入口:service、repository、api/${FEATURE}s.py。调用方直接依赖实际模块,不保留旧路径转发。
+
+依赖方向:本模块依赖 core(db / auth / infra / harness registry);core 不导入本模块,只通过 extension.setup()
+注册的钩子接入。模型由安装清单加载,不依赖运行开关。
+
+版本差异:如 CE/EE 行为不同,统一放 edition.py,CE 只覆盖该入口。"
+
+render "$TEMPLATE_DIR/extension.py" "$PKG/extension.py"
+render "$TEMPLATE_DIR/model.py" "$PKG/models.py"
+render "$TEMPLATE_DIR/repository.py" "$PKG/repository.py"
+render "$TEMPLATE_DIR/service.py" "$PKG/service.py"
+render "$TEMPLATE_DIR/route.py" "$PKG/api/${FEATURE}s.py"
+render "$TEMPLATE_DIR/test.py" "$TESTS/test_${FEATURE}.py"
+
+cat < bool called once at startup; register hooks/routers/workers and return True,
+ or register nothing and return False when switched off (env flag etc.)
+ - Optional hooks the host looks up by name:
+ register_tools() agent tool contributions (product tool composition)
+ register_gateway_contributions() desktop gateway upload/receipt channels
+ register_catalog_options() capability catalog options
+ register_retrieval_context() retrieval context providers
+ - models.py is imported for every INSTALLED_FEATURES entry regardless of setup() — schema registration
+ must not depend on runtime switches
+
+Core never imports a feature's implementation: features call core.harness.registry.* to plug in.
+"""
+
+import os
+
+NAME = "${feature}"
+
+
+def enabled() -> bool:
+ return os.getenv("${FEATURE}_ENABLED", "true").strip().lower() in ("1", "true", "yes", "on")
+
+
+def setup() -> bool:
+ if not enabled():
+ return False
+ # Routes that must disappear when the module is off are mounted here instead of CE_ROUTERS.
+ # Always-on routes go into api/routes/v1/__init__.py::CE_ROUTERS (ordering is explicit there).
+ from core.harness.registry import register_router
+ from features.${feature}.api.${feature}s import router
+
+ register_router(router) # optional policy_key="content_admin" etc. applies the EE edition gate
+ return True
diff --git a/.claude/skills/hugagent-backend-dev/templates/model.py b/.claude/skills/hugagent-backend-dev/templates/model.py
index b591cc07..d3ad9d55 100644
--- a/.claude/skills/hugagent-backend-dev/templates/model.py
+++ b/.claude/skills/hugagent-backend-dev/templates/model.py
@@ -1,74 +1,58 @@
"""ORM model template.
Replace ${Feature}, ${feature}, ${table_name} with actual names.
-Copy relevant parts into core/db/models/<领域>.py (并在包 __init__.py re-export).
+
+Where it goes:
+ - Feature module → features/${feature}/models.py (loaded via INSTALLED_FEATURES by app and alembic)
+ - Core domain → core/db/models/.py and re-export in core/db/models/__init__.py
+ - EE-only table → edition_ee/db/models/… and add the table name to edition_ee/db/edition_tables.py::EE_ONLY_TABLES
+
+Conventions enforced by tests (tests/test_utc_conventions.py, tests/test_db_import_order.py):
+ - Timestamps: UTCDateTime(timezone=True) + default=utc_now (never datetime.utcnow / naive datetimes)
+ - JSON: JSONType (JSON on SQLite, JSONB on PostgreSQL) — models must work on both
+ - Soft delete via deleted_at; FKs declare ondelete
+ - Model modules import only core.db / core.infra leaves, never services or routes
"""
-from datetime import datetime
-from sqlalchemy import (
- Column, String, Integer, Boolean, Text, TIMESTAMP,
- ForeignKey, CheckConstraint, Index,
-)
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import relationship
from core.db.engine import Base
+from core.db.models.column_types import JSONType
+from core.db.utc_datetime import UTCDateTime
+from core.infra.time import utc_now
+from sqlalchemy import Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text
class ${Feature}(Base):
- """${Feature} table."""
+ """${Feature} — one sentence on what a row represents and who owns it."""
+
__tablename__ = "${table_name}"
- # --- Primary key ---
id = Column(String(64), primary_key=True)
-
- # --- Foreign key (to users_shadow) ---
user_id = Column(
String(64),
ForeignKey("users_shadow.user_id", ondelete="CASCADE"),
nullable=False,
)
+ # Optional link that must survive parent deletion → SET NULL
+ project_id = Column(
+ String(64),
+ ForeignKey("projects.project_id", ondelete="SET NULL"),
+ nullable=True,
+ )
- # --- Business fields ---
name = Column(String(200), nullable=False)
description = Column(Text)
- status = Column(String(20), nullable=False, default="active")
- count = Column(Integer, default=0)
- is_active = Column(Boolean, default=True)
-
- # --- Flexible JSONB ---
- extra_data = Column("metadata", JSONB, default={})
+ position = Column(Integer, nullable=False, default=0)
+ is_enabled = Column(Boolean, nullable=False, default=True)
- # --- Soft delete ---
- deleted_at = Column(TIMESTAMP(timezone=True))
-
- # --- Timestamps (required) ---
- created_at = Column(TIMESTAMP(timezone=True), default=datetime.utcnow)
- updated_at = Column(
- TIMESTAMP(timezone=True),
- default=datetime.utcnow,
- onupdate=datetime.utcnow,
- )
+ # Python attribute extra_data, DB column "metadata" (metadata is reserved on declarative models)
+ extra_data = Column("metadata", JSONType, default=dict)
- # --- Relationships ---
- user = relationship("UserShadow", back_populates="${feature}s")
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ deleted_at = Column(UTCDateTime(timezone=True))
- # --- Indexes & Constraints ---
__table_args__ = (
- CheckConstraint("count >= 0", name="${table_name}_count_check"),
+ CheckConstraint("position >= 0", name="${table_name}_position_check"),
Index("idx_${table_name}_user_id", "user_id"),
Index("idx_${table_name}_updated_at", "updated_at"),
- Index(
- "idx_${table_name}_user_updated",
- "user_id", "updated_at",
- ),
- Index(
- "idx_${table_name}_deleted",
- "deleted_at",
- postgresql_where=Column("deleted_at").isnot(None),
- ),
- Index(
- "idx_${table_name}_metadata_gin",
- "metadata",
- postgresql_using="gin",
- ),
)
diff --git a/.claude/skills/hugagent-backend-dev/templates/repository.py b/.claude/skills/hugagent-backend-dev/templates/repository.py
index fdebaa87..fa006a57 100644
--- a/.claude/skills/hugagent-backend-dev/templates/repository.py
+++ b/.claude/skills/hugagent-backend-dev/templates/repository.py
@@ -1,83 +1,57 @@
-"""Repository template.
+"""Repository template — data access only, no business rules.
Replace ${Feature}, ${feature} with actual names.
-Add to core/db/repository/<领域>.py.
+Feature module: features/${feature}/repository.py. Core domain: core/db/repository/.py.
+
+Rules:
+ - Synchronous; takes a Session. Called from sync routes or inside core.db.threads.run_in_session
+ - Every read filters deleted_at IS NULL
+ - Time via core.infra.time.utc_now
"""
-from typing import Optional, List, Dict, Any, Tuple
-from datetime import datetime
-from sqlalchemy.orm import Session
+from typing import Any, Dict, List, Optional, Tuple
+
+from core.infra.time import utc_now
+from features.${feature}.models import ${Feature}
from sqlalchemy import desc
-from core.db.models import ${Feature}
+from sqlalchemy.orm import Session
class ${Feature}Repository:
- """Repository for ${feature} operations."""
-
def __init__(self, db: Session):
self.db = db
- # --- Read ---
+ def _live(self):
+ return self.db.query(${Feature}).filter(${Feature}.deleted_at.is_(None))
- def get_by_id(self, id: str) -> Optional[${Feature}]:
- """Get by primary key (respects soft delete)."""
- return self.db.query(${Feature}).filter(
- ${Feature}.id == id,
- ${Feature}.deleted_at.is_(None),
- ).first()
+ def get_by_id(self, item_id: str) -> Optional[${Feature}]:
+ return self._live().filter(${Feature}.id == item_id).first()
- def list_by_user(
- self,
- user_id: str,
- page: int = 1,
- page_size: int = 20,
- ) -> Tuple[List[${Feature}], int]:
- """Paginated list for user."""
- query = self.db.query(${Feature}).filter(
- ${Feature}.user_id == user_id,
- ${Feature}.deleted_at.is_(None),
- )
+ def list_by_user(self, user_id: str, page: int = 1, page_size: int = 20) -> Tuple[List[${Feature}], int]:
+ query = self._live().filter(${Feature}.user_id == user_id)
total = query.count()
items = (
- query
- .order_by(desc(${Feature}.updated_at))
+ query.order_by(desc(${Feature}.updated_at))
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
return items, total
- # --- Create ---
-
def create(self, data: Dict[str, Any]) -> ${Feature}:
- """Create and commit."""
item = ${Feature}(**data)
self.db.add(item)
self.db.commit()
self.db.refresh(item)
return item
- # --- Update ---
-
- def update(self, id: str, data: Dict[str, Any]) -> Optional[${Feature}]:
- """Update fields and commit."""
- item = self.get_by_id(id)
- if not item:
- return None
+ def update(self, item: ${Feature}, data: Dict[str, Any]) -> ${Feature}:
for key, value in data.items():
setattr(item, key, value)
- item.updated_at = datetime.utcnow()
self.db.commit()
self.db.refresh(item)
return item
- # --- Delete (soft) ---
-
- def soft_delete(self, id: str) -> bool:
- """Mark as deleted."""
- item = self.get_by_id(id)
- if not item:
- return False
- item.deleted_at = datetime.utcnow()
+ def soft_delete(self, item: ${Feature}) -> None:
+ item.deleted_at = utc_now()
self.db.commit()
- return True
diff --git a/.claude/skills/hugagent-backend-dev/templates/route.py b/.claude/skills/hugagent-backend-dev/templates/route.py
index 279a688d..d584e541 100644
--- a/.claude/skills/hugagent-backend-dev/templates/route.py
+++ b/.claude/skills/hugagent-backend-dev/templates/route.py
@@ -1,136 +1,126 @@
-"""${FEATURE_NAME} API routes (v1).
-
-Replace ${FEATURE_NAME}, ${feature}, ${Feature} with actual names.
+"""API route template.
+
+Replace ${Feature}, ${feature} with actual names.
+Feature module: features/${feature}/api/${feature}s.py, registered in api/routes/v1/__init__.py::CE_ROUTERS as
+("features.${feature}.api.${feature}s", "router") — or in edition_ee/routes/registry.py::EE_ROUTERS with a policy key,
+or via register_router() from features/${feature}/extension.py when the module can be switched off.
+
+Event-loop rules (enforced by tests/test_no_blocking_async_routes.py, tests/test_no_event_loop_in_sync_routes.py,
+tests/infra/test_no_session_on_event_loop.py):
+ 1. Endpoint uses a Session and awaits nothing → declare it `def` (FastAPI runs it in the thread pool).
+ 2. Endpoint must await network I/O → `async def` WITHOUT `db: Session = Depends(get_db)`; put each DB step in a
+ sync helper and call it with `await run_in_session(helper, ...)`. Never hold a Session across an await.
+ 3. A `def` endpoint has no event loop: no asyncio.create_task / get_running_loop. Durable work → persist and let a
+ worker pick it up (orchestration/workers/_worker_base.py); loop-bound fire-and-forget → core.infra.background.spawn.
+
+Module must import on its own without composing the app (tests/test_route_import_order.py).
"""
-from typing import Any, Dict, List, Optional
+from typing import Optional
+from core.auth.backend import UserContext, get_current_user
+from core.db.engine import get_db
+from core.db.threads import run_in_session
+from core.infra.responses import created_response, paginated_response, success_response
from fastapi import APIRouter, Depends, Query, status
-from pydantic import BaseModel, Field
+from features.${feature}.service import ${Feature}Service
+from pydantic import BaseModel, Field, field_validator
from sqlalchemy.orm import Session
-from api.deps import require_admin
-from core.auth.backend import get_current_user, UserContext
-from core.db.engine import get_db
-from core.infra.exceptions import ResourceNotFoundError, BadRequestError
-from core.infra.logging import get_logger
-from core.infra.responses import success_response, created_response, paginated_response
-from core.services.${feature}_service import ${Feature}Service
-
-logger = get_logger(__name__)
+router = APIRouter(prefix="/v1/${feature}s", tags=["${Feature}"])
-router = APIRouter(prefix="/v1/${feature}s", tags=["${Feature}s"])
-
-
-# ---------------------------------------------------------------------------
-# Request / Response Models
-# ---------------------------------------------------------------------------
class Create${Feature}Request(BaseModel):
- """创建 ${Feature} 请求体."""
- name: str = Field(..., description="名称", min_length=1, max_length=200)
- description: Optional[str] = Field(None, description="描述")
- metadata: Optional[dict] = Field(default_factory=dict, description="扩展信息")
+ name: str = Field(..., min_length=1, max_length=200, description="名称")
+ description: Optional[str] = Field(None, max_length=2000, description="说明")
+ @field_validator("name")
+ @classmethod
+ def _strip(cls, v: str) -> str:
+ return v.strip()
-class Update${Feature}Request(BaseModel):
- """更新 ${Feature} 请求体."""
- name: Optional[str] = Field(None, description="名称", max_length=200)
- description: Optional[str] = Field(None, description="描述")
- metadata: Optional[dict] = Field(None, description="扩展信息")
+class Update${Feature}Request(BaseModel):
+ name: Optional[str] = Field(None, min_length=1, max_length=200)
+ description: Optional[str] = Field(None, max_length=2000)
+ is_enabled: Optional[bool] = None
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
def _to_dict(item) -> dict:
- """ORM → API response dict."""
return {
"id": item.id,
"name": item.name,
"description": item.description,
- "metadata": item.extra_data or {},
+ "is_enabled": item.is_enabled,
"created_at": item.created_at.isoformat() if item.created_at else None,
"updated_at": item.updated_at.isoformat() if item.updated_at else None,
}
-# ---------------------------------------------------------------------------
-# Endpoints
-# ---------------------------------------------------------------------------
+# --- Pure DB endpoints: plain `def` -----------------------------------------------------------
+
-@router.get("", summary="获取${Feature}列表")
-async def list_items(
+@router.get("", summary="列出我的${Feature}")
+def list_items(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- service = ${Feature}Service(db)
- items, total, total_pages = service.list_items(user.user_id, page, page_size)
+ items, total = ${Feature}Service(db).list_items(str(user.user_id), page, page_size)
return paginated_response(
- items=[_to_dict(i) for i in items],
- page=page,
- page_size=page_size,
- total_items=total,
+ items=[_to_dict(i) for i in items], page=page, page_size=page_size, total_items=total
)
@router.get("/{item_id}", summary="获取${Feature}详情")
-async def get_item(
- item_id: str,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- service = ${Feature}Service(db)
- item = service.get_item(item_id, user.user_id)
- if not item:
- raise ResourceNotFoundError("${feature}", item_id)
- return success_response(data=_to_dict(item))
+def get_item(item_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
+ return success_response(data=_to_dict(${Feature}Service(db).get_owned(item_id, str(user.user_id))))
-@router.post("", status_code=status.HTTP_201_CREATED, summary="创建${Feature}")
-async def create_item(
+@router.post("", status_code=status.HTTP_201_CREATED, summary="新建${Feature}")
+def create_item(
body: Create${Feature}Request,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- service = ${Feature}Service(db)
- item = service.create(
- user_id=user.user_id,
- name=body.name,
- description=body.description,
- metadata=body.metadata,
- )
+ item = ${Feature}Service(db).create(str(user.user_id), body.name, body.description)
return created_response(data=_to_dict(item))
-@router.put("/{item_id}", summary="更新${Feature}")
-async def update_item(
+@router.patch("/{item_id}", summary="修改${Feature}")
+def update_item(
item_id: str,
body: Update${Feature}Request,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- service = ${Feature}Service(db)
- update_data = body.model_dump(exclude_none=True)
- if not update_data:
- raise BadRequestError("No fields to update")
- item = service.update(item_id, user.user_id, update_data)
- if not item:
- raise ResourceNotFoundError("${feature}", item_id)
+ item = ${Feature}Service(db).update(item_id, str(user.user_id), body.model_dump(exclude_unset=True))
return success_response(data=_to_dict(item))
@router.delete("/{item_id}", summary="删除${Feature}")
-async def delete_item(
- item_id: str,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- service = ${Feature}Service(db)
- ok = service.delete(item_id, user.user_id)
- if not ok:
- raise ResourceNotFoundError("${feature}", item_id)
- return success_response(message="Deleted")
+def delete_item(item_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
+ ${Feature}Service(db).delete(item_id, str(user.user_id))
+ return success_response(data={"id": item_id})
+
+
+# --- Endpoint that awaits network I/O: async, DB steps off the loop ---------------------------
+
+
+def _load_for_sync(db: Session, item_id: str, user_id: str) -> dict:
+ return _to_dict(${Feature}Service(db).get_owned(item_id, user_id))
+
+
+def _record_sync_result(db: Session, item_id: str, user_id: str, remote: dict) -> dict:
+ item = ${Feature}Service(db).update(item_id, user_id, {"description": remote.get("summary")})
+ return _to_dict(item)
+
+
+@router.post("/{item_id}/sync", summary="与外部系统同步")
+async def sync_item(item_id: str, user: UserContext = Depends(get_current_user)):
+ user_id = str(user.user_id)
+ current = await run_in_session(_load_for_sync, item_id, user_id) # DB step 1, session closed after
+ remote = {"summary": current["name"]} # await some_client.fetch(...) here
+ data = await run_in_session(_record_sync_result, item_id, user_id, remote) # DB step 2
+ return success_response(data=data)
diff --git a/.claude/skills/hugagent-backend-dev/templates/service.py b/.claude/skills/hugagent-backend-dev/templates/service.py
index a2e22666..5d02f65f 100644
--- a/.claude/skills/hugagent-backend-dev/templates/service.py
+++ b/.claude/skills/hugagent-backend-dev/templates/service.py
@@ -1,108 +1,60 @@
-"""Service layer template.
+"""Service template — business rules and permission checks.
Replace ${Feature}, ${feature} with actual names.
-Create as core/services/${feature}_service.py.
+Feature module: features/${feature}/service.py. Core domain: core//.py
+(there is no core/services/ package any more — services live with the domain that owns them).
+
+Rules:
+ - Synchronous; constructor takes a Session and builds its repositories
+ - Raises core.infra.exceptions.* (never HTTPException, never returns error dicts)
+ - Ownership / permission checks live here, not in the route
+ - No network I/O inside a method that holds the Session: do DB work, return, await network
+ in the async caller, then open a new DB step (core.db.threads)
+ - Module ≤ 500 lines; split by responsibility before it grows past that
"""
import uuid
-from typing import Optional, List, Dict, Any, Tuple
-from sqlalchemy.orm import Session
+from typing import List, Optional, Tuple
-from core.db.repository import ${Feature}Repository, AuditLogRepository
-from core.db.models import ${Feature}
-from core.infra.exceptions import ResourceOwnershipError
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError, ResourceOwnershipError
+from features.${feature}.models import ${Feature}
+from features.${feature}.repository import ${Feature}Repository
+from sqlalchemy.orm import Session
class ${Feature}Service:
- """Business logic for ${feature}s."""
-
def __init__(self, db: Session):
self.db = db
self.repo = ${Feature}Repository(db)
- self.audit_repo = AuditLogRepository(db)
-
- # --- Create ---
-
- def create(
- self,
- user_id: str,
- name: str,
- description: str = None,
- metadata: dict = None,
- ) -> ${Feature}:
- """Create a new ${feature}."""
- item = self.repo.create({
- "id": f"${feature}_{uuid.uuid4().hex[:16]}",
- "user_id": user_id,
- "name": name,
- "description": description,
- "extra_data": metadata or {},
- })
-
- self.audit_repo.create({
- "user_id": user_id,
- "action": "${feature}.created",
- "resource_type": "${feature}",
- "resource_id": item.id,
- "status": "success",
- })
-
- return item
-
- # --- Read ---
- def get_item(self, id: str, user_id: str) -> Optional[${Feature}]:
- """Get item with ownership check."""
- item = self.repo.get_by_id(id)
- if item and item.user_id != user_id:
- raise ResourceOwnershipError("${feature}", id)
- return item
-
- def list_items(
- self,
- user_id: str,
- page: int = 1,
- page_size: int = 20,
- ) -> Tuple[List[${Feature}], int, int]:
- """List with pagination. Returns (items, total, total_pages)."""
- items, total = self.repo.list_by_user(user_id, page, page_size)
- total_pages = (total + page_size - 1) // page_size
- return items, total, total_pages
-
- # --- Update ---
-
- def update(
- self,
- id: str,
- user_id: str,
- update_data: Dict[str, Any],
- ) -> Optional[${Feature}]:
- """Update with ownership check."""
- item = self.repo.get_by_id(id)
- if not item:
- return None
+ def get_owned(self, item_id: str, user_id: str) -> ${Feature}:
+ item = self.repo.get_by_id(item_id)
+ if item is None:
+ raise ResourceNotFoundError("${feature}", item_id)
if item.user_id != user_id:
- raise ResourceOwnershipError("${feature}", id)
- return self.repo.update(id, update_data)
-
- # --- Delete ---
-
- def delete(self, id: str, user_id: str) -> bool:
- """Soft delete with ownership check."""
- item = self.repo.get_by_id(id)
- if not item:
- return False
- if item.user_id != user_id:
- raise ResourceOwnershipError("${feature}", id)
- return self.repo.soft_delete(id)
-
- # --- Idempotent ensure ---
+ raise ResourceOwnershipError("${feature}", item_id)
+ return item
- def ensure(self, id: str, user_id: str, **defaults) -> Optional[${Feature}]:
- """Get existing or create (idempotent)."""
- existing = self.repo.get_by_id(id)
- if existing:
- if existing.user_id != user_id:
- return None
- return existing
- return self.create(user_id=user_id, **defaults)
+ def list_items(self, user_id: str, page: int, page_size: int) -> Tuple[List[${Feature}], int]:
+ return self.repo.list_by_user(user_id, page, page_size)
+
+ def create(self, user_id: str, name: str, description: Optional[str] = None) -> ${Feature}:
+ name = name.strip()
+ if not name:
+ raise BadRequestError("名称不能为空")
+ return self.repo.create(
+ {
+ "id": f"${feature}_{uuid.uuid4().hex[:16]}",
+ "user_id": user_id,
+ "name": name,
+ "description": description,
+ }
+ )
+
+ def update(self, item_id: str, user_id: str, changes: dict) -> ${Feature}:
+ item = self.get_owned(item_id, user_id)
+ allowed = {k: v for k, v in changes.items() if k in {"name", "description", "is_enabled"} and v is not None}
+ return self.repo.update(item, allowed)
+
+ def delete(self, item_id: str, user_id: str) -> None:
+ self.repo.soft_delete(self.get_owned(item_id, user_id))
diff --git a/.claude/skills/hugagent-backend-dev/templates/test.py b/.claude/skills/hugagent-backend-dev/templates/test.py
index aa456b7b..6f3b1a33 100644
--- a/.claude/skills/hugagent-backend-dev/templates/test.py
+++ b/.claude/skills/hugagent-backend-dev/templates/test.py
@@ -1,115 +1,86 @@
"""Test template.
Replace ${Feature}, ${feature} with actual names.
-Create as tests/test_${feature}.py.
-Run: PYTHONPATH=src/backend pytest src/backend/tests/test_${feature}.py -v
+Create as src/backend/tests/features/${feature}/test_${feature}.py (core domains: tests//test_*.py).
+Run: PYTHONPATH=src/backend pytest src/backend/tests/features/${feature} -q
+
+Rules:
+ - Test behaviour at a real boundary (service against a real DB session, or HTTP through the router);
+ do not assert internal call order
+ - The shared `db_session` fixture (tests/conftest.py) gives a fresh database per test (SQLite unless
+ TEST_DATABASE_URL points at PostgreSQL); schema changes should pass on both
+ - Write the failing test first; never weaken an assertion to make it pass
"""
import pytest
-from unittest.mock import MagicMock, patch
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError, ResourceOwnershipError
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from features.${feature}.service import ${Feature}Service
-from core.db.engine import Base
-from core.db.models import ${Feature}
-from core.db.repository import ${Feature}Repository
-from core.services.${feature}_service import ${Feature}Service
+@pytest.fixture
+def service(db_session):
+ return ${Feature}Service(db_session)
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-@pytest.fixture
-def db_session():
- """In-memory SQLite session for testing."""
- engine = create_engine("sqlite:///:memory:")
- Base.metadata.create_all(engine)
- Session = sessionmaker(bind=engine)
- session = Session()
- yield session
- session.close()
- Base.metadata.drop_all(engine)
+def test_create_and_list(service):
+ created = service.create("user_a", " 周报 ")
+ assert created.name == "周报"
+ items, total = service.list_items("user_a", 1, 20)
+ assert total == 1 and items[0].id == created.id
-@pytest.fixture
-def service(db_session):
- """${Feature}Service with test DB session."""
- return ${Feature}Service(db_session)
+def test_rejects_blank_name(service):
+ with pytest.raises(BadRequestError):
+ service.create("user_a", " ")
+
+
+def test_other_users_cannot_read_or_delete(service):
+ item = service.create("user_a", "私有")
+ with pytest.raises(ResourceOwnershipError):
+ service.get_owned(item.id, "user_b")
+ with pytest.raises(ResourceOwnershipError):
+ service.delete(item.id, "user_b")
+
+
+def test_soft_delete_hides_item(service):
+ item = service.create("user_a", "待删")
+ service.delete(item.id, "user_a")
+ with pytest.raises(ResourceNotFoundError):
+ service.get_owned(item.id, "user_a")
+ assert service.list_items("user_a", 1, 20)[1] == 0
+
+
+# --- HTTP boundary: mount only this router, override auth and DB --------------------------------
@pytest.fixture
-def sample_data():
- """Sample creation data."""
- return {
- "user_id": "test_user_001",
- "name": "Test ${Feature}",
- "description": "Test description",
- }
-
-
-# ---------------------------------------------------------------------------
-# Repository Tests
-# ---------------------------------------------------------------------------
-
-class TestRepository:
- def test_create(self, db_session, sample_data):
- repo = ${Feature}Repository(db_session)
- item = repo.create({
- "id": "${feature}_test001",
- **sample_data,
- })
- assert item.id == "${feature}_test001"
- assert item.name == sample_data["name"]
- assert item.deleted_at is None
-
- def test_get_by_id(self, db_session):
- repo = ${Feature}Repository(db_session)
- repo.create({"id": "${feature}_001", "user_id": "u1", "name": "A"})
- found = repo.get_by_id("${feature}_001")
- assert found is not None
- assert found.name == "A"
-
- def test_get_by_id_respects_soft_delete(self, db_session):
- repo = ${Feature}Repository(db_session)
- repo.create({"id": "${feature}_del", "user_id": "u1", "name": "D"})
- repo.soft_delete("${feature}_del")
- assert repo.get_by_id("${feature}_del") is None
-
- def test_list_by_user_pagination(self, db_session):
- repo = ${Feature}Repository(db_session)
- for i in range(5):
- repo.create({"id": f"${feature}_{i}", "user_id": "u1", "name": f"Item {i}"})
- items, total = repo.list_by_user("u1", page=1, page_size=3)
- assert total == 5
- assert len(items) == 3
-
-
-# ---------------------------------------------------------------------------
-# Service Tests
-# ---------------------------------------------------------------------------
-
-class TestService:
- def test_create(self, service, sample_data):
- item = service.create(**sample_data)
- assert item.name == sample_data["name"]
- assert item.user_id == sample_data["user_id"]
- assert item.id.startswith("${feature}_")
-
- def test_get_item_ownership(self, service):
- item = service.create(user_id="u1", name="Mine")
- # Owner can access
- assert service.get_item(item.id, "u1") is not None
- # Non-owner raises
- with pytest.raises(Exception):
- service.get_item(item.id, "u2")
-
- def test_delete(self, service):
- item = service.create(user_id="u1", name="ToDelete")
- assert service.delete(item.id, "u1") is True
- assert service.get_item(item.id, "u1") is None
-
- def test_ensure_idempotent(self, service):
- item1 = service.create(user_id="u1", name="First")
- item2 = service.ensure(item1.id, "u1")
- assert item1.id == item2.id
+def client(db_session, monkeypatch):
+ from core.auth.backend import UserContext, get_current_user
+ from core.db.engine import get_db
+ from core.infra.exceptions import AppException
+ from features.${feature}.api.${feature}s import router
+ from fastapi.responses import JSONResponse
+ from tests.db_support import share_session
+
+ app = FastAPI()
+ app.include_router(router)
+ app.dependency_overrides[get_db] = lambda: db_session
+ # async routes reach the database through run_in_session, not get_db
+ share_session(monkeypatch, db_session)
+ app.dependency_overrides[get_current_user] = lambda: UserContext(user_id="user_a", user_center_id="uc_a", username="a")
+
+ @app.exception_handler(AppException)
+ async def _envelope(_, exc: AppException):
+ return JSONResponse(status_code=exc.status_code, content={"code": exc.code, "message": exc.message})
+
+ return TestClient(app)
+
+
+def test_http_create_returns_envelope(client):
+ resp = client.post("/v1/${feature}s", json={"name": "周报"})
+ assert resp.status_code == 201
+ body = resp.json()
+ assert body["code"] == 10001
+ assert body["data"]["name"] == "周报"
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/attachment.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/attachment.svg
new file mode 100644
index 00000000..029d1c30
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/attachment.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/automation.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/automation.svg
new file mode 100644
index 00000000..0663b72e
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/automation.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/check.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/check.svg
new file mode 100644
index 00000000..1bfd4b84
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/check.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/chevron-right.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/chevron-right.svg
new file mode 100644
index 00000000..068cb596
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/chevron-right.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/close.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/close.svg
new file mode 100644
index 00000000..b5d6b13d
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/close.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/knowledge.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/knowledge.svg
new file mode 100644
index 00000000..1221ffaa
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/knowledge.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/my-space.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/my-space.svg
new file mode 100644
index 00000000..d7b07e6a
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/my-space.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/new-chat.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/new-chat.svg
new file mode 100644
index 00000000..98a36618
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/new-chat.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/search.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/search.svg
new file mode 100644
index 00000000..263a0e98
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/search.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/send.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/send.svg
new file mode 100644
index 00000000..1eee11cf
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/send.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/settings.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/settings.svg
new file mode 100644
index 00000000..6d957f24
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/settings.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/skill.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/skill.svg
new file mode 100644
index 00000000..48244686
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/skill.svg
@@ -0,0 +1,3 @@
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/favicon.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/favicon.svg
new file mode 100644
index 00000000..77e0a10a
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/favicon.svg
@@ -0,0 +1,5 @@
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-black.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-black.svg
new file mode 100644
index 00000000..4ac67167
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-black.svg
@@ -0,0 +1,4 @@
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-blue.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-blue.svg
new file mode 100644
index 00000000..f1b49091
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-blue.svg
@@ -0,0 +1,5 @@
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-white.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-white.svg
new file mode 100644
index 00000000..5c8116ff
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-white.svg
@@ -0,0 +1,4 @@
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-black.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-black.svg
new file mode 100644
index 00000000..2fa9be89
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-black.svg
@@ -0,0 +1,5 @@
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-white.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-white.svg
new file mode 100644
index 00000000..198f5f3e
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-white.svg
@@ -0,0 +1,5 @@
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-reverse.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-reverse.svg
new file mode 100644
index 00000000..902c6bf0
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-reverse.svg
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-small.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-small.svg
new file mode 100644
index 00000000..77e0a10a
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-small.svg
@@ -0,0 +1,5 @@
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark.svg b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark.svg
new file mode 100644
index 00000000..67a8da85
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark.svg
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/appTheme.v2.ts b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/appTheme.v2.ts
new file mode 100644
index 00000000..599b20a0
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/appTheme.v2.ts
@@ -0,0 +1,39 @@
+// HugAgentOS Ant Design tokens v2 — reference for src/frontend/src/appTheme.ts.
+// Differences from the current file: hover/active darken instead of lighten; dark primary stays #126DFF;
+// semantic text colours and placeholder meet 4.5:1.
+import type { ThemeConfig } from 'antd';
+import { theme as antdTheme } from 'antd';
+
+const shared = {
+ borderRadius: 10, borderRadiusLG: 16, borderRadiusSM: 6,
+ fontFamily: 'var(--font-family)', fontSize: 14, fontSizeSM: 12, fontSizeLG: 16,
+ colorWarning: '#F8AB42',
+};
+
+export const LIGHT_V2: ThemeConfig = {
+ algorithm: antdTheme.defaultAlgorithm,
+ token: {
+ ...shared,
+ colorPrimary: '#126DFF', colorPrimaryHover: '#0F62E6', colorPrimaryActive: '#0A56CC',
+ colorLink: '#0F62E6', colorInfo: '#126DFF',
+ colorSuccess: '#02B589', colorError: '#D92D20',
+ colorSuccessText: '#067A5E', colorWarningText: '#B45309', colorErrorText: '#D92D20',
+ colorText: '#101828', colorTextSecondary: '#475467', colorTextTertiary: '#667085', colorTextPlaceholder: '#667085',
+ colorBorder: '#C5CCD8', colorBorderSecondary: '#E3E8EF',
+ colorBgContainer: '#FFFFFF', colorBgLayout: '#F5F7FB', colorBgElevated: '#FFFFFF',
+ },
+};
+
+export const DARK_V2: ThemeConfig = {
+ algorithm: antdTheme.darkAlgorithm,
+ token: {
+ ...shared,
+ colorPrimary: '#126DFF', colorPrimaryHover: '#0F62E6', colorPrimaryActive: '#0A56CC',
+ colorLink: '#5C9BFF', colorInfo: '#126DFF',
+ colorSuccess: '#22C79D', colorError: '#FF6B6B',
+ colorSuccessText: '#3DDBB0', colorWarningText: '#FFC26B', colorErrorText: '#FF8A80',
+ colorText: '#E8ECF4', colorTextSecondary: '#B3BDCD', colorTextTertiary: '#8792A4', colorTextPlaceholder: '#8792A4',
+ colorBorder: '#3A4556', colorBorderSecondary: '#2B3442',
+ colorBgContainer: '#161C25', colorBgLayout: '#12171F', colorBgElevated: '#1C2330', colorBgSpotlight: '#1C2330',
+ },
+};
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.css b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.css
new file mode 100644
index 00000000..a74c2600
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.css
@@ -0,0 +1,123 @@
+/* HugAgentOS design tokens v2 — generated with the design guidelines (2026-10).
+ * Additive to src/frontend/src/styles/variables.css: new names only, existing names keep their values
+ * except where the guidelines' migration list says otherwise. */
+:root{
+ --color-primary:#126DFF;
+ --color-on-primary:#FFFFFF;
+ --color-primary-text:#0F62E6;
+ --color-primary-bg:#DBE9FF;
+ --color-primary-light:#EBF2FF;
+ --state-hover:rgba(16,24,40,.04);
+ --state-press:rgba(16,24,40,.08);
+ --state-on-fill-hover:rgba(0,0,0,.08);
+ --state-on-fill-press:rgba(0,0,0,.16);
+ --color-success:#02B589;
+ --color-success-text:#067A5E;
+ --color-success-bg:#E6FFF9;
+ --color-warning:#F8AB42;
+ --color-warning-text:#B45309;
+ --color-warning-bg:#FFF5E8;
+ --color-error:#FC5D5D;
+ --color-error-text:#D92D20;
+ --color-error-bg:#FFF0F0;
+ --color-danger-fill:#D92D20;
+ --color-text:#101828;
+ --color-text-secondary:#475467;
+ --color-text-tertiary:#667085;
+ --color-text-placeholder:#667085;
+ --color-separator:#E3E8EF;
+ --color-control-border:#C5CCD8;
+ --color-fill-quiet:#F2F4F7;
+ --color-bg-base:#FFFFFF;
+ --color-bg-container:#FFFFFF;
+ --color-bg-elevated:#FFFFFF;
+ --color-bg-layout:#F5F7FB;
+ --color-bg-chat:#FDFDFC;
+ --color-brand-spark:#19D3A7;
+ --material:rgba(255,255,255,.82);
+ --scrim:rgba(16,24,40,.32);
+ --color-tint-blue-bg:#DBEAFE;
+ --color-tint-blue:#1E40AF;
+ --color-tint-purple-bg:#EDE9FE;
+ --color-tint-purple:#6D28D9;
+ --color-tint-teal-bg:#CCFBF1;
+ --color-tint-teal:#0F766E;
+ --color-tint-amber-bg:#FEF3C7;
+ --color-tint-amber:#B45309;
+ --shadow-raised:0 1px 2px rgba(16,24,40,.05), 0 8px 24px rgba(16,24,40,.06);
+ --shadow-floating:0 2px 8px rgba(16,24,40,.06), 0 20px 48px rgba(16,24,40,.14);
+ --space-1:4px;
+ --space-2:8px;
+ --space-3:12px;
+ --space-4:16px;
+ --space-6:24px;
+ --space-8:32px;
+ --space-12:48px;
+ --space-16:64px;
+ --radius-xs:6px;
+ --radius-sm:10px;
+ --radius-md:16px;
+ --radius-lg:24px;
+ --radius-full:999px;
+ --font-size-display:2.5rem;--line-height-display:3rem;--font-weight-display:600;--letter-spacing-display:-0.02em;
+ --font-size-title-1:1.75rem;--line-height-title-1:2.25rem;--font-weight-title-1:600;--letter-spacing-title-1:-0.01em;
+ --font-size-title-2:1.375rem;--line-height-title-2:1.875rem;--font-weight-title-2:600;--letter-spacing-title-2:0;
+ --font-size-title-3:1.125rem;--line-height-title-3:1.625rem;--font-weight-title-3:600;--letter-spacing-title-3:0;
+ --font-size-headline:0.9375rem;--line-height-headline:1.375rem;--font-weight-headline:600;--letter-spacing-headline:0;
+ --font-size-body-lg:1rem;--line-height-body-lg:1.75rem;--font-weight-body-lg:400;--letter-spacing-body-lg:0;
+ --font-size-body:0.875rem;--line-height-body:1.375rem;--font-weight-body:400;--letter-spacing-body:0;
+ --font-size-footnote:0.8125rem;--line-height-footnote:1.25rem;--font-weight-footnote:400;--letter-spacing-footnote:0;
+ --font-size-caption:0.75rem;--line-height-caption:1.125rem;--font-weight-caption:400;--letter-spacing-caption:0;
+ --motion-duration-instant:100ms;
+ --motion-duration-fast:160ms;
+ --motion-duration-normal:240ms;
+ --motion-duration-slow:320ms;
+ --motion-ease-brand-out:cubic-bezier(0.16,1,0.3,1);
+ --motion-ease-standard:cubic-bezier(0.4,0,0.2,1);
+ --motion-ease-exit:cubic-bezier(0.4,0,1,1);
+ --motion-stagger-step:60ms;
+}
+
+:root[data-theme="dark"]{
+ --color-primary:#126DFF;
+ --color-primary-text:#5C9BFF;
+ --color-primary-bg:rgba(62,139,255,.20);
+ --color-primary-light:rgba(62,139,255,.12);
+ --state-hover:rgba(255,255,255,.06);
+ --state-press:rgba(255,255,255,.10);
+ --state-on-fill-hover:rgba(0,0,0,.12);
+ --state-on-fill-press:rgba(0,0,0,.22);
+ --color-success:#22C79D;
+ --color-success-text:#3DDBB0;
+ --color-success-bg:rgba(34,199,157,.12);
+ --color-warning-text:#FFC26B;
+ --color-warning-bg:rgba(248,171,66,.12);
+ --color-error:#FF6B6B;
+ --color-error-text:#FF8A80;
+ --color-error-bg:rgba(255,107,107,.12);
+ --color-text:#E8ECF4;
+ --color-text-secondary:#B3BDCD;
+ --color-text-tertiary:#8792A4;
+ --color-text-placeholder:#8792A4;
+ --color-separator:#2B3442;
+ --color-control-border:#3A4556;
+ --color-fill-quiet:#1F2631;
+ --color-bg-base:#0F141B;
+ --color-bg-container:#161C25;
+ --color-bg-elevated:#1C2330;
+ --color-bg-layout:#12171F;
+ --color-bg-chat:#111720;
+ --material:rgba(28,35,48,.80);
+ --scrim:rgba(0,0,0,.56);
+ --color-tint-blue-bg:rgba(62,139,255,.20);
+ --color-tint-blue:#8FB9FF;
+ --color-tint-purple-bg:rgba(149,120,255,.18);
+ --color-tint-purple:#B9A5FF;
+ --color-tint-teal-bg:rgba(45,212,191,.18);
+ --color-tint-teal:#5EE7D5;
+ --color-tint-amber-bg:rgba(248,171,66,.18);
+ --color-tint-amber:#FFC97A;
+ --shadow-raised:0 1px 2px rgba(0,0,0,.40), 0 8px 24px rgba(0,0,0,.32);
+ --shadow-floating:0 2px 8px rgba(0,0,0,.45), 0 20px 48px rgba(0,0,0,.55);
+ color-scheme:dark;
+}
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.json b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.json
new file mode 100644
index 00000000..d925c2e3
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.json
@@ -0,0 +1,223 @@
+{
+ "name": "HugAgentOS",
+ "version": "2.0.0",
+ "color": {
+ "light": {
+ "color-primary": "#126DFF",
+ "color-on-primary": "#FFFFFF",
+ "color-primary-text": "#0F62E6",
+ "color-primary-bg": "#DBE9FF",
+ "color-primary-light": "#EBF2FF",
+ "state-hover": "rgba(16,24,40,.04)",
+ "state-press": "rgba(16,24,40,.08)",
+ "state-on-fill-hover": "rgba(0,0,0,.08)",
+ "state-on-fill-press": "rgba(0,0,0,.16)",
+ "color-success": "#02B589",
+ "color-success-text": "#067A5E",
+ "color-success-bg": "#E6FFF9",
+ "color-warning": "#F8AB42",
+ "color-warning-text": "#B45309",
+ "color-warning-bg": "#FFF5E8",
+ "color-error": "#FC5D5D",
+ "color-error-text": "#D92D20",
+ "color-error-bg": "#FFF0F0",
+ "color-danger-fill": "#D92D20",
+ "color-text": "#101828",
+ "color-text-secondary": "#475467",
+ "color-text-tertiary": "#667085",
+ "color-text-placeholder": "#667085",
+ "color-separator": "#E3E8EF",
+ "color-control-border": "#C5CCD8",
+ "color-fill-quiet": "#F2F4F7",
+ "color-bg-base": "#FFFFFF",
+ "color-bg-container": "#FFFFFF",
+ "color-bg-elevated": "#FFFFFF",
+ "color-bg-layout": "#F5F7FB",
+ "color-bg-chat": "#FDFDFC",
+ "color-brand-spark": "#19D3A7",
+ "material": "rgba(255,255,255,.82)",
+ "scrim": "rgba(16,24,40,.32)",
+ "color-tint-blue-bg": "#DBEAFE",
+ "color-tint-blue": "#1E40AF",
+ "color-tint-purple-bg": "#EDE9FE",
+ "color-tint-purple": "#6D28D9",
+ "color-tint-teal-bg": "#CCFBF1",
+ "color-tint-teal": "#0F766E",
+ "color-tint-amber-bg": "#FEF3C7",
+ "color-tint-amber": "#B45309",
+ "shadow-raised": "0 1px 2px rgba(16,24,40,.05), 0 8px 24px rgba(16,24,40,.06)",
+ "shadow-floating": "0 2px 8px rgba(16,24,40,.06), 0 20px 48px rgba(16,24,40,.14)"
+ },
+ "dark": {
+ "color-primary": "#126DFF",
+ "color-on-primary": "#FFFFFF",
+ "color-primary-text": "#5C9BFF",
+ "color-primary-bg": "rgba(62,139,255,.20)",
+ "color-primary-light": "rgba(62,139,255,.12)",
+ "state-hover": "rgba(255,255,255,.06)",
+ "state-press": "rgba(255,255,255,.10)",
+ "state-on-fill-hover": "rgba(0,0,0,.12)",
+ "state-on-fill-press": "rgba(0,0,0,.22)",
+ "color-success": "#22C79D",
+ "color-success-text": "#3DDBB0",
+ "color-success-bg": "rgba(34,199,157,.12)",
+ "color-warning": "#F8AB42",
+ "color-warning-text": "#FFC26B",
+ "color-warning-bg": "rgba(248,171,66,.12)",
+ "color-error": "#FF6B6B",
+ "color-error-text": "#FF8A80",
+ "color-error-bg": "rgba(255,107,107,.12)",
+ "color-danger-fill": "#D92D20",
+ "color-text": "#E8ECF4",
+ "color-text-secondary": "#B3BDCD",
+ "color-text-tertiary": "#8792A4",
+ "color-text-placeholder": "#8792A4",
+ "color-separator": "#2B3442",
+ "color-control-border": "#3A4556",
+ "color-fill-quiet": "#1F2631",
+ "color-bg-base": "#0F141B",
+ "color-bg-container": "#161C25",
+ "color-bg-elevated": "#1C2330",
+ "color-bg-layout": "#12171F",
+ "color-bg-chat": "#111720",
+ "color-brand-spark": "#19D3A7",
+ "material": "rgba(28,35,48,.80)",
+ "scrim": "rgba(0,0,0,.56)",
+ "color-tint-blue-bg": "rgba(62,139,255,.20)",
+ "color-tint-blue": "#8FB9FF",
+ "color-tint-purple-bg": "rgba(149,120,255,.18)",
+ "color-tint-purple": "#B9A5FF",
+ "color-tint-teal-bg": "rgba(45,212,191,.18)",
+ "color-tint-teal": "#5EE7D5",
+ "color-tint-amber-bg": "rgba(248,171,66,.18)",
+ "color-tint-amber": "#FFC97A",
+ "shadow-raised": "0 1px 2px rgba(0,0,0,.40), 0 8px 24px rgba(0,0,0,.32)",
+ "shadow-floating": "0 2px 8px rgba(0,0,0,.45), 0 20px 48px rgba(0,0,0,.55)"
+ }
+ },
+ "space": {
+ "space-1": 4,
+ "space-2": 8,
+ "space-3": 12,
+ "space-4": 16,
+ "space-6": 24,
+ "space-8": 32,
+ "space-12": 48,
+ "space-16": 64
+ },
+ "radius": {
+ "radius-xs": 6,
+ "radius-sm": 10,
+ "radius-md": 16,
+ "radius-lg": 24,
+ "radius-full": 999
+ },
+ "type": {
+ "display": {
+ "fontSize": 40,
+ "lineHeight": 48,
+ "fontWeight": 600,
+ "letterSpacing": "-0.02em"
+ },
+ "title-1": {
+ "fontSize": 28,
+ "lineHeight": 36,
+ "fontWeight": 600,
+ "letterSpacing": "-0.01em"
+ },
+ "title-2": {
+ "fontSize": 22,
+ "lineHeight": 30,
+ "fontWeight": 600,
+ "letterSpacing": "0"
+ },
+ "title-3": {
+ "fontSize": 18,
+ "lineHeight": 26,
+ "fontWeight": 600,
+ "letterSpacing": "0"
+ },
+ "headline": {
+ "fontSize": 15,
+ "lineHeight": 22,
+ "fontWeight": 600,
+ "letterSpacing": "0"
+ },
+ "body-lg": {
+ "fontSize": 16,
+ "lineHeight": 28,
+ "fontWeight": 400,
+ "letterSpacing": "0"
+ },
+ "body": {
+ "fontSize": 14,
+ "lineHeight": 22,
+ "fontWeight": 400,
+ "letterSpacing": "0"
+ },
+ "footnote": {
+ "fontSize": 13,
+ "lineHeight": 20,
+ "fontWeight": 400,
+ "letterSpacing": "0"
+ },
+ "caption": {
+ "fontSize": 12,
+ "lineHeight": 18,
+ "fontWeight": 400,
+ "letterSpacing": "0"
+ }
+ },
+ "font": {
+ "ui": "PingFang SC / Microsoft YaHei UI / Noto Sans CJK SC (system)",
+ "number": "DingTalk Sans",
+ "mono": "ui-monospace, SF Mono, Menlo, Consolas",
+ "marketing": "Source Han Sans SC / Noto Sans SC"
+ },
+ "motion": {
+ "duration": {
+ "instant": 100,
+ "fast": 160,
+ "normal": 240,
+ "slow": 320
+ },
+ "easing": {
+ "brand-out": [
+ 0.16,
+ 1,
+ 0.3,
+ 1
+ ],
+ "standard": [
+ 0.4,
+ 0,
+ 0.2,
+ 1
+ ],
+ "exit": [
+ 0.4,
+ 0,
+ 1,
+ 1
+ ]
+ },
+ "spring": {
+ "smooth": {
+ "visualDuration": 0.35,
+ "bounce": 0,
+ "stiffness": 322,
+ "damping": 36
+ },
+ "snappy": {
+ "visualDuration": 0.25,
+ "bounce": 0.15,
+ "stiffness": 632,
+ "damping": 43
+ }
+ },
+ "staggerStep": 60,
+ "staggerCap": 8,
+ "layoutAnimMaxItems": 20,
+ "freshEnterWindowMs": 2000
+ }
+}
diff --git a/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/motionTokens.v2.ts b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/motionTokens.v2.ts
new file mode 100644
index 00000000..da835955
--- /dev/null
+++ b/.claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/motionTokens.v2.ts
@@ -0,0 +1,32 @@
+// HugAgentOS motion tokens v2 — drop-in superset of src/frontend/src/utils/motionTokens.ts.
+// New: SPRING.smooth / SPRING.snappy described as visualDuration + bounce (Motion >= 11 accepts both forms).
+// Durations / easings are unchanged; keep them in sync with styles/variables.css (--motion-*).
+
+export const DUR = { instant: 0.1, fast: 0.16, normal: 0.24, slow: 0.32 } as const;
+
+export const EASE = {
+ /** Enter / expand / rise */
+ brandOut: [0.16, 1, 0.3, 1] as const,
+ /** Two-way state changes (colour, opacity) */
+ standard: [0.4, 0, 0.2, 1] as const,
+ /** Exit: accelerate away, ~0.7x the enter duration */
+ exit: [0.4, 0, 1, 1] as const,
+};
+
+export const SPRING = {
+ /** Panels, drawers, collapse/expand, layout changes. No overshoot. (≈ stiffness 322, damping 36) */
+ smooth: { type: 'spring', visualDuration: 0.35, bounce: 0 },
+ /** Tab ink, toggles, badge confirmation. Slight overshoot. (≈ stiffness 632, damping 43) */
+ snappy: { type: 'spring', visualDuration: 0.25, bounce: 0.15 },
+ /** @deprecated use SPRING.snappy — 33% overshoot */
+ pop: { type: 'spring', stiffness: 500, damping: 30 },
+ /** @deprecated use SPRING.snappy */
+ ink: { type: 'spring', stiffness: 480, damping: 38 },
+ /** @deprecated use SPRING.smooth — 33% overshoot makes panels wobble */
+ panel: { type: 'spring', stiffness: 320, damping: 24 },
+} as const;
+
+export const STAGGER_STEP = 0.06;
+export const STAGGER_CAP = 8;
+export const LAYOUT_ANIM_MAX_ITEMS = 20;
+export const FRESH_ENTER_WINDOW_MS = 2000;
diff --git a/.claude/skills/hugagent-frontend-dev/references/css-variables.md b/.claude/skills/hugagent-frontend-dev/references/css-variables.md
index afe227db..4fa848a1 100644
--- a/.claude/skills/hugagent-frontend-dev/references/css-variables.md
+++ b/.claude/skills/hugagent-frontend-dev/references/css-variables.md
@@ -1,153 +1,174 @@
-# CSS 变量与样式规范参考
+# CSS 变量、命名与样式模式
-> **重要**: 颜色、圆角、间距等视觉规范以 `references/ui-design-spec.md` 为权威来源。本文件侧重 CSS 实践模式。
+真源:`src/frontend/src/styles/variables.css`(`:root` 浅色 + 文件末尾 `:root[data-theme="dark"]` 深色覆盖 +
+`:root[data-font-platform=…]` 字体优先级)。TS 镜像:`utils/motionTokens.ts`(动效)、`appTheme.ts`(antd token)。
+三处同步改。设计规则见 `ui-design-spec.md`(v2)。
-## 全局变量 (variables.css)
+---
-以下变量需与 UI 设计规范对齐(参见 `ui-design-spec.md` 第 12 节的完整映射)。
+## 1. v1 → v2 令牌对照(写新代码先查这张表)
-```css
-:root {
- /* 主色 */
- --color-primary: #126DFF;
- --color-primary-hover: #3C87FF;
- --color-primary-active: #0862F3;
- --color-primary-disabled: #A0C5FF;
- --color-primary-bg: #DBE9FF;
- --color-primary-light: #EBF2FF;
-
- /* 边框与填充 */
- --color-fill-heavy: #8C94A2;
- --color-fill-deep: #B3BAC8;
- --color-fill: #D8DBE2;
- --color-border: #E3E6EA;
- --color-fill-hover: #EBEDEE;
- --color-bg-gray: #F5F6F7;
-
- /* 文字 */
- --color-text: #101828;
- --color-text-secondary: #475467;
- --color-text-tertiary: #667085;
- --color-text-placeholder: #98A2B3;
- --color-text-white: #FFFFFF;
-
- /* 语义色 */
- --color-success: #02B589;
- --color-warning: #F8AB42;
- --color-error: #FC5D5D;
-
- /* 圆角 */
- --radius-xs: 6px; /* 标签等小面积 */
- --radius-sm: 10px; /* 按钮、输入框、下拉框 */
- --radius-md: 16px; /* 卡片视图、背景卡片 */
- --radius-lg: 24px; /* 弹窗、大输入框 */
-
- /* 间距 (4px 基准) */
- --space-xs: 4px;
- --space-sm: 8px;
- --space-md: 16px;
- --space-lg: 20px;
- --space-xl: 40px;
-
- /* 字体(后一条定义生效:系统栈优先,中文回退 PingFang/雅黑) */
- --font-family: -apple-system, BlinkMacSystemFont, "SF Pro Text", "Helvetica Neue",
- "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei UI", "Microsoft YaHei", sans-serif;
- --font-family-number: "DingTalk Sans", "PingFang SC", "Microsoft YaHei", sans-serif;
-
- /* Motion 令牌(TS 侧镜像在 utils/motionTokens.ts) */
- --motion-duration-instant: 100ms;
- --motion-duration-fast: 160ms;
- --motion-duration-normal: 240ms;
- --motion-duration-slow: 320ms;
- /* easing: --motion-ease-brand-out cubic-bezier(.16,1,.3,1) / --motion-ease-standard cubic-bezier(.4,0,.2,1) */
-}
-```
+v2 令牌**尚未全部合入** `variables.css`。写代码前先 `grep -n -- '--令牌名' src/frontend/src/styles/variables.css` 确认:
-> 阴影令牌:`--shadow-card` / `--shadow-card-hover` / `--shadow-dropdown`(双层 ambient + key-light)。
-> 底部还有 legacy 别名(`--primary` / `--bg` / `--text` / `--muted` / `--border` 等),新代码不要使用。
+- **已存在** → 直接用;
+- **不存在、而你的改动需要它** → 按迁移清单第 1 项把它加进 `variables.css`(浅色 `:root` 和深色覆盖块**两处都加**,
+ 值取自 `assets/design-v2/tokens/hugagentos-tokens.v2.css`),同步 `appTheme.ts`(如涉及 antd),然后再用。
+ **不要**写死色值、**不要**引用未定义的变量名、**不要**写 `var(--x, #fff)` 兜底(深色门禁会拦)。
-## CSS 类名前缀
+| 用途 | v1(现行代码) | v2(目标) | 说明 |
+|---|---|---|---|
+| 强调填充 | `--color-primary` #126DFF(深色 #3E8BFF) | `--color-primary`(深色也 #126DFF) | 深色值待迁移 #6 |
+| 强调文字 / 链接 | `--color-primary` | `--color-primary-text` #0F62E6 / 深 #5C9BFF | 新增 |
+| 主色上的文字 | `--color-text-white` | `--color-on-primary` | 新增 |
+| 悬停 / 按下 | `--color-*-hover` / `--color-*-active` 色阶 | `--state-hover` / `--state-press` / `--state-on-fill-hover` / `--state-on-fill-press` 叠加层 | **新代码禁用色阶** |
+| 禁用 | `--color-*-disabled` | `opacity: .4` | |
+| 成功/警告/错误 文字 | `--color-success` 等直接当文字 | `--color-success-text` / `--color-warning-text` / `--color-error-text` | P0 对比度 |
+| 成功/警告/错误 浅底 | `--color-*-light`(v1 的 `*-bg` 是更深一档) | `--color-success-bg` #E6FFF9 等 | **注意:v2 的 `*-bg` 取 v1 `*-light` 的值** |
+| 破坏性实心按钮 | — | `--color-danger-fill` #D92D20 | 新增 |
+| 占位符 | `--color-text-placeholder` #98A2B3 | 同名,值改为 #667085(= 三级文字) | 迁移 #7 |
+| 分隔线 | `--color-border` #E3E8EF | `--color-separator` | 改名,值相同 |
+| 输入框边框 | `--color-border` / `--color-fill` | `--color-control-border` #C5CCD8 | 新增 |
+| 次要按钮 / 胶囊 / 用户气泡 | `--color-bg-gray` / `--color-fill-hover` | `--color-fill-quiet` #F2F4F7 | 新增 |
+| 浮层材质 | 各处自写半透明 | `--material` + `backdrop-filter: blur(20px)` | 新增 |
+| 模态遮罩 | antd 默认 | `--scrim` | 新增 |
+| 静止卡片投影 | `--shadow-card` | **无投影** | 迁移 #9 |
+| 悬停卡片 / 对话输入区 | `--shadow-card-hover` / `--shadow-card` | `--shadow-raised` | 调整 |
+| 菜单 / 弹窗 | `--shadow-dropdown` | `--shadow-floating` | 调整 |
+| 分类色 | tint × 7(purple/cyan/teal/amber/green/red/blue) | tint × 4(blue/purple/teal/amber) | cyan/green/red 不新增引用 |
+| 胶囊圆角 | 写 `999px` | `--radius-full` | 新增 |
+| 间距 | `--space-xs/sm/md/lg/xl` = 4/8/16/**20**/**40** | `--space-1/2/3/4/6/8/12/16` = 4/8/12/16/24/32/48/64 | 20/40 不在栅格上 |
+| 字号 | `--font-size-xs/sm/md/lg/xl/xxl` = 12/14/16/18/22/44(px) | 九级角色:display / title-1/2/3 / headline / body-lg / body / footnote / caption(rem) | 见 ui-design-spec §2 |
+| 品牌火花绿 | — | `--color-brand-spark` | 仅标志,界面禁用 |
+| 弹簧 | `SPRING.pop / ink / panel` | `SPRING.smooth / snappy` | 迁移 #2 |
-所有自定义类使用 `.jx-` 前缀,避免与 Ant Design 冲突。
+在 v2 间距令牌合入前:4/8/16 用 `--space-xs/sm/md`,12/24/32/48/64 可直接写 px(门禁只管颜色),
+**不要**在新代码里用 `--space-lg`(20) / `--space-xl`(40)。
-## 命名规则 (BEM-like)
+---
-| 类型 | 格式 | 示例 |
-|------|------|------|
-| Block | `.jx-blockName` | `.jx-chatArea` |
-| Element | `.jx-blockName-element` | `.jx-chatArea-header` |
-| Modifier | `.jx-blockName--modifier` | `.jx-chatArea--empty` |
-| Sub-element | `.jx-blockName-element-sub` | `.jx-fileCard-icon-bg` |
+## 2. 现行令牌速查(variables.css 当前值)
-## 常用尺寸
+### 背景四级 + 对话底
-| 用途 | 值 |
-|------|-----|
-| 侧边栏宽度 | 260px (展开) / 0 (收起) |
-| 输入框最大宽度 | 840px |
-| 消息最大宽度 | 840px |
-| 面板内边距 | 20px (桌面) / 16px (移动) |
-| 卡片间距 | 8px - 16px |
-| 卡片圆角 | 16px (`--radius-md`) |
-| 输入框圆角 | 10px (`--radius-sm`) |
-| 按钮圆角 | 10px (`--radius-sm`) |
-| 弹窗圆角 | 24px (`--radius-lg`) |
-| 按钮最小高度 | 36px |
+| 令牌 | 浅色 | 深色 |
+|---|---|---|
+| `--color-bg-base` | #FFFFFF | #0F141B |
+| `--color-bg-container` | #FFFFFF | #161C25 |
+| `--color-bg-elevated` | #FFFFFF | #1C2330 |
+| `--color-bg-layout` | #F5F7FB | #12171F |
+| `--color-bg-chat` | #FDFDFC | #111720 |
-## 常用样式模式
+### 文字
-### 卡片
+`--color-text` #101828 · `--color-text-secondary` #475467 · `--color-text-tertiary` #667085 ·
+`--color-text-placeholder` #98A2B3(v2 改 #667085)· `--color-text-white` #FFFFFF
-```css
-.jx-card {
- padding: var(--space-md) var(--space-lg); /* 16px 20px */
- border-radius: var(--radius-md); /* 16px */
- border: 1px solid var(--color-border);
- background: #fff;
- transition: box-shadow 0.2s, border-color 0.2s;
-}
-.jx-card:hover {
- border-color: var(--color-primary-hover);
- box-shadow: 0 4px 12px rgba(18, 109, 255, 0.08);
-}
-```
+### 主色与语义
+
+- 主色:`--color-primary` #126DFF、`-hover` #3C87FF、`-active` #0862F3、`-disabled` #A0C5FF、`-bg` #DBE9FF、`-light` #EBF2FF
+- 语义:`--color-{success|warning|error}` + `-hover/-active/-disabled/-bg/-light`(v1 六档,新代码按上表换 v2)
+- 线与填充:`--color-border` #E3E8EF、`--color-fill` #DDE3EC、`--color-fill-hover` #EEF1F5、`--color-bg-gray` #F4F6F9、
+ `--color-fill-deep` #98A2B3、`--color-fill-heavy` #8C94A2
+- 分类 tint:`--color-tint-{purple,cyan,teal,amber,green,red,blue}-bg` + `--color-tint-{…}`(深浅两档已定义)
+
+### 圆角 / 间距 / 字号 / 字体
+
+- `--radius-xs/sm/md/lg` = 6 / 10 / 16 / 24
+- `--space-xs/sm/md/lg/xl` = 4 / 8 / 16 / 20 / 40
+- `--font-size-xs/sm/md/lg/xl/xxl` = 12 / 14 / 16 / 18 / 22 / 44 px
+- `--font-family`(`--font-family-cjk` 按 `data-font-platform` 切换 PingFang SC / Microsoft YaHei UI / Noto Sans CJK SC)、
+ `--font-family-mono`、`--font-family-number`(DingTalk Sans)
+
+### 投影
+
+`--shadow-card` / `--shadow-card-hover` / `--shadow-dropdown`(深浅两档)
+
+### 动效
+
+`--motion-duration-instant/fast/normal/slow` = 100 / 160 / 240 / 320ms ·
+`--motion-ease-brand-out` / `--motion-ease-standard` / `--motion-ease-exit` · `--motion-stagger-step` 60ms
-### 按钮
+### 局部域令牌(只在对应模块内用)
+
+- `--delivery-*`:交付文件卡片(文件类型图标色等)
+- `--color-personal-*` / `--shadow-personal-composer`:Personal Agent 对话皮肤
+- `--module-*`:模块导航轨与二级边栏
+- `--scrollbar-*`:滚动条
+- `--bg / --text / --muted / --border / --primary / --primary2 / --accent / --shadow1`:**旧别名,新代码禁用**
+
+---
+
+## 3. 命名规则
```css
-.jx-btn {
- display: inline-flex;
- align-items: center;
- gap: 6px;
- padding: 8px 18px;
- border-radius: var(--radius-sm); /* 10px */
- font-size: 14px;
- cursor: pointer;
- transition: all 0.2s;
- min-height: 36px;
-}
+.jx-chatArea { } /* Block:.jx- 前缀 + camelCase */
+.jx-chatArea-header { } /* Element:连字符 */
+.jx-chatArea--empty { } /* Modifier:双连字符 */
```
-### 输入区
+- 所有自定义类 `.jx-` 前缀;关键帧 `jx-kf-名称`,动效工具类 `jx-anim-名称`
+- 组件私有的自定义属性用 `--jx--`,**值也必须引令牌**(`css-var-literal-color` 规则)
+- 样式按功能放 `styles/` 对应文件或子目录(`styles/chat/`、`styles/catalog/`、`styles/myspace/`、`styles/automation/`、
+ `styles/mobile/`);`styles/index.ts` 统一 import,组件专属样式可在组件内直接 import
+
+---
+
+## 4. 常用样式模式(v2 写法)
```css
-.jx-inputArea {
- width: 100%;
- max-width: 840px;
- margin: 0 auto;
- border-radius: var(--radius-lg); /* 24px */
- border: 1.5px solid var(--color-border);
- background: #fff;
+/* L1 卡片:静止无投影,悬停浮起 */
+.jx-xxx-card {
+ background: var(--color-bg-container);
+ border-radius: var(--radius-md);
+ padding: var(--space-md);
+ transition: transform var(--motion-duration-fast) var(--motion-ease-standard),
+ box-shadow var(--motion-duration-fast) var(--motion-ease-standard);
+}
+.jx-xxx-card:hover { transform: translateY(-1px); box-shadow: var(--shadow-raised); }
+
+/* 中性控件悬停:叠加层而非换色 */
+.jx-xxx-item:hover { background: var(--state-hover); }
+.jx-xxx-item:active { background: var(--state-press); }
+
+/* L2 浮层:材质 + 模糊,从触发点展开 */
+.jx-xxx-menu {
+ background: var(--material);
+ backdrop-filter: blur(20px);
+ border-radius: var(--radius-md); /* 外 16 */
+ padding: 6px; /* 内边距 6 */
+ box-shadow: var(--shadow-floating);
+ transform-origin: var(--jx-menu-origin, top left);
}
+.jx-xxx-menu-item { border-radius: var(--radius-sm); } /* 内 10 = 16 − 6,同心 */
+
+/* 语义提示条:浅底 + 前景文字成对 */
+.jx-xxx-hint--warning { background: var(--color-warning-bg); color: var(--color-warning-text); }
+
+/* 分类标签:tint 成对 */
+.jx-xxx-tag--skill { background: var(--color-tint-purple-bg); color: var(--color-tint-purple); border-radius: var(--radius-full); }
+
+/* 半透明面 / 墨色浮层(深色下自动正确) */
+.jx-xxx-glass { background: color-mix(in srgb, var(--color-bg-container) 60%, transparent); }
+.jx-xxx-ink { background: color-mix(in srgb, var(--color-text) 5%, transparent); }
+
+/* 数字列 */
+.jx-xxx-num { font-family: var(--font-family-number); font-variant-numeric: tabular-nums; }
+
+/* 小图标按钮:视觉 28,点击区外扩到 32 */
+.jx-xxx-iconBtn { position: relative; width: 28px; height: 28px; border-radius: var(--radius-full); }
+.jx-xxx-iconBtn::after { content: ''; position: absolute; inset: -2px; }
```
-## 文件索引
+---
+
+## 5. 布局约定(模块页面与容器响应式)
-| 文件 | 内容 |
-|------|------|
-| `variables.css` | CSS 变量 + body 背景 + AntD 覆写 |
-| `chat.css` | 聊天区域、输入框、消息气泡、空状态 |
-| `sidebar.css` | 侧边栏、导航、历史列表 |
-| `catalog.css` | Catalog 面板、搜索、列表项 |
-| `tool.css` | 工具输出面板、展开/折叠 |
-| `common.css` | 全局工具类、弹窗覆写 |
+- **模块主页面**(项目、我的空间、站点、应用中心、能力中心、定时任务、设置、更新记录、实验室):公共内容区是全宽滚动视口,
+ 几何集中在 `styles/workspace-layout.css`。模块面板用正常文档流,**不要**再给面板自己设 `height:100%` + `overflow:auto`
+ 造第二个滚动容器,也不要用 `overflow-x:hidden`(会隐式把 `overflow-y` 变成 auto)。限宽只给模块内容。
+ 回归:`npm run test:module-pages`。
+- **容器响应式**:聊天工具栏、项目 / 能力目录 / 知识库卡片按所在面板的可用宽度排版——容器提供宽度,后代用容器查询响应;
+ 不要让元素查询自身,不要用 JS 重复维护 CSS 断点。入口 `components/chat/ComposerToolbar.tsx`、
+ `styles/chat/toolbar-layout.css`、`styles/projects.css`、`styles/catalog/`。
+ 回归:`npm run test:container-layout-browser`、`npm run test:chat-width-browser`。
+- 聊天、Canvas、插件工作区和真正的编辑器 / 限高预览保留专用布局。
diff --git a/.claude/skills/hugagent-frontend-dev/references/icon-assets.md b/.claude/skills/hugagent-frontend-dev/references/icon-assets.md
index c781434b..f0ae7e10 100644
--- a/.claude/skills/hugagent-frontend-dev/references/icon-assets.md
+++ b/.claude/skills/hugagent-frontend-dev/references/icon-assets.md
@@ -1,7 +1,38 @@
# 图标资产清单
-本技能内置了项目 UI 设计稿中的全部 SVG 图标,位于 `assets/` 目录。
-当需要在前端使用图标时,优先从此清单中选取,保持视觉一致性。
+本技能带两批图标:
+
+- `assets/design-v2/`:**v2 设计规范**的界面图标风格样例、标志与网页图标(新代码以此为风格基准)
+- `assets/icons/`、`assets/mcp-icons/`:v1 设计稿导出的图标(存量界面在用,部分带硬编码色值,复用前先改成 `currentColor`)
+
+## v2 资产(`assets/design-v2/`)
+
+| 目录 | 内容 | 规则 |
+|---|---|---|
+| `icons-ui/` | 12 个界面图标:new-chat / search / knowledge / skill / automation / my-space / attachment / settings / send / chevron-right / check / close | 24px 画板、留白 2px、线宽 1.5、圆头圆角、`stroke="currentColor"` |
+| `logo/hugagentos-mark*.svg` | 方形标:标准 / small(≤24px 用)/ reverse(深色底)/ mono-black / mono-white | 只用于方形容器;禁止与标准字并排 |
+| `logo/hugagentos-glyph-*.svg` | 环线(无底板):蓝 / 白 / 黑 | 头像、App 图标内自行合成 |
+| `logo/favicon.svg` | 网页图标(小尺寸版) | 16/32px 使用 |
+| `tokens/` | v2 令牌参考实现(CSS / JSON / motionTokens / appTheme) | 见 `css-variables.md` |
+
+图标规则(v2,详见 `ui-design-spec.md` §6):图标是文字的一部分——尺寸跟随字号、线宽跟随字重(13px→1.5,15/20px→2)、
+颜色继承文字色;默认线性,选中态换面性。**不要给图标单独上品牌色,火花绿 `#19D3A7` 不进界面。**
+
+```tsx
+// v2 写法:currentColor + 跟随字号
+export function SearchIcon({ size = '1em' }: { size?: string | number }) {
+ return (
+
+
+
+ );
+}
+```
+
+---
+
+## v1 图标
## 使用方式
@@ -89,10 +120,9 @@ import sendIcon from '../assets/icons/send.svg';
---
-## 设计约束
+## 设计约束(v2 替代 v1 的约定)
-- **主色**: 图标主色为 `#126DFF`(蓝色),与设计规范一致
-- **灰色**: 非激活态图标用 `#808080`
-- **尺寸**: 常用 16px / 28px / 32px / 34px / 38px(均为 4 的倍数或接近)
-- **风格**: 线性为主,部分填充,圆角端点
-- 详见 `ui-design-spec.md` 第 3 节「图标」规范
+- v1 约定"图标主色 `#126DFF`、非激活 `#808080`"**已废止**:v2 图标继承文字色,激活态用面性而不是换色
+- v1 SVG 里的硬编码 `fill` / `stroke` 色值在深色模式下不会反相;复用前改成 `currentColor`,否则过不了视觉走查
+- 尺寸跟随字号(13 / 15 / 20px 常用),点击区按桌面 ≥32、触屏 ≥44 外扩
+- 详见 `ui-design-spec.md` §6「图标」
diff --git a/.claude/skills/hugagent-frontend-dev/references/sse-events.md b/.claude/skills/hugagent-frontend-dev/references/sse-events.md
index 8764623b..4b31de68 100644
--- a/.claude/skills/hugagent-frontend-dev/references/sse-events.md
+++ b/.claude/skills/hugagent-frontend-dev/references/sse-events.md
@@ -1,246 +1,126 @@
# SSE 事件类型参考
-统一回复端点 `POST /v1/agents/responses` 在请求体 `stream: true` 时返回 Server-Sent Events(默认 false 返回完整 JSON)。后端把 run 跑在后台
-(ChatRun + Redis Stream),SSE 只是"跟随"——断线后可用 `GET /v1/chats/stream/{run_id}`
-重新接上同一条流续播。
+统一回复端点 `POST /v1/agents/responses`(`stream: true` 返回 SSE,默认返回完整 JSON)。后端把每一轮跑成后台 run
+(`core/runs` 持有租约、事件日志与续播;`core/chat/runtime/worker.py` 消费事件并持久化输出),SSE 只是"跟随"。
-**前端解析单一真源是 `hooks/chatStream.ts`(`processChatStream()`)**;`hooks/useStreaming.ts`
-只做发送/中止/续播/排队编排;通用分帧器在 `utils/sse.ts`(`parseSSE`)。后端事件产出主体在
-`orchestration/streaming.py`(StreamingAgent)与 `orchestration/chat_run_executor.py`。
+| 端 | 位置 |
+|---|---|
+| 前端解析单一真源 | `hooks/chatStream.ts` → `hooks/chatStream/`(`consume.ts` 入口;`payload.ts` 分帧与分派;`lifecycleAndTools.ts` / `toolResultsAndMeta.ts` / `controls.ts` 分领域 reducer;`snapshot*.ts` 快照恢复) |
+| 前端发送 / 续播编排 | `hooks/useStreaming.ts` + `hooks/streamSend.ts` / `streamFollowing.ts` / `streamRecovery.ts` / `streamQueue.ts` |
+| 通用分帧器 | `utils/sse.ts`(`parseSSE`) |
+| 订阅客户端 | `runSubscriptionClient.ts`(`GET /v1/chats/stream/{run_id}/subscription`) |
+| 后端事件产出 | `core/agent_runtime/stream_main.py` / `stream_direct.py`、`core/llm/runtime/stream_events.py`;SSE 输出 `api/transports/run_sse.py` |
-## 流格式
-
-```
-data: {"type": "", ...payload} # JSON 事件
-data: [DONE] # 流终止标记
-: heartbeat # SSE 注释行心跳(直接忽略)
-```
-
-> 没有独立的 `done` 事件——结束是 `end` 事件或 `[DONE]`。正文事件规范名是 `content`,
-> 但 `ai_message` / `text` / `delta` 都会被当作正文别名接受。
-
-## 事件类型
-
-### `run_started` — 首帧
-
-```json
-{"type": "run_started", "run_id": "run_xxx", "message_id": "msg_xxx"}
-```
-
-- 必须记录到 `useChatStore.setActiveRun(chatId, { runId, messageId })`
-- `run_id` 是断线续播的凭据
-
-### `content` — 文本增量
-
-```json
-{"type": "content", "delta": "你好"}
-```
-
-- 取 `delta || content || text` 追加到当前消息内容
-- 兼容别名:`ai_message` / `text` / `delta`
-- 内嵌 `... ` 标签的模型由 parseBuffer 解析为 thinking(structuredReasoning=false 时)
-
-### `thinking` — 思考过程
-
-```json
-{"type": "thinking", "delta": "让我分析一下..."}
-```
-
-- 取 `content || text || delta`;有 `delta` 字段时按增量追加到最近的 thinking segment
-- 收到带 `delta` 的 thinking 即标记 structuredReasoning(思考走独立通道,如 reasoning_content),关闭 `` 内嵌解析
-- 兼容别名:`thought`
-
-### `content_replace` — 整段替换正文
-
-```json
-{"type": "content_replace", "content": "重写后的完整正文"}
-```
-
-- 本体治理评审后重写答案时使用:**整体替换**当前消息正文,而不是追加
-
-### `tool_call_start` / `tool_call_delta` — 工具参数流式展示
-
-```json
-{"type": "tool_call_start", "id": "call_abc123", "name": "internet_search"}
-{"type": "tool_call_delta", "id": "call_abc123", "delta": "{\"query\": \"北京"}
-```
-
-- 参数尚未生成完时先出卡片(status='running'),delta 增量拼接参数 JSON 实时展示
-- 随后的 `tool_call` 带完整参数,按 id 更新同一张卡片
-
-### `tool_call` — 工具调用(完整参数)
-
-```json
-{"type": "tool_call", "id": "call_abc123", "name": "internet_search", "input": {"query": "北京 天气"}}
-```
-
-- 参数字段取 `input ?? args ?? tool_args ?? arguments`
-- 按工具 id 去重:已存在则更新(status='running'),否则 push 到 `toolCalls[]` 并追加 `{ type: 'tool', toolIndex }` segment
-- 兼容别名:`tool_use` / `tool_start`
-
-### `tool_result` — 工具执行结果
-
-```json
-{
- "type": "tool_result",
- "id": "call_abc123",
- "output": "搜索结果内容...",
- "citations": [{"title": "...", "url": "...", "source": "internet_search"}]
-}
-```
+**新增事件处理只写在 `hooks/chatStream/` 对应的 reducer 里**,不要在别处再解析 SSE。
-- 结果字段取 `output ?? result`;`error` 字段存在 → status='error',否则 'success'
-- `citations` 追加到 `allCitations[]`
-- `subagent_name` 存在时改写 displayName 为「调用子智能体:xxx」
-- 兼容别名:`tool_end`
-
-### `tool_pending` — 工具等待批复
+## 流格式
-```json
-{"type": "tool_pending"}
```
-
-- UI 进入 pending 态;收到下一个非 heartbeat 事件自动解除
-
-### `batch_confirm` — 批量计划确认
-
-```json
-{"type": "batch_confirm", "plan_id": "...", "total": 20, "source_type": "xlsx",
- "preview": [...], "default_template": "...", "placeholder_keys": [...], "chat_id": "...", "warnings": [...]}
+data: {"type": "", "server_ts": 1730000000000, "event_offset": 42, ...payload}
+data: [DONE] # 流终止
+: heartbeat # 注释行心跳,忽略
```
-- batch_runner MCP 返回了执行计划,后端已暂停 agent
-- 调 `useBatchStore.setPendingConfirm(...)` 打开确认弹窗,用户审阅/编辑模板后才执行
+- `server_ts` 更新最后活动时间(停滞检测);`event_offset` 记录续播偏移。
+- 无 `type` 的裸文本帧当作正文追加。
+- 正文规范名 `content`,`ai_message` / `text` / `delta` 均作别名接受。
-### `file_confirm` — 我的空间写确认
+## 订阅帧(续播 / 刷新后)
-```json
-{"type": "file_confirm", "confirm_id": "...", ...}
-{"type": "file_confirm", "confirm_id": "...", "expired": true}
-```
+| 事件 | 说明 |
+|---|---|
+| `run_snapshot` | 订阅首帧,携带 `state`:对齐的原生快照(已产生的正文/思考/工具段),`installRunSnapshot()` 直接装入 |
+| `run_terminal` | run 已终态,结束跟随 |
-- 某工具协程已挂起等用户确认 /myspace 写。**本 SSE 流不结束**——用户点允许/拒绝走带外
- `POST /file-confirm`,挂起的工具原地续跑,后续 tool_result/meta 仍从同一条流来
-- 入队 `useUIStore.enqueuePendingConfirm(chatId, info)`;`expired: true` 表示该项超时回收,
- 调 `resolvePendingConfirm` 只摘掉这一个 confirm_id
+## 生命周期
-### `design_pick` — 建站方案三选一
+| 事件 | 载荷要点 | 处理 |
+|---|---|---|
+| `run_started` | `run_id`, `message_id` | 首帧,存 activeRun(续播凭据) |
+| `queued_run_started` | 排队消息交接信息 | 排队中的下一条消息开始执行(`parseQueuedRunHandoff`) |
+| `steer_applied` | 插话边界 | 运行中插话被采纳,切分正文边界 |
+| `meta` | `message_id`, `citations`, `workspace_files`, … | 设置 messageId;`citations` 非空时整体替换 |
+| `follow_up` | `follow_up_questions[]` | 追问建议;也可能流结束后后台生成写入消息 `extra_data` |
+| `error` | `error` | 终止本次流处理 |
+| `end` / `[DONE]` | — | 收尾所有 running 工具,标记流结束 |
-- 与 `file_confirm` 同机制:流不结束,用户选择走带外接口,工具协程原地续跑
+## 正文与思考
-### `plan_update` — 计划清单更新
+| 事件 | 处理 |
+|---|---|
+| `content`(别名 `ai_message` / `text` / `delta`) | 取 `delta || content || text` 追加;内嵌 `` 的模型由 parseBuffer 拆出思考 |
+| `thinking`(别名 `thought`) | 带 `delta` 时按增量追加到最近的 thinking 段,并标记 structuredReasoning |
+| `content_replace` | **整体替换**当前正文(本体治理重写答案) |
-```json
-{"type": "plan_update", "plan": [{"content": "...", "status": "completed"}, ...]}
-```
+## 工具
-- `update_plan` 工具产生的轻量计划清单;run 正常结束时后端自动补发全部 completed 的收尾帧
+| 事件 | 载荷要点 | 处理 |
+|---|---|---|
+| `tool_call_start`(兼容 `tool_use` / `tool_start`) | `id`, `name` | 先出运行中的工具行 |
+| `tool_call_delta` | `id`, `arguments_delta` | 参数 JSON 增量拼接实时展示 |
+| `tool_call` | `id`, `name`, `input`(兼容 `args` / `tool_args` / `arguments`) | 按 id 更新同一项 |
+| `tool_progress` | `id`, `progress`, `total`, `message` | 运行中工具的进度注记(`progressNote`,如"3/10") |
+| `tool_result`(兼容 `tool_end`) | `id`, `output`/`result`, `error`, `citations` | 置 success / error;`subagent_name` 存在时改写展示名 |
+| `tool_pending` | — | 工具等待批复,下一个非心跳事件自动解除 |
+| `subagent_event` | `subType`: tool_call / tool_call_delta / tool_result / thinking / content / error | 渲染为父工具的 `subSteps` |
+| `vision_progress` | `status`, `count` | 视觉桥在模型开口前读图,写 `chatStore.setVisionReading` |
-### `subagent_event` — 子智能体嵌套事件
+> **UI 规范(v2)**:工具调用默认是**一行安静的文字**(做了什么 · 结果多少 · 用了多久),点击展开;运行中用文字微光,
+> 完成用 `statusIn` 落定图标。只有文件、表格、网页等产物做成卡片。见 `ui-design-spec.md` §7.3 / §8.6。
-```json
-{"type": "subagent_event", "subType": "tool_call", ...}
-```
+## 需要用户介入(流不结束,带外批复后原地续跑)
-- `subType` 取值:`tool_call` / `tool_call_delta` / `tool_result` / `thinking` / `content` / `error`
-- 渲染为父工具卡片的 `subSteps` 子步骤
+| 事件 | 处理 |
+|---|---|
+| `file_confirm`(`confirm_id`;`expired: true` 表示超时回收) | 入队 `useUIStore.enqueuePendingConfirm`;用户允许/拒绝走带外接口 |
+| `design_pick` / `choose_design` 工具 | 建站三选一卡片,同 file_confirm 机制;超时由助手自行选择 |
+| `user_question`(`request_id`, `questions[]`) | `ask_user_question` 挂起 run,常驻输入区被问答面板替换;`enqueuePendingUserQuestion`,按 `request_id` 去重 |
+| `user_question_resolved`(`request_id`, `outcome`) | **后端拥有终态**:在这里摘除;`outcome === 'timeout'` 时提示采用默认方案 |
-### `steer_applied` — 运行中插话生效
+> v2:这些"等你确认"项是**唯一允许扩散涟漪**的状态。
-- 用户在 run 进行中追加的指令(steering)被后端采纳的边界标记
+## 上下文与压缩
-### `compaction_notice` — 上下文压缩提示
+| 事件 | 处理 |
+|---|---|
+| `context_usage` | 上下文用量快照 → `chatStore.setContextUsage`(ContextGauge) |
+| `compaction_progress` | 压缩进度(`phase` 如 `post_turn` = 本轮结束后执行的压缩)→ upsert `compaction` 段 |
+| `compaction_notice` | 上一轮后台压缩完成的提示 → `chatStore.setContextCompaction` |
-- 长会话触发上下文压缩时的提示帧
+## 计划 / 循环
-### `model_progress` — 模型活性信号
+- `plan_update`:`update_plan` 工具产生的轻量计划清单;run 正常结束时后端补发全部 completed 的收尾帧。
+- Plan 模式(`plan_step_*` / `plan_complete` 等)与自主循环(`loop_*`)事件经 `ChatStreamOptions.onEvent` 钩子交给
+ `usePlanMode` / `useLoopMode` 先处理,钩子返回 true 则不再进通用 reducer。
-- 表示"模型还在飞行中",用于喂 run 卡死看门狗/停滞检测,UI 不直接渲染
+## 本体治理
-### `ontology_*` — 本体治理事件族
+`ontology_activation` / `ontology_gate` / `ontology_review` / `ontology_repair` / `ontology_revision` /
+`ontology_revision_thinking`;`scope === 'ontology_revision'` 的工具事件单独分流(重写结果经 `content_replace` 落地)。
-`ontology_activation` / `ontology_gate` / `ontology_review` / `ontology_repair` /
-`ontology_revision` / `ontology_revision_thinking`——本体激活、工具闸、输出评审、修复与重写过程
-(重写结果经 `content_replace` 落地)。另有 `eventObj.scope === 'ontology_revision'` 的分流。
+## 已退役(不要再处理或产出)
-### `follow_up` — 追问建议
-
-```json
-{"type": "follow_up", "follow_up_questions": ["追问1", "追问2"]}
-```
-
-- 追问也可能不走 SSE:流结束后由后台任务生成,写进消息 `extra_data.follow_up_questions`,
- 前端轮询/刷新历史时取到
-
-### `meta` — 元信息
-
-```json
-{"type": "meta", "message_id": "msg_xxx", "citations": [...], "workspace_files": [...], "artifacts": [...]}
-```
-
-- 设置 `messageId`;`citations` 非空时**整体替换** allCitations
-- `artifacts` 追加为下载类 toolCall 展示
-
-### `error` — 错误
-
-```json
-{"type": "error", "error": "错误描述"}
-```
-
-- 直接 throw,终止本次流处理
-
-### `end` / `[DONE]` — 流结束
-
-- `{"type": "end"}` 或 `data: [DONE]` 均表示结束
-- finalize 所有 running 状态的 toolCalls,触发后续操作(摘要、分类等由后端负责)
+- `batch_confirm`(批量执行已退役)
+- `model_progress`(活性信号改由 `server_ts` / 心跳承担)
## Segment 渲染顺序
-消息通过 `segments[]` 按顺序渲染:
+`segments[]` 按到达顺序渲染:`thinking` / `tool` / `text` / `plan` / `compaction`。
-```
-[thinking] → [tool] → [thinking] → [tool] → [text]
-```
-
-每个 segment 类型对应不同的 UI 组件:
-- `thinking` → ThinkingBlock (可折叠)
-- `tool` → ToolCall (可展开,显示名称/参数/结果;子智能体带 subSteps)
-- `text` → Markdown 渲染 + 引用锚点胶囊(`[锚文本](cite:eN)`)
+- `thinking` → 一行"已思考 N 秒",可折叠
+- `tool` → 一行过程记录,可展开(子智能体带 subSteps)
+- `text` → Markdown + 引用锚点胶囊(`[锚文本](cite:eN)`)
- `plan` → 计划清单条
-
-## 中断流
-
-```typescript
-// AbortController 中断
-const abortController = new AbortController();
-
-// 发送请求时
-const r = await authFetch(url, {
- method: 'POST',
- signal: abortController.signal,
- body: ...,
-});
-
-// 用户点击停止
-abortController.abort();
-```
+- `compaction` → 压缩过程行
## 断线续播
-```typescript
-// run_started 时已存下 activeRun
-const { runId } = useChatStore.getState().activeRuns[chatId];
+`hooks/streamRecovery.ts`:
-// 重连:从后台 run 的 Redis Stream 重放 + 继续跟随
-const r = await authFetch(`${apiUrl}/v1/chats/stream/${runId}`);
-// 事件格式与 POST /v1/agents/responses 完全一致
-```
-
-实际编排在 `useStreaming.ts::resumeRunIfAny`:
-
-- `getActiveChatRun` 探测活跃 run;已终态则清僵尸状态(`cleanupZombieRunState`)
-- 普通对话从 `last_event_offset` **断点续传**;`plan_generate` / `plan_execute` / `autonomous_loop`
- 从 offset 0 **全量重放**(loop 会先截掉尾部 assistant 占位再重建)
-- 跨标签页互斥:Web Locks `hugagent_run_follow_${run_id}`(`ifAvailable: true`),
- 只允许一个标签页跟随同一 run
+- 探测活跃 run(`last_event_offset`);已终态则清理僵尸状态
+- 普通对话:`openRunSubscription(runId)` → `run_snapshot` 首帧恢复已产生内容,再继续跟随
+- `plan_generate` / `plan_execute` / `autonomous_loop`:从 offset 0 全量重放
+- 跨标签页互斥:Web Locks `hugagent_run_follow_${run_id}`,只允许一个标签页跟随同一 run
- 运行中插话:`steerChatRun` / `withdrawChatRunSteer`,失败时降级为下一轮普通消息
+
+中断:`AbortController.abort()` 只断开跟随;取消 run 走显式取消接口(`markRunCancelledByUser`)。
diff --git a/.claude/skills/hugagent-frontend-dev/references/ui-design-spec.md b/.claude/skills/hugagent-frontend-dev/references/ui-design-spec.md
index 8acca712..081f9362 100644
--- a/.claude/skills/hugagent-frontend-dev/references/ui-design-spec.md
+++ b/.claude/skills/hugagent-frontend-dev/references/ui-design-spec.md
@@ -1,448 +1,423 @@
-# UI 设计规范参考
+# HugAgentOS UI 设计规范 v2(开发落地版)
-本文档基于 UI 团队出品的设计规范(`img/设计规范/`),是前端开发的视觉对齐依据。
+> 来源:《HugAgentOS 设计规范 v2.0 · 2026.10》(`HugAgentOS-UI设计规范/HugAgentOS设计规范-v2.html` + `assets/`)。
+> v2 相对 v1 经过一次对抗性审查,修了 25 项问题(6 项 P0 可访问性/语义错误、14 项 P1 层级一致性、5 项 P2 品牌表达)。
+> 本文只保留写代码要用的规则与数值;品牌物料、宣传版式见原规范。
+>
+> **代码是唯一事实来源。** 改令牌时 `styles/variables.css`、`utils/motionTokens.ts`、`appTheme.ts` 三处同步改。
+> v2 令牌参考实现在本 skill 的 `assets/design-v2/tokens/`,**尚未全部合入仓库**——合入状态和落地方式见
+> `css-variables.md` 的「v1 → v2 对照」与本文末尾「迁移清单」。
---
-## 1. 颜色系统
-
-### 1.1 主色 (Primary Blue)
-
-| 状态 | 色值 | 用途 |
-|------|------|------|
-| 常规 | `#126DFF` | 主按钮、链接、选中态 |
-| 悬浮 | `#3C87FF` | hover 状态 |
-| 按下 | `#0862F3` | active/pressed 状态 |
-| 禁用、描边 | `#A0C5FF` | disabled 按钮描边 |
-| 背景 | `#DBE9FF` | 选中项背景、badge 背景 |
-| 浅色/白色悬浮 | `#EBF2FF` | 列表 hover、浅色背景 |
-
-### 1.2 边框与填充 (Neutral Gray)
-
-| 状态 | 色值 | 用途 |
-|------|------|------|
-| 重(填充或 icon) | `#8C94A2` | 实心 icon 填充 |
-| 深/悬浮 | `#B3BAC8` | 深边框、placeholder |
-| 一般 | `#D8DBE2` | 常规分割线、边框 |
-| 描边 | `#E3E6EA` | 轻量描边 |
-| 灰色背景上的 hover | `#EBEDEE` | 灰背景区域的 hover |
-| 背景灰 | `#F5F6F7` | 次级背景、侧边栏底色 |
-
-### 1.3 文字色
-
-| 色值 | 用途 |
-|------|------|
-| `#101828` | 强调文字、正文标题 |
-| `#475467` | 次强调、大段正文内容 |
-| `#667085` | 次要信息 |
-| `#98A2B3` | 置灰/说明文字 |
-| `#FFFFFF` | 纯白文字(深色背景上) |
-
-### 1.4 语义色 — 成功 (Success Green)
-
-| 状态 | 色值 |
-|------|------|
-| 常规 | `#02B589` |
-| 悬浮 | `#11D4A4` |
-| 按下 | `#04A87F` |
-| 禁用、描边 | `#80DAC4` |
-| 背景 | `#CFF8EE` |
-| 浅色/白色悬浮 | `#E6FFF9` |
-
-### 1.5 语义色 — 警告 (Warning Orange)
-
-| 状态 | 色值 |
-|------|------|
-| 常规 | `#F8AB42` |
-| 悬浮 | `#FFC16C` |
-| 按下 | `#F69D24` |
-| 禁用、描边 | `#FBD5A0` |
-| 背景 | `#FFECD1` |
-| 浅色/白色悬浮 | `#FFF5E8` |
-
-### 1.6 语义色 — 失败/危险 (Error Red)
-
-| 状态 | 色值 |
-|------|------|
-| 常规 | `#FC5D5D` |
-| 悬浮 | `#FF7676` |
-| 按下 | `#F95555` |
-| 禁用、描边 | `#FCAAAA` |
-| 背景 | `#FFDFDF` |
-| 浅色/白色悬浮 | `#FFF0F0` |
-
-### 1.7 Icon 专用色
-
-按需取色,保证页面**蓝色系占比最多**。
-
-| 颜色 | 色值 |
-|------|------|
-| 蓝色 | `#126DFF` |
-| 红色 | `#F25149` |
-| 紫色 | `#7655FA` |
-| 绿色 | `#02B589` |
-| 橙色 | `#F8AB42` |
+## 0. 设计原则(冲突时按顺序取舍)
+
+1. **内容为先**:界面是容器,报告、表格、答案才是主角。界面元素用中性色,颜色只表达含义和**唯一**的主要操作。能用留白分组就不加边框和底色。
+2. **清晰的层级**:每个视图只有一个主要操作。层级靠字号、字重、间距建立,不靠盒子套盒子。
+3. **一致而自然**:同样的东西永远长一样、动一样。动效有来处、有去处、可打断。照顾深浅外观、字号放大、减弱动效。
+4. **过程安静可见**:智能体在思考/检索/调用工具时,用**一行安静的文字**说清在做什么、用了多久;可展开看细节,默认不打扰。不要多处同时闪动的指示器。
---
-## 2. 文字排版
+## 1. 色彩
+
+### 1.1 品牌色
+
+| 名称 | 色值 | 用途 |
+|---|---|---|
+| 环抱蓝 Hug Blue | `#126DFF`(白字 4.51:1) | **唯一强调色**:主要操作、链接、选中、焦点 |
+| 火花绿 Spark | `#19D3A7` | **只属于标志和品牌动画,界面中禁用**(与成功色亮度对比仅 1.37:1,分不清) |
+| 墨 | `#0B0D12` | 文字、深色底(宣传) |
+| 纸 | `#FFFFFF` | 浅色底 |
+
+- 深色模式主色**填充仍用 `#126DFF`**(v1 的 `#3E8BFF` 白字只有 3.31:1,P0);深色下的链接/强调文字改用 `--color-primary-text`(`#5C9BFF`)。
+
+### 1.2 语义色:每种只有三个令牌
+
+| 语义 | 填充(状态点、实心按钮) | 前景(文字/图标,≥4.5:1) | 浅底(提示条、标签) |
+|---|---|---|---|
+| 强调 | `--color-primary` #126DFF | `--color-primary-text` #0F62E6 | `--color-primary-light` #EBF2FF |
+| 成功 | `--color-success` #02B589 | `--color-success-text` #067A5E | `--color-success-bg` #E6FFF9 |
+| 警告 | `--color-warning` #F8AB42 | `--color-warning-text` #B45309 | `--color-warning-bg` #FFF5E8 |
+| 错误 | `--color-error` #FC5D5D | `--color-error-text` #D92D20 | `--color-error-bg` #FFF0F0 |
+| 破坏性实心按钮 | `--color-danger-fill` #D92D20(白字 4.83:1) | — | — |
+
+规则:
+- **语义色不能直接当文字色用**(v1 的 success/warning/error 当文字只有 2.63/1.93/3.06:1,P0)。文字、图标一律用 `*-text`。
+- **取消 hover/active/disabled 色阶**(v1 的 `--color-*-hover/-active/-disabled` 不再用于新代码)。状态改为统一叠加层,见 1.3。
+- 浅底上的文字用对应的 `*-text`,不要用填充色压浅底(v1 危险按钮红字压浅红底 2.76:1,P0)。
-### 2.1 字体栈
+### 1.3 状态叠加层(hover / press / disabled)
-| 平台 | 字体 | 字重 |
-|------|------|------|
-| macOS | PingFang SC | 正常 (400) / 加粗 (600) |
-| Windows | 微软雅黑 | 正常 (400) / 加粗 (700) |
-| 特殊数字/汉字 | 钉钉进步体 / DingTalk Sans | 正常 (400) |
+颜色只会**变深**,白字对比度不会下降:
+
+| 状态 | 中性控件(透明/浅底) | 彩色填充(主按钮等) |
+|---|---|---|
+| 悬停 | 叠 `--state-hover`(浅 `rgba(16,24,40,.04)` / 深 `rgba(255,255,255,.06)`) | 叠 `--state-on-fill-hover`(黑 8% / 深色黑 12%) |
+| 按下 | 叠 `--state-press`(.08 / .10),并 `scale(.97)` | 叠 `--state-on-fill-press`(黑 16% / 深色黑 22%),并 `scale(.97)` |
+| 禁用 | 整体 `opacity: .4` | 整体 `opacity: .4` |
+
+写法(叠加而不是换色):
-**CSS font-family 推荐写法(与 variables.css 实现一致,系统栈优先):**
```css
-font-family: -apple-system, BlinkMacSystemFont, "SF Pro Text", "Helvetica Neue",
- "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei UI", "Microsoft YaHei", sans-serif;
+.jx-xxx-btn { background: var(--color-primary); color: var(--color-on-primary); }
+.jx-xxx-btn:hover { background-image: linear-gradient(var(--state-on-fill-hover), var(--state-on-fill-hover)); }
+.jx-xxx-btn:active { background-image: linear-gradient(var(--state-on-fill-press), var(--state-on-fill-press)); transform: scale(.97); }
+.jx-xxx-btn:disabled { opacity: .4; }
```
-### 2.2 字号层级
+antd 侧:`colorPrimaryHover` / `colorPrimaryActive` 设为**更深**的 `#0F62E6` / `#0A56CC`(见 `assets/design-v2/tokens/appTheme.v2.ts`)。
-| 字号 | 字重 | 用途 |
-|------|------|------|
-| 12px | 常规 | 辅助文案、标签 |
-| 14px | 常规 | 正文 - 常规(默认正文) |
-| 14px | 加粗 | 小标题 |
-| 16px | 常规 | 正文 - 大字号 |
-| 16px | 加粗 | 常规标题 |
-| 18px | 加粗 | 特殊标题和强调 |
-| 22px | 加粗 | 页面一级标题 |
-| 44px | 加粗 | 页面 banner、运营等标题 |
+### 1.4 文字与背景
-### 2.3 日期格式
+文字三级(**占位符 = 三级文字色**,不再用 2.58:1 的 `#98A2B3`):
-- 用 `/` 分隔:`2025/08/18 14:08`
-- 到最末右对齐
+| 令牌 | 浅色 | 深色 | 用途 |
+|---|---|---|---|
+| `--color-text` | #101828 | #E8ECF4 | 正文、标题 |
+| `--color-text-secondary` | #475467 | #B3BDCD | 说明 |
+| `--color-text-tertiary` | #667085 | #8792A4 | 辅助、时间戳、**占位符** |
----
+背景与线(深色模式层级越高表面越亮,不靠加重投影):
+
+| 令牌 | 浅色 | 深色 | 用途 |
+|---|---|---|---|
+| `--color-bg-base` | #FFFFFF | #0F141B | 页面底色 |
+| `--color-bg-layout` | #F5F7FB | #12171F | 侧栏、分区底 |
+| `--color-bg-container` | #FFFFFF | #161C25 | 卡片、面板 |
+| `--color-bg-elevated` | #FFFFFF | #1C2330 | 浮层、弹窗 |
+| `--color-bg-chat` | #FDFDFC | #111720 | 对话区底 |
+| `--color-separator` | #E3E8EF | #2B3442 | 分隔线(装饰性;即 v1 的 `--color-border`) |
+| `--color-control-border` | #C5CCD8 | #3A4556 | 输入框边框(配合可见标签) |
+| `--color-fill-quiet` | #F2F4F7 | #1F2631 | 次要按钮、胶囊、用户气泡 |
+| `--material` | rgba(255,255,255,.82) | rgba(28,35,48,.80) | 浮层材质,配合 `backdrop-filter: blur(20px)` |
+| `--scrim` | rgba(16,24,40,.32) | rgba(0,0,0,.56) | 模态遮罩 |
-## 3. 图标
+### 1.5 分类色:从 7 组收敛到 4 组
-- **推荐图标库**: IconPark(字节跳动)
-- **最小切图尺寸**: 16px,其他尺寸为 4 的倍数
+只用于区分类别,**类别优先用图标和文字区分,颜色只做辅助**:
-### 3.1 线性图标
+| 组 | 示例 | 令牌(浅底 + 前景成对使用) |
+|---|---|---|
+| 蓝 | MCP / 连接器 | `--color-tint-blue-bg` + `--color-tint-blue` |
+| 紫 | 技能 | `--color-tint-purple-bg` + `--color-tint-purple` |
+| 青 | 自动化 | `--color-tint-teal-bg` + `--color-tint-teal` |
+| 琥珀 | 计划模式 | `--color-tint-amber-bg` + `--color-tint-amber` |
+
+新代码不要再用 `cyan / green / red` 三组 tint(存量保留,不新增引用)。
+
+---
-| 尺寸 | 描边宽度 | 圆角 | 端点 |
-|------|---------|------|------|
-| 32px | 2.4px 居中 | 2px | 圆形 |
-| 16px | 1.2px 居中 | 1px | 圆形 |
+## 2. 文字层级
-> 使用时只根据 32px / 16px 两种基准尺寸,通过缩放功能来变化大小。
+按**角色**命名(参考 Apple 文字样式),九级覆盖所有场景。**字重只用 400 / 500 / 600**(不再出现 700/900)。中文不加负字距;西文 28px 以上收紧。**字号一律用 rem**,跟随浏览器缩放。
-### 3.2 面性图标 — 单色
+| 角色 | 令牌前缀 | 字号 / 行高 / 字重 / 字距 | 用途 | 与 v1 |
+|---|---|---|---|---|
+| 展示 display | `--font-size-display` | 2.5rem(40) / 48 / 600 / -0.02em | 欢迎语、宣传 | 调整(原 xxl 44) |
+| 标题一 title-1 | `--font-size-title-1` | 1.75rem(28) / 36 / 600 / -0.01em | 页面标题 | 新增 |
+| 标题二 title-2 | `--font-size-title-2` | 1.375rem(22) / 30 / 600 | 分区标题 | 沿用 xl |
+| 标题三 title-3 | `--font-size-title-3` | 1.125rem(18) / 26 / 600 | 弹窗、卡片标题 | 沿用 lg |
+| 重点 headline | `--font-size-headline` | 0.9375rem(15) / 22 / 600 | 列表项标题 | 新增 |
+| 阅读 body-lg | `--font-size-body-lg` | 1rem(16) / 28 / 400 | 对话、文档正文 | 沿用 md |
+| 正文 body | `--font-size-body` | 0.875rem(14) / 22 / 400 | **界面默认**:按钮、输入、列表、表格 | 沿用 sm |
+| 注释 footnote | `--font-size-footnote` | 0.8125rem(13) / 20 / 400 | 过程行、说明 | 新增 |
+| 说明 caption | `--font-size-caption` | 0.75rem(12) / 18 / 400 | 时间戳、标签 | 沿用 xs |
-- 尺寸 32px,背景圆角 8px
+每级同时有 `--line-height-*` / `--font-weight-*` / `--letter-spacing-*` 四件套(见 v2 token CSS)。
-### 3.3 面性图标 — 双色
+字体(已在代码中落地,`platformTypography.ts` 在挂载前给 `` 打 `data-font-platform`):
-- 同色系,主颜色 100%,浅色 50%
+| 场景 | 字体 |
+|---|---|
+| 界面中文 | Apple: PingFang SC · Windows: Microsoft YaHei UI · 其他: Noto Sans CJK SC(`--font-family`) |
+| 数字 | DingTalk Sans → 系统字体(`--font-family-number`);**表格、计时、金额一律 `font-variant-numeric: tabular-nums`** |
+| 代码 | ui-monospace, SF Mono, Menlo, Consolas(`--font-family-mono`) |
+| 宣传 | 思源黑体(不进产品界面) |
---
-## 4. 间距
+## 3. 间距与尺寸
-以 **4px 为基本单位**,使用 4 的倍数作为间距。
+**8 栅格**:`4 / 8 / 12 / 16 / 24 / 32 / 48 / 64`。4 只用于图标和文字之间的半步。v1 的 20 与 40 不在栅格上,**新代码不再使用**。
-| 值 | 常见用途 |
-|-----|---------|
-| 4px | 紧凑间距(标签内边距、小图标间距) |
-| 8px | 元素间常规间距 |
-| 16px | 卡片内间距、段落间距 |
-| 20px | 模块间距、表单行距 |
-| 40px | 大区域分隔 |
+| v2 令牌 | 值 | 对应 v1 |
+|---|---|---|
+| `--space-1` | 4 | `--space-xs` |
+| `--space-2` | 8 | `--space-sm` |
+| `--space-3` | 12 | (v1 缺失) |
+| `--space-4` | 16 | `--space-md` |
+| `--space-6` | 24 | 替代 `--space-lg` 20 |
+| `--space-8` | 32 | (v1 缺失) |
+| `--space-12` | 48 | 替代 `--space-xl` 40 |
+| `--space-16` | 64 | — |
+
+尺寸规则:
+- **点击区**:桌面 ≥ 32×32,触屏 ≥ 44×44(真实尺寸)。紧凑控件最小 28,用伪元素/padding 外扩点击区。
+- **控件高度**:28(紧凑,仅桌面表格和工具栏)· 32(默认)· 40(主要操作、移动端)。不再出现 24px 高的按钮。
+- **阅读宽度**:对话与文档正文最大 **720px**(每行约 40 个汉字)。
+- **页面边距**:桌面 24 / 32,移动端 16 + 安全区(`env(safe-area-inset-*)`)。
---
-## 5. 圆角
+## 4. 圆角
+
+值不变,规则变严:**圆角跟随尺寸,且同心——内层圆角 = 外层圆角 − 内边距**。
-系统使用**大圆角**表达 AI、时尚、亲近的品牌感。
+| 令牌 | 值 | 用于 |
+|---|---|---|
+| `--radius-xs` | 6 | 28px 控件、代码块 |
+| `--radius-sm` | 10 | 32–40px 控件、菜单项 |
+| `--radius-md` | 16 | 卡片、菜单、弹窗 |
+| `--radius-lg` | 24 | 对话输入区、面板 |
+| `--radius-full` | 999px | 标签、头像、开关、独立图标按钮(胶囊) |
-| 圆角值 | 变量 | 用途 | 示例 |
-|--------|------|------|------|
-| 6px | `--radius-xs` | 小面积元素 | 标签、小卡片 |
-| 10px | `--radius-sm` | 常用组件 | 按钮、输入框、下拉框、菜单 |
-| 16px | `--radius-md` | 分割模块 | 卡片视图、背景卡片 |
-| 24px | `--radius-lg` | 大容器 | 弹窗、大输入框 |
+例:菜单外 16、内边距 6 → 选项 10(v1 外 10 / 内边距 6 / 选项 6 不同心,P1)。
---
-## 6. 投影
+## 5. 层级与材质
-为整体性统一限制投影层级,分为两种状态:
+一共四层。**投影只给浮在内容之上的东西。**
-- **常态**: 轻微投影(几乎无感知)
-- **悬停/hover**: 加深投影(凸显层级)
+| 层 | 背景 | 内容 | 投影 |
+|---|---|---|---|
+| L0 底面 | `--color-bg-layout` / `--color-bg-base` | 页面、侧栏 | 无 |
+| L1 内容 | `--color-bg-container` | 卡片、面板 | **静止无投影**,悬停出现 `--shadow-raised` 并上浮 1px |
+| L2 浮层 | `--material` + `backdrop-filter: blur(20px)` | 菜单、弹出框、提示 | `--shadow-floating` |
+| L3 模态 | `--color-bg-elevated` + `--scrim` 遮罩 | 对话框、表单抽屉 | `--shadow-floating` |
-可根据具体场景微调投影颜色。
+- 静止卡片不加投影,用底色和留白分组(替代 v1 `--shadow-card`)。
+- **对话输入区只保留投影(`--shadow-raised`),不再同时加边框**。
+- 投影令牌:`--shadow-raised`(原 card)、`--shadow-floating`(原 dropdown),深浅两档已定义。
---
-## 7. 按钮
+## 6. 图标
-最小切图尺寸 **36px**,其他为 4 的倍数。
+参考 SF Symbols:**图标是文字的一部分**。
-### 7.1 按钮类型
+- 24px 画板,四周留 2px,**线宽 1.5px**,圆头圆角(`stroke-linecap/linejoin: round`),`stroke="currentColor"`。
+- 尺寸跟随字号、线宽跟随字重、颜色继承文字色:13px 常规 → 线宽 1.5;15px 中粗 → 2;20px 标题 → 2。
+- 默认线性,**选中态换面性**。
+- 风格样例:`assets/design-v2/icons-ui/`(12 个:new-chat / search / knowledge / skill / automation / my-space / attachment / settings / send / chevron-right / check / close)。
+- 既有 IconPark / `@ant-design/icons` 可继续用,但要满足上面的线宽与 currentColor 规则;不要给图标单独上品牌色。
-| 类型 | 说明 |
-|------|------|
-| 主要按钮 | 蓝色填充(`#126DFF`),白色文字 |
-| 次要按钮 | 白色填充、蓝色描边 |
-| 文字按钮 | 无边框、无背景,文字色即操作色 |
-| 图标按钮 | 仅图标,无文字 |
-| 成功按钮 | 绿色填充(`#02B589`) |
-| 危险按钮 | 红色填充(`#FC5D5D`) |
+---
-### 7.2 按钮尺寸
+## 7. 组件
-| 尺寸 | 高度(参考) |
-|------|------------|
-| 大 | 40px |
-| 中(默认) | 36px |
-| 小 | 28px |
+以 Ant Design 为底,用 v2 令牌覆盖(ConfigProvider token,不覆写组件内部 DOM 样式)。
-### 7.3 按钮状态
+### 7.1 按钮
-每种按钮均有 4 种状态:常规 → 悬浮(hover) → 按下(active) → 禁用(disabled)。
+- **一个视图只有一个主要按钮**(`type="primary"`)。
+- 次要操作用**浅灰填充**(`--color-fill-quiet`),**不再用描边按钮**。
+- 低频操作用文字按钮(`type="text"` / `type="link"`)。
+- 破坏性操作:**确认框里用实心红**(`--color-danger-fill` + 白字),**列表里用红色文字按钮**(`--color-error-text`)。
+- 尺寸 28 / 32 / 40;加载中按钮文案写进行时("生成中")。
+- 文案写动作:"生成报告",不写"立即开启智能分析之旅"。
----
+### 7.2 输入框与对话输入区
-## 8. 导航
+- 输入框**必须有可见标签**,不能只靠占位符说明用途。占位符写示例("例如:Q3 华东区销售分析")。
+- 出错时**写清原因和改法**:"请输入 00:00 到 23:59 之间的时间"。
+- 边框用 `--color-control-border`。
+- **对话输入区**是最重要的控件:`--radius-lg` 大圆角,**只有投影没有边框**;内容为空时发送按钮是中性色,**有内容后才变主色**(主色只在能操作时出现)。
-### 8.1 导航菜单
+### 7.3 对话与过程
-- 垂直列表,图标 + 文字
-- 选中项高亮
+- 用户消息:浅灰气泡(`--color-fill-quiet`)。助手回复**不加气泡**。
+- 思考、工具调用:**默认一行安静的文字**——做了什么、结果多少、用了多久,点击展开细节。
+ 例:"已思考 12 秒"、"已检索知识库「销售数据 2026」14 条 · 0.8s"、"正在生成 Excel 报表"。
+- **只有用户会打开的产物**(文件、表格、网页)才做成卡片。v1 把每次工具调用做成卡片(过程比答案显眼,P1),新代码不要这样。
+- 正文宽度 ≤ 720px,阅读字号 body-lg(16/28)。
-### 8.2 Tabs / 标签页
+### 7.4 状态
-- **一级**: 页面或模块的一级导航,有选中样式、默认样式、hover 样式
-- **二级**: 页面或模块的二级导航
-- **三级**: 紧凑标签切换
+状态**同时用颜色和形状**表达。**一屏只有一个持续动画。**
-### 8.3 面包屑 / 返回
+| 状态 | 表现 |
+|---|---|
+| 运行中(当前那一项) | 文字微光(shimmer),一屏一个 |
+| 运行中(其他项) | 静止蓝点 |
+| 已完成 | 成功色勾,`statusIn` 落定 |
+| 等你确认 | 警告色点 + **扩散涟漪**(唯一允许涟漪的状态,因为它在叫人) |
+| 失败 | 错误色 |
+| 排队中 / 已暂停 | 中性色静止 |
+| 计数徽标 | 超过 99 显示 `99+` |
-- **面包屑**: 显示页面在系统层级结构中的位置,并提供上返回,本系统中较少使用
-- **返回**: 用于二级页面,能向上返回
+### 7.5 卡片与浮层
+
+- 卡片静止无投影,悬停出现 `--shadow-raised` 并 `translateY(-1px)`。
+- 菜单用材质背景,**从触发点展开**(`transform-origin` 指向触发点),`--radius-md` + 选项 `--radius-sm`。
+- 提示条(toast):**反色胶囊**,出现在**底部中间**,3 秒后自动消失,新提示替换旧提示。
+- 快捷键提示(⌘N / ⌘K)用 caption 字号、三级文字色。
---
-## 9. 数据录入
+## 8. 动效
-### 9.1 输入框
+### 8.1 原则
-- 标签对齐方式在页面内统一(左对齐 / 右对齐 / 顶对齐)
-- 默认高度 36px
-- 前后间距 20px(页面范围统一)
-- 前后间距需要在整个页面范围统一
-- 可附带前缀 icon
+1. **有来处**:菜单从按钮展开,面板从边缘滑入,消息从输入框升起;返回沿原路离开。
+2. **可打断**:手势、拖拽、展开收起、布局变化都用弹簧;中途打断从当前位置和速度继续。动画期间界面始终可操作。
+3. **克制**:产品界面弹簧不回弹或只轻微回弹;一次性过渡 ≤ 320ms;除折叠展开外**只动 `opacity` 和 `transform`**,不动 filter(含 blur)和阴影。
+4. **进行中常在**:表示"正在运行"的动画在减弱动效时也继续;装饰性位移在减弱动效时换成淡入淡出。
-### 9.2 下拉选择
+### 8.2 弹簧(Motion 库,用时长 + 回弹描述)
-- 包含下拉选择 + 搜索
-- 基础样式有正文类型、hover 状态
-- 有单选和多选模式
-- 支持复选框列表
+| 预设 | 参数 | 用于 |
+|---|---|---|
+| `SPRING.smooth` | `visualDuration: .35, bounce: 0` | 面板、抽屉、折叠展开、布局变化(替代 v1 `panel`) |
+| `SPRING.snappy` | `visualDuration: .25, bounce: .15` | 标签页指示条、开关、徽标确认(替代 v1 `pop` / `ink`) |
-### 9.3 日期选择器
+v1 的 `pop` / `panel` 回弹 33%(面板会晃,P1),`ink` 一并标记弃用;**新代码只用 smooth / snappy**,`bouncy` 不进产品界面。
-- 标准日历面板
-- 支持日期范围选择
-- 支持快捷选项(本周/上周/本月等)
+### 8.3 时长与曲线(不适合弹簧的一次性过渡)
-### 9.4 多选 / 单选
+| 令牌 | 时长 | 用于 |
+|---|---|---|
+| `--motion-duration-instant` | 100ms | 悬停着色、按下反馈 |
+| `--motion-duration-fast` | 160ms | 状态切换、菜单弹出、图标落定 |
+| `--motion-duration-normal` | 240ms | 入场、出场、消息发送、列表编排 |
+| `--motion-duration-slow` | 320ms | 面板转场、折叠展开(能用弹簧优先弹簧) |
-- 单选: 圆形 radio,选中为蓝色填充
-- 多选: 方形 checkbox,选中为蓝色勾选
-- 有横排和竖排两种布局
+曲线三条:`--motion-ease-brand-out`(入场/展开/升起)、`--motion-ease-standard`(双向状态切换)、`--motion-ease-exit`(退场)。
+**退场时长 = 入场 × 0.7**(如菜单 160 → 112ms,提示条 240 → 168ms)。
+**禁止**在代码里写 `ease` / `ease-out` / 新的 cubic-bezier 值,TS 侧用 `utils/motionTokens.ts` 的 `DUR` / `EASE`。
-### 9.5 开关
+### 8.4 编排
-- 默认灰色背景
-- 开启状态为主色蓝(`#126DFF`)
-- 带有开/关文字标识
+- 静态列表入场逐项错开 60ms(`--motion-stagger-step` / `STAGGER_STEP`),**第 8 项起不再增加延迟**(`STAGGER_CAP`)。
+- 超过 20 项的动态列表不做布局动画(`LAYOUT_ANIM_MAX_ITEMS`)。
+- 翻看历史记录不播任何入场动画,只对 **2 秒内新产生**的内容播放(`FRESH_ENTER_WINDOW_MS`)。
----
+### 8.5 原语库(新增动画前先找同语义的原语)
-## 10. 数据展示
+命名:关键帧 `jx-kf-名称`,工具类 `jx-anim-名称`(定义在 `styles/motion.css`)。**入场类一律 `animation-fill-mode: backwards`。**
-### 10.1 标签 (Tag)
+| 原语 | 工具类 | 参数 | 用于 | 减弱动效时 |
+|---|---|---|---|---|
+| fadeInUp | `.jx-anim-fadeInUp` | 240ms · brand-out · 上移 8px | 卡片、确认条入场 | 淡入 |
+| fadeIn | `.jx-anim-fadeIn` | 240ms · standard | 大面积区域 | 淡入 |
+| scaleIn | `.jx-anim-scaleIn` | 160ms · brand-out · .97 → 1 | 菜单、弹出框(原点指向触发点) | 淡入 |
+| statusIn | `.jx-anim-statusIn` | 160ms · brand-out · .7 → 1 | 状态图标落定 | 淡入 |
+| flash | `.jx-anim-flash` | 1.2s · standard | 跳转定位后的一次性高亮 | 保持 |
+| shake | `.jx-anim-shake`(`jx-kf-shakeX`) | 300ms · ±3px | 校验失败 | 停止 |
+| shimmer 文字 | 关键帧 `jx-kf-shimmer` | 2s linear 循环 | 运行中的那一行(一屏一个) | 保持(`motion-keep`) |
+| ripple | `.jx-anim-ripple` | 1.6s 循环 | **只用于"等你确认"** | 停止 |
+| stagger | `.jx-anim-stagger` + `staggerStyle(i)` | 60ms 步进,封顶第 8 项 | 静态列表入场 | 淡入 |
-- 慎重使用标签,保证页面干净有序
-- 同一信息前台展现阶段对比最多使用 2 种颜色样式
-- 类型:基础标签、可关闭标签、表态标签
+> 代码里另有 `.jx-anim-pulse`(`motion-keep` 的通用"进行中"脉冲)与 `.jx-anim-flash-row`。v2 要求一屏只有一个持续动画:
+> 运行中的当前项优先用文字微光,其他运行项保持静止蓝点,不要为每一项都挂 pulse。
-### 10.2 卡片 (Card)
+### 8.6 智能体动效(一次任务的完整过程)
-- padding: 16–20px(可根据内容调整,特殊情况为 4 的倍数)
-- 类型一:图标 + 标题 + 描述 + 标签
-- 类型二:大面积内容卡片(标题 + 正文段落)
-- 圆角 12px
+同一时刻只有一个微光在动;文字按词块淡入,不做模糊,不逐字排队。
-### 10.3 气泡提示卡片
+| 环节 | v2 | v1(不要再这样写) |
+|---|---|---|
+| 发送 | 用户气泡 `translateY(8px) + scale(.98) → 1`,240ms brand-out,原点右下 | scale .92 · 300ms ease-out |
+| 思考 | 一行"正在思考 · 8 秒",文字微光 2s linear,计时每秒更新;骨架屏只在加载历史时出现,不与微光同屏 | 跳点 + 呼吸 + 骨架三处同时动 |
+| 调用工具 | 进行中:文字微光;完成:图标 `statusIn` 160ms,只留下一行可展开的记录 | 卡片 + 无限涟漪 |
+| 流式输出 | **按词块淡入**,opacity 180ms standard,落后模型 ≤ 300ms | 逐字 350ms + 65ms 错开 + blur 3px(每秒约 15 字,越积越多) |
-- 点击/鼠标移入元素时弹出气泡式卡片浮层
+### 8.7 交互控件
-### 10.4 Tips 提示
+| 控件 | 动效 |
+|---|---|
+| 开关 | 滑块 snappy 弹簧;底色 160ms standard |
+| 标签页 | 指示条位置与宽度 snappy 弹簧;文字颜色 160ms standard |
+| 面板 / 抽屉 | smooth 弹簧从边缘滑入;遮罩透明度跟随同一进度 |
+| 菜单 | 打开 160ms brand-out `scale(.97 → 1)`;关闭 112ms exit;原点在按钮处 |
+| 提示条 | 入场 240ms brand-out 上移 16px;退场 168ms exit;3 秒后消失 |
+| 折叠展开 | 高度与透明度 smooth 弹簧;箭头旋转 240ms brand-out |
-- 用于展示不完全的内容,鼠标移入时展示完整内容
-- 有浅色和深色两种样式
+### 8.8 减弱动效(`prefers-reduced-motion: reduce`)
-### 10.5 表格 (Table)
+v2 改为 Apple 做法:**位移和缩放换成淡入淡出,时长不变**;循环装饰停止;运行指示器继续。
-- 标准表格:表头 + 数据行 + 分页
-- 特殊表格:带颜色标记行
+| 类型 | 减弱动效时 | 写法 |
+|---|---|---|
+| 入场位移 / 缩放 | 淡入,时长不变 | 同文件 `@media (prefers-reduced-motion: reduce)` 内写**同名 `@keyframes` 覆盖版,只保留 opacity**,标注 `motion-fade` |
+| 弹簧(Motion 库) | 换成透明度过渡 | `useReducedMotion()` 为真时只动 opacity,时长取 `DUR.fast` |
+| 循环装饰(涟漪、抖动) | 停止 | 覆盖版定格,标注 `motion-stop` |
+| 工作指示器(微光、转圈、骨架) | 继续 | 使用它的规则上标注 `motion-keep: 理由` |
+| CSS transition | 归零 | 保持现状(改成正数会让 antd 浮层定位也产生过渡) |
----
+> 构建门禁 `scripts/check-loading-motion.mjs` 保留。**当前门禁仍按 v1 规则检查**:装饰性关键帧必须有同名
+> 覆盖版(v1 为"定格终态")或在使用处标注 `motion-keep:`。按 v2 写"仅 opacity"的同名覆盖版同样满足门禁;
+> `motion-fade` / `motion-stop` 标注随迁移清单第 3 项一起被门禁识别。
-## 11. 数据提示
+---
-### 11.1 Alert 警告提示
+## 9. 文案语气
-4 种语义,每种有可关闭和不可关闭两种形态:
+像一位靠谱的同事说话:直接、具体、不卖弄。
-| 类型 | 颜色 |
-|------|------|
-| 成功 | 绿色系 (`#02B589`) |
-| 注意 | 橙色系 (`#F8AB42`) |
-| 失败 | 红色系 (`#FC5D5D`) |
-| 提示信息 | 蓝色系 (`#126DFF`) |
+| 场景 | ✅ | ❌ |
+|---|---|---|
+| 按钮 | 写动作:生成报告 | 立即开启智能分析之旅 |
+| 进行中 | 写正在做什么:正在检索 3 个知识库 | AI 正在努力思考中… |
+| 出错 | 写原因和办法:文件超过 50MB,请压缩后重新上传 | 哎呀,出错了! |
+| 完成 | 写结果在哪:报告已保存到我的空间 | 操作成功 |
-### 11.2 全局提示 (Message)
+所有用户可见文案仍须包 `t()`(见 SKILL.md i18n 节)。
-- 顶部居中弹出,自动消失
-- 类型:成功 / 注意 / 失败
+---
-### 11.3 弹窗 (Modal)
+## 10. 品牌标志(界面内用到时)
-- 标题 + 内容区 + 底部操作栏(取消 + 确认/保存)
-- 确认按钮为主色蓝,危险操作为红色
-- 圆角 20px
+- **主标志是标准字**(A 里已含环线和火花点)。**禁止方形标与标准字并排**。产品顶栏这类横向空间用标准字。
+- 方形标只用于方形容器(应用图标、头像、网页图标)。**24px 及以下用小尺寸版** `hugagentos-mark-small.svg` / `favicon.svg`。
+- 彩色稿里火花点永远是火花绿;只有单色稿可以让火花点跟随主体变色。
+- 禁止:拉伸压扁、换品牌色、旋转倾斜、投影/描边/渐变、火花点改色或移位。
+- 对话中的助手头像:品牌蓝底 + 白色环线 + 火花绿点,环线占圆形头像宽度 60%。
+- 标志动画 ≤ 1 秒(描边 0–600ms standard → 填实 500–700ms → 火花点 650ms 起 snappy 弹出);启动页只播一次;减弱动效时整体 240ms 淡入。
+- 素材:`assets/design-v2/logo/`(mark / mark-small / reverse / mono / glyph / favicon)。
---
-## 12. CSS 变量映射(推荐)
+## 11. Ant Design 主题映射(v2 目标值)
-以下为设计规范到 CSS 变量的推荐映射,应用在 `variables.css` 中:
+真源 `src/frontend/src/appTheme.ts`,参考实现 `assets/design-v2/tokens/appTheme.v2.ts`:
-```css
-:root {
- /* 主色 */
- --color-primary: #126DFF;
- --color-primary-hover: #3C87FF;
- --color-primary-active: #0862F3;
- --color-primary-disabled: #A0C5FF;
- --color-primary-bg: #DBE9FF;
- --color-primary-light: #EBF2FF;
-
- /* 边框与填充 */
- --color-fill-heavy: #8C94A2;
- --color-fill-deep: #B3BAC8;
- --color-fill: #D8DBE2;
- --color-border: #E3E6EA;
- --color-fill-hover: #EBEDEE;
- --color-bg-gray: #F5F6F7;
-
- /* 文字 */
- --color-text: #101828;
- --color-text-secondary: #475467;
- --color-text-tertiary: #667085;
- --color-text-placeholder: #98A2B3;
- --color-text-white: #FFFFFF;
-
- /* 成功 */
- --color-success: #02B589;
- --color-success-hover: #11D4A4;
- --color-success-active: #04A87F;
- --color-success-disabled: #80DAC4;
- --color-success-bg: #CFF8EE;
- --color-success-light: #E6FFF9;
-
- /* 警告 */
- --color-warning: #F8AB42;
- --color-warning-hover: #FFC16C;
- --color-warning-active: #F69D24;
- --color-warning-disabled: #FBD5A0;
- --color-warning-bg: #FFECD1;
- --color-warning-light: #FFF5E8;
-
- /* 错误/危险 */
- --color-error: #FC5D5D;
- --color-error-hover: #FF7676;
- --color-error-active: #F95555;
- --color-error-disabled: #FCAAAA;
- --color-error-bg: #FFDFDF;
- --color-error-light: #FFF0F0;
-
- /* Icon 色 */
- --color-icon-blue: #126DFF;
- --color-icon-red: #F25149;
- --color-icon-purple: #7655FA;
- --color-icon-green: #02B589;
- --color-icon-orange: #F8AB42;
-
- /* 圆角 */
- --radius-xs: 6px;
- --radius-sm: 10px;
- --radius-md: 16px;
- --radius-lg: 24px;
-
- /* 间距 */
- --space-xs: 4px;
- --space-sm: 8px;
- --space-md: 16px;
- --space-lg: 20px;
- --space-xl: 40px;
-
- /* 字体(系统栈优先) */
- --font-family: -apple-system, BlinkMacSystemFont, "SF Pro Text", "Helvetica Neue",
- "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei UI", "Microsoft YaHei", sans-serif;
- --font-family-number: "DingTalk Sans", "PingFang SC", "Microsoft YaHei", sans-serif;
-
- /* 字号 */
- --font-size-xs: 12px;
- --font-size-sm: 14px;
- --font-size-md: 16px;
- --font-size-lg: 18px;
- --font-size-xl: 22px;
- --font-size-xxl: 44px;
-}
-```
+| Token | 浅色 | 深色 |
+|---|---|---|
+| colorPrimary | #126DFF | **#126DFF**(v1 为 #3E8BFF) |
+| colorPrimaryHover / Active | #0F62E6 / #0A56CC(变深) | 同左 |
+| colorLink | #0F62E6 | #5C9BFF |
+| colorSuccess / colorError | #02B589 / #D92D20 | #22C79D / #FF6B6B |
+| colorSuccessText / WarningText / ErrorText | #067A5E / #B45309 / #D92D20 | #3DDBB0 / #FFC26B / #FF8A80 |
+| colorText / Secondary / Tertiary | #101828 / #475467 / #667085 | #E8ECF4 / #B3BDCD / #8792A4 |
+| colorTextPlaceholder | #667085 | #8792A4 |
+| colorBorder / BorderSecondary | #C5CCD8 / #E3E8EF | #3A4556 / #2B3442 |
+| colorBgContainer / Layout / Elevated | #FFFFFF / #F5F7FB / #FFFFFF | #161C25 / #12171F / #1C2330 |
+| borderRadius / LG / SM | 10 / 16 / 6 | 同左 |
+| fontSize / SM / LG | 14 / 12 / 16 | 同左 |
+
+`AppThemeProvider.tsx` 在系统开启减弱动效时额外给 antd 传 `motion: false`(已实现,保持)。
---
-## 13. Ant Design 主题映射
-
-使用 Ant Design ConfigProvider 时,按以下规范配置 token:
-
-```typescript
-const theme = {
- token: {
- colorPrimary: '#126DFF',
- colorSuccess: '#02B589',
- colorWarning: '#F8AB42',
- colorError: '#FC5D5D',
- colorInfo: '#126DFF',
- colorText: '#101828',
- colorTextSecondary: '#475467',
- colorTextTertiary: '#667085',
- colorTextQuaternary: '#98A2B3',
- colorBorder: '#E3E6EA',
- colorBorderSecondary: '#D8DBE2',
- colorBgContainer: '#FFFFFF',
- colorBgLayout: '#F5F6F7',
- borderRadius: 10,
- borderRadiusLG: 16,
- borderRadiusSM: 6,
- fontFamily: '-apple-system, BlinkMacSystemFont, "SF Pro Text", "Helvetica Neue", "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei UI", "Microsoft YaHei", sans-serif',
- fontSize: 14,
- fontSizeSM: 12,
- fontSizeLG: 16,
- },
-};
-```
+## 12. 迁移清单(按风险从低到高)
+
+前四项**只加不删、无视觉变化**,可以先合入;后面几项会改变现有界面,需要逐页走查 / 产品评审。
+
+| # | 改动 | 文件 | 影响 |
+|---|---|---|---|
+| 1 | 新增 `*-text`、`danger-fill`、`state-*`、`fill-quiet`、`brand-spark`、`material`、`scrim`、`radius-full`、`space-1…16`、文字层级、`shadow-raised/floating` 令牌 | variables.css · appTheme.ts | 只加不删 |
+| 2 | SPRING 增加 smooth / snappy,旧 pop / ink / panel 标记 `@deprecated` | utils/motionTokens.ts | 新代码用新预设 |
+| 3 | 减弱动效:位移类覆盖版从"定格"改为"仅 opacity",门禁识别 `motion-fade` / `motion-stop` | styles/motion.css · scripts/check-loading-motion.mjs | 只影响开启减弱动效的用户 |
+| 4 | 新增小尺寸方形标,网页图标 16 / 32 换用 | public/ · index.html | 网页图标更清晰 |
+| 5 | 语义色当文字处换成 `*-text` | 全局搜 `color: var(--color-(success|warning|error))` | 提示文字变深 |
+| 6 | 主按钮悬停 / 按下改为叠加层;深色主色改回 #126DFF | appTheme.ts · variables.css | 悬停由变浅改为变深 |
+| 7 | 占位符改为三级文字色;输入框补齐可见标签 | variables.css · 各表单 | 占位符变深 |
+| 8 | `--space-lg` 20 → 24、`--space-xl` 40 → 48 | variables.css · 引用处 | 需逐页走查 |
+| 9 | 静止卡片去掉 `shadow-card`;对话输入区去掉边框 | 各卡片样式 · composer | 需走查 |
+| 10 | 用户气泡入场 .98 + 8px / 240ms;流式输出改为词块淡入,去掉 blur | chat/messages.css · chat/grapheme-reveal.css | 需体验评审 |
+| 11 | 工具调用默认折叠为一行;运行中改为文字微光,涟漪只留给待确认 | tool.css · 工具调用组件 | 需产品评审 |
+| 12 | 菜单圆角 16、选项 10(同心) | antd Dropdown 主题 · 自研菜单 | 视觉微调 |
+
+做第 1、2 项时直接以 `assets/design-v2/tokens/` 下的 CSS / TS 为准,深浅两档一起加;第 5 项起每一项都是**会改变现有界面**的改动,没有用户明确要求不要顺手做。
diff --git a/.claude/skills/hugagent-frontend-dev/scripts/scaffold_component.sh b/.claude/skills/hugagent-frontend-dev/scripts/scaffold_component.sh
index dfe21e1c..b357a282 100755
--- a/.claude/skills/hugagent-frontend-dev/scripts/scaffold_component.sh
+++ b/.claude/skills/hugagent-frontend-dev/scripts/scaffold_component.sh
@@ -75,7 +75,10 @@ fi
echo ""
echo "=== 手动步骤 ==="
-echo "1. 在 styles/index.ts 中 import './${COMPONENT_CAMEL}.css'"
+echo "1. 在 styles/index.ts 中 import './${COMPONENT_CAMEL}.css'(或在组件内直接 import)"
echo "2. 按需在父组件中引入 <$COMPONENT />"
+echo "3. 新文案的英文翻译加进 src/i18n/en/ 对应分片(EE 专属进 edition-ee/i18n/)"
+echo "4. 模板里标 (v2) 的令牌先 grep variables.css;缺失则按 references/css-variables.md 补齐深浅两档"
+echo "5. 运行门禁: cd src/frontend && npm run check:i18n && npm run check:dark && npm run check:motion"
echo ""
echo "Done!"
diff --git a/.claude/skills/hugagent-frontend-dev/templates/component.tsx b/.claude/skills/hugagent-frontend-dev/templates/component.tsx
index 153cc176..5c12f6cb 100644
--- a/.claude/skills/hugagent-frontend-dev/templates/component.tsx
+++ b/.claude/skills/hugagent-frontend-dev/templates/component.tsx
@@ -2,92 +2,73 @@
* Component template.
*
* Replace ${ComponentName}, ${componentName}, ${group} with actual names.
- * Create as components/${group}/${ComponentName}.tsx
+ * Create as components/${group}/${ComponentName}.tsx and export it from components/${group}/index.ts.
+ *
+ * Rules (see SKILL.md + references/ui-design-spec.md):
+ * - Every user-visible string goes through t() (npm run build fails otherwise)
+ * - Copy: buttons say the action, errors say cause + fix, success says where the result is
+ * - One primary button per view; secondary actions use the quiet fill, not outlined buttons
+ * - No colors in inline style — put them in the CSS file with tokens
*/
-import React, { useState, useMemo, useCallback, useRef } from 'react';
-import { Button, Space, message } from 'antd';
-// import { useChatStore } from '../../stores';
-// import type { SomeType } from '../../types';
+import { useCallback, useMemo, useState } from 'react';
+import { Button, message } from 'antd';
+import { t } from '../../i18n';
+// import { useSomeStore } from '../../stores';
+// import type { SomeItem } from '../../types';
-// ---------------------------------------------------------------------------
-// Props
-// ---------------------------------------------------------------------------
+interface ${ComponentName}Item {
+ id: string;
+ name: string;
+ active: boolean;
+}
interface ${ComponentName}Props {
- /** 必填属性示例 */
- itemId: string;
- /** 回调示例 */
- onClose: () => void;
- /** 可选属性示例 */
- items?: any[];
+ /** 所属对象 id */
+ ownerId: string;
+ items?: ${ComponentName}Item[];
+ onSaved?: () => void;
}
-// ---------------------------------------------------------------------------
-// Component
-// ---------------------------------------------------------------------------
-
-export function ${ComponentName}({ itemId, onClose, items = [] }: ${ComponentName}Props) {
- // --- Zustand stores (global state) ---
- // const { someState, someAction } = useSomeStore();
-
- // --- Local state ---
- const [loading, setLoading] = useState(false);
- const [editMode, setEditMode] = useState(false);
-
- // --- Refs ---
- const inputRef = useRef(null);
+export function ${ComponentName}({ ownerId, items = [], onSaved }: ${ComponentName}Props) {
+ const [saving, setSaving] = useState(false);
- // --- Derived state (useMemo) ---
- const filteredItems = useMemo(() => {
- return items.filter((item) => item.active);
- }, [items]);
+ const activeItems = useMemo(() => items.filter((item) => item.active), [items]);
- // --- Callbacks (useCallback) ---
- const handleSubmit = useCallback(async () => {
- setLoading(true);
+ const handleSave = useCallback(async () => {
+ setSaving(true);
try {
- // await someApiCall();
- message.success('操作成功');
- onClose();
+ // await saveSomething(ownerId);
+ message.success(t('已保存到我的空间'));
+ onSaved?.();
} catch (e) {
- message.error(`操作失败:${(e as Error).message}`);
+ // Error text: cause + what to do. Prefer the backend message when it is user-facing.
+ message.error(t('保存失败:{reason}', { reason: (e as Error).message }));
} finally {
- setLoading(false);
+ setSaving(false);
}
- }, [itemId, onClose]);
+ }, [ownerId, onSaved]);
- // --- Early return for empty state ---
- if (!filteredItems.length) {
- return (
-
- );
+ if (!activeItems.length) {
+ return {t('还没有内容')}
;
}
- // --- Render ---
return (
-
-
-
{filteredItems.length} 项
-
- setEditMode(!editMode)}>
- {editMode ? '取消' : '编辑'}
-
-
- 提交
-
-
-
+
+
+ {t('{n} 项', { n: activeItems.length })}
+
+ {saving ? t('保存中') : t('保存')}
+
+
-
- {filteredItems.map((item) => (
-
+
+ {activeItems.map((item) => (
+
{item.name}
-
+
))}
-
-
+
+
);
}
diff --git a/.claude/skills/hugagent-frontend-dev/templates/css-module.css b/.claude/skills/hugagent-frontend-dev/templates/css-module.css
index a1c2e9b8..6701c6c8 100644
--- a/.claude/skills/hugagent-frontend-dev/templates/css-module.css
+++ b/.claude/skills/hugagent-frontend-dev/templates/css-module.css
@@ -1,96 +1,126 @@
/**
- * CSS module template.
+ * CSS template (UI design spec v2).
*
- * Replace ${componentName} with actual name.
- * Add to styles/ or alongside the component.
- * Import in styles/index.ts or component file.
+ * Replace ${componentName} with the actual name. Put it in styles/ (or a styles/ / subfolder) and import it
+ * from styles/index.ts, or import it from the component file.
*
- * Rules:
- * - All classes prefixed with .jx-
- * - Colors use CSS variables (--color-*), see references/ui-design-spec.md
- * - BEM-like: .jx-name (block), .jx-name-child (element), .jx-name--mod (modifier)
- * - Spacing: 4px grid (--space-xs/sm/md/lg/xl)
- * - Border-radius: --radius-xs(4) / --radius-sm(8) / --radius-md(12) / --radius-lg(20)
- * - Font size: 12/14(default)/16/18/22/44px
+ * Rules — see references/ui-design-spec.md and references/css-variables.md:
+ * - Classes prefixed .jx- ; BEM-like: .jx-name / .jx-name-child / .jx-name--mod
+ * - Colors only through tokens (npm run check:dark fails on literals, rgba(), and var(--x, #fff) fallbacks)
+ * - 8-point grid: 4 / 8 / 12 / 16 / 24 / 32 / 48 / 64. Do not use --space-lg (20) / --space-xl (40) in new code
+ * - Concentric radii: inner radius = outer radius − padding. --radius-xs 6 / sm 10 / md 16 / lg 24 / full 999px
+ * - Resting cards have no shadow; shadow only on hover (raised) and on floating layers
+ * - Hover/press = overlay that darkens, never a lighter color; disabled = opacity .4
+ * - Motion: only opacity/transform, tokens for duration/easing, never `ease` / `ease-out` / new cubic-bezier
+ * - Font weights 400 / 500 / 600 only; sizes in rem
+ *
+ * v2 tokens marked (v2) may not be in variables.css yet: grep first; if missing, add them to BOTH the light
+ * :root block and the dark block (values from assets/design-v2/tokens/hugagentos-tokens.v2.css) before use.
*/
-/* --- Container --- */
+/* --- Block --- */
.jx-${componentName} {
display: flex;
flex-direction: column;
- gap: var(--space-md); /* 16px */
- padding: var(--space-lg); /* 20px */
+ gap: var(--space-md); /* 16 */
+ padding: 24px; /* --space-6 (v2) */
+ color: var(--color-text);
}
-/* --- Header --- */
+/* --- Header: hierarchy from size/weight/spacing, not boxes --- */
.jx-${componentName}-header {
display: flex;
align-items: center;
justify-content: space-between;
+ gap: 12px; /* --space-3 (v2) */
}
-.jx-${componentName}-header h3 {
+.jx-${componentName}-title {
margin: 0;
- font-size: 16px;
+ font-size: 1.125rem; /* title-3: --font-size-title-3 (v2) */
+ line-height: 1.625rem;
font-weight: 600;
- color: var(--color-text);
}
/* --- List --- */
.jx-${componentName}-list {
display: flex;
flex-direction: column;
- gap: var(--space-sm); /* 8px */
+ gap: var(--space-sm); /* 8 */
+ margin: 0;
+ padding: 0;
+ list-style: none;
}
-/* --- Item --- */
+/* --- L1 item: resting = no border, no shadow; grouped by background + spacing --- */
.jx-${componentName}-item {
display: flex;
align-items: center;
- gap: var(--space-md); /* 16px */
- padding: var(--space-md); /* 16px */
- border-radius: var(--radius-md); /* 12px - card level */
- border: 1px solid var(--color-border);
- background: #fff;
- transition: box-shadow 0.2s, border-color 0.2s;
+ gap: 12px;
+ min-height: 32px; /* default control height; touch targets ≥ 44 */
+ padding: 12px var(--space-md);
+ border-radius: var(--radius-md);
+ background: var(--color-bg-container);
+ font-size: 0.875rem; /* body */
+ line-height: 1.375rem;
cursor: pointer;
+ transition:
+ transform var(--motion-duration-fast) var(--motion-ease-standard),
+ box-shadow var(--motion-duration-fast) var(--motion-ease-standard);
}
.jx-${componentName}-item:hover {
- border-color: var(--color-primary-hover);
- box-shadow: 0 4px 12px rgba(18, 109, 255, 0.08);
+ transform: translateY(-1px);
+ box-shadow: var(--shadow-raised); /* (v2) — until merged: var(--shadow-card-hover) */
}
-/* --- Item active modifier --- */
+.jx-${componentName}-item:active {
+ transform: scale(.97);
+}
+
+/* Selected: primary light background + primary text; no extra border */
.jx-${componentName}-item--active {
- border-color: var(--color-primary);
background: var(--color-primary-light);
+ color: var(--color-primary-text); /* (v2) — until merged: var(--color-primary) */
}
-/* --- Item disabled modifier --- */
.jx-${componentName}-item--disabled {
- opacity: 0.5;
+ opacity: .4;
cursor: not-allowed;
}
-/* --- Empty state --- */
+/* Secondary metadata: footnote/caption + tertiary text; numbers tabular */
+.jx-${componentName}-meta {
+ font-size: 0.8125rem; /* footnote */
+ color: var(--color-text-tertiary);
+ font-variant-numeric: tabular-nums;
+}
+
+/* Semantic hint: light background + *-text foreground, never the fill color as text */
+.jx-${componentName}-hint--error {
+ padding: var(--space-sm) 12px;
+ border-radius: var(--radius-sm);
+ background: var(--color-error-bg); /* v2 value = v1 --color-error-light */
+ color: var(--color-error-text); /* (v2) */
+}
+
+/* --- Empty state: one plain sentence, no illustration box --- */
.jx-${componentName}-empty {
- display: flex;
- flex-direction: column;
- align-items: center;
- justify-content: center;
- padding: 60px 0;
+ padding: 48px 0; /* --space-12 (v2) */
+ text-align: center;
+ font-size: 0.875rem;
color: var(--color-text-tertiary);
- font-size: 14px;
}
-/* --- Responsive --- */
+/* --- Entrance: reuse motion primitives (.jx-anim-fadeInUp etc.) instead of new keyframes --- */
+
+/* --- Narrow container / mobile --- */
@media (max-width: 768px) {
.jx-${componentName} {
- padding: var(--space-md); /* 16px */
+ padding: var(--space-md); /* 16 + safe area where relevant */
}
.jx-${componentName}-item {
- padding: 10px var(--space-md);
+ min-height: 44px;
}
}
diff --git a/.claude/skills/hugagent-frontend-dev/templates/hook.ts b/.claude/skills/hugagent-frontend-dev/templates/hook.ts
index 9b09e939..284e8301 100644
--- a/.claude/skills/hugagent-frontend-dev/templates/hook.ts
+++ b/.claude/skills/hugagent-frontend-dev/templates/hook.ts
@@ -2,87 +2,35 @@
* Custom hook template.
*
* Replace ${Feature}, ${feature} with actual names.
- * Create as hooks/use${Feature}.ts
+ * Create as hooks/use${Feature}.ts. hooks/index.ts only re-exports the most common hooks;
+ * import others by deep path.
+ *
+ * Rules:
+ * - Network calls go through API functions (api.ts / src/api/.ts), which use apiRequest()
+ * - Every request started by the hook is aborted on unmount or when superseded
+ * - Effects are split by concern; no conditional hooks
*/
-import { useState, useEffect, useCallback, useRef } from 'react';
-// import { use${Feature}Store } from '../stores';
-// import { authFetch } from '../api';
-
-// ---------------------------------------------------------------------------
-// Hook
-// ---------------------------------------------------------------------------
-
-export function use${Feature}(apiUrl: string) {
- // --- Store state ---
- // const { items, setItems, loading, setLoading } = use${Feature}Store();
+import { useCallback, useEffect, useRef } from 'react';
+import { use${Feature}Store } from '../stores';
- // --- Local state ---
- const [initialized, setInitialized] = useState(false);
-
- // --- Refs ---
+export function use${Feature}() {
+ const fetchItems = use${Feature}Store((s) => s.fetchItems);
+ const loading = use${Feature}Store((s) => s.loading);
const abortRef = useRef(null);
- const timerRef = useRef | null>(null);
-
- // --- Initialization ---
- useEffect(() => {
- if (initialized) return;
- const init = async () => {
- try {
- // const r = await authFetch(`${apiUrl}/v1/${feature}s`);
- // const { data } = await r.json();
- // setItems(data.items);
- setInitialized(true);
- } catch (e) {
- console.error('Failed to init ${feature}:', e);
- }
- };
-
- init();
- }, [apiUrl, initialized]);
-
- // --- Cleanup ---
- useEffect(() => {
- return () => {
- abortRef.current?.abort();
- if (timerRef.current) clearTimeout(timerRef.current);
- };
- }, []);
-
- // --- Actions ---
const refresh = useCallback(async () => {
- // setLoading(true);
- try {
- // const r = await authFetch(`${apiUrl}/v1/${feature}s`);
- // const { data } = await r.json();
- // setItems(data.items);
- } catch (e) {
- console.error('Refresh failed:', e);
- } finally {
- // setLoading(false);
- }
- }, [apiUrl]);
+ abortRef.current?.abort();
+ const controller = new AbortController();
+ abortRef.current = controller;
+ await fetchItems(controller.signal);
+ }, [fetchItems]);
- const doSomething = useCallback(async (id: string) => {
- abortRef.current = new AbortController();
- try {
- // await authFetch(`${apiUrl}/v1/${feature}s/${id}/action`, {
- // method: 'POST',
- // signal: abortRef.current.signal,
- // });
- } catch (e) {
- if ((e as Error).name !== 'AbortError') {
- console.error('Action failed:', e);
- }
- }
- }, [apiUrl]);
+ // Initial load + cleanup.
+ useEffect(() => {
+ void refresh();
+ return () => abortRef.current?.abort();
+ }, [refresh]);
- // --- Return ---
- return {
- initialized,
- refresh,
- doSomething,
- abortRef,
- };
+ return { loading, refresh };
}
diff --git a/.claude/skills/hugagent-frontend-dev/templates/store.ts b/.claude/skills/hugagent-frontend-dev/templates/store.ts
index e4fd65a6..6b6defa7 100644
--- a/.claude/skills/hugagent-frontend-dev/templates/store.ts
+++ b/.claude/skills/hugagent-frontend-dev/templates/store.ts
@@ -2,130 +2,101 @@
* Zustand store template.
*
* Replace ${Feature}, ${feature} with actual names.
- * Create as stores/${feature}Store.ts
+ * Create as stores/${feature}Store.ts and re-export it from stores/index.ts.
+ *
+ * Rules:
+ * - Server data comes from API functions (api.ts or src/api/.ts), never raw fetch here
+ * - Business data cached in localStorage MUST be keyed per account with userScopedKey(base, userId)
+ * (a null key means "not signed in yet": do not read or write)
+ * - Clear account-scoped state on logout / account switch (reset())
*/
import { create } from 'zustand';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
+import { loadJsonPref, removeLocal, saveJsonPref, userScopedKey } from '../storage';
+// import { list${Feature}s, create${Feature}, delete${Feature} } from '../api/${feature}';
interface ${Feature}Item {
id: string;
name: string;
- // ... add fields
}
-// ---------------------------------------------------------------------------
-// State Interface (state + actions)
-// ---------------------------------------------------------------------------
-
interface ${Feature}State {
- // --- State ---
+ userId: string | null;
items: ${Feature}Item[];
loading: boolean;
selectedId: string | null;
error: string | null;
- // --- Actions ---
- setItems: (items: ${Feature}Item[]) => void;
- setLoading: (loading: boolean) => void;
+ bindUser: (userId: string | null) => void;
selectItem: (id: string | null) => void;
- fetchItems: () => Promise;
- createItem: (data: Partial<${Feature}Item>) => Promise;
+ fetchItems: (signal?: AbortSignal) => Promise;
+ createItem: (name: string) => Promise<${Feature}Item>;
deleteItem: (id: string) => Promise;
reset: () => void;
}
-// ---------------------------------------------------------------------------
-// localStorage persistence (optional)
-// ---------------------------------------------------------------------------
-
-const STORAGE_KEY = 'hugagent_ui_${feature}_v1';
+const STORAGE_BASE = 'hugagent_ui_${feature}_v1';
-function loadFromStorage(): ${Feature}Item[] {
- try {
- const raw = localStorage.getItem(STORAGE_KEY);
- return raw ? JSON.parse(raw) : [];
- } catch {
- return [];
- }
+function persist(userId: string | null, items: ${Feature}Item[]) {
+ const key = userScopedKey(STORAGE_BASE, userId);
+ if (key) saveJsonPref(key, { items });
}
-function saveToStorage(items: ${Feature}Item[]) {
- localStorage.setItem(STORAGE_KEY, JSON.stringify(items));
-}
-
-// ---------------------------------------------------------------------------
-// Store
-// ---------------------------------------------------------------------------
-
export const use${Feature}Store = create<${Feature}State>((set, get) => ({
- // --- Initial state ---
- items: loadFromStorage(),
+ userId: null,
+ items: [],
loading: false,
selectedId: null,
error: null,
- // --- Simple setters ---
- setItems: (items) => {
- set({ items });
- saveToStorage(items);
+ bindUser: (userId) => {
+ const key = userScopedKey(STORAGE_BASE, userId);
+ const cached = key ? loadJsonPref<{ items: ${Feature}Item[] }>(key, { items: [] }) : { items: [] };
+ set({ userId, items: cached.items, selectedId: null, error: null });
},
- setLoading: (loading) => set({ loading }),
+
selectItem: (id) => set({ selectedId: id }),
- // --- Async: fetch ---
- fetchItems: async () => {
+ fetchItems: async (signal) => {
set({ loading: true, error: null });
try {
- // const r = await authFetch(`${apiUrl}/v1/${feature}s`);
- // const { data } = await r.json();
- // set({ items: data.items });
- // saveToStorage(data.items);
+ const items: ${Feature}Item[] = []; // = await list${Feature}s(signal);
+ void signal;
+ set({ items });
+ persist(get().userId, items);
} catch (e) {
+ if ((e as Error).name === 'AbortError') return;
set({ error: (e as Error).message });
- console.error('Failed to fetch ${feature}s:', e);
} finally {
set({ loading: false });
}
},
- // --- Async: create ---
- createItem: async (data) => {
- try {
- // const r = await authFetch(`${apiUrl}/v1/${feature}s`, {
- // method: 'POST',
- // headers: { 'Content-Type': 'application/json' },
- // body: JSON.stringify(data),
- // });
- // const { data: created } = await r.json();
- // const items = [...get().items, created];
- // set({ items });
- // saveToStorage(items);
- } catch (e) {
- console.error('Failed to create ${feature}:', e);
- throw e;
- }
+ createItem: async (name) => {
+ const created: ${Feature}Item = { id: '', name }; // = await create${Feature}(name);
+ const items = [created, ...get().items];
+ set({ items });
+ persist(get().userId, items);
+ return created;
},
- // --- Async: delete ---
+ // Optimistic delete with rollback: the list stays truthful if the request fails.
deleteItem: async (id) => {
+ const previous = get().items;
+ const items = previous.filter((item) => item.id !== id);
+ set({ items, selectedId: get().selectedId === id ? null : get().selectedId });
try {
- // await authFetch(`${apiUrl}/v1/${feature}s/${id}`, { method: 'DELETE' });
- const items = get().items.filter((i) => i.id !== id);
- set({ items, selectedId: get().selectedId === id ? null : get().selectedId });
- saveToStorage(items);
+ // await delete${Feature}(id);
+ persist(get().userId, items);
} catch (e) {
- console.error('Failed to delete ${feature}:', e);
+ set({ items: previous });
throw e;
}
},
- // --- Reset ---
reset: () => {
- set({ items: [], loading: false, selectedId: null, error: null });
- localStorage.removeItem(STORAGE_KEY);
+ const key = userScopedKey(STORAGE_BASE, get().userId);
+ if (key) removeLocal(key);
+ set({ userId: null, items: [], loading: false, selectedId: null, error: null });
},
}));
diff --git a/.env.example b/.env.example
index 68b1ad2c..054c0a41 100644
--- a/.env.example
+++ b/.env.example
@@ -41,7 +41,7 @@ VITE_DEFAULT_LANGUAGE=en
# ── 存储(社区版:本地文件) ─────────────────────────────────────────────────
STORAGE_TYPE=s3
-COMPOSE_FILE=docker-compose.yml:docker/compose.object-storage.yml
+COMPOSE_FILE=docker-compose.yml:docker/object-storage/compose.object-storage.yml
S3_ENDPOINT=http://object-storage:9000
S3_BUCKET=hugagent-files
S3_REGION=us-east-1
@@ -86,10 +86,10 @@ REDIS_HOST_PORT=6380
# Personal Agent (CE/EE): optional module, off unless enabled here.
# CE allows authenticated users; EE also applies account/role/team permission.
# No Personal Agent license feature is required. Start the dedicated
-# controller with docker/compose.personal-agent.yml and --profile personal-agent.
+# controller with docker/personal-agent/compose.personal-agent.yml and --profile personal-agent.
PERSONAL_AGENT_ENABLED=false
# Build a versioned graphical image before use.
-# docker build -f docker/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .
+# docker build -f docker/personal-agent/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .
# Do not overwrite a tag used by a running permanent computer.
PERSONAL_AGENT_IMAGE=hugagent-personal-agent:v5
# Backend API reachable from the permanent computer. No credential value here.
diff --git a/.gitignore b/.gitignore
index f99ef493..ac636bd6 100755
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,7 @@
# Runtime-generated catalog (synced from DB + filesystem at startup)
src/backend/configs/catalog.json
-# OpenSandbox config — generated from docker/opensandbox-config.toml.tpl by the
+# OpenSandbox config — generated from docker/opensandbox-server/opensandbox-config.toml.tpl by the
# opensandbox-config-init service at compose-up (sed 把 @@HOST_REPO_PATH@@
# 替换成各机 .env 里的实际仓库路径)。.tpl 入库,实文件不入库。
opensandbox-config.toml
@@ -60,6 +60,8 @@ spike/
.DS_Store
# Project specific
+# 根目录 examples/(示例工程)仅本地使用,不入库
+/examples/
output/
images/
logs/
@@ -164,10 +166,12 @@ deploy_kit/
node_modules/
# 建站模板必须完整入库(覆盖上面的 lib/ 与 AGENTS.md 规则),但依赖树除外
-!docker/site-template/**
-docker/site-template/react-vite/node_modules/
+!docker/sandbox/site-template/**
+docker/sandbox/site-template/react-vite/node_modules/
.jobtest/
# Local scratch scripts and verification artifacts
/.scratch/
+# macOS native release archives (desktop/scripts/workbench-macos-native.mjs)
+/release-archive/
.ageval
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 365c5fa5..f3ae4788 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
.PHONY: help install install-dev dev test selftest lint format type-check security-scan \
- migrate migrate-new migrate-down migrate-history db-reset db-seed \
+ migrate migrate-new migrate-down migrate-history db-reset \
build build-no-cache up up-recreate down down-volumes restart logs logs-backend logs-frontend ps \
health-check clean clean-docker clean-all
@@ -32,7 +32,6 @@ help:
@echo " make migrate-down - Rollback one migration"
@echo " make migrate-history - Show migration history"
@echo " make db-reset - Reset DB to latest (destructive)"
- @echo " make db-seed - Seed sample data"
@echo ""
@echo "Docker:"
@echo " make build - Build images"
@@ -105,9 +104,6 @@ db-reset:
alembic downgrade base
alembic upgrade head
-db-seed:
- PYTHONPATH=$(PYTHONPATH_BACKEND) $(PYTHON) src/backend/scripts/seed_data.py
-
build:
$(COMPOSE) build
diff --git a/README.md b/README.md
index c359056f..636b9550 100644
--- a/README.md
+++ b/README.md
@@ -3,16 +3,12 @@
- HugAgentOS: The Enterprise AgentOS for Ontology-Grounded Trustworthy Reasoning
+ HugAgentOS: the open-source Agent Harness that runs 24/7
- The open-source, self-hosted foundation for enterprise AI agents
-
-
-
- Give models the context and tools to retrieve knowledge, work with files,
- run code, and carry real tasks through to completion.
+ Hand it your question, then close your laptop —
+ the agent keeps working in the cloud and tells you when it's done.
@@ -37,8 +33,11 @@
-
-
+
+
+
+
+
@@ -48,11 +47,18 @@
-HugAgentOS is an enterprise-grade AgentOS that treats domain ontology as a
-control plane for agent reasoning, decisions, and actions. Its open-source
-Community Edition combines agentic chat, private knowledge-base RAG,
-sub-agents, MCP tools, Agent Skills, sandboxed execution, long-term memory,
-automation, and a data canvas in one self-hosted workspace.
+Models keep getting stronger, but getting an agent to **actually finish the job**
+depends on the layer around the model: run lifecycle, context assembly, tools and
+sandboxes, human approvals, memory, and failure recovery. That layer is the
+**agent harness**.
+
+HugAgentOS is an open-source, self-hosted agent harness built around the
+**durable agent**: your request runs **in the cloud**, not in your browser or on your
+computer. Close the tab, shut the laptop or go offline, and the task keeps moving;
+you can also hand off tasks and get results in the IM app you use. On top of
+that, HugAgentOS provides an always-on **Personal Agent** and runs in two forms —
+**HugAgent Cloud** (on your server) and **HugAgent Local** (on your desktop) — which
+can also be combined.
> [!NOTE]
> This Community repository is generated from the upstream main repository for
@@ -60,6 +66,18 @@ automation, and a data canvas in one self-hosted workspace.
> Issue or Discussion. Pull requests for documentation and examples are
> welcome. See [CONTRIBUTING.md](./CONTRIBUTING.md) for details.
+## At a glance
+
+| Situation | A typical chat assistant | HugAgentOS |
+|---|---|---|
+| You close your laptop | The task stops with it | The task keeps running in the cloud; come back to the result |
+| The server restarts or fails | In-flight work is lost | Another instance takes over from the last saved progress |
+| A step fails and retries | It may send the email twice | Operations with an uncertain outcome are never redone automatically |
+| A very long task | Bound by timeouts and turn limits | No business time limit; it runs until the job is done |
+| Reaching you | Waits for you to open the page | IM notifications, schedules and heartbeat reminders |
+| Where it runs | Cloud or local, pick one | HugAgent Cloud, HugAgent Local, or a local + cloud hybrid |
+| Over time | Starts from zero every time | Memory, procedures and skills accumulate, each approved and reversible |
+
## Demo
A 60-second tour: the same task run twice, with audited self-evolution in between —
@@ -68,140 +86,190 @@ approve it.
https://github.com/user-attachments/assets/635c5b35-10c0-47b9-a6f1-c73df9510e90
-## Quick start
+## ⏱️ Durable agent: close the laptop, the work goes on
-Use the one-command install to try it out, or Docker Compose for a long-running
-service with isolation. Either way you need an OpenAI-compatible or local model.
-**The initial account and password are both `admin` and must be changed on first
-sign-in; the Community Edition has no self-registration.**
+Once you hand a task to HugAgentOS, it runs in cloud services and sandboxes — **your
+device is just a window onto it**. Leave whenever you like, or come back on another
+device; you always see the same progress and results.
-### Option 1: one-command install (Linux / macOS / WSL2)
-
-Requires Python 3.11+, Node.js 20+, Git and `curl`. No Docker, PostgreSQL or
-Redis needed.
-
-```bash
-curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash
+```text
+09:00 In Feishu, tell the agent: "Research pricing for competitors A, B and C, put it in a table and write a comparison report"
+09:01 The agent replies "Started"; the task runs in a cloud sandbox → you close the laptop and go to a meeting
+10:40 It needs to send an email: an approval reminder arrives in Feishu, and you approve it on the web from your phone
+11:30 The report arrives in the same Feishu chat
+11:35 Reply "Add D as well" → the same task picks it up, no restart
```
-The installer fetches the source into `~/.hugagent/source`, creates an isolated
-Python environment, builds the web application and walks you through first-run
-setup; it then opens [http://127.0.0.1:3001](http://127.0.0.1:3001). Start it
-again later with `~/.hugagent/venv/bin/hugagent`.
+**Why it keeps going**
+
+- **Runs in the cloud, not on your device**: background tasks, schedules and heartbeats
+ run on the server, and working files live in a dedicated persistent sandbox. Closing
+ the page doesn't stop anything, and any device can pick up the live progress.
+- **Survives server failures**: progress is saved to the database step by step and
+ executed by leased workers. If a process or machine dies, another instance takes over
+ from the last saved progress instead of silently dropping the task.
+- **Never repeats risky actions**: emails, writes to external systems and other
+ operations record their intent and outcome. On recovery, anything that may or may not
+ have happened is not redone automatically — you confirm "executed" or "not executed".
+- **Runs until it's done**: long tasks and the
+ [autonomous loop](./document/en/modules/autonomous-loop.md) have no total time limit or
+ business turn cap; stall detection, failure circuit breakers and a hard backstop prevent
+ runaways. Waiting for your approval or answer doesn't consume the execution budget.
+- **Change your mind mid-run**: add requirements at any time and the agent takes them at
+ its next step, with no cancel-and-restart.
+- **Finds you when it needs you**: actions that need approval are approved on the web, with
+ a reminder in IM; until you answer, the task waits instead of acting on its own.
+
+> [!IMPORTANT]
+> Continuous cloud operation relies on PostgreSQL, Redis and OpenSandbox — deploy
+> **HugAgent Cloud** with Docker Compose. The one-command install uses SQLite and
+> in-process state and is meant for trials and development.
+
+## 🤖 Personal Agent: an agent that is always on
+
+Personal Agent is built on the durable harness above: **one conversation, one
+persistent identity, one permanent cloud computer**. You just say what you need; it
+decides whether to answer right away or start a background task, and reports back
+when the work is done.
+
+| Capability | Description |
+|---|---|
+| 💬 **One entry, parallel background tasks** | Small talk and status questions are answered on the spot; research, reports, code and computer work become background tasks (3 in parallel by default, adjustable from 1 to 8). Follow-ups go in the same chat and it decides which task they belong to |
+| 🖥️ **Permanent sandbox and cloud computer** | Each user gets a dedicated OpenSandbox with no expiry, with separate persistent volumes for the workspace and browser profile; a lost container is rebuilt on the same volumes |
+| ⏰ **Schedules and heartbeats** | "Remind me to check email at 8 every morning" creates a schedule; heartbeats check `HEARTBEAT.md` at an interval and only speak up when something needs you |
+| 📱 **Reachable on your channels** | Bind Feishu, DingTalk, WeChat or WeCom bots; each contact or group gets its own session and results go back where the request came from |
+| ✅ **Execution you control** | Email, computer input and session writes need approval; every email is approved individually, other actions can be allowed for the rest of a task; a rejection makes it find another way |
+| 🧠 **Persona and memory as files** | `SOUL.md`, `USER.md`, `MEMORY.md` and daily logs live in the workspace and can be viewed and edited in My Space; Dreaming periodically consolidates logs into long-term memory |
+| 💻 **Connect your own computer** | In the desktop client's local + cloud mode, connect a Mac, Windows or Linux machine in one click and grant screen control or local session management; the device only polls outbound and opens no inbound ports |
+
+Personal Agent is off by default in the Community Edition. In a HugAgent Cloud
+deployment, set `PERSONAL_AGENT_ENABLED=true` and enable the `personal-agent` Compose
+profile. See the [Personal Agent guide](./document/en/modules/personal-agent.md).
+
+## ☁️ HugAgent Cloud and 💻 HugAgent Local
+
+The same harness, placed wherever your data and compute should live.
+
+| | **HugAgent Cloud** | **HugAgent Local** | **Local + cloud (hybrid)** |
+|---|---|---|---|
+| Runs on | Your server or private cloud | Your computer | Cloud as control plane, local machine as execution plane |
+| Access | Browser, from any device | Desktop client (Windows / macOS / Linux) | Desktop client |
+| Data and state | PostgreSQL, Redis, S3-compatible storage | Local SQLite, Milvus Lite, local files | Cloud projects stay in the cloud; local projects stay on the machine |
+| Execution | Isolated OpenSandbox sandboxes with snapshots | Your real project directories, gated by folder grants and permission profiles | Both |
+| Best for | 24/7 Personal Agent, long runs, IM channels | Offline work, sensitive data, local code and files | An always-on cloud agent that can also operate your machine and projects |
+| Install with | Docker Compose | Desktop offline installer, or the one-command script | Choose "Local + cloud" in the desktop client |
+
+- **HugAgent Cloud** is the complete durable harness, for work that must stay online,
+ be reachable from any device and connect to IM channels. Each Docker sandbox uses a
+ separate bridge with the trusted backend and controller; sandbox host ports are not
+ published and peer sandboxes cannot reach each other's execd.
+- **HugAgent Local** ships its own private Python runtime and services, so **no Python,
+ Docker or database install is needed**. It can run fully offline with local models;
+ once a local project is bound, Bash and file tools work directly in the project's real
+ directory. A failed upgrade rolls back to the previous version and data backup.
+- **Hybrid mode** uses one cloud account: cloud conversations are stored in the cloud,
+ while only local projects' conversations, files and execution stay on the machine.
+ This is also how Personal Agent "connects this computer" so an always-on cloud agent
+ can operate your desktop.
+
+See [Desktop local mode](./document/en/deployment/desktop-local-mode.md) and
+[Local project workspace](./document/en/deployment/local-project-workspace.md).
+
+## 🧬 Self-evolution: better with real work, audited at every step
+
+HugAgentOS does not make the agent start from zero every time — and does not let it
+quietly learn the wrong things.
+
+- **Learning procedures from trajectories**: the post-response memory pipeline keeps
+ outcome-verified trajectories such as "failure → user correction → success" as
+ reusable **procedural** memory instead of facts that go stale; stable procedures can
+ be compiled further into skills.
+- **Three memory layers**: L1 personal profile, L2 vector procedural memory (Milvus,
+ optional) and L3 relationship graph (Neo4j, optional). Writes go through a durable
+ outbox so crash replays never add duplicates, and a sanitizer filters credentials.
+- **Approved one by one, reversible**: evolution proposals for memory and skills appear
+ as cards and only take effect after you confirm them; you can switch it off at any time.
+- **Dreaming for Personal Agent**: a daily job consolidates logs into `MEMORY.md` /
+ `USER.md`, with `DREAMS.md` recording sources, deduplication and conflicts; only what
+ you explicitly asked it to remember is promoted automatically.
+- **Traceable ontology evolution**: approvals, rejections, evidence and outcomes are
+ recorded and organized into versioned ontology proposals that apply after human review.
+
+## 🧰 More harness capabilities
+
+| Capability | Description |
+|---|---|
+| 🔁 **Autonomous loop** | Goals survive across turns until done; maker ≠ checker — a worker does the work, a read-only reviewer verifies the real output, and command-checkable requirements are verified by the driver itself |
+| 🗺️ **Plan Mode** | Plan first, then execute step by step with per-step MCP, skill and sub-agent bindings; resumable like any run |
+| 🤝 **Sub-agents** | Built-in explorer, worker and reviewer, plus your own sub-agents, routed automatically or by `@` mention |
+| 🔧 **MCP, Skills and plugins** | Built-in web search, page fetching, knowledge retrieval, charts and automation tools; an MCP marketplace, private MCP servers, Agent Skills and plugins |
+| 🧪 **Sandbox and artifacts** | Run code in OpenSandbox or a local subprocess sandbox, producing charts, reports, Office files, web pages and data canvases |
+| 📚 **Private knowledge-base RAG** | Document chunking, hybrid vector and keyword retrieval, optional reranking |
+| 🛡️ **Optional ontology control plane** | With a Domain Pack, deterministic rule checks run before tool execution and high-risk answers are reviewed by an independent reviewer or committee; switched off, the original flow is unchanged |
+| 🔌 **Replaceable models** | Connect cloud or local OpenAI-compatible models through one model service; reviewer, planner, memory and other roles can each use their own model |
-> [!WARNING]
-> The installation listens on `127.0.0.1` only. If you genuinely need remote
-> access, use `hugagent serve --host 0.0.0.0 --port 3001 --no-browser`, and set a
-> strong password, a firewall and HTTPS first. Do not expose it on an untrusted
-> network.
+## Quick start
-This path uses SQLite, in-process state and a local subprocess sandbox, which
-suits personal use and development. Options and troubleshooting are in the
-[one-command install guide](./document/en/deployment/quick-install.md).
+Every option needs an OpenAI-compatible or local model. **The initial account and
+password are both `admin` and must be changed on first sign-in; the Community Edition
+has no self-registration.**
-### Option 2: Docker Compose
+### HugAgent Cloud: Docker Compose (recommended for 24/7)
-Use this when you need PostgreSQL, Redis, an isolated sandbox and persistent
-volumes. Requires Git, Docker and Compose v2.
+Install Git, Docker, and Compose v2 first.
```bash
git clone https://github.com/ZJU-REAL/HugAgentOS.git
cd HugAgentOS
-python3 scripts/configure-ce-docker.py
-docker build -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+cp .env.example .env
+mkdir -p data/storage
docker compose build backend
docker compose up -d --build
```
-Docker CE uses OpenSandbox by default. Initialization creates private controller credentials and absolute host paths. It refuses to overwrite an existing .env. Local installation keeps subprocess execution.
-
-Open [http://localhost:3002](http://localhost:3002), then connect a model under
-**Settings → System → Model services**. Profiles, persistence and production
-configuration are covered in the
-[Docker Compose deployment guide](./document/en/deployment/docker-compose.md).
-
-## Why HugAgentOS
-
-The point is not another chat wrapper. It is putting the context, the execution
-capability, and the artifact management an agent needs to finish real work on a
-single path — and raising the domain ontology from a knowledge base to a
-**machine-executable control plane**, so that governed concepts, relations, rules
-and action contracts give the skill, memory and orchestration engines one shared
-business vocabulary.
-
-
-
-
- 🔌 Model-agnostic
- Connect cloud or local models through one model-service configuration,
- without locking the application to a single vendor.
-
-
- 🛠️ Actually does the work
- ReAct orchestrates MCP servers, skills and sandboxes, so the model can
- search, analyse, produce files and call external capabilities.
-
-
-
-
- 🧠 Knowledge and memory
- Private knowledge bases and layered memory supply long-term context across
- files and conversations.
-
-
- 🏠 Your data stays yours
- Application, database and file storage all run on your own infrastructure.
-
-
-
-
- 🛡️ Gated, trustworthy execution
- Candidate plans pass deterministic rule checks, risk-tiered evidence review
- and a gate. A violating action returns with the rule, the evidence and a
- correction — it is never waved through silently.
-
-
- 🔎 Traceable evolution
- Approvals, rejections, evidence and outcomes are all recorded, distilled
- into versioned ontology proposals that take effect only after human review,
- and can be rolled back.
-
-
-
+Open [http://localhost:3002](http://localhost:3002), sign in, and connect a model
+under **Settings → System administration → Model services**. See the
+[Docker Compose guide](./document/en/deployment/docker-compose.md) for persistence
+and production settings, and [Deploy Personal Agent](./document/en/modules/personal-agent.md#deploy)
+to enable it.
-> [!NOTE]
-> The ontology trust control plane is an enterprise target architecture being
-> integrated into the existing harness in stages. It strengthens structured
-> compliance and evidence-based review; it does not promise "zero hallucination"
-> for free text.
+### HugAgent Local: desktop client
-## Core capabilities
+Download the installer for your platform from
+[Releases](https://github.com/ZJU-REAL/HugAgentOS/releases) and choose "Local only",
+"Cloud only" or "Local + cloud" on first launch. The full offline package bundles its
+runtime, so nothing else needs to be installed.
-The Community Edition covers the full loop for a personal agent — conversation,
-execution, consolidation and reuse. Optional components are enabled as needed.
+### Developer trial: one-command install (Linux / macOS / WSL2)
-| Capability | What it gives you |
-|---|---|
-| 💬 **Conversation and plan mode** | SSE streaming, ReAct tool orchestration, deep thinking, plan mode, citations, resumable runs |
-| 📚 **Private knowledge base RAG** | Document chunking, hybrid vector and keyword retrieval, optional reranking, per-user isolation |
-| 🤝 **Personal sub-agents** | Sub-agents with distinct roles, reached by automatic routing or an `@` mention |
-| 🔧 **MCP tool ecosystem** | Built-in web search, page fetch, knowledge retrieval, batch runs, automation, skill management |
-| 🧩 **Agent Skills** | Extend the agent with standardised skill definitions and scripts — built-in, marketplace and personal |
-| ⚙️ **Automation and batch runs** | Create scheduled tasks in natural language; run one process across an Excel sheet, a Word file or a file list |
-| 💬 **Group chat channels** | Feishu / DingTalk / WeCom bots, with optional group listening and history retrieval so the agent can see the conversation around it |
-| 🧪 **Sandbox and artifacts** | Run code locally or in an OpenSandbox container, producing charts, reports, Office files, web pages and data canvases |
-| 🧠 **Three-layer personal memory** | L1 profile in the relational store; optional Milvus vector memory and Neo4j graph memory |
-| 🧬 **Personal evolution** | Settle memory and skills out of your real work, each approved individually before it applies to you, and switchable off |
-| 🗂️ **Personal workspace** | Projects, folders, favourites, conversation sharing and an artifact center |
-| 📊 **Data canvas** | Inspect and edit structured data inside the conversation, keeping analysis and result in one workspace |
+Requires Python 3.11+, Node.js 20+, Git and `curl`. No Docker, PostgreSQL or
+Redis needed.
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash
+```
+
+The installer clones the source into `~/.hugagent/source`, creates an isolated
+Python environment, builds the web app and runs the first-time setup wizard, then
+opens [http://127.0.0.1:3001](http://127.0.0.1:3001). Start it again later with
+`~/.hugagent/venv/bin/hugagent`.
+
+> [!WARNING]
+> It listens on `127.0.0.1` only by default. If you really need remote access, use
+> `hugagent serve --host 0.0.0.0 --port 3001 --no-browser`, and set up a strong
+> password, a firewall and HTTPS first. Don't expose it directly on an untrusted network.
+
+The one-command install uses SQLite, in-process state and a local subprocess
+sandbox, which suits trials and development. See the
+[quick install guide](./document/en/deployment/quick-install.md) for options and
+troubleshooting.
## Architecture
-HugAgentOS separates user channels, agent workflows, reusable capability
-engines, ontology contracts, data governance, and infrastructure into clear
-layers. Action contracts connect the ontology layer to planning, validation,
-and gated execution, while security and platform governance span the complete
-stack.
+HugAgentOS separates user channels, agent workflows, reusable capability engines,
+ontology contracts, data governance, and infrastructure into clear layers. Durable
+runs, leases and recovery run through the execution path, while security and
+platform governance span the complete stack.

@@ -212,35 +280,35 @@ stack.
### Technology stack
-The project combines mature, replaceable open-source components behind clear
-service boundaries.
-
| Layer | Main technologies |
|---|---|
| Agent runtime | AgentScope 2.0, ReAct, Model Context Protocol |
| Backend | Python, FastAPI, SQLAlchemy, Alembic |
| Frontend | React 19, TypeScript, Vite, Zustand, Ant Design |
-| Data and state | SQLite or PostgreSQL 15, in-process state or Redis 7, local file storage |
+| Desktop | Tauri 2, private CPython 3.11 runtime |
+| Data and state | SQLite or PostgreSQL 15, in-process state or Redis 7, local or S3-compatible storage |
+| Sandbox | OpenSandbox, local subprocess |
| Optional memory | Milvus 2.4, Neo4j 5 Community, mem0 |
-| Deployment | One-command local installer, Docker Compose, Nginx |
+| Deployment | Desktop installer, one-command installer, Docker Compose, Nginx |
See the [architecture overview](./document/en/architecture/overview.md) for the
full request lifecycle, container topology, and design decisions.
## Community and Enterprise editions
-Community Edition gives an individual a complete agent workspace. Enterprise
-Edition adds the governance, collaboration, and delivery capabilities needed
-to operate the same experience across an organization. Enterprise-only source
-is physically absent from the Community tree.
+Community Edition lets an individual run the complete harness and Personal Agent.
+Enterprise Edition adds the governance, collaboration, and delivery capabilities
+needed to operate the same experience across an organization. Enterprise-only
+source is physically absent from the Community tree.
| Community Edition | Enterprise Edition adds |
|---|---|
-| Agentic chat, Plan Mode, and personal sub-agents | Teams, organization agents, and permission matrices |
-| 8 general MCP tools, personal skills, and a skill marketplace | Industry data tools, organization governance, and skill review |
-| Private knowledge bases and three-tier personal memory | Public knowledge administration and memory auditing |
+| Durable runs, autonomous loop, Plan Mode, and personal sub-agents | Teams, organization agents, and permission matrices |
+| Personal Agent: background tasks, dedicated computer, channels, schedules, and file memory | User, role, and team authorization, plus cross-device session management |
+| General MCP tools, personal skills, and a skill marketplace | Industry data tools, organization governance, and skill review |
+| Private knowledge bases, three-tier personal memory, and personal evolution | Public knowledge administration and memory auditing |
| Automation, and a personal data canvas | Organization billing, usage reports, and canvas collaboration |
-| OpenSandbox sessions, snapshots, and S3 file storage | Team workspace mounts, Cube, additional cloud providers, and offline delivery |
+| OpenSandbox sessions, snapshots, and S3 file storage; desktop local mode | Team workspace mounts, Cube, additional cloud providers, and offline delivery |
| Local accounts and branding with Powered-by attribution | SSO, compliance auditing, and full white-labeling |
See the [edition overview](./document/en/editions/overview.md) for the complete
@@ -248,26 +316,27 @@ feature boundary and upgrade path.
## Documentation
-The repository includes complete English and Chinese documentation for
-operators, users, and contributors, and you can read it offline.
-
| Goal | English | 中文文档 |
|---|---|---|
| Understand the product | [Introduction](./document/en/getting-started/introduction.md) | [产品简介](./document/zh-CN/getting-started/introduction.md) |
| Run it in 10 minutes | [Quick start](./document/en/getting-started/quick-start.md) | [快速开始](./document/zh-CN/getting-started/quick-start.md) |
+| Run lifecycle and resumable streams | [Chat](./document/en/modules/chat.md) | [智能对话](./document/zh-CN/modules/chat.md) |
+| The always-on personal agent | [Personal Agent](./document/en/modules/personal-agent.md) | [Personal Agent](./document/zh-CN/modules/personal-agent.md) |
+| Long-running self-driven tasks | [Autonomous loop](./document/en/modules/autonomous-loop.md) | [自主循环](./document/zh-CN/modules/autonomous-loop.md) |
+| Desktop local and hybrid modes | [Desktop local mode](./document/en/deployment/desktop-local-mode.md) | [桌面离线本机模式](./document/zh-CN/deployment/desktop-local-mode.md) |
+| Memory and self-evolution | [Memory](./document/en/modules/memory.md) | [记忆系统](./document/zh-CN/modules/memory.md) |
| Configure a deployment | [Deployment](./document/en/deployment/README.md) | [部署指南](./document/zh-CN/deployment/README.md) |
| Explore the system design | [Architecture](./document/en/architecture/overview.md) | [架构总览](./document/zh-CN/architecture/overview.md) |
-| Build a domain ontology | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) | [快速构建领域本体](./document/zh-CN/getting-started/domain-ontology-quickstart.md) |
-| Learn MCP, skills, memory, and sandboxing | [Modules](./document/en/README.md#modules) | [功能模块](./document/zh-CN/README.md#功能模块) |
+| The ontology control plane | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) | [快速构建领域本体](./document/zh-CN/getting-started/domain-ontology-quickstart.md) |
| Build backend or frontend features | [Development](./document/en/README.md#development) | [开发指南](./document/zh-CN/README.md#开发指南) |
Start from [document/README.md](./document/README.md) to browse every guide.
## Roadmap
-1. **Seamless cloud/local switching across clients** — one server deployment
- serving multiple clients, keeping conversations, agents, skills, files and task
- state in sync.
+1. **Seamless Cloud/Local switching across clients** — building on today's hybrid
+ mode, one server deployment serving multiple clients, keeping conversations,
+ agents, skills, files and task state in sync.
2. **Adaptive model routing based on Mixture of Agents** — select or combine models
by task complexity, modality, latency and cost: light models for simple work,
stronger ones only when the task warrants it.
@@ -299,5 +368,3 @@ supplementary terms. The terms restrict operating the software as a competing
multi-tenant SaaS offering and require the UI's Powered-by attribution to
remain visible. [LICENSE](./LICENSE) and [NOTICE](./NOTICE) define the complete
rights and obligations for internal use, modification, and distribution.
-
-Each Docker sandbox uses a separate bridge with the trusted backend and controller. Sandbox host ports are not published; peer sandboxes cannot access each other's execd. Use the bundled custom controller image.
diff --git a/README_CN.md b/README_CN.md
index 0502ea1b..a96fd4e2 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -3,16 +3,12 @@
- HugAgentOS:面向企业的本体驱动可信推理 AgentOS
+ HugAgentOS:7×24 小时持久运行的开源 Agent Harness
- 面向企业智能体的开源、自托管底座
-
-
-
- 让模型不只回答问题,还能检索知识、调用工具、处理文件、运行代码,
- 并持续完成真实任务。
+ 把问题交给它,然后合上电脑——
+ 智能体在云端继续干活,做完再告诉你。
@@ -37,8 +33,11 @@
-
-
+
+
+
+
+
@@ -48,16 +47,33 @@
-HugAgentOS 是面向企业级智能体场景的 AgentOS,把领域本体提升为推理、决策与
-行动的控制平面。其开源社区版将智能对话、私有知识库 RAG、子智能体、MCP 工具、
-Agent Skills、沙箱执行、长期记忆、自动化和数据画布整合到一个可私有部署的
-工作空间中。
+模型能力越来越强,但让智能体**真正把一件事干完**,靠的是模型之外的那一层:
+运行生命周期、上下文装配、工具与沙箱、人工审批、记忆、故障恢复。这一层就是
+**Agent Harness**。
+
+HugAgentOS 是一个开源、可自托管的 Agent Harness,核心是 **Durable Agent**:
+你提出的问题在**云端**执行,而不是依赖你的浏览器或电脑。关掉网页、合上笔记本、
+断网下班,任务都会继续推进;你也可以在常用的 IM 里下达任务、接收结果。在此之上,
+我们提供一个 7×24 小时在线的 **Personal Agent**,并以 **HugAgent Cloud**(服务器)
+和 **HugAgent Local**(桌面本机)两种形态运行,二者还可以混合使用。
> [!NOTE]
> 本社区仓库由上游主仓按发布版本自动生成,并标记为 `generated`。
> `src/**` 的修改请通过 Issue 或 Discussion 反馈;文档与示例欢迎直接提交 PR。
> 详细规则见 [CONTRIBUTING.md](./CONTRIBUTING.md)。
+## 一眼看懂
+
+| 场景 | 普通对话式 AI | HugAgentOS |
+|---|---|---|
+| 你合上电脑 | 任务跟着停下 | 任务在云端继续,回来直接看结果 |
+| 服务器重启或宕机 | 进行中的任务丢失 | 其他实例接管,从上次保存的进度继续 |
+| 中途失败重试 | 可能重复发邮件、重复写入 | 不确定是否成功的操作不会自动重做 |
+| 很长的任务 | 受超时和轮数上限约束 | 不设业务时限,跑到做完为止 |
+| 怎么找到你 | 只能等你打开网页 | IM 推送、定时任务、心跳主动提醒 |
+| 在哪运行 | 云端或本地二选一 | HugAgent Cloud / HugAgent Local / 本机 + 云端混合 |
+| 用得越久 | 每次从零开始 | 记忆、做法与技能持续沉淀,逐条审批、可回滚 |
+
## 演示
60 秒速览:同一个任务做两次——中间是受审计的自进化,记忆、技能与编排逐条沉淀,
@@ -65,36 +81,115 @@ Agent Skills、沙箱执行、长期记忆、自动化和数据画布整合到
https://github.com/user-attachments/assets/a268e371-a11f-488b-98d4-08598e2fdf89
-## 快速开始
+## ⏱️ Durable Agent:合上电脑,任务继续
-个人试用用一键安装,长期服务或需要服务隔离用 Docker Compose。两者都需要准备
-OpenAI 兼容模型或本地模型。**初始账号与密码均为 `admin`,首次登录必须修改;
-CE 不提供自助注册。**
+把问题交给 HugAgentOS 后,任务运行在云端的服务和沙箱里,**你的设备只是一个入口**。
+你可以随时离开,也可以换一台设备回来,看到的始终是同一份进度和结果。
-### 方式一:一键安装(Linux / macOS / WSL2)
+```text
+09:00 在飞书里对智能体说:「调研竞品 A、B、C 的定价,整理成表格并写一份对比报告」
+09:01 智能体回复「已开始」,任务在云端沙箱里运行 → 你合上电脑去开会
+10:40 要发一封邮件?飞书里收到审批提醒,你在手机上打开网页点「批准」
+11:30 同一个飞书对话里收到汇报:报告完成
+11:35 接着回一句「再加上 D」→ 同一个任务继续做,不用从头来
+```
-需要 Python 3.11+、Node.js 20+、Git 和 `curl`;不需要 Docker、PostgreSQL 或 Redis。
+**为什么能一直跑下去**
-```bash
-curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash
-```
+- **执行在云端,不依赖你的设备**:后台任务、定时任务和心跳都在服务端运行,任务的工作文件
+ 保存在专属的持久沙箱中。关掉页面不会停止任务,任意设备打开都能接着看实时进度。
+- **服务器出问题也不丢**:任务进度按步写入数据库,并由带租约的 worker 执行。进程或机器中断后,
+ 其他实例接管,从最后保存的进度继续,而不是悄悄消失。
+- **不会重复做有风险的事**:发邮件、写入外部系统等操作都会记录意图与结果;恢复时,不确定是否
+ 已经执行的操作不会自动重做,而是请你确认「已执行 / 未执行」。
+- **跑到做完为止**:长任务和[自主循环](./document/zh-CN/modules/autonomous-loop.md)不设总时长和
+ 业务轮次上限;停滞检测、异常熔断和硬性后备防止失控。等你审批或回答期间,不消耗执行预算。
+- **中途可以改主意**:运行中随时补充要求,智能体在下一步采纳,不用取消重来。
+- **需要你时会找你**:需要审批的操作会在 IM 中提醒、在网页上批准;你没回复之前,任务等待而不是擅自行动。
-安装器会把源码拉到 `~/.hugagent/source`,创建隔离 Python 环境、构建 Web 应用并进入
-首次配置向导,完成后打开 [http://127.0.0.1:3001](http://127.0.0.1:3001)。之后用
-`~/.hugagent/venv/bin/hugagent` 再次启动。
+> [!IMPORTANT]
+> 云端持续运行依赖 PostgreSQL、Redis 与 OpenSandbox,请使用 **HugAgent Cloud**
+> (Docker Compose)部署。一键安装使用 SQLite 与进程内状态,适合个人试用与开发。
-> [!WARNING]
-> 默认仅监听 `127.0.0.1`。确需远程访问时用
-> `hugagent serve --host 0.0.0.0 --port 3001 --no-browser`,并先配置强密码、防火墙
-> 与 HTTPS,不要直接暴露在不受信任的网络中。
+## 🤖 Personal Agent:一个一直在线的个人智能体
-一键安装使用 SQLite、进程内状态与本地子进程沙箱,适合个人试用与开发;选项与故障
-排查见[一键安装指南](./document/zh-CN/deployment/quick-install.md)。
+Personal Agent 构建在上述 Durable Harness 之上:**一个对话入口、一个持续存在的身份、
+一台永久的云端电脑**。你只管说事,它决定是当场回答,还是开一个后台任务去做,
+做完再回来汇报。
+
+| 能力 | 说明 |
+|---|---|
+| 💬 **单一入口,并行后台任务** | 闲聊、查进度当场回答;调研、写报告、跑代码、操作电脑自动转为后台任务(默认并行 3 个,可调 1–8)。补充要求直接说,它自己判断归属哪个任务 |
+| 🖥️ **永久沙盒与云电脑** | 每个用户一台不设到期时间的专属 OpenSandbox,工作目录与浏览器资料独立持久卷;容器丢失时复用卷重建 |
+| ⏰ **定时任务与心跳** | 「每天 8 点提醒我看邮件」创建定时任务;心跳按间隔对照 `HEARTBEAT.md` 自查,无事不打扰,有事才提醒 |
+| 📱 **多渠道在线** | 绑定飞书、钉钉、微信、企业微信机器人;每个联系人或群拥有独立会话,结果发回来源会话 |
+| ✅ **可控的执行权** | 邮件、电脑输入、会话写入需要批准;邮件逐封批准,其他可「本任务内同类操作都允许」;拒绝后改用其他方法 |
+| 🧠 **文件化的人格与记忆** | 工作区里的 `SOUL.md`、`USER.md`、`MEMORY.md` 与每日日志,可在「我的空间」直接查看与编辑;Dreaming 定期把日志整理为长期记忆 |
+| 💻 **连接你的电脑** | 在桌面客户端「本机 + 云端」模式下一键连接 Mac / Windows / Linux,按授权接管屏幕或管理本机会话;设备只出站轮询,不开放入站端口 |
+
+Personal Agent 在社区版中默认关闭,在 HugAgent Cloud 部署中设置
+`PERSONAL_AGENT_ENABLED=true` 并启用 `personal-agent` Compose profile 后即可使用。
+完整说明见 [Personal Agent 文档](./document/zh-CN/modules/personal-agent.md)。
+
+## ☁️ HugAgent Cloud 与 💻 HugAgent Local
+
+同一套 Harness,按你的数据和算力放在哪里来选择运行形态。
+
+| | **HugAgent Cloud** | **HugAgent Local** | **本机 + 云端(混合)** |
+|---|---|---|---|
+| 运行位置 | 你的服务器 / 私有云 | 你的电脑 | 云端为控制面,本机为执行面 |
+| 入口 | 浏览器,多端访问 | 桌面客户端(Windows / macOS / Linux) | 桌面客户端 |
+| 数据与状态 | PostgreSQL、Redis、S3 兼容存储 | 本地 SQLite、Milvus Lite、本地文件 | 云端项目存云端,本地项目留本机 |
+| 执行环境 | OpenSandbox 隔离沙箱,快照恢复 | 本机真实项目目录,按目录授权与权限档执行 | 两者兼有 |
+| 适合 | 7×24 Personal Agent、长跑任务、多渠道接入 | 离线、敏感数据、本地代码与文件 | 云端常驻 + 操作本机电脑与项目 |
+| 部署方式 | Docker Compose | 桌面离线安装包;或一键安装脚本 | 桌面客户端选择「本机 + 云端」 |
+
+- **HugAgent Cloud**:完整的 Durable Harness。适合需要一直在线、跨设备访问、接入 IM 渠道的场景。
+ 每个 Docker 沙箱与受信任的后端和控制器使用独立网桥,不发布沙箱宿主端口,沙箱之间不能互访 execd。
+- **HugAgent Local**:桌面安装包自带私有 Python 运行时与服务,**不需要安装 Python、Docker
+ 或数据库**,可完全离线运行并接入本地模型;绑定本地项目后,Bash 与文件工具直接在项目真实
+ 目录中工作。升级失败自动回滚到上一版本与数据备份。
+- **混合模式**:一套云端账号,云端会话存云端;只有本地项目的对话、文件与执行留在本机。
+ Personal Agent 可以借此「连接这台电脑」,让云端常驻的智能体操作你的本机。
+
+详见[桌面离线本机模式](./document/zh-CN/deployment/desktop-local-mode.md)与
+[本机项目工作目录](./document/zh-CN/deployment/local-project-workspace.md)。
+
+## 🧬 自进化:从真实工作中变强,每一步都可审计
+
+HugAgentOS 不让智能体每次从零开始,也不让它「悄悄学坏」。
+
+- **从轨迹中学做法**:后置记忆流水线会识别「失败 → 用户纠正 → 再次成功」这类经过结果验证
+ 的轨迹,沉淀为可复用的**做法(procedural)**,而不是记住会过期的事实;稳定做法可以进一步
+ 编译为技能。
+- **三层记忆**:L1 个人画像、L2 向量做法记忆(Milvus,可选)、L3 关系图谱(Neo4j,可选);
+ 写入走持久 Outbox,崩溃重放不会重复新增,脱敏闸门过滤凭据。
+- **逐条审批、可回滚**:记忆与技能的进化建议以卡片形式出现,经你确认才生效,可随时关闭。
+- **Personal Agent 的 Dreaming**:每天定时把日志整理为 `MEMORY.md` / `USER.md` 的更新,
+ `DREAMS.md` 记录来源、去重与冲突;只自动提升你明确说「记住」的内容。
+- **可溯的本体进化**:审批、驳回、证据与执行结果留痕,整理为版本化的本体建议,人工审核后生效。
+
+## 🧰 更多 Harness 能力
+
+| 能力 | 说明 |
+|---|---|
+| 🔁 **自主循环** | 目标跨轮存活、做完为止;maker≠checker:worker 干活、只读评审员亲验产出,命令可判定的需求由 driver 机检 |
+| 🗺️ **计划模式** | 先出计划再逐步执行,步骤级绑定 MCP / 技能 / 子智能体,同样支持断线续播 |
+| 🤝 **子智能体** | 内置 explorer / worker / reviewer,可创建个人子智能体,自动路由或 `@` 提及协同 |
+| 🔧 **MCP · Skills · 插件** | 内置联网搜索、网页抓取、知识检索、图表、自动化等工具;MCP 市场、私有 MCP、Agent Skills 与插件 |
+| 🧪 **沙箱与产物** | OpenSandbox 或子进程沙箱执行代码,产出图表、报告、Office 文件、网页与数据画布 |
+| 📚 **私有知识库 RAG** | 文档分块,向量与关键词混合检索,可选重排 |
+| 🛡️ **可选的本体控制平面** | 导入 Domain Pack 后,工具执行前做确定性规则检查,高风险答案由独立评审员或委员会复核;关闭时完全不改变原流程 |
+| 🔌 **模型可替换** | 统一的模型服务配置接入云端或本地 OpenAI 兼容模型,评审、规划、记忆等角色可分别指定模型 |
-### 方式二:Docker Compose
+## 快速开始
-需要 PostgreSQL、Redis、隔离沙箱与持久化卷时使用。需先安装 Git、Docker 与
-Compose v2。
+所有方式都需要准备 OpenAI 兼容模型或本地模型。**初始账号与密码均为 `admin`,首次登录必须修改;
+CE 不提供自助注册。**
+
+### HugAgent Cloud:Docker Compose(推荐用于 7×24)
+
+需先安装 Git、Docker 与 Compose v2。
```bash
git clone https://github.com/ZJU-REAL/HugAgentOS.git
@@ -106,112 +201,75 @@ docker compose up -d --build
```
打开 [http://localhost:3002](http://localhost:3002),登录后进入「设置 → 系统管理 →
-模型服务」接入模型。Profiles、持久化与生产配置见
-[Docker Compose 部署指南](./document/zh-CN/deployment/docker-compose.md)。
-
-## 为什么是 HugAgentOS
-
-重点不是再包装一个聊天界面,而是把智能体完成任务所需的上下文、执行能力和产物管理
-放在同一条链路里;并把领域本体从「知识库」提升为**机器可执行的控制平面**——受控的
-概念、关系、铁律与 Action 契约,让技能、记忆和编排三大引擎共享同一套业务语言。
-
-
-
-
- 🔌 模型可替换
- 统一的模型服务配置接入云端或本地模型,不锁定单一供应商。
-
-
- 🛠️ 能执行任务
- ReAct 编排 MCP、技能与沙箱,让模型能搜索、分析、生成文件并调用外部能力。
-
-
-
-
- 🧠 有知识与记忆
- 私有知识库与分层记忆提供跨文件、跨会话的长期上下文。
-
-
- 🏠 数据可自持
- 应用、数据库与文件存储都可运行在自己的基础设施中。
-
-
-
-
- 🛡️ 门控可信执行
- 候选计划依次经过确定性规则检查、风险分级证据评审与门控执行;违规动作会带着
- 规则、证据和修正建议返回,不会静默放行。
-
-
- 🔎 可溯的持续进化
- 审批、驳回、证据与执行结果均留痕,整理为版本化本体建议,人工审核后生效并可回滚。
-
-
-
+模型服务」接入模型。启用 Personal Agent、持久化与生产配置见
+[Docker Compose 部署指南](./document/zh-CN/deployment/docker-compose.md)和
+[Personal Agent 部署](./document/zh-CN/modules/personal-agent.md#部署)。
-> [!NOTE]
-> 本体可信控制平面是正在现有 Harness 上分阶段集成的企业级目标架构。它强化结构化
-> 合规和基于证据的评审,但不对自由文本作「零幻觉」承诺。
+### HugAgent Local:桌面客户端
-## 核心能力
+从 [Releases](https://github.com/ZJU-REAL/HugAgentOS/releases) 下载对应平台的桌面安装包,
+首次启动选择「仅本机」「仅云端」或「本机 + 云端」。完整离线包自带运行时,无需额外安装依赖。
-社区版覆盖个人智能体从对话、执行到沉淀复用的完整闭环,可选组件按需启用。
+### 开发者试用:一键安装(Linux / macOS / WSL2)
-| 能力 | 说明 |
-|---|---|
-| 💬 **智能对话与计划模式** | SSE 流式、ReAct 工具编排、深度思考、计划模式、引用溯源、断线续播 |
-| 📚 **私有知识库 RAG** | 文档分块、向量与关键词混合检索、可选重排、个人知识库隔离 |
-| 🤝 **个人子智能体** | 创建不同角色的子智能体,自动路由或 `@` 提及协同 |
-| 🔧 **MCP 工具生态** | 内置联网搜索、网页抓取、知识检索、图表、报告、批量、自动化、技能管理 |
-| 🧩 **Agent Skills** | 标准化技能说明 + 脚本扩展,支持内置、市场与个人技能 |
-| ⚙️ **定时自动化** | 自然语言创建定时任务 |
-| 💬 **群聊接入** | 飞书 / 钉钉 / 企微机器人;可选群聊旁听与历史回溯,让智能体看得到群里的上下文 |
-| 🧪 **沙箱与产物** | 子进程或轻量容器沙箱执行代码,产出图表、报告、Office 文件、网页与数据画布 |
-| 🧠 **三层个人记忆** | L1 画像存关系库;可选 Milvus 向量记忆与 Neo4j 图谱记忆 |
-| 🧬 **个人进化** | 从你的真实工作沉淀记忆与技能,逐条审批后才对你生效,可随时关闭 |
-| 🗂️ **个人工作空间** | 项目、文件夹、收藏、会话分享与产物中心 |
-| 📊 **数据画布** | 会话内查看与编辑结构化数据,分析过程与结果同处一个工作区 |
+需要 Python 3.11+、Node.js 20+、Git 和 `curl`;不需要 Docker、PostgreSQL 或 Redis。
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash
+```
+
+安装器会把源码拉到 `~/.hugagent/source`,创建隔离 Python 环境、构建 Web 应用并进入
+首次配置向导,完成后打开 [http://127.0.0.1:3001](http://127.0.0.1:3001)。之后用
+`~/.hugagent/venv/bin/hugagent` 再次启动。
+
+> [!WARNING]
+> 默认仅监听 `127.0.0.1`。确需远程访问时用
+> `hugagent serve --host 0.0.0.0 --port 3001 --no-browser`,并先配置强密码、防火墙
+> 与 HTTPS,不要直接暴露在不受信任的网络中。
+
+一键安装使用 SQLite、进程内状态与本地子进程沙箱,适合试用与开发;选项与故障
+排查见[一键安装指南](./document/zh-CN/deployment/quick-install.md)。
## 系统架构
HugAgentOS 将用户渠道、智能体工作流、可复用能力引擎、本体契约、数据治理和
-基础设施分层组织。Action 契约把本体层与计划生成、审批校验和门控执行连接起来,
-安全管理与平台监管则贯穿完整技术栈。
+基础设施分层组织;Durable Run、租约与恢复机制贯穿执行链路,安全管理与平台监管
+贯穿完整技术栈。

> [!NOTE]
-> 架构图展示 HugAgentOS 的完整产品架构。其中部分治理、协作、网关和持久沙箱
-> 能力仅在商业版中提供。
+> 架构图展示 HugAgentOS 的完整产品架构。其中部分治理、协作和网关能力仅在商业版中提供。
### 技术栈
-项目选择成熟、可替换的开源组件,并通过清晰的服务边界组合为完整产品。
-
| 层级 | 主要技术 |
|---|---|
| 智能体运行时 | AgentScope 2.0、ReAct、Model Context Protocol |
| 后端 | Python、FastAPI、SQLAlchemy、Alembic |
| 前端 | React 19、TypeScript、Vite、Zustand、Ant Design |
-| 数据与状态 | SQLite 或 PostgreSQL 15、进程内状态或 Redis 7、本地文件存储 |
+| 桌面端 | Tauri 2、私有 CPython 3.11 运行时 |
+| 数据与状态 | SQLite 或 PostgreSQL 15、进程内状态或 Redis 7、本地或 S3 兼容存储 |
+| 沙箱 | OpenSandbox、本地子进程 |
| 可选记忆 | Milvus 2.4、Neo4j 5 Community、mem0 |
-| 部署 | 本地一键安装、Docker Compose、Nginx |
+| 部署 | 桌面安装包、一键安装、Docker Compose、Nginx |
更详细的请求生命周期、容器拓扑和关键设计决策见
[架构总览](./document/zh-CN/architecture/overview.md)。
## 社区版与商业版
-社区版让个人把智能体能力完整运行起来;商业版在相同产品体验上补充组织级治理、
-协作和交付能力。商业版能力不会以关闭开关的形式混入社区版源码。
+社区版让个人把完整的 Harness 与 Personal Agent 运行起来;商业版在相同产品体验上补充
+组织级治理、协作和交付能力。商业版能力不会以关闭开关的形式混入社区版源码。
| 社区版(CE) | 商业版(EE)新增 |
|---|---|
-| 智能对话、计划模式与个人子智能体 | 团队、成员、组织级智能体库与权限矩阵 |
-| 7 个通用 MCP、个人技能与技能市场 | 行业数据工具、组织级能力治理与技能审核 |
-| 私有知识库、三层个人记忆 | 公共知识库管理与记忆审计 |
+| Durable Run、自主循环、计划模式与个人子智能体 | 团队、成员、组织级智能体库与权限矩阵 |
+| Personal Agent:后台任务、专属电脑、渠道、调度与文件记忆 | 用户、角色、团队授权治理与跨设备会话管理 |
+| 通用 MCP、个人技能与技能市场 | 行业数据工具、组织级能力治理与技能审核 |
+| 私有知识库、三层个人记忆与个人进化 | 公共知识库管理与记忆审计 |
| 自动化、个人数据画布 | 组织计费、用量汇总与画布多人协同 |
-| OpenSandbox、快照与 S3 文件存储 | 团队挂载、其他云存储与离线商业交付 |
+| OpenSandbox、快照与 S3 文件存储;桌面本机模式 | 团队挂载、其他云存储与离线商业交付 |
| 本地账号与保留 Powered-by 的品牌配置 | SSO、审计合规与完整 white-label |
功能边界与升级路径以
@@ -219,24 +277,26 @@ HugAgentOS 将用户渠道、智能体工作流、可复用能力引擎、本体
## 文档
-仓库内提供完整的中英文文档,从首次部署到架构和二次开发均可离线阅读。
-
| 你想了解 | 中文文档 | English |
|---|---|---|
| 项目定位与整体能力 | [产品简介](./document/zh-CN/getting-started/introduction.md) | [Introduction](./document/en/getting-started/introduction.md) |
| 10 分钟运行项目 | [快速开始](./document/zh-CN/getting-started/quick-start.md) | [Quick start](./document/en/getting-started/quick-start.md) |
+| Run 生命周期与断线续播 | [智能对话](./document/zh-CN/modules/chat.md) | [Chat](./document/en/modules/chat.md) |
+| 7×24 个人智能体 | [Personal Agent](./document/zh-CN/modules/personal-agent.md) | [Personal Agent](./document/en/modules/personal-agent.md) |
+| 长时运行的自驱动任务 | [自主循环](./document/zh-CN/modules/autonomous-loop.md) | [Autonomous loop](./document/en/modules/autonomous-loop.md) |
+| 桌面本机与混合模式 | [桌面离线本机模式](./document/zh-CN/deployment/desktop-local-mode.md) | [Desktop local mode](./document/en/deployment/desktop-local-mode.md) |
+| 记忆与自进化 | [记忆系统](./document/zh-CN/modules/memory.md) | [Memory](./document/en/modules/memory.md) |
| 生产部署与环境变量 | [部署指南](./document/zh-CN/deployment/README.md) | [Deployment](./document/en/deployment/README.md) |
| 系统设计与请求链路 | [架构总览](./document/zh-CN/architecture/overview.md) | [Architecture](./document/en/architecture/overview.md) |
-| 快速构建领域本体 | [领域本体快速入门](./document/zh-CN/getting-started/domain-ontology-quickstart.md) | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) |
-| MCP、技能、记忆与沙箱 | [功能模块](./document/zh-CN/README.md#功能模块) | [Modules](./document/en/README.md#modules) |
+| 领域本体控制平面 | [领域本体快速入门](./document/zh-CN/getting-started/domain-ontology-quickstart.md) | [Domain ontology quickstart](./document/en/getting-started/domain-ontology-quickstart.md) |
| 后端与前端开发 | [开发指南](./document/zh-CN/README.md#开发指南) | [Development](./document/en/README.md#development) |
所有文档入口见 [document/README.md](./document/README.md)。
## 路线图
-1. **云端 / 本地无缝切换与多端互通** —— 一套服务端连接多客户端,同步会话、智能体、
- 技能、文件与任务状态。
+1. **Cloud / Local 无缝切换与多端互通** —— 在现有混合模式基础上,一套服务端连接多客户端,
+ 同步会话、智能体、技能、文件与任务状态。
2. **基于 MoA 的自适应模型路由** —— 按任务复杂度、模态、时延与成本选择或组合模型,
简单任务用轻量模型,复杂任务再升级。
3. **更丰富的扩展生态** —— 持续扩充智能体、技能、MCP 服务与插件,完善发现、安装、
diff --git a/desktop/requirements-desktop-linux-aarch64-py311.lock b/desktop/requirements-desktop-linux-aarch64-py311.lock
index 27cb3904..1420921a 100644
--- a/desktop/requirements-desktop-linux-aarch64-py311.lock
+++ b/desktop/requirements-desktop-linux-aarch64-py311.lock
@@ -1,8 +1,8 @@
# This file is generated. Do not edit it by hand.
# Regenerate with: npm --prefix desktop run lock:desktop
-# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990
+# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046
# Target: CPython 3.11 on Linux aarch64 (UOS 1070 / manylinux 2.28)
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# via -r desktop/requirements-desktop.txt
aiofiles==25.1.0
# via agentscope
@@ -105,8 +105,6 @@ defusedxml==0.7.1
# via
# -r desktop/requirements-desktop.txt
# markitdown
-deprecated==1.3.1
- # via limits
dingtalk-stream==0.24.3
# via -r desktop/requirements-desktop.txt
distro==1.9.0
@@ -183,18 +181,18 @@ httpx==0.28.1
# huggingface-hub
# lark-oapi
# litellm
- # mcp
# mem0ai
# ollama
# openai
# qdrant-client
# tavily-python
httpx-sse==0.4.3
+ # via dashscope
+httpx2==2.12.0
# via
- # dashscope
+ # agentscope
+ # anthropic
# mcp
-httpx2==2.12.0
- # via anthropic
huggingface-hub==1.28.0
# via tokenizers
hyperframe==6.1.0
@@ -220,7 +218,7 @@ jmespath==1.1.0
# via
# boto3
# botocore
-json-repair==0.63.3
+json-repair==0.64.1
# via agentscope
json5==0.15.0
# via agentscope
@@ -228,6 +226,7 @@ jsonschema==4.26.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+ # json-repair
# litellm
# mcp
jsonschema-specifications==2025.9.1
@@ -236,8 +235,6 @@ kiwisolver==1.5.0
# via matplotlib
lark-oapi==1.7.3
# via -r desktop/requirements-desktop.txt
-limits==5.8.0
- # via slowapi
linkify-it-py==2.1.0
# via markdown-it-py
litellm==1.98.0
@@ -270,10 +267,12 @@ matplotlib==3.11.1
# via
# -r desktop/requirements-desktop.txt
# seaborn
-mcp==1.29.0
+mcp==2.3.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+mcp-types==2.3.0
+ # via mcp
mdit-py-plugins==0.6.1
# via textual
mdurl==0.1.2
@@ -316,6 +315,7 @@ openpyxl==3.1.5
opentelemetry-api==1.44.0
# via
# agentscope
+ # mcp
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# opentelemetry-sdk
@@ -348,7 +348,6 @@ packaging==26.3
# via
# faiss-cpu
# huggingface-hub
- # limits
# matplotlib
# onnxruntime
# pikepdf
@@ -415,8 +414,10 @@ pydantic==2.13.4
# dashscope
# fastapi
# google-genai
+ # json-repair
# litellm
# mcp
+ # mcp-types
# mem0ai
# ollama
# openai
@@ -425,9 +426,7 @@ pydantic==2.13.4
pydantic-core==2.46.4
# via pydantic
pydantic-settings==2.15.0
- # via
- # litellm
- # mcp
+ # via litellm
pyee==13.0.1
# via playwright
pygments==2.21.0
@@ -441,7 +440,9 @@ pymilvus==2.5.18
pyparsing==3.3.2
# via matplotlib
pypdf==6.16.2
- # via -r desktop/requirements-desktop.txt
+ # via
+ # -r desktop/requirements-desktop.txt
+ # agentscope
pypdf2==3.0.1
# via -r desktop/requirements-desktop.txt
pypdfium2==5.13.0
@@ -518,11 +519,10 @@ requests-toolbelt==1.0.0
# via lark-oapi
rich==15.0.0
# via
+ # agentscope
# dashscope
# textual
# typer
-ripgrep==14.1.0
- # via agentscope
rpds-py==2026.6.3
# via
# jsonschema
@@ -549,8 +549,6 @@ six==1.17.0
# via
# markdownify
# python-dateutil
-slowapi==0.1.10
- # via -r desktop/requirements-desktop.txt
sniffio==1.3.1
# via
# anthropic
@@ -564,7 +562,9 @@ sqlalchemy==2.0.52
# alembic
# mem0ai
sse-starlette==3.0.3
- # via mcp
+ # via
+ # -r desktop/requirements-desktop.txt
+ # mcp
starlette==0.46.2
# via
# fastapi
@@ -615,8 +615,8 @@ typing-extensions==4.16.0
# grpcio
# httpx2
# huggingface-hub
- # limits
# mcp
+ # mcp-types
# openai
# opentelemetry-api
# opentelemetry-exporter-otlp-proto-grpc
@@ -639,6 +639,8 @@ typing-inspection==0.4.4
# mcp
# pydantic
# pydantic-settings
+tzdata==2026.5
+ # via agentscope
uc-micro-py==2.0.0
# via linkify-it-py
ujson==5.13.0
@@ -667,8 +669,6 @@ werkzeug==3.1.8
# via -r desktop/requirements-desktop.txt
wheel==0.48.0
# via -r desktop/requirements-desktop.txt
-wrapt==2.3.0
- # via deprecated
wsproto==1.3.2
# via simple-websocket
xlsxwriter==3.2.9
diff --git a/desktop/requirements-desktop-linux-x86_64-py311.lock b/desktop/requirements-desktop-linux-x86_64-py311.lock
index 1ae48d54..a71239c2 100644
--- a/desktop/requirements-desktop-linux-x86_64-py311.lock
+++ b/desktop/requirements-desktop-linux-x86_64-py311.lock
@@ -1,8 +1,8 @@
# This file is generated. Do not edit it by hand.
# Regenerate with: npm --prefix desktop run lock:desktop
-# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990
+# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046
# Target: CPython 3.11 on Linux x86_64
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# via -r desktop/requirements-desktop.txt
aiofiles==25.1.0
# via agentscope
@@ -31,6 +31,7 @@ anyio==4.14.2
# anthropic
# google-genai
# httpx
+ # httpx2
# mcp
# openai
# sse-starlette
@@ -98,8 +99,6 @@ defusedxml==0.7.1
# via
# -r desktop/requirements-desktop.txt
# markitdown
-deprecated==1.3.1
- # via limits
dingtalk-stream==0.24.3
# via -r desktop/requirements-desktop.txt
distro==1.9.0
@@ -155,6 +154,7 @@ grpcio==1.83.0
h11==0.16.0
# via
# httpcore
+ # httpcore2
# uvicorn
# wsproto
h2==4.4.0
@@ -165,6 +165,8 @@ hpack==4.2.0
# via h2
httpcore==1.0.9
# via httpx
+httpcore2==2.16.0
+ # via httpx2
httpx==0.28.1
# via
# -r desktop/requirements-desktop.txt
@@ -175,15 +177,16 @@ httpx==0.28.1
# huggingface-hub
# lark-oapi
# litellm
- # mcp
# mem0ai
# ollama
# openai
# qdrant-client
# tavily-python
httpx-sse==0.4.3
+ # via dashscope
+httpx2==2.16.0
# via
- # dashscope
+ # agentscope
# mcp
huggingface-hub==1.26.0
# via tokenizers
@@ -193,6 +196,7 @@ idna==3.18
# via
# anyio
# httpx
+ # httpx2
# requests
# yarl
importlib-metadata==8.9.0
@@ -205,7 +209,7 @@ jiter==0.16.0
# via
# anthropic
# openai
-json-repair==0.61.7
+json-repair==0.64.1
# via agentscope
json5==0.15.0
# via agentscope
@@ -213,6 +217,7 @@ jsonschema==4.26.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+ # json-repair
# litellm
# mcp
jsonschema-specifications==2025.9.1
@@ -221,8 +226,6 @@ kiwisolver==1.5.0
# via matplotlib
lark-oapi==1.7.1
# via -r desktop/requirements-desktop.txt
-limits==5.8.0
- # via slowapi
litellm==1.94.1
# via -r desktop/requirements-desktop.txt
lxml==6.1.1
@@ -250,10 +253,12 @@ matplotlib==3.11.1
# via
# -r desktop/requirements-desktop.txt
# seaborn
-mcp==1.29.0
+mcp==2.3.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+mcp-types==2.3.0
+ # via mcp
mdurl==0.1.2
# via markdown-it-py
mem0ai==2.0.14
@@ -294,6 +299,7 @@ openpyxl==3.1.5
opentelemetry-api==1.44.0
# via
# agentscope
+ # mcp
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# opentelemetry-sdk
@@ -326,7 +332,6 @@ packaging==26.2
# via
# faiss-cpu
# huggingface-hub
- # limits
# matplotlib
# onnxruntime
# pikepdf
@@ -388,17 +393,16 @@ pydantic==2.13.4
# anthropic
# fastapi
# google-genai
+ # json-repair
# litellm
# mcp
+ # mcp-types
# mem0ai
# ollama
# openai
- # pydantic-settings
# qdrant-client
pydantic-core==2.46.4
# via pydantic
-pydantic-settings==2.14.2
- # via mcp
pyee==13.0.1
# via playwright
pygments==2.20.0
@@ -410,7 +414,9 @@ pymilvus==2.5.18
pyparsing==3.3.2
# via matplotlib
pypdf==6.14.2
- # via -r desktop/requirements-desktop.txt
+ # via
+ # -r desktop/requirements-desktop.txt
+ # agentscope
pypdf2==3.0.1
# via -r desktop/requirements-desktop.txt
pypdfium2==5.12.1
@@ -429,7 +435,6 @@ python-dotenv==1.2.2
# -r desktop/requirements-desktop.txt
# litellm
# magika
- # pydantic-settings
# pymilvus
python-engineio==4.13.3
# via python-socketio
@@ -485,10 +490,9 @@ requests-toolbelt==1.0.0
# via lark-oapi
rich==15.0.0
# via
+ # agentscope
# dashscope
# typer
-ripgrep==14.1.0
- # via agentscope
rpds-py==2026.6.3
# via
# jsonschema
@@ -513,8 +517,6 @@ six==1.17.0
# via
# markdownify
# python-dateutil
-slowapi==0.1.10
- # via -r desktop/requirements-desktop.txt
sniffio==1.3.1
# via
# anthropic
@@ -528,7 +530,9 @@ sqlalchemy==2.0.51
# alembic
# mem0ai
sse-starlette==3.0.3
- # via mcp
+ # via
+ # -r desktop/requirements-desktop.txt
+ # mcp
starlette==0.46.2
# via
# fastapi
@@ -556,6 +560,10 @@ tree-sitter==0.26.0
# via agentscope
tree-sitter-bash==0.25.1
# via agentscope
+truststore==0.10.5
+ # via
+ # httpcore2
+ # httpx2
typer==0.27.0
# via dashscope
typing-extensions==4.16.0
@@ -569,9 +577,10 @@ typing-extensions==4.16.0
# fastapi
# google-genai
# grpcio
+ # httpx2
# huggingface-hub
- # limits
# mcp
+ # mcp-types
# openai
# opentelemetry-api
# opentelemetry-exporter-otlp-proto-grpc
@@ -592,7 +601,8 @@ typing-inspection==0.4.2
# via
# mcp
# pydantic
- # pydantic-settings
+tzdata==2026.5
+ # via agentscope
ujson==5.13.0
# via pymilvus
urllib3==2.7.0
@@ -616,8 +626,6 @@ werkzeug==3.1.8
# via -r desktop/requirements-desktop.txt
wheel==0.47.0
# via -r desktop/requirements-desktop.txt
-wrapt==2.3.0
- # via deprecated
wsproto==1.3.2
# via simple-websocket
xlsxwriter==3.2.9
diff --git a/desktop/requirements-desktop-macos-aarch64-py311.lock b/desktop/requirements-desktop-macos-aarch64-py311.lock
index 887a7ba2..9df1e05d 100644
--- a/desktop/requirements-desktop-macos-aarch64-py311.lock
+++ b/desktop/requirements-desktop-macos-aarch64-py311.lock
@@ -1,8 +1,8 @@
# This file is generated. Do not edit it by hand.
# Regenerate with: npm --prefix desktop run lock:desktop
-# input-sha256: 207ac56d3accd481bbd7d5fdd7de4ab076f3184f1dc6c7d0bd6de1d1b958d38a
+# input-sha256: 2580e67756923a30f416529f071ff7372044a681fb55cd05ef896e558846dd73
# Target: CPython 3.11 on macOS Apple Silicon
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# via -r desktop/requirements-desktop.txt
aiofiles==25.1.0
# via agentscope
@@ -31,6 +31,7 @@ anyio==4.14.2
# anthropic
# google-genai
# httpx
+ # httpx2
# mcp
# openai
# sse-starlette
@@ -101,9 +102,7 @@ defusedxml==0.7.1
# -r desktop/requirements-desktop.txt
# markitdown
deprecated==1.3.1
- # via
- # limits
- # pikepdf
+ # via pikepdf
dingtalk-stream==0.24.3
# via -r desktop/requirements-desktop.txt
distro==1.9.0
@@ -157,6 +156,7 @@ grpcio==1.83.0
h11==0.16.0
# via
# httpcore
+ # httpcore2
# uvicorn
# wsproto
h2==4.4.0
@@ -167,6 +167,8 @@ hpack==4.2.0
# via h2
httpcore==1.0.9
# via httpx
+httpcore2==2.16.0
+ # via httpx2
httpx==0.28.1
# via
# -r desktop/requirements-desktop.txt
@@ -177,15 +179,16 @@ httpx==0.28.1
# huggingface-hub
# lark-oapi
# litellm
- # mcp
# mem0ai
# ollama
# openai
# qdrant-client
# tavily-python
httpx-sse==0.4.3
+ # via dashscope
+httpx2==2.16.0
# via
- # dashscope
+ # agentscope
# mcp
huggingface-hub==1.26.0
# via tokenizers
@@ -197,6 +200,7 @@ idna==3.18
# via
# anyio
# httpx
+ # httpx2
# requests
# yarl
importlib-metadata==8.9.0
@@ -209,7 +213,7 @@ jiter==0.16.0
# via
# anthropic
# openai
-json-repair==0.61.7
+json-repair==0.64.1
# via agentscope
json5==0.15.0
# via agentscope
@@ -217,6 +221,7 @@ jsonschema==4.26.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+ # json-repair
# litellm
# mcp
jsonschema-specifications==2025.9.1
@@ -225,8 +230,6 @@ kiwisolver==1.5.0
# via matplotlib
lark-oapi==1.7.1
# via -r desktop/requirements-desktop.txt
-limits==5.8.0
- # via slowapi
litellm==1.91.4
# via -r desktop/requirements-desktop.txt
lxml==6.1.1
@@ -254,10 +257,12 @@ matplotlib==3.11.1
# via
# -r desktop/requirements-desktop.txt
# seaborn
-mcp==1.29.0
+mcp==2.3.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+mcp-types==2.3.0
+ # via mcp
mdurl==0.1.2
# via markdown-it-py
mem0ai==2.0.14
@@ -300,6 +305,7 @@ openpyxl==3.1.5
opentelemetry-api==1.44.0
# via
# agentscope
+ # mcp
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# opentelemetry-sdk
@@ -332,7 +338,6 @@ packaging==26.2
# via
# faiss-cpu
# huggingface-hub
- # limits
# matplotlib
# onnxruntime
# pikepdf
@@ -396,17 +401,16 @@ pydantic==2.13.4
# anthropic
# fastapi
# google-genai
+ # json-repair
# litellm
# mcp
+ # mcp-types
# mem0ai
# ollama
# openai
- # pydantic-settings
# qdrant-client
pydantic-core==2.46.4
# via pydantic
-pydantic-settings==2.14.2
- # via mcp
pyee==13.0.1
# via playwright
pygments==2.20.0
@@ -418,7 +422,9 @@ pymilvus==2.5.18
pyparsing==3.3.2
# via matplotlib
pypdf==6.14.2
- # via -r desktop/requirements-desktop.txt
+ # via
+ # -r desktop/requirements-desktop.txt
+ # agentscope
pypdf2==3.0.1
# via -r desktop/requirements-desktop.txt
pypdfium2==5.12.1
@@ -437,7 +443,6 @@ python-dotenv==1.2.2
# -r desktop/requirements-desktop.txt
# litellm
# magika
- # pydantic-settings
# pymilvus
python-engineio==4.13.3
# via python-socketio
@@ -493,10 +498,9 @@ requests-toolbelt==1.0.0
# via lark-oapi
rich==15.0.0
# via
+ # agentscope
# dashscope
# typer
-ripgrep==15.1.0
- # via agentscope
rpds-py==2026.6.3
# via
# jsonschema
@@ -521,8 +525,6 @@ six==1.17.0
# via
# markdownify
# python-dateutil
-slowapi==0.1.10
- # via -r desktop/requirements-desktop.txt
sniffio==1.3.1
# via
# anthropic
@@ -536,7 +538,9 @@ sqlalchemy==2.0.51
# alembic
# mem0ai
sse-starlette==3.0.3
- # via mcp
+ # via
+ # -r desktop/requirements-desktop.txt
+ # mcp
starlette==0.46.2
# via
# fastapi
@@ -566,6 +570,10 @@ tree-sitter==0.26.0
# via agentscope
tree-sitter-bash==0.25.1
# via agentscope
+truststore==0.10.5
+ # via
+ # httpcore2
+ # httpx2
typer==0.27.0
# via dashscope
typing-extensions==4.16.0
@@ -579,9 +587,10 @@ typing-extensions==4.16.0
# fastapi
# google-genai
# grpcio
+ # httpx2
# huggingface-hub
- # limits
# mcp
+ # mcp-types
# openai
# opentelemetry-api
# opentelemetry-exporter-otlp-proto-grpc
@@ -602,7 +611,8 @@ typing-inspection==0.4.2
# via
# mcp
# pydantic
- # pydantic-settings
+tzdata==2026.5
+ # via agentscope
ujson==5.13.0
# via pymilvus
urllib3==2.7.0
diff --git a/desktop/requirements-desktop-macos-x86_64-py311.lock b/desktop/requirements-desktop-macos-x86_64-py311.lock
index 4118f46d..39a17b34 100644
--- a/desktop/requirements-desktop-macos-x86_64-py311.lock
+++ b/desktop/requirements-desktop-macos-x86_64-py311.lock
@@ -1,8 +1,8 @@
# This file is generated. Do not edit it by hand.
# Regenerate with: npm --prefix desktop run lock:desktop
-# input-sha256: 207ac56d3accd481bbd7d5fdd7de4ab076f3184f1dc6c7d0bd6de1d1b958d38a
+# input-sha256: 2580e67756923a30f416529f071ff7372044a681fb55cd05ef896e558846dd73
# Target: CPython 3.11 on macOS Intel
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# via -r desktop/requirements-desktop.txt
aiofiles==25.1.0
# via agentscope
@@ -31,6 +31,7 @@ anyio==4.14.2
# anthropic
# google-genai
# httpx
+ # httpx2
# mcp
# openai
# sse-starlette
@@ -101,9 +102,7 @@ defusedxml==0.7.1
# -r desktop/requirements-desktop.txt
# markitdown
deprecated==1.3.1
- # via
- # limits
- # pikepdf
+ # via pikepdf
dingtalk-stream==0.24.3
# via -r desktop/requirements-desktop.txt
distro==1.9.0
@@ -159,6 +158,7 @@ grpcio==1.83.0
h11==0.16.0
# via
# httpcore
+ # httpcore2
# uvicorn
# wsproto
h2==4.4.0
@@ -169,6 +169,8 @@ hpack==4.2.0
# via h2
httpcore==1.0.9
# via httpx
+httpcore2==2.16.0
+ # via httpx2
httpx==0.28.1
# via
# -r desktop/requirements-desktop.txt
@@ -179,15 +181,16 @@ httpx==0.28.1
# huggingface-hub
# lark-oapi
# litellm
- # mcp
# mem0ai
# ollama
# openai
# qdrant-client
# tavily-python
httpx-sse==0.4.3
+ # via dashscope
+httpx2==2.16.0
# via
- # dashscope
+ # agentscope
# mcp
huggingface-hub==1.26.0
# via tokenizers
@@ -199,6 +202,7 @@ idna==3.18
# via
# anyio
# httpx
+ # httpx2
# requests
# yarl
importlib-metadata==8.9.0
@@ -211,7 +215,7 @@ jiter==0.16.0
# via
# anthropic
# openai
-json-repair==0.61.7
+json-repair==0.64.1
# via agentscope
json5==0.15.0
# via agentscope
@@ -219,6 +223,7 @@ jsonschema==4.26.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+ # json-repair
# litellm
# mcp
jsonschema-specifications==2025.9.1
@@ -227,8 +232,6 @@ kiwisolver==1.5.0
# via matplotlib
lark-oapi==1.7.1
# via -r desktop/requirements-desktop.txt
-limits==5.8.0
- # via slowapi
litellm==1.91.4
# via -r desktop/requirements-desktop.txt
lxml==6.1.1
@@ -256,10 +259,12 @@ matplotlib==3.11.1
# via
# -r desktop/requirements-desktop.txt
# seaborn
-mcp==1.29.0
+mcp==2.3.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+mcp-types==2.3.0
+ # via mcp
mdurl==0.1.2
# via markdown-it-py
mem0ai==2.0.14
@@ -302,6 +307,7 @@ openpyxl==3.1.5
opentelemetry-api==1.44.0
# via
# agentscope
+ # mcp
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# opentelemetry-sdk
@@ -334,7 +340,6 @@ packaging==26.2
# via
# faiss-cpu
# huggingface-hub
- # limits
# matplotlib
# onnxruntime
# pikepdf
@@ -398,17 +403,16 @@ pydantic==2.13.4
# anthropic
# fastapi
# google-genai
+ # json-repair
# litellm
# mcp
+ # mcp-types
# mem0ai
# ollama
# openai
- # pydantic-settings
# qdrant-client
pydantic-core==2.46.4
# via pydantic
-pydantic-settings==2.14.2
- # via mcp
pyee==13.0.1
# via playwright
pygments==2.20.0
@@ -420,7 +424,9 @@ pymilvus==2.5.18
pyparsing==3.3.2
# via matplotlib
pypdf==6.14.2
- # via -r desktop/requirements-desktop.txt
+ # via
+ # -r desktop/requirements-desktop.txt
+ # agentscope
pypdf2==3.0.1
# via -r desktop/requirements-desktop.txt
pypdfium2==5.12.1
@@ -439,7 +445,6 @@ python-dotenv==1.2.2
# -r desktop/requirements-desktop.txt
# litellm
# magika
- # pydantic-settings
# pymilvus
python-engineio==4.13.3
# via python-socketio
@@ -495,10 +500,9 @@ requests-toolbelt==1.0.0
# via lark-oapi
rich==15.0.0
# via
+ # agentscope
# dashscope
# typer
-ripgrep==14.1.0
- # via agentscope
rpds-py==2026.6.3
# via
# jsonschema
@@ -523,8 +527,6 @@ six==1.17.0
# via
# markdownify
# python-dateutil
-slowapi==0.1.10
- # via -r desktop/requirements-desktop.txt
sniffio==1.3.1
# via
# anthropic
@@ -538,7 +540,9 @@ sqlalchemy==2.0.51
# alembic
# mem0ai
sse-starlette==3.0.3
- # via mcp
+ # via
+ # -r desktop/requirements-desktop.txt
+ # mcp
starlette==0.46.2
# via
# fastapi
@@ -568,6 +572,10 @@ tree-sitter==0.26.0
# via agentscope
tree-sitter-bash==0.25.1
# via agentscope
+truststore==0.10.5
+ # via
+ # httpcore2
+ # httpx2
typer==0.27.0
# via dashscope
typing-extensions==4.16.0
@@ -581,9 +589,10 @@ typing-extensions==4.16.0
# fastapi
# google-genai
# grpcio
+ # httpx2
# huggingface-hub
- # limits
# mcp
+ # mcp-types
# openai
# opentelemetry-api
# opentelemetry-exporter-otlp-proto-grpc
@@ -604,7 +613,8 @@ typing-inspection==0.4.2
# via
# mcp
# pydantic
- # pydantic-settings
+tzdata==2026.5
+ # via agentscope
ujson==5.13.0
# via pymilvus
urllib3==2.7.0
diff --git a/desktop/requirements-desktop-windows-py311.lock b/desktop/requirements-desktop-windows-py311.lock
index 36895fe1..48953834 100644
--- a/desktop/requirements-desktop-windows-py311.lock
+++ b/desktop/requirements-desktop-windows-py311.lock
@@ -1,8 +1,8 @@
# This file is generated. Do not edit it by hand.
# Regenerate with: npm --prefix desktop run lock:windows
-# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990
+# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046
# Target: CPython 3.11 on Windows x86_64
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# via -r desktop/requirements-desktop.txt
aiofiles==25.1.0
# via agentscope
@@ -31,6 +31,7 @@ anyio==4.14.2
# anthropic
# google-genai
# httpx
+ # httpx2
# mcp
# openai
# sse-starlette
@@ -106,8 +107,6 @@ defusedxml==0.7.1
# via
# -r desktop/requirements-desktop.txt
# markitdown
-deprecated==1.3.1
- # via limits
dingtalk-stream==0.24.3
# via -r desktop/requirements-desktop.txt
distro==1.9.0
@@ -163,6 +162,7 @@ grpcio==1.83.0
h11==0.16.0
# via
# httpcore
+ # httpcore2
# uvicorn
# wsproto
h2==4.4.0
@@ -173,6 +173,8 @@ hpack==4.2.0
# via h2
httpcore==1.0.9
# via httpx
+httpcore2==2.16.0
+ # via httpx2
httpx==0.28.1
# via
# -r desktop/requirements-desktop.txt
@@ -183,15 +185,16 @@ httpx==0.28.1
# huggingface-hub
# lark-oapi
# litellm
- # mcp
# mem0ai
# ollama
# openai
# qdrant-client
# tavily-python
httpx-sse==0.4.3
+ # via dashscope
+httpx2==2.16.0
# via
- # dashscope
+ # agentscope
# mcp
huggingface-hub==1.26.0
# via tokenizers
@@ -203,6 +206,7 @@ idna==3.18
# via
# anyio
# httpx
+ # httpx2
# requests
# yarl
importlib-metadata==8.9.0
@@ -215,7 +219,7 @@ jiter==0.16.0
# via
# anthropic
# openai
-json-repair==0.61.7
+json-repair==0.64.1
# via agentscope
json5==0.15.0
# via agentscope
@@ -223,6 +227,7 @@ jsonschema==4.26.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+ # json-repair
# litellm
# mcp
jsonschema-specifications==2025.9.1
@@ -231,8 +236,6 @@ kiwisolver==1.5.0
# via matplotlib
lark-oapi==1.7.1
# via -r desktop/requirements-desktop.txt
-limits==5.8.0
- # via slowapi
litellm==1.94.1
# via -r desktop/requirements-desktop.txt
lxml==6.1.1
@@ -260,10 +263,12 @@ matplotlib==3.11.1
# via
# -r desktop/requirements-desktop.txt
# seaborn
-mcp==1.29.0
+mcp==2.3.0
# via
# -r desktop/requirements-desktop.txt
# agentscope
+mcp-types==2.3.0
+ # via mcp
mdurl==0.1.2
# via markdown-it-py
mem0ai==2.0.14
@@ -304,6 +309,7 @@ openpyxl==3.1.5
opentelemetry-api==1.44.0
# via
# agentscope
+ # mcp
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# opentelemetry-sdk
@@ -336,7 +342,6 @@ packaging==26.2
# via
# faiss-cpu
# huggingface-hub
- # limits
# matplotlib
# onnxruntime
# pikepdf
@@ -398,17 +403,16 @@ pydantic==2.13.4
# anthropic
# fastapi
# google-genai
+ # json-repair
# litellm
# mcp
+ # mcp-types
# mem0ai
# ollama
# openai
- # pydantic-settings
# qdrant-client
pydantic-core==2.46.4
# via pydantic
-pydantic-settings==2.14.2
- # via mcp
pyee==13.0.1
# via playwright
pygments==2.20.0
@@ -420,7 +424,9 @@ pymilvus==2.5.18
pyparsing==3.3.2
# via matplotlib
pypdf==6.14.2
- # via -r desktop/requirements-desktop.txt
+ # via
+ # -r desktop/requirements-desktop.txt
+ # agentscope
pypdf2==3.0.1
# via -r desktop/requirements-desktop.txt
pypdfium2==5.12.1
@@ -441,7 +447,6 @@ python-dotenv==1.2.2
# -r desktop/requirements-desktop.txt
# litellm
# magika
- # pydantic-settings
# pymilvus
python-engineio==4.13.3
# via python-socketio
@@ -501,10 +506,9 @@ requests-toolbelt==1.0.0
# via lark-oapi
rich==15.0.0
# via
+ # agentscope
# dashscope
# typer
-ripgrep==14.1.0
- # via agentscope
rpds-py==2026.6.3
# via
# jsonschema
@@ -529,8 +533,6 @@ six==1.17.0
# via
# markdownify
# python-dateutil
-slowapi==0.1.10
- # via -r desktop/requirements-desktop.txt
sniffio==1.3.1
# via
# anthropic
@@ -544,7 +546,9 @@ sqlalchemy==2.0.51
# alembic
# mem0ai
sse-starlette==3.0.3
- # via mcp
+ # via
+ # -r desktop/requirements-desktop.txt
+ # mcp
starlette==0.46.2
# via
# fastapi
@@ -574,6 +578,10 @@ tree-sitter==0.26.0
# via agentscope
tree-sitter-bash==0.25.1
# via agentscope
+truststore==0.10.5
+ # via
+ # httpcore2
+ # httpx2
typer==0.27.0
# via dashscope
typing-extensions==4.16.0
@@ -587,9 +595,10 @@ typing-extensions==4.16.0
# fastapi
# google-genai
# grpcio
+ # httpx2
# huggingface-hub
- # limits
# mcp
+ # mcp-types
# openai
# opentelemetry-api
# opentelemetry-exporter-otlp-proto-grpc
@@ -610,9 +619,10 @@ typing-inspection==0.4.2
# via
# mcp
# pydantic
- # pydantic-settings
tzdata==2026.3
- # via pandas
+ # via
+ # agentscope
+ # pandas
ujson==5.13.0
# via pymilvus
urllib3==2.7.0
@@ -636,8 +646,6 @@ werkzeug==3.1.8
# via -r desktop/requirements-desktop.txt
wheel==0.47.0
# via -r desktop/requirements-desktop.txt
-wrapt==2.3.0
- # via deprecated
wsproto==1.3.2
# via simple-websocket
xlsxwriter==3.2.9
diff --git a/desktop/requirements-desktop.txt b/desktop/requirements-desktop.txt
index e2937d7a..9b2d2192 100644
--- a/desktop/requirements-desktop.txt
+++ b/desktop/requirements-desktop.txt
@@ -18,17 +18,19 @@ uv==0.11.33
# Core web service and orchestration.
fastapi==0.115.12
uvicorn>=0.27.0
-pydantic>=2.6.0
-anyio>=4.2.0
+pydantic>=2.12.0
+anyio>=4.9.0
python-dotenv>=1.0.0
requests>=2.31.0
httpx>=0.27.0
jsonschema>=4.20.0
-agentscope==2.0.0
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
litellm>=1.50.0
google-genai>=0.3.0
ollama>=0.3.0
-mcp>=1.0.0,<2.0.0
+mcp==2.3.0
+# Keep the SSE dependency within FastAPI 0.115 Starlette constraints.
+sse-starlette==3.0.3
tavily-python>=0.5.0
beautifulsoup4>=4.12.0
@@ -40,7 +42,6 @@ redis>=5.0.0
werkzeug>=3.0.0
python-multipart>=0.0.9
structlog>=23.2.0
-slowapi>=0.1.9
argon2-cffi>=23.1.0
croniter>=2.0.0
pytz>=2024.1
diff --git a/desktop/scripts/build-runtime.mjs b/desktop/scripts/build-runtime.mjs
index bb4a1430..dc159fc4 100644
--- a/desktop/scripts/build-runtime.mjs
+++ b/desktop/scripts/build-runtime.mjs
@@ -112,6 +112,9 @@ export function buildDesktopRuntime({ desktopDir, repoRoot, sourceRoot, python }
"--break-system-packages",
"--only-binary",
":all:",
+ // AgentScope is pinned to a reviewed source commit; other packages require wheels.
+ "--no-binary",
+ "agentscope",
join(repoRoot, config.lockFile),
],
{ cwd: repoRoot },
diff --git a/desktop/scripts/desktop-dependencies.test.mjs b/desktop/scripts/desktop-dependencies.test.mjs
index 2f28f373..c6540fbc 100644
--- a/desktop/scripts/desktop-dependencies.test.mjs
+++ b/desktop/scripts/desktop-dependencies.test.mjs
@@ -18,11 +18,26 @@ import {
WINDOWS_DESKTOP_LOCK_FILE,
desktopDependencyFingerprint,
readAndValidateWindowsDesktopLock,
+ readAndValidateDesktopLock,
} from "./desktop-dependencies.mjs";
const desktopDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const repoRoot = resolve(desktopDir, "..");
+function assertExactLock(lock) {
+ const source = readFileSync(join(repoRoot, DESKTOP_REQUIREMENTS_FILE), "utf8")
+ .match(/^agentscope @ https:\/\/github\.com\/agentscope-ai\/agentscope\/archive\/[a-f0-9]{40}\.tar\.gz$/m)?.[0];
+ assert.ok(source, "AgentScope must use an immutable commit archive");
+ assert.ok(readFileSync(join(repoRoot, "requirements.txt"), "utf8").split(/\r?\n/).includes(source));
+ assert.ok(lock.split(/\r?\n/).includes(source));
+ for (const line of lock.split(/\r?\n/)) {
+ if (/^[A-Za-z0-9_.-]+/.test(line) && line !== source) {
+ assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/);
+ }
+ }
+ assert.match(lock, /^mcp==2\.3\.0$/m);
+}
+
function requirementNames(contents) {
return new Set(
contents
@@ -73,14 +88,7 @@ test("Windows Python 3.11 lock is exact and matches its desktop input", () => {
const lock = readAndValidateWindowsDesktopLock(repoRoot);
const names = requirementNames(lock);
assert.ok(names.size > 100, "the transitive desktop lock must be complete");
- for (const line of lock.split(/\r?\n/)) {
- if (/^[A-Za-z0-9_.-]+/.test(line)) {
- assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/);
- }
- }
- const mcpVersion = lock.match(/^mcp==([^\s]+)$/m)?.[1];
- assert.ok(mcpVersion, "the lock must contain mcp");
- assert.equal(Number(mcpVersion.split(".")[0]), 1);
+ assertExactLock(lock);
assert.match(lock, /^pymilvus==2\.5\.18$/m);
assert.match(lock, /^milvus-lite==3\.2\.2$/m);
// Some platform locks pull boto3 through LiteLLM; it is not a storage backend.
@@ -124,14 +132,9 @@ test("desktop dependency hash is independent of checkout line endings", () => {
test("all supported desktop targets have exact Python 3.11 locks", () => {
for (const [target, config] of Object.entries(DESKTOP_TARGETS)) {
- const lock = readFileSync(join(repoRoot, config.lockFile), "utf8");
- assert.match(lock, /# input-sha256: [a-f0-9]{64}/);
+ const lock = readAndValidateDesktopLock(repoRoot, target);
assert.ok(requirementNames(lock).size > 100, `${target} lock is incomplete`);
- for (const line of lock.split(/\r?\n/)) {
- if (/^[A-Za-z0-9_.-]+/.test(line)) {
- assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/);
- }
- }
+ assertExactLock(lock);
}
});
diff --git a/desktop/scripts/test-native-tools.py b/desktop/scripts/test-native-tools.py
index fa8e40ee..9e076108 100644
--- a/desktop/scripts/test-native-tools.py
+++ b/desktop/scripts/test-native-tools.py
@@ -174,7 +174,7 @@ def test_backend_prefers_relocated_private_tools_without_host_path(self):
import importlib.util
from unittest.mock import patch
spec = importlib.util.spec_from_file_location(
- "office_helpers", SCRIPT.parents[2] / "src/backend/core/content/office.py")
+ "office_helpers", SCRIPT.parents[2] / "src/backend/core/files/content/office.py")
office = importlib.util.module_from_spec(spec)
spec.loader.exec_module(office)
with tempfile.TemporaryDirectory() as folder:
diff --git a/docker-compose.yml b/docker-compose.yml
index c8d35b9b..c6c2cc8e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -73,8 +73,7 @@ services:
BRAND_POWERED_BY: ${BRAND_POWERED_BY:-true}
DATABASE_URL: postgresql://hugagent_user:${DB_PASSWORD:-hugagent_dev_password}@postgres:5432/hugagent
APPLICATION_DATABASE_URL: ${APPLICATION_DATABASE_URL:-}
- DB_POOL_SIZE: ${DB_POOL_SIZE:-100}
- DB_POOL_MAX_OVERFLOW: ${DB_POOL_MAX_OVERFLOW:-20}
+ WORKER_THREADS: ${WORKER_THREADS:-40}
DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-30}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
ENV: ${ENV:-dev}
@@ -128,8 +127,6 @@ services:
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-}
S3_SECRET_KEY: ${S3_SECRET_KEY:-}
CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173}
- RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-true}
- RATE_LIMIT_DEFAULT: ${RATE_LIMIT_DEFAULT:-100/minute}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
LOG_FORMAT: ${LOG_FORMAT:-json}
LOG_TO_FILE: ${LOG_TO_FILE:-false}
@@ -270,8 +267,7 @@ services:
environment:
TZ: ${TZ:-Asia/Shanghai}
DATABASE_URL: postgresql://hugagent_user:${DB_PASSWORD:-hugagent_dev_password}@postgres:5432/hugagent
- DB_POOL_SIZE: ${DB_POOL_SIZE:-100}
- DB_POOL_MAX_OVERFLOW: ${DB_POOL_MAX_OVERFLOW:-20}
+ WORKER_THREADS: ${WORKER_THREADS:-40}
DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-30}
STORAGE_TYPE: ${STORAGE_TYPE:-s3}
STORAGE_PATH: /app/storage
@@ -369,7 +365,7 @@ services:
- script_runner
build:
context: .
- dockerfile: docker/Dockerfile.script-runner
+ dockerfile: docker/sandbox/Dockerfile.script-runner
container_name: hugagent-script-runner
logging: *id001
environment:
@@ -414,8 +410,8 @@ services:
COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME must be explicit}
OPENSANDBOX_API_KEY: ${OPENSANDBOX_API_KEY:-}
volumes:
- - ./docker/opensandbox-config.toml.tpl:/tpl/config.toml.tpl:ro
- - ./docker/render-opensandbox-config.py:/tpl/render.py:ro
+ - ./docker/opensandbox-server/opensandbox-config.toml.tpl:/tpl/config.toml.tpl:ro
+ - ./docker/opensandbox-server/render-opensandbox-config.py:/tpl/render.py:ro
- opensandbox_config:/out
command:
- python
@@ -429,7 +425,7 @@ services:
image: hugagent-opensandbox-server:v0.1.13-3
build:
context: .
- dockerfile: docker/Dockerfile.opensandbox-server
+ dockerfile: docker/opensandbox-server/Dockerfile.opensandbox-server
container_name: hugagent-opensandbox
logging: *id001
depends_on:
@@ -438,6 +434,7 @@ services:
environment:
TZ: ${TZ:-Asia/Shanghai}
OPENSANDBOX_INSECURE_SERVER: 'NO'
+ OPENSANDBOX_POOL_MAX_TOTAL: ${OPENSANDBOX_POOL_MAX_TOTAL:-20}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- opensandbox_config:/etc/opensandbox:ro
diff --git a/docker/Dockerfile.opensandbox-server.dockerignore b/docker/Dockerfile.opensandbox-server.dockerignore
deleted file mode 100644
index 2b3b7e78..00000000
--- a/docker/Dockerfile.opensandbox-server.dockerignore
+++ /dev/null
@@ -1,16 +0,0 @@
-**
-!docker/
-!docker/patch_opensandbox_server.py
-!docker/opensandbox_expirations.py
-!docker/opensandbox_mount_worker.py
-!docker/opensandbox_mount_recovery.py
-!docker/opensandbox_object_mounts.py
-!src/
-!src/backend/
-!src/backend/core/
-!src/backend/core/sandbox/
-!src/backend/core/sandbox/sessions/
-!src/backend/core/sandbox/sessions/metadata.py
-!src/backend/tests/
-!src/backend/tests/sandbox/
-!src/backend/tests/sandbox/test_opensandbox_controller_patch.py
diff --git a/docker/Dockerfile.personal-agent-test b/docker/Dockerfile.personal-agent-test
deleted file mode 100644
index d689f1ab..00000000
--- a/docker/Dockerfile.personal-agent-test
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM hugagent-backend:latest
-USER root
-COPY --chown=appuser:appuser src/backend /app/src/backend
-COPY --chown=appuser:appuser alembic.ini /app/alembic.ini
-USER appuser
-CMD ["bash", "/app/src/backend/scripts/backend_entrypoint.sh"]
diff --git a/docker-compose.application-hosting.yml b/docker/application-hosting/compose.application-hosting.yml
similarity index 90%
rename from docker-compose.application-hosting.yml
rename to docker/application-hosting/compose.application-hosting.yml
index 74c1e7c7..3f8c5808 100644
--- a/docker-compose.application-hosting.yml
+++ b/docker/application-hosting/compose.application-hosting.yml
@@ -1,5 +1,5 @@
# Opt-in application data database, separate from the platform postgres service.
-# docker compose -f docker-compose.yml -f docker-compose.application-hosting.yml up -d
+# docker compose -f docker-compose.yml -f docker/application-hosting/compose.application-hosting.yml up -d
services:
application-postgres:
image: postgres:15-alpine
@@ -29,7 +29,7 @@ services:
target: production
volumes:
- ./src/backend:/app/src/backend:ro
- command: ["python", "-m", "core.services.application_hosting_setup"]
+ command: ["python", "-m", "features.hosted_apps.hosting_setup"]
environment:
APPLICATION_DATABASE_URL: postgresql://application_owner:${APPLICATION_OWNER_DB_PASSWORD:?Set a separate APPLICATION_OWNER_DB_PASSWORD}@application-postgres:5432/applications
APPLICATION_PROVISION_DATABASE_URL: postgresql://application_admin:${APPLICATION_DB_PASSWORD}@application-postgres:5432/applications
diff --git a/docker/Dockerfile.object-storage b/docker/object-storage/Dockerfile.object-storage
similarity index 100%
rename from docker/Dockerfile.object-storage
rename to docker/object-storage/Dockerfile.object-storage
diff --git a/docker/compose.object-storage.yml b/docker/object-storage/compose.object-storage.yml
similarity index 91%
rename from docker/compose.object-storage.yml
rename to docker/object-storage/compose.object-storage.yml
index b0c46c20..ba0b8334 100644
--- a/docker/compose.object-storage.yml
+++ b/docker/object-storage/compose.object-storage.yml
@@ -4,7 +4,7 @@ services:
image: ${COMPOSE_PROJECT_NAME:-hugagent}-object-storage:RELEASE.2023-03-13T19-46-17Z
build:
context: ..
- dockerfile: docker/Dockerfile.object-storage
+ dockerfile: docker/object-storage/Dockerfile.object-storage
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${S3_ACCESS_KEY:?Set a random S3_ACCESS_KEY}
@@ -31,7 +31,7 @@ services:
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
volumes:
- - ./docker/init-object-storage.py:/init-object-storage.py:ro
+ - ./docker/object-storage/init-object-storage.py:/init-object-storage.py:ro
depends_on:
object-storage: {condition: service_healthy}
networks: [hugagent-network]
diff --git a/docker/init-object-storage.py b/docker/object-storage/init-object-storage.py
similarity index 100%
rename from docker/init-object-storage.py
rename to docker/object-storage/init-object-storage.py
diff --git a/docker/Dockerfile.opensandbox-server b/docker/opensandbox-server/Dockerfile.opensandbox-server
similarity index 52%
rename from docker/Dockerfile.opensandbox-server
rename to docker/opensandbox-server/Dockerfile.opensandbox-server
index 95b92d71..9ff52a54 100644
--- a/docker/Dockerfile.opensandbox-server
+++ b/docker/opensandbox-server/Dockerfile.opensandbox-server
@@ -2,12 +2,13 @@
FROM opensandbox/server:v0.1.13
USER root
RUN apt-get update && apt-get install -y --no-install-recommends s3fs fuse && rm -rf /var/lib/apt/lists/*
-COPY docker/opensandbox_expirations.py /app/opensandbox_server/services/docker/expirations.py
-COPY docker/opensandbox_mount_worker.py /app/opensandbox_server/services/docker/mount_worker.py
+COPY docker/opensandbox-server/opensandbox_capacity.py /app/opensandbox_server/services/docker/capacity.py
+COPY docker/opensandbox-server/opensandbox_expirations.py /app/opensandbox_server/services/docker/expirations.py
+COPY docker/opensandbox-server/opensandbox_mount_worker.py /app/opensandbox_server/services/docker/mount_worker.py
COPY src/backend/core/sandbox/sessions/metadata.py /app/opensandbox_server/services/docker/sandbox_metadata.py
-COPY docker/opensandbox_mount_recovery.py /app/opensandbox_server/services/docker/mount_recovery.py
-COPY docker/opensandbox_object_mounts.py /app/opensandbox_server/services/docker/object_mounts.py
-COPY docker/patch_opensandbox_server.py /tmp/hugagent-controller/patch.py
+COPY docker/opensandbox-server/opensandbox_mount_recovery.py /app/opensandbox_server/services/docker/mount_recovery.py
+COPY docker/opensandbox-server/opensandbox_object_mounts.py /app/opensandbox_server/services/docker/object_mounts.py
+COPY docker/opensandbox-server/patch_opensandbox_server.py /tmp/hugagent-controller/patch.py
COPY src/backend/tests/sandbox/test_opensandbox_controller_patch.py /tmp/hugagent-controller/test_patch.py
RUN OPENSANDBOX_PATCH_SCRIPT=/tmp/hugagent-controller/patch.py \
OPENSANDBOX_TEST_SOURCE=/app/opensandbox_server \
diff --git a/docker/opensandbox-server/Dockerfile.opensandbox-server.dockerignore b/docker/opensandbox-server/Dockerfile.opensandbox-server.dockerignore
new file mode 100644
index 00000000..9f99abbe
--- /dev/null
+++ b/docker/opensandbox-server/Dockerfile.opensandbox-server.dockerignore
@@ -0,0 +1,18 @@
+**
+!docker/
+!docker/opensandbox-server/
+!docker/opensandbox-server/opensandbox_capacity.py
+!docker/opensandbox-server/patch_opensandbox_server.py
+!docker/opensandbox-server/opensandbox_expirations.py
+!docker/opensandbox-server/opensandbox_mount_worker.py
+!docker/opensandbox-server/opensandbox_mount_recovery.py
+!docker/opensandbox-server/opensandbox_object_mounts.py
+!src/
+!src/backend/
+!src/backend/core/
+!src/backend/core/sandbox/
+!src/backend/core/sandbox/sessions/
+!src/backend/core/sandbox/sessions/metadata.py
+!src/backend/tests/
+!src/backend/tests/sandbox/
+!src/backend/tests/sandbox/test_opensandbox_controller_patch.py
diff --git a/docker/opensandbox-config.toml.tpl b/docker/opensandbox-server/opensandbox-config.toml.tpl
similarity index 100%
rename from docker/opensandbox-config.toml.tpl
rename to docker/opensandbox-server/opensandbox-config.toml.tpl
diff --git a/docker/opensandbox-server/opensandbox_capacity.py b/docker/opensandbox-server/opensandbox_capacity.py
new file mode 100644
index 00000000..8c1c0cbb
--- /dev/null
+++ b/docker/opensandbox-server/opensandbox_capacity.py
@@ -0,0 +1,31 @@
+"""Serialize Docker admission, not sandbox initialization or mount recovery."""
+import fcntl
+import hashlib
+import os
+from contextlib import contextmanager
+from pathlib import Path
+
+from fastapi import HTTPException
+
+
+@contextmanager
+def sandbox_capacity(service):
+ # File locking also fences multiple controller processes sharing the volume.
+ # Docker inventory survives controller/backend restart; no reservation ledger.
+ root = Path("/var/lib/opensandbox")
+ root.mkdir(parents=True, exist_ok=True)
+ namespace = hashlib.sha256(service.network_mode.encode()).hexdigest()[:24]
+ limit = int(os.environ.get("OPENSANDBOX_POOL_MAX_TOTAL", "20"))
+ if limit < 1:
+ raise HTTPException(503, "Sandbox capacity must be positive")
+ with (root / ("capacity-" + namespace + ".lock")).open("a") as lock:
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ raise HTTPException(429, "Sandbox creation busy; retry later") from None
+ try:
+ if len(service._owned_sandboxes()) >= limit:
+ raise HTTPException(429, "Deployment sandbox capacity reached")
+ yield
+ finally:
+ fcntl.flock(lock, fcntl.LOCK_UN)
diff --git a/docker/opensandbox_expirations.py b/docker/opensandbox-server/opensandbox_expirations.py
similarity index 100%
rename from docker/opensandbox_expirations.py
rename to docker/opensandbox-server/opensandbox_expirations.py
diff --git a/docker/opensandbox_mount_recovery.py b/docker/opensandbox-server/opensandbox_mount_recovery.py
similarity index 91%
rename from docker/opensandbox_mount_recovery.py
rename to docker/opensandbox-server/opensandbox_mount_recovery.py
index b7efa6b0..09c608a2 100644
--- a/docker/opensandbox_mount_recovery.py
+++ b/docker/opensandbox-server/opensandbox_mount_recovery.py
@@ -55,7 +55,7 @@ def _specifications(service, container, request):
def _healthy(container, specs):
if not specs:
return True
- script = """import json,os,sys
+ script = """import errno,json,os,sys
from pathlib import Path
mounts = {}
for line in Path('/proc/self/mountinfo').read_text().splitlines():
@@ -63,14 +63,23 @@ def _healthy(container, specs):
separator = fields.index('-')
mounts[fields[4]] = (fields[separator + 1], fields[5].split(','))
for target, readonly in json.loads(sys.argv[1]):
+ if target not in mounts:
+ sys.exit(10)
filesystem, options = mounts[target]
- assert filesystem.startswith('fuse'), 'Object filesystem is absent'
- assert ('ro' in options) == readonly, 'Mount access mode differs'
- os.stat(target)
+ if not filesystem.startswith('fuse') or ('ro' in options) != readonly:
+ sys.exit(10)
+ try:
+ os.stat(target)
+ except OSError as exc:
+ sys.exit(10 if exc.errno == errno.ENOTCONN else 11)
"""
result = container.exec_run(["timeout", "4", "python3", "-c", script,
json.dumps([(s["target"], s["read_only"]) for s in specs])], user="0")
- return result.exit_code == 0
+ if result.exit_code == 0:
+ return True
+ if result.exit_code == 10:
+ return False
+ raise HTTPException(503, "Object mount health is uncertain; existing connection retained")
def remove_helper(service, sandbox_id):
from docker.errors import NotFound
@@ -127,7 +136,8 @@ def ensure_mounts(service, sandbox_id, request):
if api.exec_inspect(execution["Id"])["ExitCode"] != 0 or not _healthy(container, specs):
raise HTTPException(503, "Object mount recovery failed; workspace retained")
except Exception:
- remove_helper(service, sandbox_id)
+ # The helper may own live writes even if verification timed out.
+ # A later confirmed-broken probe can replace it safely.
raise
return {"epoch": EPOCH, "recovered": True}
diff --git a/docker/opensandbox_mount_worker.py b/docker/opensandbox-server/opensandbox_mount_worker.py
similarity index 100%
rename from docker/opensandbox_mount_worker.py
rename to docker/opensandbox-server/opensandbox_mount_worker.py
diff --git a/docker/opensandbox_object_mounts.py b/docker/opensandbox-server/opensandbox_object_mounts.py
similarity index 100%
rename from docker/opensandbox_object_mounts.py
rename to docker/opensandbox-server/opensandbox_object_mounts.py
diff --git a/docker/patch_opensandbox_server.py b/docker/opensandbox-server/patch_opensandbox_server.py
similarity index 96%
rename from docker/patch_opensandbox_server.py
rename to docker/opensandbox-server/patch_opensandbox_server.py
index 5460b21a..9bec849e 100644
--- a/docker/patch_opensandbox_server.py
+++ b/docker/opensandbox-server/patch_opensandbox_server.py
@@ -110,6 +110,8 @@ def patched_sources(root: Path) -> dict[str, str]:
" async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse:\n"
" return await run_in_threadpool(self._create_sandbox_sync, request)\n\n"
" def _create_sandbox_sync(self, request: CreateSandboxRequest) -> CreateSandboxResponse:\n")
+ source = replace_once(source, "import inspect\n",
+ "import inspect\nfrom .capacity import sandbox_capacity\n")
sources[name] = source
name = "api/lifecycle.py"
@@ -152,6 +154,15 @@ def patched_sources(root: Path) -> dict[str, str]:
patch_shared_lifecycle(sources)
patch_durable_expirations(sources)
patch_namespace_mounts(sources)
+ name = "services/docker/docker_service.py"
+ source = sources[name]
+ start = source.index(" created_container = self._create_and_start_container(")
+ end = source.index("\n )", start) + len("\n )")
+ block = source[start:end]
+ source = (source[:start] + " with sandbox_capacity(self):\n"
+ + "\n".join(" " + line for line in block.splitlines()) + source[end:])
+ sources[name] = source
+
name = "services/docker/docker_service.py"
sources[name] = replace_once(sources[name], "import inspect\n",
"import inspect\nfrom .object_mounts import install as install_object_mounts\ninstall_object_mounts()\n")
diff --git a/docker/render-opensandbox-config.py b/docker/opensandbox-server/render-opensandbox-config.py
similarity index 100%
rename from docker/render-opensandbox-config.py
rename to docker/opensandbox-server/render-opensandbox-config.py
diff --git a/docker/Dockerfile.personal-agent b/docker/personal-agent/Dockerfile.personal-agent
similarity index 86%
rename from docker/Dockerfile.personal-agent
rename to docker/personal-agent/Dockerfile.personal-agent
index 3eecda26..b9106b79 100644
--- a/docker/Dockerfile.personal-agent
+++ b/docker/personal-agent/Dockerfile.personal-agent
@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends xfce4-terminal
# Live screen for the owner: VNC on a private socket, behind a token-checked WebSocket.
RUN apt-get update && apt-get install -y --no-install-recommends x11vnc && rm -rf /var/lib/apt/lists/*
# A browser launcher with the flags this container needs, registered as the default browser.
-COPY docker/personal-agent-desktop/ /opt/personal-agent/desktop/
+COPY docker/personal-agent/desktop/ /opt/personal-agent/desktop/
RUN install -m 755 /opt/personal-agent/desktop/pa-browser /usr/local/bin/pa-browser \
&& install -m 644 /opt/personal-agent/desktop/pa-browser.desktop /usr/share/applications/pa-browser.desktop \
&& install -d /usr/share/xfce4/helpers \
@@ -17,13 +17,13 @@ RUN install -m 755 /opt/personal-agent/desktop/pa-browser /usr/local/bin/pa-brow
&& install -m 644 /opt/personal-agent/desktop/mimeapps.list /etc/xdg/mimeapps.list \
&& install -m 755 /opt/personal-agent/desktop/pa-stream.py /usr/local/bin/pa-stream
RUN useradd --create-home --uid 1001 --shell /bin/bash pa-desktop && chmod 700 /home/pa-desktop && mkdir -p /opt/personal-agent
-COPY src/backend/core/device_link/driver.py /opt/personal-agent/computer-driver.py
-COPY docker/personal-agent-entrypoint.sh /opt/personal-agent/start.sh
+COPY src/backend/personal_agent/device_link/driver.py /opt/personal-agent/computer-driver.py
+COPY docker/personal-agent/personal-agent-entrypoint.sh /opt/personal-agent/start.sh
RUN chmod 755 /opt/personal-agent/start.sh /opt/personal-agent/computer-driver.py
RUN find / -xdev -type f -perm /6000 -exec chmod a-s {} + 2>/dev/null && rm -f /etc/sudoers.d/ubuntu
# Lightweight client only. The cloud computer does not run another backend.
COPY src/backend/hugagent_cli /opt/hugagent-cli/hugagent_cli
-COPY docker/hugagent-cli-entrypoint.sh /usr/local/bin/hugagent
+COPY docker/personal-agent/hugagent-cli-entrypoint.sh /usr/local/bin/hugagent
RUN chmod 755 /usr/local/bin/hugagent
diff --git a/docker/compose.personal-agent.yml b/docker/personal-agent/compose.personal-agent.yml
similarity index 91%
rename from docker/compose.personal-agent.yml
rename to docker/personal-agent/compose.personal-agent.yml
index 86b8962a..d0afeb85 100644
--- a/docker/compose.personal-agent.yml
+++ b/docker/personal-agent/compose.personal-agent.yml
@@ -14,14 +14,14 @@ services:
PERSONAL_AGENT_OPENSANDBOX_API_KEY: ${PERSONAL_AGENT_OPENSANDBOX_API_KEY:?Set a private controller key}
PERSONAL_AGENT_COMPUTE_NETWORK: ${COMPOSE_PROJECT_NAME:-hugagent}_personal-agent-compute
volumes:
- - ./docker/render-personal-agent-config.py:/render.py:ro
+ - ./docker/personal-agent/render-personal-agent-config.py:/render.py:ro
- personal_agent_config:/out
command: ["python", "/render.py"]
personal-agent-opensandbox:
image: hugagent-opensandbox-server:v0.1.13-1
build:
context: .
- dockerfile: docker/Dockerfile.opensandbox-server
+ dockerfile: docker/opensandbox-server/Dockerfile.opensandbox-server
profiles: [personal-agent]
restart: unless-stopped
depends_on:
diff --git a/docker/personal-agent-desktop/helpers.rc b/docker/personal-agent/desktop/helpers.rc
similarity index 100%
rename from docker/personal-agent-desktop/helpers.rc
rename to docker/personal-agent/desktop/helpers.rc
diff --git a/docker/personal-agent-desktop/mimeapps.list b/docker/personal-agent/desktop/mimeapps.list
similarity index 100%
rename from docker/personal-agent-desktop/mimeapps.list
rename to docker/personal-agent/desktop/mimeapps.list
diff --git a/docker/personal-agent-desktop/pa-browser b/docker/personal-agent/desktop/pa-browser
similarity index 100%
rename from docker/personal-agent-desktop/pa-browser
rename to docker/personal-agent/desktop/pa-browser
diff --git a/docker/personal-agent-desktop/pa-browser-helper.desktop b/docker/personal-agent/desktop/pa-browser-helper.desktop
similarity index 100%
rename from docker/personal-agent-desktop/pa-browser-helper.desktop
rename to docker/personal-agent/desktop/pa-browser-helper.desktop
diff --git a/docker/personal-agent-desktop/pa-browser.desktop b/docker/personal-agent/desktop/pa-browser.desktop
similarity index 100%
rename from docker/personal-agent-desktop/pa-browser.desktop
rename to docker/personal-agent/desktop/pa-browser.desktop
diff --git a/docker/personal-agent-desktop/pa-stream.py b/docker/personal-agent/desktop/pa-stream.py
similarity index 100%
rename from docker/personal-agent-desktop/pa-stream.py
rename to docker/personal-agent/desktop/pa-stream.py
diff --git a/docker/hugagent-cli-entrypoint.sh b/docker/personal-agent/hugagent-cli-entrypoint.sh
similarity index 100%
rename from docker/hugagent-cli-entrypoint.sh
rename to docker/personal-agent/hugagent-cli-entrypoint.sh
diff --git a/docker/personal-agent-entrypoint.sh b/docker/personal-agent/personal-agent-entrypoint.sh
similarity index 94%
rename from docker/personal-agent-entrypoint.sh
rename to docker/personal-agent/personal-agent-entrypoint.sh
index 33865ada..dde1f819 100644
--- a/docker/personal-agent-entrypoint.sh
+++ b/docker/personal-agent/personal-agent-entrypoint.sh
@@ -31,4 +31,6 @@ su - pa-desktop -c 'export DISPLAY=:99 XAUTHORITY=/run/pa-desktop/.Xauthority; s
# Live screen: VNC listens only on a socket in the private directory; pa-stream checks the token.
su - pa-desktop -c 'sleep 2; while true; do x11vnc -display :99 -auth /run/pa-desktop/.Xauthority -rfbport 0 -unixsock /run/pa-desktop/vnc.sock -forever -shared -nopw -quiet -nowf -noscr -ncache 0 -cursor most -xkb -wait 10 -defer 10; sleep 1; done' >/run/pa-desktop/vnc.log 2>&1 &
su - pa-desktop -c 'while true; do /opt/python/current/bin/python3 -I /usr/local/bin/pa-stream; sleep 1; done' >/run/pa-desktop/stream.log 2>&1 &
-exec sleep infinity
+# PID 1 reaps every orphan and kills what task commands (UID 1000) leave behind;
+# the desktop above runs as UID 1001 and is only reaped.
+exec sandbox-init 1000
diff --git a/docker/render-personal-agent-config.py b/docker/personal-agent/render-personal-agent-config.py
similarity index 100%
rename from docker/render-personal-agent-config.py
rename to docker/personal-agent/render-personal-agent-config.py
diff --git a/docker/Dockerfile.opensandbox b/docker/sandbox/Dockerfile.opensandbox
similarity index 92%
rename from docker/Dockerfile.opensandbox
rename to docker/sandbox/Dockerfile.opensandbox
index b84926ab..957ebb3f 100644
--- a/docker/Dockerfile.opensandbox
+++ b/docker/sandbox/Dockerfile.opensandbox
@@ -5,7 +5,7 @@
#
# 构建:
# docker build -t hugagent-opensandbox-custom:latest \
-# -f Dockerfile.opensandbox .
+# -f docker/sandbox/Dockerfile.opensandbox .
#
# 启用:在 .env 中设置
# OPENSANDBOX_IMAGE=hugagent-opensandbox-custom:latest
@@ -69,7 +69,7 @@ RUN /opt/node/v22.2.0/bin/npm install -g @larksuite/cli@1.0.78 @larksuite/whiteb
RUN /opt/node/v22.2.0/bin/npm install -g openyida@2026.7.12-1
# ─── Python 数据/文档/网络全家桶 ───────────────────────────────────
-COPY docker/requirements-script-runner.txt /tmp/req-script-runner.txt
+COPY docker/sandbox/requirements-script-runner.txt /tmp/req-script-runner.txt
# Interactive package workers use FastAPI/uvicorn inside the sandbox.
RUN $PY_BIN -m pip install --no-cache-dir -r /tmp/req-script-runner.txt \
&& rm /tmp/req-script-runner.txt
@@ -77,7 +77,7 @@ RUN $PY_BIN -m pip install --no-cache-dir -r /tmp/req-script-runner.txt \
# Skill 依赖(由 backend admin UI 触发 rebuild 前聚合写入 requirements-skills.txt)。
# 依赖**只**在此 bake 进镜像;沙盒启动不再跑 pip。新增依赖须重新构建本镜像
# (管理台 → 沙盒依赖 → 重建)。/opt/skill-deps/ 副本仅留作排查参考。
-COPY docker/requirements-skills.txt /opt/skill-deps/requirements-skills.txt
+COPY docker/sandbox/requirements-skills.txt /opt/skill-deps/requirements-skills.txt
RUN $PY_BIN -m pip install --no-cache-dir -r /opt/skill-deps/requirements-skills.txt || \
(echo "[opensandbox] skill-deps install failed; continuing with empty set"; true)
@@ -141,17 +141,17 @@ RUN HIMALAYA_ARCH="$([ "${TARGETARCH:-amd64}" = "arm64" ] && echo aarch64 || ech
# 目录、node_modules 硬链接物化到 /workspace 临时区(秒级、离线、不进 myspace)。
# 构建期跑一次 smoke build:模板坏了在镜像构建阶段立刻暴露,而不是用户对话里。
# manifest 单独成层先 npm ci:模板源码微调不再击穿依赖层缓存(lock 不变即命中)
-COPY docker/site-template/react-vite/package.json \
- docker/site-template/react-vite/package-lock.json \
- docker/site-template/react-vite/.npmrc \
+COPY docker/sandbox/site-template/react-vite/package.json \
+ docker/sandbox/site-template/react-vite/package-lock.json \
+ docker/sandbox/site-template/react-vite/.npmrc \
/opt/site-template/react-vite/
RUN cd /opt/site-template/react-vite \
&& export PATH=/opt/node/v22.2.0/bin:$PATH \
&& npm ci --no-audit --no-fund \
&& npm cache clean --force \
&& rm -rf /root/.npm
-COPY docker/site-template/init-react-site.sh /opt/site-template/init-react-site.sh
-COPY docker/site-template/react-vite/ /opt/site-template/react-vite/
+COPY docker/sandbox/site-template/init-react-site.sh /opt/site-template/init-react-site.sh
+COPY docker/sandbox/site-template/react-vite/ /opt/site-template/react-vite/
RUN cd /opt/site-template/react-vite \
&& export PATH=/opt/node/v22.2.0/bin:$PATH \
&& SITE_DIST=/tmp/site-smoke SITE_CACHE=/tmp/site-cache npm run build \
@@ -221,5 +221,14 @@ RUN mkdir -p /home/ubuntu/.local /home/ubuntu/.jupyter /home/ubuntu/.npm \
done
# Real launch detects driver/browser revision mismatches before delivery.
-COPY docker/verify-browser-runtime.py /opt/verify-browser-runtime.py
+COPY docker/sandbox/verify-browser-runtime.py /opt/verify-browser-runtime.py
RUN runuser -u ubuntu -- $PY_BIN /opt/verify-browser-runtime.py --container-isolated
+
+# ─── sandbox-init(PID 1 + 命令的 subreaper)─────────────────────────────
+# 由 provider 的 entrypoint 作为 PID 1 启动:回收所有孤儿进程,并杀掉沙盒用户
+# (SANDBOX_RUN_UID)在命令结束后留下的进程;每条用户命令经 `sandbox-init
+# --subreaper` 运行,setsid / 两次 fork 也逃不出命令的进程树。放在末尾,
+# 改动不击穿上面的大层缓存。
+COPY docker/sandbox/sandbox-init.c /tmp/sandbox-init.c
+RUN gcc -O2 -Wall -Werror -o /usr/local/bin/sandbox-init /tmp/sandbox-init.c \
+ && rm /tmp/sandbox-init.c
diff --git a/docker/Dockerfile.script-runner b/docker/sandbox/Dockerfile.script-runner
similarity index 96%
rename from docker/Dockerfile.script-runner
rename to docker/sandbox/Dockerfile.script-runner
index 40f0103a..f921e735 100644
--- a/docker/Dockerfile.script-runner
+++ b/docker/sandbox/Dockerfile.script-runner
@@ -4,7 +4,7 @@
#
# 构建:
# docker build -t hugagent-script-runner:latest \
-# -f Dockerfile.script-runner .
+# -f docker/sandbox/Dockerfile.script-runner .
# 由 docker-compose 在 SANDBOX_PROVIDER=script_runner(COMPOSE_PROFILES=script_runner)
# 时自动构建启动。
@@ -77,13 +77,13 @@ RUN HIMALAYA_ARCH="$([ "${TARGETARCH:-amd64}" = "arm64" ] && echo aarch64 || ech
fi && rm -f /tmp/himalaya.tgz
# Python deps
-COPY docker/requirements-script-runner.txt /tmp/
+COPY docker/sandbox/requirements-script-runner.txt /tmp/
RUN pip install --no-cache-dir -r /tmp/requirements-script-runner.txt \
&& rm /tmp/requirements-script-runner.txt
# Skill 依赖(由 backend admin UI 触发 rebuild 前聚合写入 requirements-skills.txt)。
# 文件可能为空(默认占位),pip 接受空 requirements,不会失败。
-COPY docker/requirements-skills.txt /opt/skill-deps/requirements-skills.txt
+COPY docker/sandbox/requirements-skills.txt /opt/skill-deps/requirements-skills.txt
RUN pip install --no-cache-dir -r /opt/skill-deps/requirements-skills.txt || \
(echo "[script-runner] skill-deps install failed; continuing with empty set"; true)
@@ -141,5 +141,5 @@ EXPOSE 8900
CMD ["python", "-m", "uvicorn", "core.sandbox.runner.server:app", "--host", "0.0.0.0", "--port", "8900"]
# Real launch detects driver/browser revision mismatches before delivery.
-COPY docker/verify-browser-runtime.py /opt/verify-browser-runtime.py
+COPY docker/sandbox/verify-browser-runtime.py /opt/verify-browser-runtime.py
RUN python3 /opt/verify-browser-runtime.py --container-isolated
diff --git a/docker/apt-skills.txt b/docker/sandbox/apt-skills.txt
similarity index 100%
rename from docker/apt-skills.txt
rename to docker/sandbox/apt-skills.txt
diff --git a/docker/package-skills.json b/docker/sandbox/package-skills.json
similarity index 100%
rename from docker/package-skills.json
rename to docker/sandbox/package-skills.json
diff --git a/docker/requirements-script-runner.txt b/docker/sandbox/requirements-script-runner.txt
similarity index 100%
rename from docker/requirements-script-runner.txt
rename to docker/sandbox/requirements-script-runner.txt
diff --git a/docker/requirements-skills.txt b/docker/sandbox/requirements-skills.txt
similarity index 100%
rename from docker/requirements-skills.txt
rename to docker/sandbox/requirements-skills.txt
diff --git a/docker/sandbox/sandbox-init.c b/docker/sandbox/sandbox-init.c
new file mode 100644
index 00000000..39224bf0
--- /dev/null
+++ b/docker/sandbox/sandbox-init.c
@@ -0,0 +1,78 @@
+/*
+ * PID 1 of an OpenSandbox container, and the subreaper every user command runs under.
+ *
+ * sandbox-init as PID 1: reap every child, and SIGKILL the ones
+ * running as
+ * sandbox-init --subreaper cmd ... become a child subreaper, then exec cmd
+ *
+ * User commands run as under --subreaper, so whatever they start stays in their
+ * subtree for as long as they run, however it detaches (setsid, double fork). A leftover
+ * only reaches PID 1 once its command has ended, and is killed there: no process of
+ * outlives the command that started it. Sandbox services run as root and are
+ * only reaped.
+ */
+#define _GNU_SOURCE
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+/* The real uid: a process cannot change it, unlike the owner of /proc/. */
+static long real_uid(long pid)
+{
+ char path[64], line[256];
+ long uid = -1;
+ snprintf(path, sizeof path, "/proc/%ld/status", pid);
+ FILE *status = fopen(path, "r");
+ if (!status)
+ return -1;
+ while (fgets(line, sizeof line, status))
+ if (sscanf(line, "Uid: %ld", &uid) == 1)
+ break;
+ fclose(status);
+ return uid;
+}
+
+static void kill_orphans(long uid)
+{
+ /* Nothing is reaped during the scan, so no PID read here can be reused before its kill. */
+ FILE *children = fopen("/proc/1/task/1/children", "r");
+ long pid;
+ if (!children)
+ return;
+ while (fscanf(children, "%ld", &pid) == 1)
+ if (real_uid(pid) == uid)
+ kill((pid_t)pid, SIGKILL);
+ fclose(children);
+}
+
+int main(int argc, char **argv)
+{
+ if (argc > 2 && strcmp(argv[1], "--subreaper") == 0) {
+ if (prctl(PR_SET_CHILD_SUBREAPER, 1) == 0)
+ execvp(argv[2], argv + 2);
+ perror("sandbox-init");
+ return 127;
+ }
+ if (argc != 2) {
+ fprintf(stderr, "usage: sandbox-init | sandbox-init --subreaper cmd ...\n");
+ return 2;
+ }
+ long uid = strtol(argv[1], NULL, 10);
+ sigset_t chld;
+ sigemptyset(&chld);
+ sigaddset(&chld, SIGCHLD);
+ sigprocmask(SIG_BLOCK, &chld, NULL);
+ /* An orphan is adopted without any signal, so look for leftovers every second too. */
+ const struct timespec tick = {1, 0};
+ for (;;) {
+ sigtimedwait(&chld, NULL, &tick);
+ while (waitpid(-1, NULL, WNOHANG) > 0)
+ ;
+ kill_orphans(uid);
+ }
+}
diff --git a/docker/site-template/init-react-site.sh b/docker/sandbox/site-template/init-react-site.sh
similarity index 100%
rename from docker/site-template/init-react-site.sh
rename to docker/sandbox/site-template/init-react-site.sh
diff --git a/docker/site-template/react-vite/.npmrc b/docker/sandbox/site-template/react-vite/.npmrc
similarity index 100%
rename from docker/site-template/react-vite/.npmrc
rename to docker/sandbox/site-template/react-vite/.npmrc
diff --git a/docker/site-template/react-vite/index.html b/docker/sandbox/site-template/react-vite/index.html
similarity index 100%
rename from docker/site-template/react-vite/index.html
rename to docker/sandbox/site-template/react-vite/index.html
diff --git a/docker/site-template/react-vite/package-lock.json b/docker/sandbox/site-template/react-vite/package-lock.json
similarity index 100%
rename from docker/site-template/react-vite/package-lock.json
rename to docker/sandbox/site-template/react-vite/package-lock.json
diff --git a/docker/site-template/react-vite/package.json b/docker/sandbox/site-template/react-vite/package.json
similarity index 100%
rename from docker/site-template/react-vite/package.json
rename to docker/sandbox/site-template/react-vite/package.json
diff --git a/docker/site-template/react-vite/src/App.jsx b/docker/sandbox/site-template/react-vite/src/App.jsx
similarity index 100%
rename from docker/site-template/react-vite/src/App.jsx
rename to docker/sandbox/site-template/react-vite/src/App.jsx
diff --git a/docker/site-template/react-vite/src/index.css b/docker/sandbox/site-template/react-vite/src/index.css
similarity index 100%
rename from docker/site-template/react-vite/src/index.css
rename to docker/sandbox/site-template/react-vite/src/index.css
diff --git a/docker/site-template/react-vite/src/lib/siteApi.js b/docker/sandbox/site-template/react-vite/src/lib/siteApi.js
similarity index 100%
rename from docker/site-template/react-vite/src/lib/siteApi.js
rename to docker/sandbox/site-template/react-vite/src/lib/siteApi.js
diff --git a/docker/site-template/react-vite/src/main.jsx b/docker/sandbox/site-template/react-vite/src/main.jsx
similarity index 100%
rename from docker/site-template/react-vite/src/main.jsx
rename to docker/sandbox/site-template/react-vite/src/main.jsx
diff --git a/docker/site-template/react-vite/src/pages/Home.jsx b/docker/sandbox/site-template/react-vite/src/pages/Home.jsx
similarity index 100%
rename from docker/site-template/react-vite/src/pages/Home.jsx
rename to docker/sandbox/site-template/react-vite/src/pages/Home.jsx
diff --git a/docker/site-template/react-vite/vite.config.mjs b/docker/sandbox/site-template/react-vite/vite.config.mjs
similarity index 100%
rename from docker/site-template/react-vite/vite.config.mjs
rename to docker/sandbox/site-template/react-vite/vite.config.mjs
diff --git a/docker/verify-browser-runtime.py b/docker/sandbox/verify-browser-runtime.py
similarity index 100%
rename from docker/verify-browser-runtime.py
rename to docker/sandbox/verify-browser-runtime.py
diff --git a/document/README.md b/document/README.md
index 90158198..f06d0e45 100644
--- a/document/README.md
+++ b/document/README.md
@@ -1,10 +1,10 @@
# HugAgentOS 文档 / Documentation
-> 最后更新:2026-07-21
+> 最后更新 / Last updated:2026-10-23
-HugAgentOS 是一个企业级 AI Agent 平台(FastAPI + React + AgentScope 2.0),以 open-core 模式提供 **开源社区版(CE)** 与 **商业版(EE)** 双形态。产品文档按语言分树维护:
+HugAgentOS 是一个开源、可自托管的 Agent Harness:你提出的问题在云端持续执行,关掉电脑也不会中断;在此之上提供 7×24 小时在线的 Personal Agent,并以 HugAgentOS Cloud(服务器)和 HugAgentOS Local(桌面本机)两种形态运行,以 open-core 模式提供 **开源社区版(CE)** 与 **商业版(EE)**。产品文档按语言分树维护:
-HugAgentOS is an enterprise-grade AI agent platform (FastAPI + React + AgentScope 2.0), shipped in an open-core model as a **Community Edition (CE)** and an **Enterprise Edition (EE)**. Product documentation is maintained per language:
+HugAgentOS is an open-source, self-hosted agent harness: your request keeps running in the cloud even after you close your computer. On top of it, HugAgentOS provides an always-on Personal Agent and runs as HugAgentOS Cloud (on your server) or HugAgentOS Local (on your desktop), shipped in an open-core model as a **Community Edition (CE)** and an **Enterprise Edition (EE)**. Product documentation is maintained per language:
| 语言 / Language | 入口 / Entry |
|---|---|
@@ -19,14 +19,13 @@ HugAgentOS is an enterprise-grade AI agent platform (FastAPI + React + AgentScop
document/
├── zh-CN/ · en/ # 双语产品文档(镜像)
│ ├── getting-started/ # 产品简介、快速开始、领域本体快速入门
-│ ├── deployment/ # 部署总览(选型) + 无 Docker 一键安装 / Docker Compose / 离线生产 / Windows(WSL2) / 环境变量参考
+│ ├── deployment/ # 部署总览、Docker Compose、桌面本机模式、一键安装、Windows / macOS / UOS、环境变量
│ ├── architecture/ # 总体架构、后端、前端、数据模型
-│ ├── modules/ # 17 个功能模块文档(对话/提示词/能力目录/模型/
-│ │ # MCP 工具/技能/沙箱/记忆/知识库/存储/项目空间/
-│ │ # 认证/管理台/自动化/画布与产物)
-│ ├── api/ # API 总览(信封/鉴权/SSE/路由清单)、错误码
+│ ├── modules/ # 功能模块:对话与 Run、Personal Agent、自主循环、后台作业、MCP、技能、沙箱、
+│ │ # 记忆、本体治理、知识库、站点与应用托管、存储、项目空间、认证、管理台等
+│ ├── api/ # API 总览(信封 / 鉴权 / SSE / 路由清单)、错误码
│ ├── editions/ # 社区版 vs 商业版、License 机制、CE 构建管线
-│ └── development/ # 后端 / 前端开发指南
+│ └── development/ # 后端 / 前端开发、界面定制、工作台(Workbench)协议与 SDK
└── internal/ # ⚠️ 内部文档(仅主仓,不随社区版分发)
├── agents/ # 开发工作流约定(development-flow / issue-tracker /
│ # triage-labels / domain),被 .agents/skills 与 CLAUDE.md 引用
@@ -44,151 +43,13 @@ document/
`zh-CN/`, `en/`, and this index are **public product docs**, shipped with the Community Edition derived tree; EE-only capabilities are explicitly labeled. `internal/` holds **internal documents** (business strategy, split blueprints, design drafts) and is excluded from the CE tree via `ce/manifest.yaml` — put any new internal document there.
-云文档在线编辑:Markdown、TXT 和代码文件的编辑、保存、权限与版本冲突处理见
-[我的空间](zh-CN/modules/projects-myspace.md#云文档在线编辑) /
-[Online cloud document editing](en/modules/projects-myspace.md#online-cloud-document-editing)。
+## 常用专题 / Frequently referenced topics
-工作台协议 / Workbench protocol:
-[中文接入说明](zh-CN/development/workbench-protocol.md) /
-[English integration guide](en/development/workbench-protocol.md)。
-
-普通外部应用:[中文](zh-CN/development/workbench-external-apps.md) · [English](en/development/workbench-external-apps.md)。
-
-工作台目录与布局策略:[中文](zh-CN/development/workbench-access-policies.md) · [English](en/development/workbench-access-policies.md)。
-
-工作台导航偏好:[中文](zh-CN/development/workbench-navigation-preferences.md) · [English](en/development/workbench-navigation-preferences.md)。
-
-独立 SDK 与插件页面:[中文](zh-CN/development/workbench-sdk.md) · [English](en/development/workbench-sdk.md)。
-
-工作台独立工程与权限预览:[中文](zh-CN/development/workbench-preview.md) · [English](en/development/workbench-preview.md)。
-工作台管理员指南:[中文](zh-CN/development/workbench-administration.md) · [English](en/development/workbench-administration.md)。
-工作台错误码:[中文](zh-CN/development/workbench-errors.md) · [English](en/development/workbench-errors.md)。
-工作台兼容与迁移:[中文](zh-CN/development/workbench-migration.md) · [English](en/development/workbench-migration.md)。
-
-工作台目录通知与撤销:[中文](zh-CN/development/workbench-invalidation.md) · [English](en/development/workbench-invalidation.md)。
-
-- [工作台构建与交付](zh-CN/development/workbench-build.md)
-
-- [workbench-sdk](en/development/workbench-sdk.md)
-
-- [workbench-access-policies](en/development/workbench-access-policies.md)
-
-- [workbench-external-apps](en/development/workbench-external-apps.md)
-
-- [workbench-plugin-upgrades](en/development/workbench-plugin-upgrades.md)
-
-- [workbench-protocol](en/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](en/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](zh-CN/development/workbench-sdk.md)
-
-- [workbench-access-policies](zh-CN/development/workbench-access-policies.md)
-
-- [workbench-external-apps](zh-CN/development/workbench-external-apps.md)
-
-- [workbench-plugin-upgrades](zh-CN/development/workbench-plugin-upgrades.md)
-
-- [workbench-protocol](zh-CN/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](zh-CN/development/workbench-navigation-preferences.md)
-
-- [workbench-origins](en/development/workbench-origins.md)
-
-- [workbench-sdk](en/development/workbench-sdk.md)
-
-- [workbench-access-policies](en/development/workbench-access-policies.md)
-
-- [workbench-external-apps](en/development/workbench-external-apps.md)
-
-- [workbench-protocol](en/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](en/development/workbench-navigation-preferences.md)
-
-- [workbench-origins](zh-CN/development/workbench-origins.md)
-
-- [workbench-sdk](zh-CN/development/workbench-sdk.md)
-
-- [workbench-access-policies](zh-CN/development/workbench-access-policies.md)
-
-- [workbench-external-apps](zh-CN/development/workbench-external-apps.md)
-
-- [workbench-protocol](zh-CN/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](zh-CN/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](en/development/workbench-sdk.md)
-
-- [workbench-platform-adapters](en/development/workbench-platform-adapters.md)
-
-- [workbench-access-policies](en/development/workbench-access-policies.md)
-
-- [workbench-external-apps](en/development/workbench-external-apps.md)
-
-- [workbench-protocol](en/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](en/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](zh-CN/development/workbench-sdk.md)
-
-- [workbench-platform-adapters](zh-CN/development/workbench-platform-adapters.md)
-
-- [workbench-access-policies](zh-CN/development/workbench-access-policies.md)
-
-- [workbench-external-apps](zh-CN/development/workbench-external-apps.md)
-
-- [workbench-protocol](zh-CN/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](zh-CN/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](en/development/workbench-sdk.md)
-
-- [workbench-access-policies](en/development/workbench-access-policies.md)
-
-- [workbench-external-apps](en/development/workbench-external-apps.md)
-
-- [workbench-activity](en/development/workbench-activity.md)
-
-- [workbench-protocol](en/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](en/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](zh-CN/development/workbench-sdk.md)
-
-- [workbench-access-policies](zh-CN/development/workbench-access-policies.md)
-
-- [workbench-external-apps](zh-CN/development/workbench-external-apps.md)
-
-- [workbench-activity](zh-CN/development/workbench-activity.md)
-
-- [workbench-protocol](zh-CN/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](zh-CN/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](en/development/workbench-sdk.md)
-
-- [workbench-access-policies](en/development/workbench-access-policies.md)
-
-- [workbench-external-apps](en/development/workbench-external-apps.md)
-
-- [workbench-diagnostics](en/development/workbench-diagnostics.md)
-
-- [workbench-protocol](en/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](en/development/workbench-navigation-preferences.md)
-
-- [workbench-sdk](zh-CN/development/workbench-sdk.md)
-
-- [workbench-access-policies](zh-CN/development/workbench-access-policies.md)
-
-- [workbench-external-apps](zh-CN/development/workbench-external-apps.md)
-
-- [workbench-diagnostics](zh-CN/development/workbench-diagnostics.md)
-
-- [workbench-protocol](zh-CN/development/workbench-protocol.md)
-
-- [workbench-navigation-preferences](zh-CN/development/workbench-navigation-preferences.md)
-
-项目与智能体独立对话链接 / Standalone project and agent chat links:[中文](zh-CN/modules/chat.md#项目和智能体的独立对话链接) · [English](en/modules/chat.md#standalone-chat-links-for-projects-and-agents)。
-
-模块页面滚动与应用中心:[中文](zh-CN/architecture/frontend.md#模块页面滚动与应用中心) · [English](en/architecture/frontend.md#module-page-scrolling-and-app-center)。
+| 专题 / Topic | 中文 | English |
+|---|---|---|
+| Personal Agent | [Personal Agent](zh-CN/modules/personal-agent.md) | [Personal Agent](en/modules/personal-agent.md) |
+| 桌面本机与混合模式 / Desktop local and hybrid modes | [桌面离线本机模式](zh-CN/deployment/desktop-local-mode.md) | [Desktop local mode](en/deployment/desktop-local-mode.md) |
+| 云文档在线编辑 / Online cloud document editing | [我的空间](zh-CN/modules/projects-myspace.md#云文档在线编辑) | [Projects & My Space](en/modules/projects-myspace.md#online-cloud-document-editing) |
+| 项目与智能体独立对话链接 / Standalone chat links | [对话](zh-CN/modules/chat.md#项目和智能体的独立对话链接) | [Chat](en/modules/chat.md#standalone-chat-links-for-projects-and-agents) |
+| 模块页面滚动与应用中心 / Module scrolling and app center | [前端架构](zh-CN/architecture/frontend.md#模块页面滚动与应用中心) | [Frontend](en/architecture/frontend.md#module-page-scrolling-and-app-center) |
+| 工作台协议与 SDK / Workbench protocol and SDK | [工作台文档](zh-CN/README.md#开发指南) | [Workbench docs](en/README.md#development) |
diff --git a/document/en/README.md b/document/en/README.md
index b2d3aa13..6c80cf3b 100644
--- a/document/en/README.md
+++ b/document/en/README.md
@@ -1,15 +1,19 @@
# HugAgentOS Documentation
-> Last updated: 2026-07-21 | [简体中文](../zh-CN/README.md)
+> Last updated: 2026-10-23 | [简体中文](../zh-CN/README.md)
-HugAgentOS is an enterprise-grade AI agent platform: a FastAPI backend, a React frontend, and an AgentScope 2.0 agent runtime, fully containerized with Docker and shipped in an open-core model as a **Community Edition (CE)** and an **Enterprise Edition (EE)**. This tree is the complete product documentation for operators, users, and integrators.
+HugAgentOS is an open-source, self-hosted agent harness: your request keeps running in the cloud
+even after you close your computer. On top of it, HugAgentOS provides an always-on Personal Agent
+and runs as HugAgentOS Cloud (on your server) or HugAgentOS Local (on your desktop). It ships as an
+open-source Community Edition (CE) and an Enterprise Edition (EE). This tree is the complete
+product documentation for operators, users, and integrators.
## Getting Started
| Document | Description |
|------|------|
-| [Introduction](getting-started/introduction.md) | What it is, feature overview, CE/EE at a glance, tech stack and architecture |
-| [Quick Start](getting-started/quick-start.md) | Run HugAgentOS with the one-command installer or Docker Compose |
+| [Introduction](getting-started/introduction.md) | The durable agent, runtime forms, feature overview, tech stack and architecture |
+| [Quick Start](getting-started/quick-start.md) | Choose HugAgentOS Cloud, the desktop client or the one-command installer, and enable Personal Agent |
| [Build a Domain Ontology Quickly](getting-started/domain-ontology-quickstart.md) | Validate, publish, test, and bind tools with a minimal Domain Pack |
## Deployment
@@ -19,42 +23,74 @@ Start with the [Deployment Guide (choosing a method)](deployment/README.md), the
| Document | Description |
|------|------|
| [Deployment Guide · Overview](deployment/README.md) | Comparison and selection of deployment methods, post-deployment verification |
-| [No-Docker Quick Install](deployment/quick-install.md) | Zero-dependency single machine: one command, SQLite + in-process event log/ephemeral state + subprocess MCP/sandbox |
-| [Docker Compose Deployment](deployment/docker-compose.md) | Team/production standard: full service topology, profiles, rebuild workflows, database migrations |
-| [Offline Production Deployment](deployment/offline-production.md) | Isolated environments: image tarball packaging / production-side loading / prompt snapshot migration (Enterprise Edition) |
-| [Windows Deployment](deployment/windows-deployment.md) | Docker Desktop / WSL2 deltas, line endings, paths and sandbox limitations |
+| [Docker Compose Deployment](deployment/docker-compose.md) | The standard HugAgentOS Cloud setup: service topology, profiles, rebuild workflows, database migrations |
+| [Desktop Local Mode](deployment/desktop-local-mode.md) | HugAgentOS Local: private runtime, local-only / cloud-only / local + cloud, upgrades and rollback |
+| [Local Project Workspace](deployment/local-project-workspace.md) | Execution directory and permission rules for bound local projects |
+| [Local and Cloud Skills and Plugins](deployment/local-capability-management.md) | Where skills and plugins come from and how they are managed in hybrid mode |
+| [No-Docker Quick Install](deployment/quick-install.md) | Single machine: SQLite + in-process state + subprocess MCP / sandbox |
+| [Windows Deployment](deployment/windows-deployment.md) | Desktop-hosted single-machine service or Docker Desktop + WSL2 |
+| [macOS Signing and Upgrades](deployment/macos-signing.md) | Desktop signing, notarization and upgrades |
+| [My Space File Mounts](deployment/space-files.md) | The file directories shared by the web app and sandboxes |
| [Environment Variables](deployment/environment-variables.md) | Complete variable reference (defaults / purpose / CE·EE relevance) |
## Architecture
| Document | Description |
|------|------|
-| [Overview](architecture/overview.md) | Layered architecture, full request lifecycle, container topology, key design decisions |
-| [Backend](architecture/backend.md) | Full `src/backend/` walkthrough: api / orchestration / 15 core submodules / router registry |
-| [Frontend](architecture/frontend.md) | Five app entries, component groups, Zustand stores, hooks, build chain |
-| [Data Model](architecture/data-model.md) | 76 tables by domain, dual alembic chains, CE/EE table boundary |
+| [Overview](architecture/overview.md) | Layered architecture, ontology control plane, full request lifecycle, container topology |
+| [Backend](architecture/backend.md) | `src/backend/` walkthrough and router registry |
+| [Frontend](architecture/frontend.md) | App entries, component groups, Zustand stores, module pages and the app center |
+| [Data Model](architecture/data-model.md) | Tables by domain, migration chains, CE/EE table boundary |
## Modules
+**Harness and long-running work**
+
+| Document | Description |
+|------|------|
+| [Chat & Orchestration](modules/chat.md) | Runs decoupled from connections, stream resume, mid-run steering, per-step context restore and compaction, plan mode, sub-agents |
+| [Personal Agent](modules/personal-agent.md) | The 24/7 personal agent: background tasks, permanent cloud computer, schedules and heartbeats, IM channels, approvals, file memory and Dreaming |
+| [Autonomous Loop](modules/autonomous-loop.md) | Goals that survive across turns, maker ≠ checker, stall guards, resumable runs |
+| [Workflow Background Jobs](modules/workflow-jobs.md) | Managed sandbox processes, lease takeover, derived main agents |
+| [Automation & Plan Mode](modules/automation.md) | Scheduled tasks and plan mode |
+
+**Capabilities and extensions**
+
| Document | Description |
|------|------|
-| [Chat & Orchestration](modules/chat.md) | End-to-end chat flow, SSE events, citations, plan mode, sub-agents, stream resume |
-| [Prompt System](modules/prompts.md) | Assembly precedence, version pool, prompt hub, cross-environment migration |
+| [MCP Tool System](modules/mcp-tools.md) | Built-in and plugin servers, connection pooling, custom MCP |
+| [Agent Skills](modules/agent-skills.md) | SKILL.md mechanics, bundle layering, skill marketplace |
+| [Browser Automation Plugin](modules/browser-automation.md) | Browse pages, operate forms and test web pages |
| [Capability Catalog](modules/catalog.md) | Catalog as single source of truth, capability gating, self-service capabilities |
-| [Model Providers](modules/model-providers.md) | Provider & role system, dynamic switching, personal API keys, billing |
-| [MCP Tool System](modules/mcp-tools.md) | Built-in and plugin servers, connection pooling, admin/user custom MCP |
-| [Agent Skills](modules/agent-skills.md) | SKILL.md mechanics, bundle layering, skill marketplace, distillation |
-| [Sandbox Execution](modules/sandbox.md) | Three providers, bash tools, snapshot persistence, MySpace bind-mount |
-| [Memory System](modules/memory.md) | L1/L2/L3 layered memory, sanitization, audit, mem0 infrastructure |
-| [Domain Ontology Governance and Validation](modules/ontology-harness.md) | CE Settings governance, Domain Packs, tool gates, pre-delivery review, and governed evolution |
-| [Knowledge Base](modules/knowledge-base.md) | Self-hosted KB (hybrid retrieval) and Dify integration, public KB admin |
-| [Knowledge Base Wiki](modules/knowledge-base-wiki.md) | LLM-extracted entities and concepts as interlinked pages and a concept graph |
-| [Object Storage](modules/storage.md) | local / s3 / oss backends, artifact store, file pipelines |
-| [Projects & My Space](modules/projects-myspace.md) | Project workspaces, personal/team folders, files into context |
-| [Authentication & Permissions](modules/auth.md) | Three AUTH_MODEs, SSO, permission bits, admin credentials |
-| [Admin Consoles](modules/admin-console.md) | The /admin and /config consoles, 19 admin route groups |
-| [Automation & Batch](modules/automation.md) | Scheduled tasks, plan mode, batch orchestration |
+| [Sandbox Execution](modules/sandbox.md) | Providers, Bash tools, snapshot persistence, My Space mounts |
+| [Model Providers](modules/model-providers.md) | Provider and role system, dynamic switching, personal API keys |
+| [Prompt System](modules/prompts.md) | Assembly precedence, version pool, cross-environment migration |
+
+**Knowledge, memory and governance**
+
+| Document | Description |
+|------|------|
+| [Memory System](modules/memory.md) | L1/L2/L3 layered memory, durable outbox, procedural learning, sanitization and audit |
+| [Domain Ontology Governance and Validation](modules/ontology-harness.md) | Domain Packs, tool gates, pre-delivery review, human review of evolution drafts |
+| [Knowledge Base](modules/knowledge-base.md) | Self-hosted knowledge bases (hybrid retrieval) and external sources |
+| [Knowledge Base Wiki](modules/knowledge-base-wiki.md) | Extracted entities and concepts as interlinked pages and a concept graph |
+
+**Workspaces and artifacts**
+
+| Document | Description |
+|------|------|
+| [Projects & My Space](modules/projects-myspace.md) | Project workspaces, personal folders, online cloud document editing, files into context |
| [Canvas & Artifacts](modules/canvas-artifacts.md) | Univer canvas, code artifacts, artifact center, chat sharing |
+| [Sites](modules/sites.md) | Build sites in chat and host them on the platform |
+| [Application Databases and MCP Hosting](modules/application-hosting.md) | Separate application databases, form data and MCP publishing |
+| [Object Storage](modules/storage.md) | Local and S3-compatible storage, artifact store, file pipelines |
+
+**Platform administration**
+
+| Document | Description |
+|------|------|
+| [Authentication & Permissions](modules/auth.md) | AUTH_MODE, SSO, permission bits, admin credentials |
+| [Admin Consoles](modules/admin-console.md) | The /admin and /config consoles |
## API Reference
@@ -67,21 +103,40 @@ Start with the [Deployment Guide (choosing a method)](deployment/README.md), the
| Document | Description |
|------|------|
-| [Community vs Enterprise](editions/overview.md) | Open-core model, feature matrix, three runtime modes, upgrade paths |
-| [License Mechanism](editions/license.md) | Offline signature verification, state machine, feature enforcement, issuing tool (EE) |
+| [Community vs Enterprise](editions/overview.md) | Open-core model, feature matrix, runtime forms, upgrade paths |
+| [License Mechanism](editions/license.md) | Offline signature verification, state machine, feature enforcement (EE) |
| [CE Build Pipeline](editions/build-ce.md) | Manifest, build_ce.py pipeline, overlay, acceptance gates |
## Development
| Document | Description |
|------|------|
-| [Backend Development](development/backend.md) | In-Docker dev model, tests, migrations, layering conventions, adding routes/MCP/skills |
-| [Frontend Development](development/frontend.md) | Build hot-swap, directory conventions, API call conventions, edition gating |
-
----
-
-Internal design and planning documents (CE/EE split blueprints, sandbox snapshot design, etc.) live in the [docs root](../README.md).
+| [Backend Development](development/backend.md) | Dev model, tests, migrations, layering conventions, adding routes / MCP / skills |
+| [Frontend Development](development/frontend.md) | Builds, directory conventions, API call conventions, edition gating |
+| [Chat UI Customization](development/frontend-customization.md) | Page configuration, customer component packages and standalone frontends |
-- [Browser automation plugin](modules/browser-automation.md)
+**Workbench applications**
-- [Personal Agent](modules/personal-agent.md)
+| Document | Description |
+|------|------|
+| [Workbench Protocol](development/workbench-protocol.md) | JSON Schema, protocol versions and integration |
+| [Workbench SDK and Plugin Pages](development/workbench-sdk.md) | SDK 1.0.0 and UI v3 plugin pages |
+| [Development Preview](development/workbench-preview.md) | Standalone project and permission preview |
+| [Build and Offline Artifacts](development/workbench-build.md) | Building and delivering the protocol, SDK and host frontend |
+| [Routing and Plugin Navigation](development/workbench-routing.md) | `/apps/` URLs and applicationKey |
+| [Application Resources](development/workbench-resources.md) | In-package resource runtime |
+| [UI State and Bounded Keep-alive](development/workbench-state.md) | State schema versions, save and restore |
+| [Badges, Notifications and Background Work](development/workbench-activity.md) | badge, notification and background capabilities |
+| [Platform Adapters](development/workbench-platform-adapters.md) | Web, Tauri and Electron adapter interfaces |
+| [Origins and Offline Installation](development/workbench-origins.md) | Entries for same-name local and cloud plugins |
+| [Plugin Upgrades and Permission Approval](development/workbench-plugin-upgrades.md) | Upgrades, reinstalls and inherited grants |
+| [Catalog and Layout Policies](development/workbench-access-policies.md) | Catalog API, availability and default entries |
+| [Module Navigation and App Catalog](development/workbench-navigation-preferences.md) | Sidebar modules and the app center |
+| [Catalog Notifications and Revocation](development/workbench-invalidation.md) | ETags and catalog invalidation |
+| [External Applications](development/workbench-external-apps.md) | Managing and using external link applications |
+| [External Application Authentication](development/workbench-external-auth.md) | Authentication setup for external applications (EE console) |
+| [External Page Embedding](development/workbench-external-embedding.md) | Origins and readiness for embedded pages |
+| [Application Diagnostics](development/workbench-diagnostics.md) | Diagnostic IDs and retries |
+| [Error Codes](development/workbench-errors.md) | Workbench error codes |
+| [Administrator Guide](development/workbench-administration.md) | Workbench administration |
+| [Compatibility and Migration](development/workbench-migration.md) | Version compatibility and migration |
diff --git a/document/en/api/overview.md b/document/en/api/overview.md
index d8c14af6..dff5828d 100644
--- a/document/en/api/overview.md
+++ b/document/en/api/overview.md
@@ -59,13 +59,13 @@ The backend runs several parallel authentication mechanisms, each covering a dif
| Mechanism | How to send the credential | Scope | Implementation |
|---|---|---|---|
| **Session cookie** | `Cookie: jx_session=` (set automatically on login) | All end-user-facing `/v1/*` endpoints (chats, projects, kb, memories, …) | `get_current_user`: cookie → Redis session lookup |
-| **Personal API key** | `Authorization: Bearer sk-jx-...` | Same user identity as a session cookie; intended for scripts / third-party integrations | Recognized by the `sk-jx-` prefix; plaintext returned only once at creation, DB stores SHA256 only (`core/services/api_key_service.py`); self-managed at `/v1/me/api-keys` |
+| **Personal API key** | `Authorization: Bearer sk-jx-...` | Same user identity as a session cookie; intended for scripts / third-party integrations | Recognized by the `sk-jx-` prefix; plaintext returned only once at creation, DB stores SHA256 only (`core/auth/api_keys.py`); self-managed at `/v1/me/api-keys` |
| **Bearer token (mock/remote mode)** | `Authorization: Bearer ` | `AUTH_MODE=mock` (development: any token, or none, resolves to the mock user) or `remote` (verified against the user center) | Fallback branch of `get_current_user`; not accepted under `AUTH_MODE=session` |
| **ADMIN_TOKEN** | `Authorization: Bearer ` | `/admin` console backend (`/v1/admin/skills`, `/v1/admin/kb`, …) and content-block write endpoints | `require_admin` (env var `ADMIN_TOKEN`) |
| **CONFIG_TOKEN** | `Authorization: Bearer ` | `/config` console backend (`/v1/config/*`, `/v1/models`, `/v1/service-configs`, some `/v1/admin/*` observability endpoints) | `require_config` (env var `CONFIG_TOKEN`) |
| **ADMIN or CONFIG (either)** | Same as above | Prompt snapshot export/import (`/v1/content/prompts/export|import`) | `require_admin_or_config` |
| **super_admin session** | Session cookie (user with `extra_data.role == "super_admin"`); a valid ADMIN_TOKEN works as fallback | A few cross-user management operations | `require_super_admin` |
-| **BACKEND_INTERNAL_TOKEN** | `Authorization: Bearer ` | Internal site, browser and security-tool service calls; requests are rejected when unset | `api/routes/v1/internal_site_auth.py`, `internal_security.py` |
+| **BACKEND_INTERNAL_TOKEN** | `Authorization: Bearer ` | Internal site, browser and security-tool service calls; requests are rejected when unset | `features/sites/api/internal_site_auth.py`, `internal_security.py` |
Resolution order in `get_current_user`: **cookie session → API-key Bearer (`sk-jx-` prefix) → mock/remote Bearer**. A Bearer that looks like an API key but fails validation gets an immediate 401 — it is never silently downgraded to anonymous.
@@ -120,7 +120,7 @@ curl -X POST 'https:///api/v1/agents/responses' \
## SSE Streaming Protocol
-The streaming chat endpoint `POST /v1/agents/responses` (`src/backend/api/routes/v1/agent_responses.py`, `stream: true`) starts a background run after validation, then follows that run over SSE; after a disconnect, the client can resume from any offset via `GET /v1/chats/stream/{run_id}`. Events are produced by `src/backend/orchestration/workflow.py` and serialized to the wire by `src/backend/orchestration/chat_run_executor.py`.
+The streaming chat endpoint `POST /v1/agents/responses` (`src/backend/api/routes/v1/agent_responses.py`, `stream: true`) starts a background run after validation, then follows that run over SSE; after a disconnect, the client can resume from any offset via `GET /v1/chats/stream/{run_id}`. Events are produced by `src/backend/core/agent_runtime/runner.py` and serialized to the wire by `src/backend/core/chat/runtime/`.
**Wire format**: `Content-Type: text/event-stream`; each event is one `data: {JSON}` line with the event type in the JSON `type` field (the SSE `event:` line is not used). The stream terminates with `data: [DONE]`. After 15 seconds of silence, an SSE comment line `: heartbeat` keeps reverse-proxy connections alive.
@@ -197,7 +197,7 @@ All endpoints use the current user identity and verify chat ownership. Pending r
The answer count, question IDs, and order must exactly match the request. Private option IDs must come from the pending response, a single-select answer cannot contain multiple options, and every question must be answered or explicitly use `skipped=true`. The backend projects those private IDs back to the original option labels before returning the DSH-compatible model result `{"answers":[{"id":"scope","selected":["Minimal"]}]}`. The first valid answer/cancel atomically claims the request; concurrent submissions return `stale=true` and cannot wake the tool twice. A successful POST represents that atomic server claim, so the submitting tab may remove the exact `request_id` immediately; `user_question_resolved` and periodic authoritative pending snapshots synchronize other tabs and disconnected clients.
-`[ref:tool_name-N]` markers in the answer text are parsed into `citations` items by `orchestration/citations.py`; the frontend renders them as citation badges (see [Chat module](../modules/chat.md)).
+`[ref:tool_name-N]` markers in the answer text are parsed into `citations` items by `core/llm/citations/`; the frontend renders them as citation badges (see [Chat module](../modules/chat.md)).
### Resume, Steer, and cancel
diff --git a/document/en/architecture/backend.md b/document/en/architecture/backend.md
index 09e4841b..18029b3a 100644
--- a/document/en/architecture/backend.md
+++ b/document/en/architecture/backend.md
@@ -128,7 +128,7 @@ The root contains only the protocol, errors, factory, and package initializer. I
The runner depends only on process and path primitives that do not load backend settings or databases. OpenSandbox-specific components stay with that provider; shared processes, paths, and account environment support serve both OpenSandbox and Cube.
-### core/agent_skills — the skill engine
+### core/capabilities/skills — the skill engine
| File | Responsibility |
|---|---|
@@ -144,12 +144,12 @@ The runner depends only on process and path primitives that do not load backend
| Submodule | Responsibilities and key files |
|---|---|
| `core/chat` | Workflow context assembly (`context.py`), SSE tool-log event construction (`tool_log.py`) |
-| `core/content` | Attachment parsing (`file_parser.py`), KB document chunking/keywords/vectorization (`kb_processing.py`), upload validation (`file_validation.py`), artifact reading and summaries (`artifact_reader/refs/summary.py`), content-block import/export (`content_blocks.py`), `svg_fit.py` |
-| `core/kb` / `edition_ee/kb` | Shared private-KB parsing and Milvus vector storage stay in `core/kb`; the Dify client and external retrieval adapter live only in `edition_ee/kb` (EE) |
-| `core/artifacts` | Artifact store `store.py`: local / OSS dual mode |
-| `core/infra` | Unified responses (`responses.py`), exceptions (`exceptions.py`), structured logging (`logging.py`), rate limiting (`rate_limit.py`), Redis singleton (`redis.py`), metrics (`metrics.py`), background-task registry (`runtime_state.py`), data masking (`data_masking.py`), distillation budget gates (`distillation_budget.py`, Enterprise Edition, EE) |
+| `core/files/content` | Attachment parsing (`file_parser.py`), KB document chunking/keywords/vectorization (`kb_processing.py`), upload validation (`file_validation.py`), artifact reading and summaries (`artifact_reader/refs/summary.py`), content-block import/export (`content_blocks.py`), `svg_fit.py` |
+| `features/knowledge` / `edition_ee/kb` | Shared private-KB parsing and Milvus vector storage stay in `features/knowledge`; the Dify client and external retrieval adapter live only in `edition_ee/kb` (EE) |
+| `core/files/artifacts` | Artifact store `store.py`: local / OSS dual mode |
+| `core/infra` | Unified responses (`responses.py`), exceptions (`exceptions.py`), structured logging (`logging.py`), Redis singleton (`redis.py`), metrics (`metrics.py`), background-task registry (`runtime_state.py`), data masking (`data_masking.py`), distillation budget gates (`distillation_budget.py`, Enterprise Edition, EE) |
| `edition_ee/licensing` | Feature enums, signature verification, clock guard, seat policy, middleware, and the manager all exist only in EE; the CE tree has no License package and gets a fixed edition probe from its `api/middleware/edition.py` overlay |
-| `core/storage` | Storage protocol `protocol.py` + factory `factory.py`; `local.py` (CE), `s3.py` / `oss.py` (Enterprise Edition, EE) |
+| `core/files/storage` | Storage protocol `protocol.py` + factory `factory.py`; `local.py` (CE), `s3.py` / `oss.py` (Enterprise Edition, EE) |
## orchestration/ — the orchestration layer
@@ -219,12 +219,12 @@ CE routes live under `api/routes/v1/`; physically split commercial routes live u
| App entry and startup hooks | `src/backend/api/app.py` |
| Router registry | `src/backend/api/routes/v1/__init__.py` |
| Agent factory | `src/backend/core/llm/factory/build.py` |
-| Run executor / workflow | `src/backend/orchestration/chat_run_executor.py`, `workflow.py` |
-| Capability catalog | `src/backend/core/config/catalog.py` |
+| Run executor / workflow | `src/backend/core/chat/runtime/`, `workflow.py` |
+| Capability catalog | `src/backend/core/capabilities/catalog/` |
| Sandbox protocol | `src/backend/core/sandbox/protocol.py` |
| Memory pipeline | `src/backend/core/memory/pipeline.py` |
| EE License implementation / CE edition middleware | `src/backend/edition_ee/licensing/`, `src/backend/api/middleware/edition.py` |
-| Skill engine | `src/backend/core/agent_skills/loader.py` |
+| Skill engine | `src/backend/core/capabilities/skills/loader.py` |
| MCP registry and ports | `src/backend/extensions/mcp_servers/registry.py`, `_ports.py` |
The space MCP interface lives in `extensions/mcp_servers/space_files_mcp/`. The registry separates installation placement from exposure. Hybrid desktops access spaces through the capability gateway; mutations use unified approval. See [space file mounts](../deployment/space-files.md).
diff --git a/document/en/architecture/data-model.md b/document/en/architecture/data-model.md
index 0c1a94c2..95f89d71 100644
--- a/document/en/architecture/data-model.md
+++ b/document/en/architecture/data-model.md
@@ -179,11 +179,11 @@ A few tables that *look* EE but are required by CE are deliberately excluded fro
|---|---|
| Shared ORM / EE ORM | `src/backend/core/db/models/`, `src/backend/edition_ee/db/models/` |
| Ontology repository | `src/backend/core/db/repository/ontology.py` |
-| Feedback model / repository | `src/backend/edition_ee/db/models/feedback.py`, `db/repository/feedback.py` |
+| Feedback model / repository | `src/backend/edition_ee/db/models/feedback.py`, `edition_ee/db/repository/feedback.py` |
| Engine and startup table creation | `src/backend/core/db/engine.py` |
| Repository layer | `src/backend/core/db/repository/` |
-| CE/EE table boundary | `src/backend/core/db/edition_tables.py` |
+| CE/EE table boundary | `src/backend/edition_ee/db/edition_tables.py` |
| EE migration chain | `src/backend/alembic/versions/` |
| CE migration baseline | `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` |
-| Prompt version pool service | `src/backend/core/services/prompt_version_service.py` |
+| Prompt version pool service | `src/backend/core/llm/prompt_versions.py` |
| Authoritative edition boundary | [Editions](../editions/overview.md) |
diff --git a/document/en/architecture/overview.md b/document/en/architecture/overview.md
index 9105c485..3fc350d6 100644
--- a/document/en/architecture/overview.md
+++ b/document/en/architecture/overview.md
@@ -1,8 +1,8 @@
# Architecture Overview
-> Last updated: 2026-07-19
+> Last updated: 2026-10-23
-HugAgentOS is an enterprise-grade AgentOS with both a lightweight local profile and a fully containerized production profile. A React single-page application provides the user experience, an async FastAPI service runs the backend, AgentScope 2.0's ReActAgent supplies the agent runtime, independent MCP (Model Context Protocol) processes expose tools, and pluggable sandbox providers isolate code execution. The platform ships in two editions — an open-source Community Edition (CE) and a commercial Enterprise Edition (EE) — built from a single codebase and separated by three runtime seams: the router registry, the table-creation boundary, and license feature gates (see [Editions](../editions/overview.md)).
+HugAgentOS is an open-source, self-hosted agent harness that runs as HugAgentOS Cloud (a fully containerized server) or HugAgentOS Local (on the desktop). A React single-page application provides the user experience, an async FastAPI service runs the backend, AgentScope 2.0's ReActAgent supplies the agent runtime, independent MCP (Model Context Protocol) processes expose tools, and pluggable sandbox providers isolate code execution. The platform ships in two editions — an open-source Community Edition (CE) and a commercial Enterprise Edition (EE) — built from a single codebase and separated by three runtime seams: the router registry, the table-creation boundary, and license feature gates (see [Editions](../editions/overview.md)).
This page describes the system from five angles: the layered architecture, the ontology-grounded enterprise trust plane, the full lifecycle of one chat request, the container topology, and the key design decisions. For deeper dives, see [Backend Architecture](./backend.md), [Frontend Architecture](./frontend.md), and the [Data Model](./data-model.md).
@@ -23,51 +23,43 @@ This page describes the system from five angles: the layered architecture, the o
```
┌────────────────────────────────────────────────────────────────────┐
-│ Browser (React SPA) │
-│ App.tsx main app · AdminApp content console (EE) │
-│ ConfigApp system console (EE) │
-│ api.ts typed client + useStreaming SSE parser │
+│ Entry: web (React SPA) · desktop (Tauri) · IM channels · API · │
+│ schedules · heartbeats │
└──────────────────────────────┬─────────────────────────────────────┘
- │ HTTP / SSE (under /api)
+ │ HTTP / SSE (/api prefix)
┌──────────────────────────────▼─────────────────────────────────────┐
-│ Nginx (frontend container, :80) │
-│ Static assets · /api → backend reverse proxy │
+│ FastAPI (src/backend/api/) │
+│ Middleware: CORS · structured logging · error handling · │
+│ edition router registry │
└──────────────────────────────┬─────────────────────────────────────┘
│
┌──────────────────────────────▼─────────────────────────────────────┐
-│ FastAPI (backend container, src/backend/api/) │
-│ Middleware: CORS · structured logging · unified error handling │
-│ Router registry: CE_ROUTERS + EE_ROUTERS (license feature gates) │
+│ Run layer │
+│ core/runs: persisted runs, database leases, startup reconciliation,│
+│ tool effect intents and receipts │
+│ core/chat/runtime: run admission, chat worker, resume, steering, │
+│ compaction │
└──────────────────────────────┬─────────────────────────────────────┘
│
┌──────────────────────────────▼─────────────────────────────────────┐
-│ Orchestration layer (src/backend/orchestration/) │
-│ chat_run_executor: decouples Runs from HTTP, resumable streams │
-│ workflow: streaming orchestration · strategy: routing │
-│ citations: reference extraction · memory_integration │
-│ subagents/plan_mode · schedulers: automation + distillation (EE) │
-└──────────────────────────────┬─────────────────────────────────────┘
- │
-┌──────────────────────────────▼─────────────────────────────────────┐
-│ Agent layer (src/backend/core/llm/) │
-│ factory/build → AgentScope 2.0 ReActAgent │
-│ ToolCollector → Toolkit (built-in tools + skills + MCP tools) │
-│ middlewares: dynamic model / file context / system reminders │
-│ offloader: oversized tool results spilled to sandbox │
-│ prompts/prompt_runtime: system prompt assembly (DB-first) │
+│ Agent execution │
+│ core/agent_runtime: runner.astream_chat_workflow, routing, │
+│ memory / ontology / evolution integration │
+│ core/llm: factory/build → AgentScope ReActAgent, tools, context │
+│ orchestration/: plans · loops (autonomous loop) · automations │
+│ personal_agent/: identity, task queue, scheduling, channels, │
+│ device links, workspace │
└───────┬──────────────────────┬──────────────────────┬──────────────┘
- │ streamable-http │ sandbox protocol │ SQL / vectors
+ │ streamable-http │ sandbox provider │ SQL / vector
┌───────▼───────────┐ ┌────────▼──────────┐ ┌─────────▼─────────────┐
-│ MCP tool pool │ │ Sandboxes │ │ Data layer │
-│ mcp container, │ │ script-runner(CE) │ │ PostgreSQL · Redis │
-│ tool processes │ │ OpenSandbox CE/EE │ │ Milvus · Neo4j (opt.) │
-│ see _ports.py │ │ CubeSandbox (EE) │ │ Local / S3 / OSS │
-│ search/fetch/tools │ │ │ │ storage │
-│ plugins/dwh/chain(EE)│ │ │ │ │
+│ MCP tool pool │ │ Sandbox and jobs │ │ Data layer │
+│ mcp processes │ │ OpenSandbox │ │ PostgreSQL / SQLite │
+│ ports: _ports.py │ │ script-runner │ │ Redis · Milvus · Neo4j│
+│ plugin tools │ │ core/jobs │ │ S3-compatible / local │
└───────────────────┘ └───────────────────┘ └───────────────────────┘
```
-Dependencies point strictly downward: `api → orchestration → core`, and within `core`, `services → db`. MCP servers are independent processes that talk to the backend exclusively over HTTP, so a crash on either side never takes down the other.
+Dependencies point strictly downward: `api → features / orchestration / personal_agent → core`; `core` never depends on upper layers, and `integrations/` holds connectors to external systems such as DingTalk, Feishu and email. MCP servers are independent processes that talk to the backend exclusively over HTTP, so a crash on either side never takes down the other.
## Ontology-Grounded Enterprise Trust Plane
@@ -150,21 +142,22 @@ The entry point is `POST /v1/agents/responses` (`src/backend/api/routes/v1/chats
### 2. Starting a Run (decoupled from HTTP)
-`orchestration/chat_run_executor.py::start_run` registers the AI workflow as a background Run (persisted to the `chat_runs` table); the HTTP response merely *follows* that Run:
+`core/chat/runtime/admission.py::start_run` registers the turn as a background Run (persisted to the `chat_runs` table), executed by a chat worker that holds a database lease; the HTTP response merely *follows* that Run:
-- After a dropped connection or page refresh, the client calls `GET /v1/chats/stream/{run_id}` to resume from a given offset;
-- On backend restart, the lifespan hook `_startup_recover_chat_runs` recovers or reaps unfinished Runs.
+- After a dropped connection or page refresh, the client calls `GET /v1/chats/stream/{run_id}?from=N` to resume from a given offset;
+- While running, `POST /v1/chat-runs/{run_id}/steer` adds an instruction that applies at the next safe ReAct boundary;
+- On startup, `core/chat/runtime/recovery.py::recover_orphan_runs` reconciles Runs whose lease has expired from their last committed safe point: recoverable ones continue, unrecoverable ones are closed explicitly.
### 3. Orchestration and agent execution
-`orchestration/workflow.py::astream_chat_workflow` drives a single turn:
+`core/agent_runtime/runner.py::astream_chat_workflow` drives a single turn:
-1. **Memory retrieval** — `orchestration/memory_integration.py` reads the L1 profile / L2 vector memories (when enabled) and injects them as a system message;
-2. **Routing decision** — `orchestration/strategy.py`; `ROUTER_STRATEGY=main_only` (default) always routes to the main agent;
+1. **Memory retrieval** — `core/agent_runtime/integrations/memory.py` reads the L1 profile / L2 vector memories (when enabled) and injects them as a system message;
+2. **Routing decision** — `core/agent_runtime/strategy.py`; `ROUTER_STRATEGY=main_only` (default) always routes to the main agent;
3. **Agent construction** — `core/llm/factory/build.py::create_agent_executor`: assembles the prompt (`prompts/prompt_runtime.py`, DB version pool first), filters MCP servers by catalog and per-user capabilities, registers built-in tools (Read/Write/Edit/Glob/Grep/bash/skill loading, see `core/llm/tools/`), then constructs the AgentScope Toolkit in one shot;
-4. **Streaming execution** — `orchestration/streaming.py::StreamingAgent` wraps the ReActAgent and yields reasoning deltas, tool calls, and tool results as they happen;
-5. **Citation extraction** — `orchestration/citations.py` parses `[ref:tool_name-N]` markers out of tool results into citation items;
-6. **Wrap-up** — the assistant message and tool-call logs are persisted, `core/memory/pipeline.py` extracts memories asynchronously in the background (off the SSE hot path), and follow-up questions are generated by a separate endpoint (`orchestration/followups.py`).
+4. **Streaming execution** — `core/llm/runtime/streaming.py::StreamingAgent` wraps the ReActAgent and yields reasoning deltas, tool calls, and tool results as they happen;
+5. **Citation extraction** — `core/llm/citations/` parses `[ref:tool_name-N]` markers out of tool results into citation items;
+6. **Wrap-up** — the assistant message and tool-call logs are persisted, `core/memory/pipeline.py` extracts memories asynchronously in the background (off the SSE hot path), and follow-up questions are generated by a separate endpoint (`core/chat/followups.py`).
### 4. The SSE event stream
@@ -177,9 +170,9 @@ Each event is `data: {json}\n\n` with a `type` discriminator; the stream termina
| `tool_call_start` | Tool invocation starts (stable ID + name) | `core/chat/tool_log.py` |
| `tool_call_delta` | Batched incremental tool-argument JSON | `core/chat/tool_log.py` |
| `tool_call` | Tool arguments are complete and execution is about to start | `core/chat/tool_log.py` |
-| `tool_result` | Tool execution result (with citations, artifact card payloads) | `core/chat/tool_log.py` + `orchestration/tool_payloads.py` |
+| `tool_result` | Tool execution result (with citations, artifact card payloads) | `core/chat/tool_log.py` + `core/chat/tool_payloads.py` |
| `tool_progress` | Progress reports from long-running tools | `chats.py` |
-| `meta` | Trailing metadata: route, citation sources, artifact list, etc. | `orchestration/workflow.py` |
+| `meta` | Trailing metadata: route, citation sources, artifact list, etc. | `core/agent_runtime/runner.py` |
| `error` | Error event (immediately followed by `[DONE]`) | `chats.py` |
The frontend's `src/frontend/src/hooks/useStreaming.ts` dispatches on `type` and merges the start, deltas, completed call, and result in place into one tool card keyed by `tool_id`.
@@ -192,13 +185,14 @@ The frontend's `src/frontend/src/hooks/useStreaming.ts` dispatches on `type` and
|---|---|---|---|
| `frontend` | hugagent-frontend | `3002:80` | Nginx: static assets + `/api` reverse proxy |
| `backend` | hugagent-backend | `3001` | FastAPI main service |
-| `mcp` | hugagent-mcp | dedicated internal ports | 9 MCP servers as long-running streamable-http processes (7 general CE tools plus 2 EE industry tools) |
+| `mcp` | hugagent-mcp | dedicated internal ports | Built-in MCP servers as long-running streamable-http processes (data warehouse and industry tools are EE) |
| `postgres` | hugagent-postgres | `5432` | Primary database |
| `redis` | hugagent-redis | `6379` | Sessions / caching |
-| `script-runner` (profile `script_runner`) | hugagent-script-runner | internal | Lightweight script-execution sidecar (no DB/secret access) |
-| `opensandbox` + `opensandbox-config-init` (profile `opensandbox`) | hugagent-opensandbox | internal | Persistent sandbox (Enterprise Edition, EE) |
+| `script-runner` (profile `script_runner`) | hugagent-script-runner | internal | Optional lightweight script-execution sidecar (no DB/secret access) |
+| `opensandbox` + `opensandbox-config-init` (profile `opensandbox`) | hugagent-opensandbox | internal | Default sandbox: persistent sessions and snapshot recovery (mutually exclusive with script-runner) |
| `etcd` / `minio` / `milvus` (profile `mem0`) | hugagent-etcd etc. | internal | L2 vector-memory infrastructure |
| `neo4j` (profile `mem0`) | hugagent-neo4j | `7474/7687` | L3 graph memory (optional) |
+| `personal-agent-opensandbox` (profile `personal-agent`, `docker/personal-agent/compose.personal-agent.yml`) | — | internal | Dedicated controller for Personal Agent permanent computers |
All services share the `hugagent-network` network; named volumes such as `postgres_data` persist across rebuilds. See [Docker Compose deployment](../deployment/docker-compose.md) for details.
@@ -206,7 +200,7 @@ All services share the `hugagent-network` network; named volumes such as `postgr
### Catalog as the single source of truth
-`src/backend/core/config/catalog.json` is the sole authority for capability switches, with four kinds of entries: `skills` / `agents` / `mcp` / `kb`. `core/config/catalog.py` exposes `is_enabled(kind, id)` / `get_enabled_ids(kind)`, gating everything from MCP tool loading and skill registration to routing decisions; admin toggles are stored as overrides in the `catalog_overrides` table layered on top of the file (`core/config/catalog_loader.py`). Adding a capability to the catalog makes it visible to both the frontend capability center and the agent factory — no orchestration changes required. See [Capability Catalog](../modules/catalog.md).
+`src/backend/core/config/catalog.json` is the sole authority for capability switches, with four kinds of entries: `skills` / `agents` / `mcp` / `kb`. `core/capabilities/catalog/` exposes `is_enabled(kind, id)` / `get_enabled_ids(kind)`, gating everything from MCP tool loading and skill registration to routing decisions; admin toggles are stored as overrides in the `catalog_overrides` table layered on top of the file (`core/capabilities/catalog/loader.py`). Adding a capability to the catalog makes it visible to both the frontend capability center and the agent factory — no orchestration changes required. See [Capability Catalog](../modules/catalog.md).
### Unified response envelope
@@ -214,19 +208,19 @@ Every v1 endpoint returns `{ code, message, data, trace_id, timestamp }`, produc
### Prompts live in the database
-System prompts are no longer hardcoded: the version pool is stored in the `content_blocks` table (`id=prompt_versions`), managed by `core/services/prompt_version_service.py` (CRUD, activation, first-boot seeding). At assembly time, `prompts/prompt_runtime.py` degrades through three levels — active DB version → file fallback under `prompts/prompt_text/default/system/` → a minimal hardcoded fallback — guaranteeing a non-empty prompt. Operators can canary-switch prompt versions from the admin console without a release. See [Prompt System](../modules/prompts.md).
+System prompts are no longer hardcoded: the version pool is stored in the `content_blocks` table (`id=prompt_versions`), managed by `core/llm/prompt_versions.py` (CRUD, activation, first-boot seeding). At assembly time, `prompts/prompt_runtime.py` degrades through three levels — active DB version → file fallback under `prompts/prompt_text/default/system/` → a minimal hardcoded fallback — guaranteeing a non-empty prompt. Operators can canary-switch prompt versions from the admin console without a release. See [Prompt System](../modules/prompts.md).
### MCP tools as independent processes
-Each tool is a standalone MCP server under `src/backend/extensions/mcp_servers//`, all running inside the dedicated `mcp` container as long-lived streamable-http listeners (the port table in `extensions/mcp_servers/_ports.py` is the single source of truth). The backend connects through the HTTP client pool in `core/llm/mcp_manager.py`. Benefits: a tool crash never affects the main process, document editing and export have moved into sandbox skills, and the tool tier can scale independently. CE keeps 7 general-purpose servers; the two industry tools, `query_database` and `ai_chain_information_mcp`, depend on intranet data sources and belong to the Enterprise Edition (EE). See [MCP Tools](../modules/mcp-tools.md).
+Each tool is a standalone MCP server under `src/backend/extensions/mcp_servers//`, all running inside the dedicated `mcp` container as long-lived streamable-http listeners (the port table in `extensions/mcp_servers/_ports.py` is the single source of truth). The backend connects through the run-owned native client in `core/llm/mcp/lifetime.py`. Benefits: a tool crash never affects the main process, document editing and export have moved into sandbox skills, and the tool tier can scale independently. The two industry tools, `query_database` and `ai_chain_information_mcp`, depend on intranet data sources and belong to the Enterprise Edition (EE). See [MCP Tools](../modules/mcp-tools.md).
-### Runs decoupled from connections
+### Runs decoupled from connections and processes
-Streaming chats are not tied to the HTTP connection lifecycle: `chat_run_executor` persists execution state to the `chat_runs` table and supports offset-based resumption — eliminating "refresh loses the reply" and flaky-network drops — and the same execution channel is reused by the automation scheduler.
+Streaming chats are tied neither to the HTTP connection nor to a particular process: run state and per-step context are persisted, workers claim execution through database leases, and a stale executor can no longer commit results or call tools; when a process dies, another worker takes over. Tool calls record an intent and complete only with a receipt (`core/runs/tool_effects/`); recovery replays only calls proven not to have taken effect, and external operations with an unknown outcome go to the user for confirmation. Automation, Plan Mode, the autonomous loop and Personal Agent reuse the same run machinery.
### One codebase, two editions
-The commercial main repository carries the full code; the Community Edition is derived by `scripts/build_ce.py` according to `ce/manifest.yaml` (exclude `edition_ee` implementations + text transforms + CE overlays). Runtime boundaries include the router registry `api/routes/v1/__init__.py`, the table-creation boundary `core/db/edition_tables.py` (CE never registers the 20 EE-only tables), EE's `edition_ee/licensing/` package versus CE's fixed edition middleware, and policy seams such as site visibility. See [Editions](../editions/overview.md).
+The commercial main repository carries the full code; the Community Edition is derived by `scripts/build_ce.py` according to `ce/manifest.yaml` (exclude `edition_ee` implementations + text transforms + CE overlays). Runtime boundaries include the router registry `api/routes/v1/__init__.py`, the table-creation boundary `edition_ee/db/edition_tables.py` (CE never registers the 20 EE-only tables), EE's `edition_ee/licensing/` package versus CE's fixed edition middleware, and policy seams such as site visibility. See [Editions](../editions/overview.md).
## Related Source
@@ -234,12 +228,13 @@ The commercial main repository carries the full code; the Community Edition is d
|---|---|
| FastAPI app and startup hooks | `src/backend/api/app.py` |
| Streaming chat entry point | `src/backend/api/routes/v1/chats/__init__.py` |
-| Run executor | `src/backend/orchestration/chat_run_executor.py` |
-| Streaming orchestration | `src/backend/orchestration/workflow.py` |
+| Run executor | `src/backend/core/chat/runtime/` |
+| Streaming orchestration | `src/backend/core/agent_runtime/runner.py` |
| Agent factory | `src/backend/core/llm/factory/build.py` |
| MCP config and ports | `src/backend/core/config/mcp_config.py`, `src/backend/extensions/mcp_servers/_ports.py` |
-| Capability catalog | `src/backend/core/config/catalog.json`, `catalog.py` |
+| Capability catalog | `src/backend/core/config/catalog.json`, `src/backend/core/capabilities/catalog/` |
+| Runs, leases and recovery | `src/backend/core/runs/`, `src/backend/core/chat/runtime/recovery.py` |
| Response envelope | `src/backend/core/infra/responses.py` |
| Prompt assembly | `src/backend/prompts/prompt_runtime.py` |
-| CE/EE seams | `src/backend/api/routes/v1/__init__.py`, `src/backend/core/db/edition_tables.py`, `src/backend/api/middleware/edition.py`, `src/backend/edition_ee/licensing/` |
+| CE/EE seams | `src/backend/api/routes/v1/__init__.py`, `src/backend/edition_ee/db/edition_tables.py`, `src/backend/api/middleware/edition.py`, `src/backend/edition_ee/licensing/` |
| Container orchestration | `docker-compose.yml` |
diff --git a/document/en/deployment/README.md b/document/en/deployment/README.md
index 85827d04..f8fdfab2 100644
--- a/document/en/deployment/README.md
+++ b/document/en/deployment/README.md
@@ -9,17 +9,14 @@ HugAgentOS supports several deployment methods, from "zero-dependency single-mac
| Method | Best for | Docker | Database | Multi-user | Doc |
|---|---|---|---|---|---|
| **Windows desktop one-click install** | Personal Windows use; install the local service with the desktop client | Not required | SQLite | No (single user) | [windows-deployment.md](windows-deployment.md) |
-| **UOS 1070 ARM64 desktop** | UOS 1070 aarch64; Electron 43 `.deb` | Not required | SQLite / remote | Local single user / cloud multi-user | [uos1070-desktop.md](uos1070-desktop.md) |
| **No-Docker quick install** | Personal single-machine trial, development experience; one command and you're running | Not required | SQLite | No (single user) | [quick-install.md](quick-install.md) |
| **Docker Compose** | The standard form for teams / production — multi-user, full features | Required | PostgreSQL | Yes | [docker-compose.md](docker-compose.md) |
-| **Offline production (Enterprise Edition)** | Air-gapped environments (government intranets); image tarball offline delivery | Required | PostgreSQL | Yes | [offline-production.md](offline-production.md) |
Cross-platform and reference:
| Doc | Description |
|---|---|
| [Windows Deployment](windows-deployment.md) | Installing the desktop-managed local service, or running Compose through Docker Desktop and WSL2 |
-| [UOS 1070 ARM64 Desktop](uos1070-desktop.md) | Electron 43, glibc 2.28, ARM64 `.deb` build and acceptance |
| [Environment Variables](environment-variables.md) | Complete variable reference (defaults / purpose / CE·EE relevance) |
## One-line comparison
@@ -31,7 +28,6 @@ Cross-platform and reference:
single-user deployment.
- **No-Docker quick install**: the fastest command-line path on Linux and macOS. Run `curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash`; the installer creates the admin account, configures the model, starts the server, and opens the browser. Data lives under `~/.hugagent/`. **Single process, single user** — ideal for personal trials and development, not for multi-user or production.
- **Docker Compose**: the recommended standard deployment. All services are orchestrated by one `docker-compose.yml` (PostgreSQL + Redis + backend + MCP + frontend + sandbox), supporting multi-user, persistent sandboxes, layered memory, and every other capability.
-- **Offline production (EE)**: for isolated environments that cannot pull images online — build image tarballs on a connected machine, copy them to production, `docker load` + `compose up`. Part of the Enterprise Edition delivery scope.
> For capability differences between the Community and Enterprise editions, see [Edition Comparison](../editions/overview.md).
diff --git a/document/en/deployment/artifact-storage-key-repair.md b/document/en/deployment/artifact-storage-key-repair.md
deleted file mode 100644
index 7370f91f..00000000
--- a/document/en/deployment/artifact-storage-key-repair.md
+++ /dev/null
@@ -1,55 +0,0 @@
-# Repair historical artifact storage keys
-
-Older releases could register `artifacts/.docx` objects using the placeholder
-`artifacts/`. Generated-file persistence now requires a real storage key.
-Personal-space metadata reads drain local writes without requiring every remote
-file to download. Import failures show an error and retry control instead of an empty list.
-
-## Safety
-
-Repair matches only `storage_key == artifacts/`. It requires confirmed
-absence of that original object, an existing object with the recorded filename suffix,
-and downloaded bytes matching the recorded size. Timeouts, authorization errors,
-missing replacements and size mismatches leave the record unchanged. Existing
-extensionless objects are preserved.
-
-Updates compare the original key, filename, size and content timestamp atomically.
-They preserve the content version. The old/new keys, SHA-256 and original content
-version are committed together in `metadata.storage_key_repair` for audit and
-conditional rollback. Failed personal-space projection uses the same verified repair;
-existing conflict protection preserves unsaved local edits.
-
-## Operation
-
-Follow repository release approval rules. Validate in staging first and obtain
-separate approval before production code or data changes. Run from the project root
-using the authorized environment runtime and configuration. Never copy credentials
-or `.env` files between environments.
-
-```bash
-# Default: verify and print candidates without changing database records.
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id
-
-# Apply after reviewing candidates; already repaired records are skipped.
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id --apply
-
-# Undo audited repairs only while the current key and content version are unchanged.
-# This restores the old key. An upgraded background projector may repair it again.
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id --rollback
-```
-
-The tool processes one explicitly selected account in ID pages and exits nonzero on
-errors. It includes historical deleted records without restoring files or changing
-folder ownership.
-
-## Verification
-
-1. Confirm candidate keys, object sizes and audit records; repeat runs make no duplicate changes.
-2. Verify successful root/folder listing APIs and downloads of valid files.
-3. In Windows desktop local mode, select a file using both `@` and `+ → Import from My Space`, then send it and verify the document content is readable.
-4. With one unavailable object, other metadata remains accessible and fetching the affected file reports failure.
-5. Failed requests expose retry; old responses cannot overwrite a newer filter or folder.
-
-Frontend changes require a desktop client release to affect installed clients.
-A successful source build does not imply publication or production data changes.
-Record staging, production and desktop verification separately.
diff --git a/document/en/deployment/desktop-local-mode.md b/document/en/deployment/desktop-local-mode.md
index 12dfe662..b56d5d1d 100644
--- a/document/en/deployment/desktop-local-mode.md
+++ b/document/en/deployment/desktop-local-mode.md
@@ -3,7 +3,7 @@
The complete desktop package includes the platform-specific CE backend and a private Python runtime.
Starting local mode verifies and extracts the bundled resources without requiring Docker, system Python,
pip or a first-run dependency download. Thin packages support remote connections only.
-See [Windows deployment](windows-deployment.md), [UOS deployment](uos1070-desktop.md) and
+See [Windows deployment](windows-deployment.md) and
the [deployment overview](README.md) for installation options.
## Persistent data
diff --git a/document/en/deployment/docker-compose.md b/document/en/deployment/docker-compose.md
index 1dac3fd7..dc75240d 100644
--- a/document/en/deployment/docker-compose.md
+++ b/document/en/deployment/docker-compose.md
@@ -14,7 +14,7 @@ For a new deployment:
```bash
python3 scripts/configure-ce-docker.py
-docker build -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+docker build -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
docker compose build backend
docker compose up -d --build
```
@@ -68,8 +68,8 @@ Enterprise Edition (EE).
| Service | Profile | Container | Image / build | Role |
|---|---|---|---|---|
-| `script-runner` | `script_runner` | hugagent-script-runner | `docker/Dockerfile.script-runner` | Lightweight sandbox: 1 GB RAM / 1 CPU / read-only rootfs + tmpfs; skill directory mounted read-only at `/workspace/skills` |
-| `opensandbox-config-init` | `opensandbox` | hugagent-opensandbox-config-init | `python:3.12-alpine` | One-shot init: renders `@@HOST_REPO_PATH@@` / `@@HOST_STORAGE_PATH@@` in `docker/opensandbox-config.toml.tpl` into the named volume `opensandbox_config` |
+| `script-runner` | `script_runner` | hugagent-script-runner | `docker/sandbox/Dockerfile.script-runner` | Lightweight sandbox: 1 GB RAM / 1 CPU / read-only rootfs + tmpfs; skill directory mounted read-only at `/workspace/skills` |
+| `opensandbox-config-init` | `opensandbox` | hugagent-opensandbox-config-init | `python:3.12-alpine` | One-shot init: renders `@@HOST_REPO_PATH@@` / `@@HOST_STORAGE_PATH@@` in `docker/opensandbox-server/opensandbox-config.toml.tpl` into the named volume `opensandbox_config` |
| `opensandbox` | `opensandbox` | hugagent-opensandbox | `hugagent-opensandbox-server:v0.1.13-3` | Shared CE / EE controller: manages containers through Docker socket; execd runs commands and session files persist. CE publishes no host port; the full repository has a configurable debug port |
### mem0 memory infrastructure (profile `mem0`, optional)
@@ -219,19 +219,19 @@ make migrate-new msg="describe change"
| Feature | Files |
|---|---|
-| Service orchestration | `docker-compose.yml` (CubeSandbox overlay: `docker-compose.cube.yml`) |
+| Service orchestration | `docker-compose.yml` (CubeSandbox overlay: `docker/sandbox/compose.cube.yml`) |
| Backend image | `docker/Dockerfile` (multi-stage, target `production`) |
| MCP image | `docker/Dockerfile.mcp` |
-| Sandbox images | `docker/Dockerfile.script-runner`, `docker/Dockerfile.opensandbox` |
+| Sandbox images | `docker/sandbox/Dockerfile.script-runner`, `docker/sandbox/Dockerfile.opensandbox` |
| Frontend image + nginx proxy | `src/frontend/Dockerfile`, `src/frontend/nginx.conf`, `src/frontend/default.conf.template` |
| Backend entrypoint (auto migration) | `src/backend/scripts/backend_entrypoint.sh` |
-| OpenSandbox config template | `docker/opensandbox-config.toml.tpl` |
+| OpenSandbox config template | `docker/opensandbox-server/opensandbox-config.toml.tpl` |
| Migrations | `alembic.ini`, `src/backend/alembic/` |
| MCP port single source of truth | `src/backend/extensions/mcp_servers/_ports.py` |
### OpenSandbox command latency and controller image
-The controller is built from `docker/Dockerfile.opensandbox-server` as
+The controller is built from `docker/opensandbox-server/Dockerfile.opensandbox-server` as
`hugagent-opensandbox-server:v0.1.13-3`. Based on a pinned upstream release,
it inspects known sandboxes by name and verifies ownership labels. Synchronous
Docker lookups in lifecycle and proxy routes run in a thread pool so other requests
@@ -270,7 +270,7 @@ Create a dedicated BuildKit builder from the WSL project directory:
BUILDX_BUILDER=hugagent-limited COMPOSE_PARALLEL_LIMIT=1 docker compose build mcp
BUILDX_BUILDER=hugagent-limited COMPOSE_PARALLEL_LIMIT=1 docker compose build frontend
docker buildx build --builder hugagent-limited --load \
- -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+ -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
docker buildx stop hugagent-limited
Reuse the builder if it already exists. Stop it after builds; its cache remains available.
diff --git a/document/en/deployment/environment-variables.md b/document/en/deployment/environment-variables.md
index 27a293aa..8a10e2b6 100644
--- a/document/en/deployment/environment-variables.md
+++ b/document/en/deployment/environment-variables.md
@@ -23,7 +23,7 @@ This reference is sourced from `.env.example` and `src/backend/core/config/setti
| `HOST_REPO_PATH` | repo absolute path | Repository root on the host; used when the backend resolves compose-relative paths through docker.sock to rebuild sandbox images | CE |
| `DOCKER_GID` | `999` | Host docker group GID granting the backend container access to `/var/run/docker.sock` (find with `stat -c '%g' /var/run/docker.sock`) | CE |
| `COMPOSE_PROFILES` | CE Docker: `opensandbox` | Compose profile selection (`script_runner` / `opensandbox`, append `,mem0` as needed); must match `SANDBOX_PROVIDER` | CE |
-| `COMPOSE_FILE` | (unset) | Compose overlay files; set `docker-compose.yml:docker-compose.cube.yml` for CubeSandbox | CE |
+| `COMPOSE_FILE` | (unset) | Compose overlay files; set `docker-compose.yml:docker/sandbox/compose.cube.yml` for CubeSandbox | CE |
| `MAX_REQUEST_SIZE` | `52428800` (code default, 50 MB) | Maximum request body size (bytes) | CE |
| `DB_MIGRATION_RETRIES` / `DB_MIGRATION_RETRY_INTERVAL` | `20` / `2` (entrypoint defaults) | Startup migration retry count / interval (seconds) | CE |
@@ -57,7 +57,7 @@ This reference is sourced from `.env.example` and `src/backend/core/config/setti
| `ENABLE_SUMMARY` | `true` | When off, titles fall back to message truncation | CE |
| `SUMMARY_MAX_ROUNDS` | `3` | Titles stop updating after this many rounds | CE |
| `OPENAI_API_KEY` / `OPENAI_API_BASE` | (empty) / `https://api.openai.com/v1` | Alternate direct OpenAI configuration (passed through by compose) | CE |
-| `ROUTER_STRATEGY` | `main_only` (code default) | Routing strategy (`orchestration/strategy.py`) | CE |
+| `ROUTER_STRATEGY` | `main_only` (code default) | Routing strategy (`core/agent_runtime/strategy.py`) | CE |
| `FOLLOWUP_ENABLED` | `true` (code default) | Follow-up suggestion generation | CE |
## Authentication and login
@@ -219,13 +219,13 @@ the database has no value.
|---|---|---|
| `OPENSANDBOX_DOMAIN` | `http://opensandbox:8080` | OpenSandbox server address |
| `OPENSANDBOX_API_KEY` | Generated by CE initialization | Shared by backend and controller; never logged |
-| `OPENSANDBOX_IMAGE` | `hugagent-opensandbox-custom:latest` (compose default; code fallback `opensandbox/code-interpreter:v1.0.2`) | Sandbox runtime image; the custom image pre-installs all project dependencies (`docker/Dockerfile.opensandbox`) |
+| `OPENSANDBOX_IMAGE` | `hugagent-opensandbox-custom:latest` (compose default; code fallback `opensandbox/code-interpreter:v1.0.2`) | Sandbox runtime image; the custom image pre-installs all project dependencies (`docker/sandbox/Dockerfile.opensandbox`) |
| `OPENSANDBOX_READY_TIMEOUT_S` | `90` | Maximum wait for sandbox readiness (s) |
| `OPENSANDBOX_REQUEST_TIMEOUT_S` | `120` | Per-HTTP-call timeout (s) |
| `OPENSANDBOX_PORT` | Full repository: `8910`; CE: no mapping | Controller uses 8080 inside the network |
| `OPENSANDBOX_POOL_JUPYTER_MIN_IDLE` / `MAX_IDLE` | `.env.example` 1/3; compose defaults 2/3 | User session pool (historical setting name); CE minimum is 0 |
| `OPENSANDBOX_POOL_LIGHT_MIN_IDLE` / `MAX_IDLE` | `2` / `5` | Light bucket (one-shot executions) |
-| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | Pool ceiling (idle + in use) |
+| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | Controller-wide ceiling (all workers, warm and active instances); backend and controller must share this value |
| `OPENSANDBOX_SNAPSHOT_ENABLED` | `true` | Snapshot persistence master switch (snapshot+kill when idle, restore on reconnect) |
| `OPENSANDBOX_SNAPSHOT_RETENTION_DAYS` | `7` | Snapshot retention (GC sweeps expired rows) |
| `OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S` | `120` | Maximum poll for snapshot readiness (s) |
@@ -323,16 +323,10 @@ estimate only when it is not.
See the [License mechanism](../editions/license.md) for details.
-## Miscellaneous (rate limiting, circuit breakers, security, alerting)
+## Miscellaneous (security, alerting)
| Variable | Default | Description | Edition |
|---|---|---|---|
-| `RATE_LIMIT_ENABLED` | `true` | API rate limiting | CE |
-| `RATE_LIMIT_STORAGE` | `memory://` | Rate-limit counter store (`redis://...` supported) | CE |
-| `RATE_LIMIT_GLOBAL` / `RATE_LIMIT_PER_USER` | `500/minute` / `50/minute` | Global / per-user limits | CE |
-| `CB_USER_CENTER_THRESHOLD` / `CB_USER_CENTER_TIMEOUT` | `5` / `60` | User-center circuit breaker | EE |
-| `CB_MODEL_API_THRESHOLD` / `CB_MODEL_API_TIMEOUT` | `10` / `30` | Model-API circuit breaker | CE |
-| `CB_STORAGE_THRESHOLD` / `CB_STORAGE_TIMEOUT` | `5` / `60` | Storage circuit breaker | CE |
| `CORS_ORIGINS` | compose default `http://localhost:3000,http://localhost:5173` | Allowed cross-origin sources | CE |
| `ENABLE_SECURITY_HEADERS` | (commented; recommended `true` in production) | Security response headers | CE |
| `AUDIT_LOG_RETENTION_DAYS` / `AUDIT_LOG_EXPORT_ENABLED` | `90` / `true` (commented samples) | Audit-log retention / export | EE |
@@ -346,7 +340,7 @@ See the [License mechanism](../editions/license.md) for details.
|---|---|
| Variable samples (deployment starting point) | `.env.example` |
| Centralized settings (single read point) | `src/backend/core/config/settings.py` |
-| Compose-level injection and defaults | `docker-compose.yml`, `docker-compose.cube.yml` |
+| Compose-level injection and defaults | `docker-compose.yml`, `docker/sandbox/compose.cube.yml` |
| Sandbox provider selection | `src/backend/core/sandbox/`, `settings.py::SandboxSettings` |
| Memory settings | `settings.py::MemorySettings`, `src/backend/core/memory/` (service.py / pipeline.py) |
| Feedback GitHub configuration | `src/backend/edition_ee/services/system_config.py`, `feedback_service.py` |
@@ -395,7 +389,7 @@ Hosting remains opt-in. See [application deployment](../modules/application-host
With the module enabled, existing authenticated CE users can use Personal Agent. EE additionally checks the **Personal Agent** permission granted per user, role or team in the Config console (default off). Revoking that permission hides the entry and stops queued, scheduled and channel work. A persistent_sandbox license is not required. Both editions enforce ownership and action approvals.
-See [Personal Agent](../modules/personal-agent.md). Use docker/compose.personal-agent.yml with the root Compose file and the personal-agent profile.
+See [Personal Agent](../modules/personal-agent.md). Use docker/personal-agent/compose.personal-agent.yml with the root Compose file and the personal-agent profile.
diff --git a/document/en/deployment/offline-production.md b/document/en/deployment/offline-production.md
deleted file mode 100644
index 2b8569c1..00000000
--- a/document/en/deployment/offline-production.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# Offline Production Deployment (Enterprise Edition)
-
-> Last updated: 2026-10-16 | [简体中文](../../zh-CN/deployment/offline-production.md) | Back to [Deployment Guide](README.md)
-
-Offline delivery uses compiled images and the matching deployment kit from the `deploy` branch.
-Build and package in the supplier's separate `deploy` checkout. Load the images and start the matching kit on the target.
-Branch synchronization, builds, uploads and production publication each follow the project authorization rules.
-
-## Connected side: prepare the complete delivery
-
-1. Verify the validated `deploy` commit, clean checkout, product identity and target CPU architecture.
-2. Read that commit's `deploy_kit/save_pack_ee.sh` and matching Compose files in full.
-3. Build and package:
-
- ```bash
- # Run at the root of the deploy checkout
- bash deploy_kit/save_pack_ee.sh --build
- # Package existing compiled images only after checking their source revision
- bash deploy_kit/save_pack_ee.sh
- ```
-
-The script checks compiled backend, MCP, frontend and required infrastructure images, then includes available optional images.
-The supplier must also verify every dependency of the target profiles. An absent optional image does not mean the target does not need it.
-Use image names, versions, output paths and filenames from the selected script and manifest without renaming them.
-Deliver the image archive, manifest and matching kit together. Deliver the License separately through a secure channel.
-Ordinary source images or an incremental backend/frontend pack cannot replace the complete EE delivery.
-
-## Target side: load and start
-
-Follow the delivered kit to configure a separate project name, data directories, License and protected environment files.
-Before upgrading, back up the database and files, and verify the manifest, CPU architecture and selected profiles.
-After authorization, load the specified archive and use the matching kit's startup entry and complete Compose overlays.
-For targets without registry access, verify that all required images are loaded and startup does not build images.
-Do not use the former source-deployment scripts or automatically select an unverified archive by modification time.
-
-Verify at least `/health`, `/ready`, login, attachments, model calls and the enabled sandbox capabilities.
-Database migrations modify the schema; persistent volumes do not mean upgrades cannot change business data.
-On failure, restore the recorded images, configuration and data backups without deleting data volumes.
-
-## Shipping prompt snapshots with the pack
-
-System prompts live in the database `content_blocks` table (the `prompt_versions` pool plus the `prompt_hub` template gallery). They are **not in the repository and not in the images**. To push new prompts to production, ship them as a separate data file alongside the image pack and import on the production side:
-
-### 1. Export on the connected side
-
-```bash
-python src/backend/scripts/export_content.py \
- --api-url http://localhost:3001 --only prompts
-# outputs src/backend/scripts/exported/prompts_snapshot_.json
-```
-
-(`--database-url` is also supported for direct-DB export; `ADMIN_TOKEN` is read from `.env` automatically.)
-
-### 2. Copy
-
-Copy `prompts_snapshot_.json` to the production machine together with the image tarball.
-
-### 3. Import on the production side
-
-Prerequisite: the production backend image already includes the `POST /v1/content/prompts/import` endpoint (`src/backend/api/routes/v1/content.py`). Once the new backend is up:
-
-```bash
-BACKEND_CONTAINER=your-backend-container
-docker cp prompts_snapshot_.json "${BACKEND_CONTAINER}:/tmp/"
-docker exec "$BACKEND_CONTAINER" curl -sS -X POST \
- 'http://localhost:3001/v1/content/prompts/import?overwrite=true' \
- -H "Authorization: Bearer " \
- -H 'Content-Type: application/json' \
- -d @/tmp/prompts_snapshot_.json
-```
-
-The API import invalidates the prompt cache automatically — **no backend restart needed**.
-
-> This is an on-demand, one-time data operation: the DB volume persists, so prompts do not need re-importing on every release — only when pushing new prompts to production.
-
-## Related source
-
-| Feature | Files |
-|---|---|
-| Complete EE image pack (deploy branch) | `deploy_kit/save_pack_ee.sh` |
-| Compiled EE images (deploy branch) | `docker/Dockerfile.backend-compiled`, `docker/Dockerfile.mcp-compiled` |
-| Compose overlays (deploy branch) | `docker-compose.prod.yml`, `docker-compose.deploy.yml`; runtime follows the delivered kit |
-| Prompt export / import | `src/backend/scripts/export_content.py`, `src/backend/scripts/import_content.py` |
-| Prompt version pool | `src/backend/core/services/prompt_version_service.py` |
diff --git a/document/en/deployment/quick-install.md b/document/en/deployment/quick-install.md
index 04d1175b..c41b178b 100644
--- a/document/en/deployment/quick-install.md
+++ b/document/en/deployment/quick-install.md
@@ -185,7 +185,7 @@ The no-Docker single-machine mode is built to be lightweight. Here is how it dif
| CLI (onboard / serve / doctor) | `src/backend/cli.py` |
| Local-mode switch | `src/backend/core/config/settings.py` (`DeploySettings`, `DEPLOY_PROFILE=local`) |
| Frontend static hosting + `/api` bridge | `src/backend/api/local_hosting.py` |
-| MCP / sandbox subprocess supervision | `src/backend/orchestration/local_subprocess.py` |
-| In-process event log / ephemeral state | `src/backend/orchestration/run_event_stream.py`, `src/backend/core/infra/ephemeral.py` (enabled by leaving `REDIS_URL` empty) |
-| Built-in MCP catalog seed | `src/backend/core/services/mcp_service.py` (`seed_builtin_mcp_servers_if_empty`) |
+| MCP / sandbox subprocess supervision | `src/backend/core/infra/local_runtime/` |
+| In-process event log / ephemeral state | `src/backend/core/runs/events/`, `src/backend/core/infra/ephemeral.py` (enabled by leaving `REDIS_URL` empty) |
+| Built-in MCP catalog seed | `src/backend/core/capabilities/mcp/service.py` (`seed_builtin_mcp_servers_if_empty`) |
| Environment variables | [environment-variables.md](environment-variables.md) (the "No-Docker local mode" section) |
diff --git a/document/en/deployment/space-files.md b/document/en/deployment/space-files.md
index 6e2e1b55..5c696ccb 100644
--- a/document/en/deployment/space-files.md
+++ b/document/en/deployment/space-files.md
@@ -25,7 +25,7 @@ The controller needs no host FUSE mount or shared-propagation directory.
## Local S3 service (CE / EE)
-Add docker/compose.object-storage.yml to COMPOSE_FILE.
+Add docker/object-storage/compose.object-storage.yml to COMPOSE_FILE.
Set S3_ENDPOINT=http://object-storage:9000 and S3_BUCKET=hugagent-files.
Generate separate random S3_ACCESS_KEY and S3_SECRET_KEY values.
Store credentials in a protected environment file with mode 600.
@@ -36,15 +36,11 @@ Before switching an existing deployment, back up configuration and database.
Copy valid file objects and attachments. Check each object's SHA-256.
Keep source files. Switch STORAGE_TYPE and recreate the application containers after verification.
-## File import
+## Existing file migration
-Back up data and pause writes. Preview one user's or team's import plan:
-
- PYTHONPATH=src/backend python -m scripts.import_space_files --user USER_ID
- PYTHONPATH=src/backend python -m scripts.import_space_files --team TEAM_ID
-
-Review destinations and conflicts. Add --apply to import.
-The tool verifies content and preserves source objects.
+The standalone file import command has been removed. Back up the database and objects,
+pause destination writes, and review a migration plan before moving existing files.
+Check destinations, name conflicts, and content. Preserve source objects and references.
## Verification
@@ -52,8 +48,6 @@ Test upload, download and listing with an editor account.
Confirm that a read-only account cannot write or delete.
Revoke team access and confirm that the directory is unavailable.
-Local S3 has passed isolated MinIO, controller, sandbox and browser tests.
-Live OSS and OBS mounts need verification in dedicated test buckets.
Object-store directory moves use multiple object operations. They are not atomic filesystem renames.
## File page
@@ -105,4 +99,4 @@ Missing, expired, or changed-argument tickets cannot send a gateway request.
Desktop rollout requires the bundled backend, cloud backend, and a refreshed
capability manifest.
-The object-storage image uses checksum-pinned binaries from the official MinIO GitHub release. Docker builds this image from docker/Dockerfile.object-storage. Existing Milvus images and volumes are separate.
+The object-storage image uses checksum-pinned binaries from the official MinIO GitHub release. Docker builds this image from docker/object-storage/Dockerfile.object-storage. Existing Milvus images and volumes are separate.
diff --git a/document/en/deployment/uos1070-desktop.md b/document/en/deployment/uos1070-desktop.md
deleted file mode 100644
index 89017148..00000000
--- a/document/en/deployment/uos1070-desktop.md
+++ /dev/null
@@ -1,101 +0,0 @@
-# UOS 1070 aarch64 Desktop Client
-
-> [简体中文](../../zh-CN/deployment/uos1070-desktop.md)
-
-UOS 1070 aarch64 uses the Electron 43 release line in `desktop-uos/`. The supported
-delivery format is a Debian ARM64 `.deb`. It does not depend on system WebKitGTK 4.1,
-so it avoids the Tauri 2 WebView dependency blocker on the Debian 10-era UOS base.
-
-## Support baseline
-
-| Item | Production baseline |
-|---|---|
-| Operating system | UOS 1070 Desktop, aarch64 |
-| libc | glibc 2.28 |
-| Desktop shell | Electron 43.4.1 |
-| Installer | Debian `arm64` `.deb` |
-| Local runtime | Private CPython 3.11 with an `aarch64-manylinux_2_28` lock |
-| Data | `~/.hugagent` |
-| Configuration | `~/.config/com.hugagent.desktop` |
-
-The full package supports local, cloud, and dual modes with the existing desktop
-capabilities: deep-link login, loopback streaming proxy, tray/quick ask, native
-notifications, local-folder projects, dual-upstream routing, identity/model bridges,
-offline local-service install/upgrade/rollback, and SHA-256 verified updates.
-
-UOS hides the in-window menu bar by default and retains native title-bar controls.
-The tray exposes File, Edit, View, and Help commands; Ctrl+Shift+N opens a new window
-and Ctrl+N starts a new chat. The workspace shares macOS navigation gradients,
-rounded sidebar/content surfaces, light/dark themes, and loading layout. Quick ask
-keeps its compact appearance. The native UOS titlebar sits outside the web area,
-so no additional top inset is reserved inside the page.
-
-A distribution brand may preset the first-launch mode for a full package. A fixed
-dual-mode build goes directly to one-action initialization. The initialization and
-installation pages use the same halo, orbit, and progress-highlight motion as the
-existing desktop client, while showing only the current stage, percentage, and
-actionable errors. Raw logs remain in the local-service log directory for diagnosis
-instead of appearing at the bottom of the page.
-
-## Build and verification
-
-Build the full package on native Linux aarch64, preferably UOS 1070 itself. An x86_64
-builder may only cross-package the remote-only shell without the Python runtime.
-
-```bash
-cd desktop-uos
-npm install
-npm test
-npm run build
-npm run verify:deb -- "dist/HugAgentOS UOS_0.5.15_uos1070_arm64.deb"
-```
-
-Release acceptance must cover:
-
-1. Debian architecture is `arm64`;
-2. the Electron ELF imports no GLIBC symbol newer than 2.28;
-3. a disconnected full-package cold start performs no pip, uv, or Python download;
-4. local, cloud, and dual modes pass login, SSE chat, file transfer, folder selection, and notifications;
-5. dual-mode cloud projects remain in the cloud while local-folder projects remain local;
-6. a staged update passes SHA-256 verification, system authorization, installation, and restart.
-
-UOS shares one release index with Windows and macOS: each client fetches the update for its
-own platform and reports "no update available for this platform" when none was published,
-while older clients that send no platform read the common release, which advances only once
-all three platforms have published that version. The desktop version is identical across the
-three platforms and the build enforces it.
-
-See [`desktop-uos/README_EN.md`](../../../desktop-uos/README_EN.md) for commands,
-artifacts, and publishing.
-
-> Electron 43 upstream support is scheduled to end in January 2027. Treat it as the
-> current UOS compatibility baseline, keep the latest 43.x patch deployed, and qualify
-> Electron 44+ continuously.
-
-## Parity with the current Tauri client
-
-UOS shares the current React frontend and CE backend sources. Rebuilding the full package includes
-the latest capability origins, local enablement, skill sync and working-directory behavior.
-Updating the cloud alone does not replace resources in an installed client.
-
-- File supports opening a folder into a local project conversation; ordinary packages can change runtime mode.
-- View offers Zoom In, Zoom Out and Actual Size (Ctrl +/-/0 or Ctrl+wheel). Tauri's zoom steps are
- saved in `prefs.json` and reused by new windows and after restart; system DPI remains native.
-- Help, tray and sidebar share update handling. Background release notices only offer an update;
- confirmation opens download progress with cancellation during download. Installation still requires
- Ed25519 signature and SHA-256 verification and UOS system authorization.
- Older servers use low-frequency checks; reconnects back off and concurrent checks cannot install twice.
-- The loopback proxy checks Origin/Fetch Metadata before injecting session and bridge credentials;
- direct requests without source metadata are refused.
-- Service logs rotate while running, keeping the current and previous files at up to 32 MiB each.
- Capability readiness checks back off from 500 ms to 15 seconds. Existing UOS data and capability
- directories are preserved without automatically migrating configuration.
-
-Full packages also support Tauri's hybrid-only build option:
-
-```bash
-JX_DEFAULT_SERVER_BASE=https://agent.example.com npm run build:hybrid-only
-```
-
-This fixes the cloud address and dual mode and requires a cloud address. Thin packages cannot be
-hybrid-only. Full offline ARM64 packages still require native aarch64 build and installation validation.
diff --git a/document/en/development/backend.md b/document/en/development/backend.md
index 3ed2f541..153eac46 100644
--- a/document/en/development/backend.md
+++ b/document/en/development/backend.md
@@ -41,7 +41,7 @@ The following targets are taken from the root `Makefile`:
| `make migrate` | `alembic upgrade head` |
| `make migrate-new msg="..."` | `alembic revision --autogenerate -m "..."` |
| `make migrate-down` / `migrate-history` | roll back one step / show history |
-| `make db-reset` / `db-seed` | reset (destructive) / seed sample data |
+| `make db-reset` | Reset the database (destructive) |
| `make build` / `up` / `down` / `logs` / `ps` etc. | docker-compose wrappers |
> ⚠️ The committed tree is not formatter-clean: **do not run black/isort over files you did not touch** (it buries the real diff in formatting noise); format only the scope you changed.
@@ -68,10 +68,10 @@ The version list pins a compatible mem0, pymilvus and Milvus Lite combination. D
## Alembic migration flow
-1. Change the ORM models in `src/backend/core/db/models.py`;
+1. Change the ORM models in `src/backend/core/db/models/`;
2. `make migrate-new msg="add xxx table"` to autogenerate the migration (**review the output by hand** — autogenerate cannot be fully trusted);
3. Apply with `make migrate` (or `alembic upgrade head` inside the container);
-4. If the table is **EE-only**: add its name to `core/db/edition_tables.py::EE_ONLY_TABLES` (CE does not create empty EE tables; names in the set are existence-asserted).
+4. If the table is **EE-only**: add its name to `edition_ee/db/edition_tables.py::EE_ONLY_TABLES` (CE does not create empty EE tables; names in the set are existence-asserted).
Note: the main-repo alembic chain is the EE chain; the CE derived tree uses its own `ce_0001` baseline chain (see [CE Build Pipeline](../editions/build-ce.md#ce-database-differences)) — new main-repo migrations never enter CE.
@@ -175,7 +175,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest
## Adding a skill (Agent Skill)
-Skill loading is multi-source (`core/agent_skills/config.py`): built-in (`extensions/skills/default/`, always-on), admin (DB / `/app/storage/admin_skills/`), user, and project sources merge by priority. `extensions/skills/marketplace/` holds install-based marketplace seeds scanned separately by the marketplace service — it is **not** part of the default loading sources.
+Skill loading is multi-source (`core/capabilities/skills/config.py`): built-in (`extensions/skills/default/`, always-on), admin (DB / `/app/storage/admin_skills/`), user, and project sources merge by priority. `extensions/skills/marketplace/` holds install-based marketplace seeds scanned separately by the marketplace service — it is **not** part of the default loading sources.
Adding a **built-in skill**:
@@ -188,7 +188,7 @@ A **marketplace skill** goes under `extensions/skills/marketplace//` w
## New-feature checklist
-- [ ] ORM model + indexes + timestamps (`core/db/models.py`); EE tables added to `EE_ONLY_TABLES`
+- [ ] ORM model + indexes + timestamps (`core/db/models/`); EE tables added to `EE_ONLY_TABLES`
- [ ] Repository filters soft deletes; Service carries business logic and permission checks
- [ ] Routes use envelope responses; errors go through `core/infra/exceptions`
- [ ] Route registered in `CE_ROUTERS` / `EE_ROUTERS` (EE with a feature bit + manifest exclusion)
@@ -205,10 +205,10 @@ A **marketplace skill** goes under `extensions/skills/marketplace//` w
| Response envelope | `src/backend/core/infra/responses.py` |
| Exception hierarchy | `src/backend/core/infra/exceptions.py` |
| Dependency injection | `src/backend/api/deps.py` |
-| ORM / table boundary | `src/backend/core/db/models.py`, `src/backend/core/db/edition_tables.py` |
+| ORM / table boundary | `src/backend/core/db/models/`, `src/backend/edition_ee/db/edition_tables.py` |
| MCP port table / connection config | `src/backend/extensions/mcp_servers/_ports.py`, `src/backend/core/config/mcp_config.py` |
| Capability catalog | `src/backend/core/config/catalog.json` |
-| Skill loading | `src/backend/core/agent_skills/` (`config.py` / `loader.py` / `registry.py`) |
-| Workflow orchestration | `src/backend/orchestration/workflow.py` |
+| Skill loading | `src/backend/core/capabilities/skills/` (`config.py` / `loader.py` / `registry.py`) |
+| Workflow orchestration | `src/backend/core/agent_runtime/runner.py` |
See also: [Frontend Development Guide](frontend.md) · [API Overview](../api/overview.md) · [MCP Tools](../modules/mcp-tools.md) · [Agent Skills](../modules/agent-skills.md)
diff --git a/document/en/development/container-layout.md b/document/en/development/container-layout.md
deleted file mode 100644
index 9911e7f1..00000000
--- a/document/en/development/container-layout.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# Container-responsive layout
-
-The chat toolbar responds to its pane's available width. Home, conversation, project,
-and split views share the same layout.
-
-- Tools and model/send controls form separate groups and wrap when necessary.
-- Labels collapse progressively while icons, accessible names, and mode exits remain.
- Model names stay visible with ellipsis for long text.
-- Touch devices use real 44px targets in normal flow, without overlapping invisible hit layers.
-- Project, capability, and knowledge cards use intrinsic grids; long names cannot widen tracks.
-- Safe areas, page height, and portaled menus retain their viewport-based constraints.
-
-Implementation: frontend components/chat/ComposerToolbar.tsx,
-styles/chat/toolbar-layout.css, styles/projects.css, and styles/catalog/.
-A container supplies width and its descendants respond. Do not query the container
-from itself or duplicate CSS breakpoints in JavaScript.
-
-From the WSL project directory, enter src/frontend with a supported Node.js version:
-
- npm run test:container-layout-browser
- npm run test:canvas-layout-browser
- npm run test:chat-width-browser
- npm run lint
- npm run build
-
-The new suite covers narrow containers inside a wide window, phone viewports,
-fractional boundaries, light/dark themes, long names, all modes, touch targets,
-keyboard exits, and send/stop callbacks.
diff --git a/document/en/development/frontend.md b/document/en/development/frontend.md
index 9df68e6b..715a69c5 100644
--- a/document/en/development/frontend.md
+++ b/document/en/development/frontend.md
@@ -132,6 +132,15 @@ Caveats:
- When adding new EE UI: ① gate it with the selector on the matching feature bit; ② if the entire page is EE (like /admin, /config), no runtime gating is needed — the CE derived tree physically excludes the component directories and removes the routes via the overlay `main.tsx` (see [CE Build Pipeline](../editions/build-ce.md));
- When handling API errors, branch on `LicenseError` (402) and show "contact your administrator to update the license" rather than a generic failure message.
+## Container-responsive layout
+
+The chat toolbar and project / capability / knowledge cards lay out by the available width of their pane; home,
+conversation, project and split views share the same rules. Containers provide width and descendants respond to the
+container; elements must not query themselves, and CSS breakpoints must not be duplicated in JavaScript. Touch targets
+are real 44px areas. Entry points: `components/chat/ComposerToolbar.tsx`, `styles/chat/toolbar-layout.css`,
+`styles/projects.css`, `styles/catalog/`. After changes, run `npm run test:container-layout-browser` and
+`npm run test:chat-width-browser`.
+
## New-feature checklist
- [ ] Components are PascalCase + named export + Props interface
diff --git a/document/en/development/workbench-migration.md b/document/en/development/workbench-migration.md
index 1449f3fd..89d6a38e 100644
--- a/document/en/development/workbench-migration.md
+++ b/document/en/development/workbench-migration.md
@@ -44,24 +44,7 @@ The SDK CLI validates declarations. It does not migrate databases or business da
# Workbench configuration migration and recovery
-Migration is an explicit operator action. It never runs during startup. Rehearse on an isolated database copy before separately authorizing a real migration. The current command accepts local rehearsal copies only.
-
-## Inventory and cutover
-
-1. Freeze every writer on the copy, including content imports, preferences, plugin installation, and background jobs. --writes-frozen confirms this operator action; it does not stop services.
-2. Set WB20_COPY_DATABASE_URL through protected configuration. SQLite must reside under the current checkout's .scratch/workbench-20 directory. PostgreSQL must use a loopback host and a database name beginning with workbench_20. Never target business data.
-3. Create a private 0700 snapshot directory. Run:
-
-```bash
-PYTHONPATH=src/backend python -m scripts.workbench_migration inventory \
- --snapshot /absolute/private/path/before.json
-PYTHONPATH=src/backend python -m scripts.workbench_migration apply \
- --snapshot /absolute/private/path/before.json --digest --writes-frozen
-```
-
-Inventory never writes the database. Reports contain counts, contract versions, redacted locations, and error codes. They omit URLs, user identities, tickets, and secrets. The 0600 snapshot retains all migration inputs: configuration, preferences, role/team relationships, plugin declarations, and workbench records. It does not replace a full database or business-file backup.
-
-A matching digest and unchanged source permit one transactional cutover. Failures roll back the transaction. A workbench_migration:v1 receipt records both snapshot digests. Cutover removes the old app_config row and navigation lists. The directory reads canonical sources. Duplicate external IDs, unknown IDs, dangerous URLs, and unknown fields are quarantined without creating entries or grants. Protected snapshots retain their raw values.
+The standalone migration command has been removed. The following sections describe internal migration and recovery constraints; they are not an executable operator procedure. Back up existing deployments, prepare a version-specific plan, validate it on an isolated copy, and obtain separate authorization before execution.
## Preserved behavior
@@ -69,7 +52,7 @@ A matching digest and unchanged source permit one transactional cutover. Failure
- External ordering and enabled state survive. Default navigation becomes layout policy. Existing modern preferences win. Confirmed old preferences.pinned_apps/app_order migrate to web preferences: null inherits and an empty list explicitly unpins. Chat order, theme, and unrelated preferences remain unchanged.
- Unmapped permission references remain unchanged and are reported. They do not grant access to new identities. Plugin UI v1/v2 is not upgraded to v3. Names and slugs never substitute for exact installations.
- Existing workbench_external_auth:legacy: bindings retain their keys. Missing target adapters produce auth_configuration_required. Administrators must approve explicit login modes. Real legacy targets remain unverified until exercised. Generic SSO token forwarding must never return.
-- Same-service recovery retains workbench_origin_identity. A service clone requires a fresh inventory and clone-origin on the copy. Existing plugin keys do not automatically grant access to the new origin. Review cross-origin state and preferences separately.
+- Same-service recovery retains workbench_origin_identity. A service clone requires a fresh inventory and a regenerated service origin identity on the copy. Existing plugin keys do not automatically grant access to the new origin. Review cross-origin state and preferences separately.
## External compatibility window
@@ -77,13 +60,7 @@ content-v1 remains supported throughout workbench protocol 1.x. Announce a separ
## Recovery and incremental writes
-Freeze every writer again:
-
-```bash
-PYTHONPATH=src/backend python -m scripts.workbench_migration rollback \
- --snapshot /absolute/private/path/before.json --digest \
- --delta /absolute/private/path/delta.json --writes-frozen
-```
+Freeze all writers and follow the reviewed recovery plan.
Recovery first exports a protected delta. Changes to already-mapped ordinary legacy links can replay into the old format. New identities, application state, resources, permissions, plugins, authentication, or policy changes block unsafe downgrade with delta_not_representable. New data stays intact. Never restore only the old snapshot. Keep the verified newer version and review the delta. Recovery receipts are idempotent. A restored batch requires a new reviewed batch before another apply.
diff --git a/document/en/development/workbench-routing.md b/document/en/development/workbench-routing.md
index 33aa5420..a7c484f6 100644
--- a/document/en/development/workbench-routing.md
+++ b/document/en/development/workbench-routing.md
@@ -10,6 +10,6 @@ The page distinguishes directory loading, directory failure, an undiscoverable a
A secondary view may reference its application's workspace module. Launch accepts only the main module and modules declared by that application's views. Other installation modules are rejected.
-No database schema migration is required. Legacy UI v1/v2, managed application APIs, and existing business URLs retain their contracts. Integrated acceptance records full platform validation. Fixture checks do not establish native client compatibility.
+No database schema migration is required. Legacy UI v1/v2, managed application APIs, and existing business URLs retain their contracts.
An empty directory retains the deep link and shows an empty state. Network and server failures offer retry. They do not appear as missing applications or denied discovery.
diff --git a/document/en/editions/build-ce.md b/document/en/editions/build-ce.md
index 5921ec28..15150f24 100644
--- a/document/en/editions/build-ce.md
+++ b/document/en/editions/build-ce.md
@@ -26,7 +26,7 @@ The manifest has the following sections, in processing order:
Glob patterns (relative to the repo root); a match means the file is never copied. Covers:
-- **Backend EE modules**: the complete `edition_ee/**` implementation root (Team/RBAC, SSO, license verification and gate, EE ORM, Dify integration), cloud storage (`core/storage/s3.py`, `oss.py`), the Cube sandbox provider, memory audit, skill distillation, and other EE services;
+- **Backend EE modules**: the complete `edition_ee/**` implementation root (Team/RBAC, SSO, license verification and gate, EE ORM, Dify integration), cloud storage (`core/files/storage/s3.py`, `oss.py`), the Cube sandbox provider, memory audit, skill distillation, and other EE services;
- **EE routes**: `api/routes/v1/admin_*.py`, `config_*.py`, `audit.py`, `auth.py`, `team_files.py`, `service_configs.py`, `data_sources.py`, `db_metadata.py`, `gateway_*.py`;
- **Industry MCP servers**: `extensions/mcp_servers/query_database_mcp/**`, `ai_chain_information_mcp/**`;
- **Industry-specific frontend code and seeds**: the industry-chain Canvas,
@@ -83,13 +83,13 @@ Industry-chain, news, and company-profile presentation remains an EE feature.
| `.hugagent-edition` | Machine-readable `ce` marker used only after derivation; it lets release tooling distinguish a derived CE checkout from a source checkout whose generator is unexpectedly missing |
| `.github/workflows/desktop-release.yml` | Public CE desktop release workflow, including the release-tag/version preflight gate |
| `src/backend/api/routes/v1/__init__.py` | CE route registry; `EE_ROUTERS` is always empty |
-| `src/backend/core/auth/permissions_iface.py` | Owner-only single-tenant authorization interface with no team permission exports |
-| `src/backend/core/services/artifact_edition.py` | Personal artifact-scope interface with no team fields, permissions, or repository methods |
+| `src/backend/core/auth/personal_permissions_iface.py` | Owner-only single-tenant authorization interface with no team permission exports |
+| `src/backend/core/files/edition.py` | Personal artifact-scope interface with no team fields, permissions, or repository methods |
| `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | Personal MySpace tool, VFS, and recovery interfaces; organization implementations do not enter CE |
-| `src/backend/core/config/edition_display_names.py` | CE tool display names with no team-tool names |
+| `src/backend/core/config/display_names.py` | CE tool display names with no team-tool names |
| `src/backend/core/memory/audit.py` | Memory-audit no-op stub (same interface, writes nothing) |
| `src/backend/alembic/versions/ce_000{1,2}_*.py` | CE independent migration-chain baseline and idempotent schema upgrade (next section) |
-| `src/backend/core/services/edition_startup.py` | CE startup seam with no-op implementations for EE data-source recovery and distillation scheduling, so pruned modules are never imported |
+| `src/backend/orchestration/workers/edition_startup.py` | CE startup seam with no-op implementations for EE data-source recovery and distillation scheduling, so pruned modules are never imported |
| `src/backend/tests/ce_release/` | Executable CE release regressions for login, private skills/MCP, database upgrades, startup, and Compose contracts |
| `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | CE API contracts with administration endpoints and organization fields removed |
| `src/backend/extensions/mcp_servers/_ports.py` | Port table for the 7 general tools (EE industry-tool and retired ports marked reserved) |
@@ -194,7 +194,7 @@ A qualifying release build must pass all of:
| Overlay directory | `ce/overlay/` |
| Derived-tree edition marker | `ce/overlay/.hugagent-edition` |
| Public desktop release workflow | `ce/overlay/.github/workflows/desktop-release.yml` |
-| CE/EE table boundary | `src/backend/core/db/edition_tables.py` |
+| CE/EE table boundary | `src/backend/edition_ee/db/edition_tables.py` |
| Idempotent schema reconciler | `src/backend/core/db/schema_reconcile.py` |
| Startup table-creation CE branch | `src/backend/core/db/engine.py::init_db` |
| CE independent migration chain | `ce/overlay/src/backend/alembic/versions/ce_000{1,2}_*.py` |
diff --git a/document/en/getting-started/introduction.md b/document/en/getting-started/introduction.md
index 10859626..74117c6e 100644
--- a/document/en/getting-started/introduction.md
+++ b/document/en/getting-started/introduction.md
@@ -1,93 +1,123 @@
# Introduction
-> Last updated: 2026-07-02
+> Last updated: 2026-10-23 | [简体中文](../../zh-CN/getting-started/introduction.md)
-HugAgentOS is an enterprise-grade AI Agent platform. Built around a ReAct agent core, it brings conversation, tool calling (MCP), skills, sandboxed code execution, long-term memory, knowledge-base RAG, automation, and batch processing together in a fully containerized (Docker Compose) system. The backend runs on FastAPI + AgentScope 2.0, the frontend on React 19 + Vite + Zustand, and every capability is registered in a unified catalog that can be toggled at will. Out of the box it is a complete, self-hostable product — not a half-finished framework.
+HugAgentOS is an open-source, self-hosted **agent harness**. The model does the thinking; the
+harness handles everything around it: where a task runs, how it picks up after an interruption,
+how tools and sandboxes are used, which actions need human approval, and how experience is kept.
-The platform ships in two editions: the **Community Edition (CE, open source)** and the **Enterprise Edition (EE, commercial)**. CE targets individuals and small teams and includes the full conversation, tool, skill, sandbox, three-layer memory, automation, batch, and data-canvas stack. EE layers organization-scale capabilities on top: team collaboration, SSO, audit & compliance, industry data tools, persistent sandboxes, cloud storage, and the full admin console. See [Edition Comparison](../editions/overview.md) for the exact boundary.
+At its core is the **durable agent**: your request runs in the cloud, not in your browser or on
+your computer. Close the tab or shut the laptop and the task keeps moving; if a server restarts or
+fails, another instance continues from the last saved progress; external operations that may or
+may not have happened are never redone automatically. On top of that, HugAgentOS provides an
+always-on **Personal Agent** and runs in two forms: **HugAgentOS Cloud** and **HugAgentOS Local**.
+
+The platform ships as a **Community Edition (CE, open source)** and an **Enterprise Edition (EE)**.
+CE lets an individual run the complete harness and Personal Agent; EE adds organization-scale
+capabilities such as team collaboration, SSO, compliance auditing and industry data tools. See the
+[edition comparison](../editions/overview.md) for the exact boundary.
+
+## Runtime forms
+
+| Form | Runs on | Data and state | Best for | Guide |
+|---|---|---|---|---|
+| **HugAgentOS Cloud** | Your server or private cloud | PostgreSQL, Redis, S3-compatible storage, OpenSandbox | 24/7 Personal Agent, long tasks, IM channels, access from any device | [Docker Compose](../deployment/docker-compose.md) |
+| **HugAgentOS Local** | Your computer (Windows / macOS / Linux) | Local SQLite, Milvus Lite, local files | Offline work, sensitive data, working directly on local projects | [Desktop local mode](../deployment/desktop-local-mode.md) |
+| **Local + cloud** | Cloud as control plane, local machine as execution plane | Cloud projects in the cloud, local projects on the machine | An always-on cloud agent that also operates your computer and projects | [Desktop local mode](../deployment/desktop-local-mode.md) |
+| One-command install | A development machine | SQLite, in-process state, subprocess sandbox | Trials and development | [No-Docker quick install](../deployment/quick-install.md) |
+
+Durable 24/7 operation needs HugAgentOS Cloud. The one-command install has no cross-process lease
+takeover and is meant for trials only.
## Feature overview
| Capability | Description | Edition |
|---|---|---|
-| Conversational agent | SSE streaming chat, AgentScope 2.0 ReActAgent, Plan Mode sub-agent, deep thinking, `[ref:tool-N]` citation tracing, automatic conversation-title summarization | CE |
-| Sub-agents | User-created sub-agents, @mention collaboration, routing strategy (`ROUTER_STRATEGY`) | CE (agent versioning / org-level agent library: EE) |
-| MCP tool ecosystem | 9 built-in MCP servers (7 CE general tools: internet search, web fetch, chart generation, batch planning, automation task management, skill management, knowledge retrieval; 2 EE industry tools: data-warehouse query and industry-chain info), served from a dedicated `mcp` container over streamable-http; users can self-register remote HTTP/SSE MCP servers | CE (industry tools such as industry-chain analysis, company profiling, data-warehouse query: EE) |
-| Skill system | Agent Skills (SKILL.md + scripts): built-in skill bundles, admin upload, skill marketplace browse & install, skill distillation | CE (skill review / org governance: EE) |
-| Sandbox execution | `bash` / `sandbox_put_artifact` / `sandbox_get_artifact` tools with three switchable providers: script_runner (lightweight, built-in), OpenSandbox (persistent sessions + Jupyter context + snapshots), CubeSandbox (E2B-compatible MicroVM) | CE for the lightweight sandbox; persistent sandboxes (session keep-alive / snapshots): EE |
-| Memory system | mem0 three-layer memory: L1 user profile, L2 vector memory (Milvus), L3 knowledge graph (Neo4j), with cross-session injection and background extraction | CE (memory audit: EE) |
-| Knowledge-base RAG | Document upload, chunking, hybrid vector + keyword retrieval, and private knowledge bases; EE can add public and external Dify knowledge bases | CE (public KB and Dify integration: EE) |
-| Automation | Scheduled tasks / cron scheduling / prompt & plan automation / failure retry (`orchestration/schedulers/`) | CE |
-| Data canvas | Online spreadsheet editing built on Univer | CE for personal editing; real-time multi-user collaboration: EE |
-| Projects / My Space | Personal projects (file capacity quota), personal drive (myspace), favorites, conversation sharing | CE (team folders / permission matrix: EE) |
-| Admin consoles | `/admin` content management (skills, capability center, release notes) + `/config` system console (prompt version pool, models, users/teams, billing, audit, security) | CE basics; full console (users/teams/billing/audit/security): EE |
-
-## CE vs. EE
-
-In one sentence: **the Community Edition lets one person push the platform to its limits; the Enterprise Edition lets an organization run it at scale** — team collaboration, SSO, RBAC, audit & compliance, industry data tools, persistent sandboxes, cloud storage, and white-labeling belong to EE. See [Edition Comparison](../editions/overview.md) and the [License mechanism](../editions/license.md) for the full matrix.
+| Durable runs | Every request is a persisted run executed in the background; resumable streams and mid-run steering; lease takeover and startup reconciliation; tool side effects record intent and receipt, and unknown outcomes are never replayed automatically | CE |
+| Personal Agent | One conversation, parallel background tasks, a permanent cloud computer, schedules and heartbeats, IM channels, action approvals, file-based memory with Dreaming, and connecting your own computer | CE (authorization governance and cross-device session management are EE) |
+| Autonomous loop | Goals survive across turns until done; a worker does the work, a read-only reviewer verifies it, and command-checkable requirements are verified by the driver; resumable after interruption | CE |
+| Chat and Plan Mode | SSE streaming, ReAct tool orchestration, deep thinking, Plan Mode, citations, context compaction checkpoints, chat forks | CE |
+| Sub-agents | Built-in explorer, worker and reviewer, user-created sub-agents, automatic routing or `@` mentions | CE (organization agent library is EE) |
+| MCP tools and plugins | Built-in web search, page fetching, knowledge retrieval, charts, automation, and skill and plugin management tools; an MCP marketplace, private MCP servers, a plugin marketplace and a browser automation plugin | CE (data warehouse and industry tools are EE) |
+| Agent Skills | SKILL.md plus scripts; built-in skills, a skill marketplace and personal skills | CE (skill review and organization governance are EE) |
+| Sandbox execution | OpenSandbox sessions with environment reuse and snapshot recovery, or a local subprocess sandbox; workflow jobs run as managed processes that can be taken over by lease | CE (CubeSandbox and team mounts are EE) |
+| Memory and personal evolution | L1 profile, L2 procedural memory (Milvus, optional), L3 relationship graph (Neo4j, optional); durable outbox writes; evolution proposals approved one by one | CE (memory auditing is EE) |
+| Knowledge base | Document chunking, hybrid vector and keyword retrieval, private knowledge bases, knowledge-base wiki | CE (public KB administration and Dify integration are EE) |
+| Domain ontology governance | Domain Pack import and versioned publishing, rule checks before tool execution, independent review of high-risk answers, human review of evolution drafts | CE |
+| Automation | Scheduled tasks in natural language, cron scheduling, retries on failure | CE |
+| Sites and application hosting | Build a site in chat and publish it in one click; separate application databases and MCP hosting | CE (site collaboration after project conversion is EE) |
+| Data canvas and artifacts | Univer spreadsheets, charts, reports, Office files, web pages and an artifact center | CE personal editing; real-time collaboration is EE |
+| Projects and My Space | Personal projects, folders, favourites and chat sharing; the desktop client can bind local project directories | CE (team folders and permission matrices are EE) |
+| Desktop client | Tauri 2 client with its own private runtime, supporting local-only, cloud-only and local + cloud modes | CE |
## Technology stack
-| Layer | Technology | Notes |
-|---|---|---|
-| Backend | FastAPI + Uvicorn (Python 3.11) | `src/backend/api/app.py`, unified response envelope |
-| Agent framework | AgentScope 2.0 (`agentscope==2.0.0`) | ReActAgent + tool registration, `core/llm/factory/build.py` |
-| Frontend | React 19 + Vite 7 + Zustand 5 + Ant Design 6 | `src/frontend/`, served by nginx with an `/api` reverse proxy |
-| Database | PostgreSQL 15 (production) / SQLite (local-debug fallback) | SQLAlchemy 2 + Alembic migrations |
-| Cache / sessions | Redis 7 | Session store, streaming follower (Redis Streams) |
-| Vector DB | Milvus 2.4 (`mem0` profile) | L2 vector memory and self-hosted KB retrieval |
-| Graph DB | Neo4j 5 Community (`mem0` profile, optional) | L3 knowledge-graph memory |
-| Sandbox | script_runner sidecar / Alibaba OpenSandbox / Tencent CubeSandbox | `core/sandbox/` provider protocol, switched via env |
-| Deployment | Docker Compose (profiles: `script_runner` / `opensandbox` / `mem0`) | Everything runs in containers; there is no local dev server |
-
-## Architecture at a glance
+| Layer | Technology |
+|---|---|
+| Agent runtime | AgentScope 2.0 (ReActAgent), Model Context Protocol |
+| Backend | Python 3.11, FastAPI, SQLAlchemy 2, Alembic |
+| Frontend | React 19, TypeScript, Vite, Zustand, Ant Design |
+| Desktop | Tauri 2, private CPython 3.11 runtime |
+| Data and state | PostgreSQL 15 or SQLite; Redis 7 or in-process state |
+| Storage | S3-compatible storage (cloud), local files (desktop and one-command install) |
+| Sandbox | OpenSandbox, local subprocess (CubeSandbox is EE) |
+| Optional memory | Milvus 2.4, Neo4j 5 Community, mem0 |
+| Deployment | Docker Compose, desktop installer, one-command installer, Nginx |
+
+## Architecture
```
- ┌──────────────────────────────────────────────┐
- Browser ──► Nginx ────►│ FastAPI backend (src/backend/api/app.py) │
- (frontend ctr, │ api/routes/v1/* · 50+ routers · envelope │
- /api proxy) └───────────────────┬──────────────────────────┘
- │
- ┌──────────────────────┼─────────────────────────┐
- ▼ ▼ ▼
- orchestration/workflow.py core/services/* core/auth/*
- (SSE streaming orchestration: (business service (local / mock /
- text / tool_call / layer) remote + SSO)
- tool_result / meta / done)
- │
- ┌─────────────┼───────────────────┬─────────────────────┐
- ▼ ▼ ▼ ▼
- core/llm/ orchestration/ orchestration/ core/memory/ (svc)
- factory/build strategy.py citations.py + memory_integration
- (AgentScope (routing (citation parsing) (mem0 retrieve/save)
- 2.0 ReAct) strategy) │
- │ ▼
- ├──► core/llm/mcp_manager ──► mcp container Milvus / Neo4j
- │ (9 MCP servers, ports 9100–9104, 9106–9108, and 9112) (mem0 profile)
- ├──► core/sandbox/* ──► script-runner / OpenSandbox / CubeSandbox
- └──► PostgreSQL · Redis · storage (local / S3 / OSS)
+ Entry points: web · desktop client · IM channels · API · schedules · heartbeats
+ │
+ ▼
+ FastAPI (api/) auth · edition routing · messages persisted before the response
+ │
+ ▼
+ Run layer (core/runs · core/chat/runtime)
+ persisted runs · database leases · startup reconciliation · resumable streams · steering · tool effect receipts
+ │
+ ▼
+ Agent execution (core/agent_runtime · core/llm)
+ ReActAgent · per-step context assembly and compaction · memory / ontology / evolution integration
+ Outer orchestration (orchestration/): Plan Mode · autonomous loop · automation
+ │
+ ┌────────────────┼──────────────────┐
+ ▼ ▼ ▼
+ MCP · skills · plugins sandbox · jobs knowledge · memory
+ │ │ │
+ └────────────────┴──────────────────┘
+ ▼
+ PostgreSQL / SQLite · Redis · S3 / local storage · Milvus · Neo4j
```
-Main request path: browser → nginx inside the frontend container (`/api` proxy) → FastAPI → `orchestration/workflow.py` orchestrates streaming → `core/llm/factory/build.py` builds the ReActAgent → MCP tools / sandbox / memory → SSE events (`text` / `tool_call` / `tool_result` / `meta` / `done`) stream back to the frontend.
+Personal Agent (`personal_agent/`) reuses the same runs, leases and sandboxes, and adds identity,
+a task queue, scheduling, channels, device links and workspace files. See the
+[architecture overview](../architecture/overview.md) for the full request lifecycle.
## Next steps
-- [Quick Start in 10 minutes](quick-start.md)
-- [Full Docker Compose deployment](../deployment/docker-compose.md)
-- [Environment variable reference](../deployment/environment-variables.md)
-- [Architecture overview](../architecture/overview.md)
+- [Quick start](quick-start.md)
+- [Personal Agent](../modules/personal-agent.md)
+- [Chat and orchestration](../modules/chat.md)
+- [Autonomous loop](../modules/autonomous-loop.md)
+- [Deployment guide](../deployment/README.md)
-## Related source
+## Source map
-| Feature | Files |
+| Area | Path |
|---|---|
-| FastAPI app & router registration | `src/backend/api/app.py`, `src/backend/api/routes/v1/` |
-| Streaming orchestration (SSE) | `src/backend/orchestration/workflow.py`, `orchestration/streaming.py` |
+| FastAPI app and routes | `src/backend/api/app.py`, `src/backend/api/routes/v1/` |
+| Run lifecycle, leases and recovery | `src/backend/core/runs/` |
+| Chat worker, stream resume and steering | `src/backend/core/chat/runtime/` |
+| Agent execution entry | `src/backend/core/agent_runtime/` (`runner.astream_chat_workflow`) |
| Agent construction | `src/backend/core/llm/factory/build.py` |
-| MCP servers & port mapping | `src/backend/extensions/mcp_servers/`, `src/backend/extensions/mcp_servers/_ports.py`, `src/backend/core/config/mcp_config.py` |
-| Capability catalog | `src/backend/core/config/catalog.json`, `core/config/catalog.py` |
-| Sandbox providers | `src/backend/core/sandbox/`, `src/backend/core/config/settings.py::SandboxSettings` |
-| Memory system | `src/backend/core/memory/` (service.py / pipeline.py), `src/backend/orchestration/memory_integration.py` |
-| Automation scheduler | `src/backend/orchestration/schedulers/automation_scheduler.py` |
-| Frontend entries (App / Admin / Config) | `src/frontend/src/main.tsx`, `App.tsx`, `AdminApp.tsx`, `ConfigApp.tsx` |
-| Edition & license settings | `src/backend/core/config/settings.py::EditionSettings / LicenseSettings` |
+| Harness contracts and execution ledger | `src/backend/core/harness/` |
+| Workflow background jobs | `src/backend/core/jobs/` |
+| Plan Mode, autonomous loop, automation | `src/backend/orchestration/plans/`, `orchestration/loops/`, `orchestration/automations/` |
+| Personal Agent | `src/backend/personal_agent/` |
+| Memory and evolution | `src/backend/core/memory/`, `src/backend/core/evolution/` |
+| Domain ontology | `src/backend/core/ontology/` |
+| Sandbox | `src/backend/core/sandbox/` |
+| Built-in MCP, skills, plugins and agents | `src/backend/extensions/` |
+| Frontend entry | `src/frontend/src/main.tsx`, `App.tsx` |
+| Desktop client | `desktop/src-tauri/` |
diff --git a/document/en/getting-started/quick-start.md b/document/en/getting-started/quick-start.md
index c3fea68f..acdd4e41 100644
--- a/document/en/getting-started/quick-start.md
+++ b/document/en/getting-started/quick-start.md
@@ -1,22 +1,91 @@
# Quick start
-> Last updated: July 21, 2026 |
+> Last updated: October 23, 2026 |
> [简体中文](../../zh-CN/getting-started/quick-start.md)
-Choose the one-command installer for a personal trial or Docker Compose for a
-server-oriented, service-isolated deployment. Both methods require access to
-an OpenAI-compatible API or local model.
+There are three ways to get started. Choose HugAgentOS Cloud when agents should keep running in the
+cloud around the clock, the HugAgentOS Local desktop client to work offline on your own computer, or
+the one-command installer for development and quick trials. Every option needs access to an
+OpenAI-compatible API or a local model.
## Choose a deployment method
-The two deployment methods use different runtime and persistence models.
-
| Method | Best for | Runtime |
|---|---|---|
-| One-command installer | Personal trials and development | SQLite, in-process state, local subprocess sandbox |
-| Docker Compose | Long-running servers and service isolation | PostgreSQL, Redis, container sandbox, persistent volumes |
+| HugAgentOS Cloud (Docker Compose) | 24/7 operation, Personal Agent, IM channels, access from any device | PostgreSQL, Redis, OpenSandbox, persistent volumes |
+| HugAgentOS Local (desktop client) | Offline use, sensitive data, local projects | Bundled private runtime, local SQLite and files |
+| One-command installer | Trials and development | SQLite, in-process state, local subprocess sandbox |
+
+**The initial account and password are both `admin` and must be changed on first sign-in; the
+Community Edition has no self-registration.**
+
+## Option 1: HugAgentOS Cloud (Docker Compose)
+
+Install Git, Docker Engine or Docker Desktop, and Docker Compose v2, then run:
+
+```bash
+git clone https://github.com/ZJU-REAL/HugAgentOS.git
+cd HugAgentOS
+cp .env.example .env
+mkdir -p data/storage
+docker compose build backend
+docker compose up -d --build
+```
+
+Open [http://localhost:3002](http://localhost:3002). Sign in with `admin` /
+`admin`, change the password, then open **Settings → System Administration →
+Model Services** to connect a model. Use `docker compose ps` to inspect the
+services and `docker compose down` to stop them without deleting data.
+
+Read the [Docker Compose deployment guide](../deployment/docker-compose.md) for
+profiles, persistence, production configuration, and rebuild workflows.
+
+### Enable Personal Agent (optional)
+
+Personal Agent is off by default. It uses a dedicated OpenSandbox controller to keep a permanent
+cloud computer for each user.
+
+1. Build the computer image:
+
+ ```bash
+ docker build -f docker/personal-agent/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .
+ ```
+
+2. Set these values in `.env`:
+
+ | Variable | Value |
+ |---|---|
+ | `PERSONAL_AGENT_ENABLED` | `true` |
+ | `HOST_STORAGE_PATH` | An absolute path on the host |
+ | `PERSONAL_AGENT_OPENSANDBOX_API_KEY` | A separate random secret of at least 32 characters, for example the output of `openssl rand -hex 32` |
+
+3. Start with the additional Compose file:
+
+ ```bash
+ docker compose -f docker-compose.yml -f docker/personal-agent/compose.personal-agent.yml --profile personal-agent up -d
+ ```
+
+Then click **Personal Agent** under **New chat**. See [Personal Agent](../modules/personal-agent.md)
+for database migrations, image upgrades and channel binding. Never commit the secret to Git.
-## Option 1: one-command installer
+## Option 2: HugAgentOS Local (desktop client)
+
+1. Download the desktop installer for your platform (Windows x86_64, macOS Apple Silicon / Intel,
+ Linux x86_64) from [Releases](https://github.com/ZJU-REAL/HugAgentOS/releases).
+2. Choose a mode on first launch:
+ - **Local only**: the service, data and execution stay on this computer and can run fully
+ offline;
+ - **Cloud only**: connect to a deployed HugAgentOS Cloud;
+ - **Local + cloud**: use your cloud account; cloud projects stay in the cloud and local
+ projects run on this machine, which also lets Personal Agent connect to this computer.
+3. In local mode, the client verifies and unpacks its bundled runtime, usually within tens of
+ seconds, with no Python, Docker or database to install.
+
+See [Desktop local mode](../deployment/desktop-local-mode.md) for installation, upgrade rollback
+and data directories, and [Local project workspace](../deployment/local-project-workspace.md) for
+how execution works in a bound local project.
+
+## Option 3: one-command installer
Use this method on Linux, macOS, or Windows through WSL2. Install the following
tools before you start.
@@ -65,36 +134,20 @@ You can add the command to your shell path:
export PATH="$HOME/.hugagent/venv/bin:$PATH"
```
-## Option 2: Docker Compose
-
-Use this method when you need durable service volumes and container isolation.
-Install Git, Docker Engine or Docker Desktop, and Docker Compose v2, then run:
-
-```bash
-git clone https://github.com/ZJU-REAL/HugAgentOS.git
-cd HugAgentOS
-cp .env.example .env
-mkdir -p data/storage
-docker compose up -d --build
-```
-
-Open [http://localhost:3002](http://localhost:3002). Sign in with `admin` /
-`admin`, change the password, then open **Settings → System Administration →
-Model Services** to connect a model. Use `docker compose ps` to inspect the
-services and `docker compose down` to stop them without deleting data.
-
-Read the [Docker Compose deployment guide](../deployment/docker-compose.md) for
-profiles, persistence, production configuration, and rebuild workflows.
+The one-command installer has no cross-process lease takeover and is not meant for 24/7 operation.
## Next steps
Continue with these guides after your first successful login:
-- Read the [complete no-Docker installation guide](../deployment/quick-install.md)
- for installer options, capability boundaries, and troubleshooting.
+- Read the [introduction](introduction.md) for the durable agent, runtime forms and feature overview.
- Configure [model providers](../modules/model-providers.md).
-- Build a governed workflow with the
- [domain ontology quickstart](domain-ontology-quickstart.md).
+- Try [Personal Agent](../modules/personal-agent.md) and the
+ [autonomous loop](../modules/autonomous-loop.md).
- Explore [MCP tools](../modules/mcp-tools.md),
[Agent Skills](../modules/agent-skills.md), and
[private knowledge bases](../modules/knowledge-base.md).
+- Build a governed workflow with the
+ [domain ontology quickstart](domain-ontology-quickstart.md).
+- Read the [complete no-Docker installation guide](../deployment/quick-install.md)
+ for installer options, capability boundaries, and troubleshooting.
diff --git a/document/en/modules/admin-console.md b/document/en/modules/admin-console.md
index ae7a6d27..141faa56 100644
--- a/document/en/modules/admin-console.md
+++ b/document/en/modules/admin-console.md
@@ -53,11 +53,11 @@ Page Configuration also owns the homepage's first-screen content. Administrators
`/v1/admin/skills` (`api/routes/v1/admin_skills.py`, `ADMIN_TOKEN`): full skill lifecycle — CRUD, zip upload, enable/disable, ordering, icons, dependency scan and editing, per-file read/write/delete inside a skill, forking built-in skills, import/export. Skills are stored in the `admin_skills` table; see [Agent Skills](agent-skills.md).
-`/v1/admin/skill-drafts` (`api/routes/v1/admin_skill_drafts.py`, `ADMIN_TOKEN`): review of auto-distilled skill drafts — list / count / detail, edit-then-approve, reject, delete, plus a manual trigger for the daily distillation scan.
+`/v1/admin/skill-drafts` (`features/distillation/api/admin_skill_drafts.py`, `ADMIN_TOKEN`): review of auto-distilled skill drafts — list / count / detail, edit-then-approve, reject, delete, plus a manual trigger for the daily distillation scan.
### Marketplace review (admin_marketplace) (Enterprise Edition: content_admin)
-`/v1/admin/marketplace` (`api/routes/v1/admin_marketplace.py`, `ADMIN_TOKEN`) does two things:
+`/v1/admin/marketplace` (`features/marketplace/api/admin_marketplace.py`, `ADMIN_TOKEN`) does two things:
1. **Install marketplace skills globally**: browse the marketplace listing (flagging "already installed globally"); installed skills have an empty owner, are available to everyone, and remain editable under Skill Management.
2. **Review user listing submissions**: submission list (filterable by status), detail (with SKILL.md preview), approve (publish, installable by all), reject / unpublish.
@@ -91,14 +91,14 @@ The sub-agent status switch uses `PUT /v1/admin/agents/{agent_id}/toggle`. While
### Knowledge base management (admin_kb) (Enterprise Edition: content_admin)
-`/v1/admin/kb` (`api/routes/v1/admin_kb.py`, `ADMIN_TOKEN`): management of self-hosted **public knowledge bases** (`KBSpace.visibility == "public"`, owned by the system owner), mirroring the user-side `kb.py` — KB CRUD, AI-generated descriptions, document upload / listing / original-file preview (Office→PDF) / deletion / reindexing, chunk preview and per-chunk editing (content / tags / questions). Public KBs are visible to all users in the capability catalog and searchable. In Dify mode, Dify datasets are shown read-only and writes return 409. See [Knowledge Base](knowledge-base.md).
+`/v1/admin/kb` (`edition_ee/routes/admin_kb.py`, `ADMIN_TOKEN`): management of self-hosted **public knowledge bases** (`KBSpace.visibility == "public"`, owned by the system owner), mirroring the user-side `kb.py` — KB CRUD, AI-generated descriptions, document upload / listing / original-file preview (Office→PDF) / deletion / reindexing, chunk preview and per-chunk editing (content / tags / questions). Public KBs are visible to all users in the capability catalog and searchable. In Dify mode, Dify datasets are shown read-only and writes return 409. See [Knowledge Base](knowledge-base.md).
### Sandbox management (admin_sandbox + config_security/sandbox)
Two complementary panels:
-- `/v1/admin/sandbox` (`api/routes/v1/admin_sandbox.py`, `ADMIN_TOKEN`, **Enterprise Edition: content_admin**): aggregated skill-dependency manifest preview, sandbox image rebuild trigger + container hot-swap, rebuild history and log queries.
-- `/v1/config/security/sandbox/*` (`api/routes/v1/config_security.py`, `CONFIG_TOKEN`, **Enterprise Edition: system_config**): read-only security view — sandbox overview, running instances list / detail, snapshots, rebuild history, redacted configuration. Capabilities a provider cannot support (e.g. ScriptRunner cannot enumerate instances) return `code=42210`, which the frontend renders as disabled.
+- `/v1/admin/sandbox` (`features/sandbox_ops/api/admin_sandbox.py`, `ADMIN_TOKEN`, **Enterprise Edition: content_admin**): aggregated skill-dependency manifest preview, sandbox image rebuild trigger + container hot-swap, rebuild history and log queries.
+- `/v1/config/security/sandbox/*` (`features/security_console/api/config_security.py`, `CONFIG_TOKEN`, **Enterprise Edition: system_config**): read-only security view — sandbox overview, running instances list / detail, snapshots, rebuild history, redacted configuration. Capabilities a provider cannot support (e.g. ScriptRunner cannot enumerate instances) return `code=42210`, which the frontend renders as disabled.
See [Sandbox](sandbox.md).
@@ -166,13 +166,13 @@ Aligned with chapter 4 of the productization plan and the route registry:
| Administrative credential dependencies | `src/backend/api/deps.py` |
| License features | `src/backend/edition_ee/licensing/features.py`, `src/backend/edition_ee/licensing/deps.py` |
| Skills / drafts / marketplace | `src/backend/api/routes/v1/admin_skills.py`, `admin_skill_drafts.py`, `admin_marketplace.py` |
-| Feedback and GitHub publishing | `src/backend/edition_ee/routes/feedbacks.py`, `admin_feedbacks.py`, `services/feedback_service.py` |
+| Feedback and GitHub publishing | `src/backend/edition_ee/routes/feedbacks.py`, `admin_feedbacks.py`, `edition_ee/services/feedback_service.py` |
| Prompts / MCP / sub-agents | `src/backend/api/routes/v1/admin_prompts.py`, `admin_mcp_servers.py`, `admin_agents.py` |
-| Knowledge base / sandbox | `src/backend/edition_ee/routes/admin_kb.py`, `src/backend/api/routes/v1/admin_sandbox.py` |
+| Knowledge base / sandbox | `src/backend/edition_ee/routes/admin_kb.py`, `src/backend/features/sandbox_ops/api/admin_sandbox.py` |
| Billing / usage / logs / chat review | `src/backend/api/routes/v1/admin_billing.py`, `admin_usage_logs.py`, `admin_logs.py`, `admin_chat_history.py` |
| Content management | `src/backend/api/routes/v1/content.py` |
| Users / teams / invites / license (EE) | `src/backend/edition_ee/routes/config_users.py`, `config_teams.py`, `config_invites.py`, `config_license.py` |
-| Security | `src/backend/api/routes/v1/config_security.py` |
+| Security | `src/backend/features/security_console/api/config_security.py` |
| Service configs | `src/backend/api/routes/v1/service_configs.py` |
Further reading: [Authentication & Permissions](auth.md) · [Prompt System](prompts.md) · [Editions & Licensing](../editions/overview.md)
diff --git a/document/en/modules/agent-skills.md b/document/en/modules/agent-skills.md
index 18846fd9..8af22ed4 100644
--- a/document/en/modules/agent-skills.md
+++ b/document/en/modules/agent-skills.md
@@ -29,11 +29,11 @@ Download integrity, path containment and account ownership checks remain enforce
└── _scripts.json # optional: script whitelist (auto-detected by extension when absent)
```
-Parsing and data structures live in `core/agent_skills/registry.py`: `AgentSkillMetadata` (lightweight, for listings) and `AgentSkillSpec` (full instructions, for execution). When `_scripts.json` is missing, `loader._auto_detect_scripts` builds the whitelist from file extensions automatically.
+Parsing and data structures live in `core/capabilities/skills/registry.py`: `AgentSkillMetadata` (lightweight, for listings) and `AgentSkillSpec` (full instructions, for execution). When `_scripts.json` is missing, `loader._auto_detect_scripts` builds the whitelist from file extensions automatically.
## Multi-source loading and priorities
-`core/agent_skills/loader.py::MultiSourceSkillLoader` loads skills from several sources through a backend abstraction (`backends/`: filesystem / database / composite); on ID conflicts the higher-priority source wins (`core/agent_skills/config.py`):
+`core/capabilities/skills/loader.py::MultiSourceSkillLoader` loads skills from several sources through a backend abstraction (`backends/`: filesystem / database / composite); on ID conflicts the higher-priority source wins (`core/capabilities/skills/config.py`):
| Source | Directory | Priority | Notes |
|---|---|---|---|
@@ -149,11 +149,11 @@ Key points:
- **Scope**: main-chat assembly and sub-agents. A sub-agent's bound plugins are deferred the same way, but activation lives only within that run and is never persisted (sub-agent runs are short-lived and isolated; writing under the parent chat's key would leak the activation into the main agent's assembly). Plugins pinned by a restricted chat mode stay eagerly loaded (a deliberate admin narrowing); plugins with stdio-transport MCP components are not deferred.
- **Kill switch**: set `PLUGIN_PROGRESSIVE_LOADING=false` to revert to eager assembly.
-Implementation: `core/plugins/runtime/__init__.py` (deferral resolution / directory rendering / activation tool) plus the assembly hook in `core/llm/factory/build.py`.
+Implementation: `core/capabilities/plugins/runtime/__init__.py` (deferral resolution / directory rendering / activation tool) plus the assembly hook in `core/llm/factory/build.py`.
## The Skill Marketplace
-The marketplace is an installable skill library with two sources — curated presets and community submissions — implemented in `core/services/marketplace_service.py`.
+The marketplace is an installable skill library with two sources — curated presets and community submissions — implemented in `features/marketplace/`.
### Browsing and installing
@@ -205,7 +205,7 @@ admin GET/PUT /v1/admin/marketplace/skills/{slug}/visibility (skill ma
GET /v1/admin/visibility/principals ← brief user/team/role lists (picker data source)
```
-Storage: `marketplace_listing_states.visibility` (missing row = public) plus the `marketplace_visibility_grants` whitelist table; user-side resolution is centralized in `core/auth/marketplace_visibility.py` (roles include both direct assignments and team-default roles inherited via team membership).
+Storage: `marketplace_listing_states.visibility` (missing row = public) plus the `marketplace_visibility_grants` whitelist table; user-side resolution is centralized in `edition_ee/auth/marketplace_visibility.py` (roles include both direct assignments and team-default roles inherited via team membership).
## Admin skill management
@@ -214,7 +214,7 @@ The `/admin` console's skill management maps to `api/routes/v1/admin_skills.py`
- **Create/edit**: hand-written SKILL.md (`POST /`, `PUT /{id}`) or **zip upload** of an entire package (`POST /upload`, auto-detects directory prefixes, base64-encodes binaries into the DB, 200 MB cap);
- **File-level management**: `GET/PUT/DELETE /{id}/files/{filename}` for inline editing of auxiliary files;
- **Operations**: toggle (`/{id}/toggle`), ordering, icons, fork, bulk export/import (cross-environment migration);
-- **Dependency management**: `POST /{id}/rescan-deps` statically scans scripts for pip/apt dependencies via `core/agent_skills/deps_detector.py`, with `PUT /{id}/dependencies` for manual correction; the aggregated manifest feeds the sandbox image rebuild (see the admin dependency rebuild in the [sandbox module](sandbox.md)).
+- **Dependency management**: `POST /{id}/rescan-deps` statically scans scripts for pip/apt dependencies via `core/capabilities/skills/deps/detector.py`, with `PUT /{id}/dependencies` for manual correction; the aggregated manifest feeds the sandbox image rebuild (see the admin dependency rebuild in the [sandbox module](sandbox.md)).
## Skill distillation (Enterprise EE)
@@ -233,7 +233,7 @@ daily cron (default 02:30, DISTILL_CRON_EXPRESSION)
approve → promoted to AdminSkill; reject / delete; manual trigger-daily-scan available
```
-Thresholds, keywords and budgets are all overridable via `DISTILL_*` environment variables (`core/config/distillation.py`). The distillation pipeline (`skill_distiller.py`, the scheduler) is **Enterprise EE**; the CE tree strips it, and the draft-review console only appears in the Enterprise Edition.
+Thresholds, keywords and budgets are all overridable via `DISTILL_*` environment variables (`features/distillation/config.py`). The distillation pipeline (`skill_distiller.py`, the scheduler) is **Enterprise EE**; the CE tree strips it, and the draft-review console only appears in the Enterprise Edition.
## User self-service skills (capability center)
@@ -281,22 +281,22 @@ only submission, never conversation or continued local editing.
| Path | Description |
|---|---|
-| `src/backend/core/agent_skills/registry.py` | SKILL.md parsing, metadata / full-spec dataclasses |
-| `src/backend/core/agent_skills/loader.py` | Multi-source loading, DB-skill materialization, sandbox path repointing |
-| `src/backend/core/agent_skills/config.py` | Source priorities, unified sandbox skills dir, built-in sync |
-| `src/backend/core/agent_skills/selector.py` | LLM-based skill selection per user intent (progressive disclosure) |
-| `src/backend/core/agent_skills/deps_detector.py` | Static pip/apt dependency detection for scripts |
-| `src/backend/core/agent_skills/backends/` | filesystem / database / composite loading backends |
+| `src/backend/core/capabilities/skills/registry.py` | SKILL.md parsing, metadata / full-spec dataclasses |
+| `src/backend/core/capabilities/skills/loader.py` | Multi-source loading, DB-skill materialization, sandbox path repointing |
+| `src/backend/core/capabilities/skills/config.py` | Source priorities, unified sandbox skills dir, built-in sync |
+| `src/backend/core/llm/skill_selection.py` | LLM-based skill selection per user intent (progressive disclosure) |
+| `src/backend/core/capabilities/skills/deps/detector.py` | Static pip/apt dependency detection for scripts |
+| `src/backend/core/capabilities/skills/backends/` | filesystem / database / composite loading backends |
| `src/backend/core/llm/tools/skill_tool.py` | Restricted view_text_file + {baseDir} substitution + Runtime Hint |
| `src/backend/extensions/skills/default/` | The 12 built-in skills (incl. the 10 OfficeCLI ones) |
| `src/backend/extensions/skills/marketplace/` | The 45 installable marketplace packages |
-| `src/backend/core/services/marketplace_service.py` | Marketplace listing / install / secret injection / submission review |
-| `src/backend/api/routes/v1/marketplace.py` | User-side marketplace API (browse/install/submit/withdraw) |
-| `src/backend/api/routes/v1/admin_marketplace.py` | Admin marketplace API (global install / submission review) |
+| `src/backend/features/marketplace/` | Marketplace listing / install / secret injection / submission review |
+| `src/backend/features/marketplace/api/marketplace.py` | User-side marketplace API (browse/install/submit/withdraw) |
+| `src/backend/features/marketplace/api/admin_marketplace.py` | Admin marketplace API (global install / submission review) |
| `src/backend/api/routes/v1/admin_skills.py` | Admin skill CRUD / zip / dependency management |
-| `src/backend/api/routes/v1/admin_skill_drafts.py` | Distillation draft review (EE) |
-| `src/backend/core/llm/skill_distiller.py` | Skill distillation LLM pipeline (EE) |
-| `src/backend/orchestration/schedulers/distillation_cron_scheduler.py` | Daily distillation scheduler (EE) |
+| `src/backend/features/distillation/api/admin_skill_drafts.py` | Distillation draft review (EE) |
+| `src/backend/features/distillation/` | Skill distillation LLM pipeline (EE) |
+| `src/backend/orchestration/workers/distillation_cron_scheduler.py` | Daily distillation scheduler (EE) |
| `src/backend/api/routes/v1/me_capabilities.py` | User self-service skill / private MCP API |
Related docs: [Sandbox execution](sandbox.md) · [MCP tool system](mcp-tools.md) · [Capability catalog](catalog.md) · [Admin console](admin-console.md) · [Editions & licensing](../editions/overview.md)
diff --git a/document/en/modules/application-hosting.md b/document/en/modules/application-hosting.md
index 2173c621..be4f44a4 100644
--- a/document/en/modules/application-hosting.md
+++ b/document/en/modules/application-hosting.md
@@ -6,11 +6,11 @@ Static sites do not create databases. Persistent forms create typed application
Set `APPLICATION_DATABASE_URL` to a separate database and run `python -m core.services.application_hosting_setup` once. Cloud installations without this setting show hosting as unavailable; local mode may use SQLite. Production uses PostgreSQL.
-The optional `docker-compose.application-hosting.yml` adds PostgreSQL, a persistent volume and an initialization job. Set different URL-safe `APPLICATION_DB_PASSWORD` (bootstrap) and `APPLICATION_OWNER_DB_PASSWORD` (runtime), or encode connection strings correctly. The initialization job uses the same Dockerfile build target as the backend and mounts current backend sources read-only. Build the backend from these sources and rebuild the frontend for the nginx `/applications-mcp/` route. Follow repository approval rules before deployment. End-to-end tests use separate databases and platform mock authentication for local testing only.
+The optional `docker/application-hosting/compose.application-hosting.yml` adds PostgreSQL, a persistent volume and an initialization job. Set different URL-safe `APPLICATION_DB_PASSWORD` (bootstrap) and `APPLICATION_OWNER_DB_PASSWORD` (runtime), or encode connection strings correctly. The initialization job uses the same Dockerfile build target as the backend and mounts current backend sources read-only. Build the backend from these sources and rebuild the frontend for the nginx `/applications-mcp/` route. Follow repository approval rules before deployment. End-to-end tests use separate databases and platform mock authentication for local testing only.
```bash
-docker compose -f docker-compose.yml -f docker-compose.application-hosting.yml config --quiet
-docker compose -f docker-compose.yml -f docker-compose.application-hosting.yml up -d
+docker compose -f docker-compose.yml -f docker/application-hosting/compose.application-hosting.yml config --quiet
+docker compose -f docker-compose.yml -f docker/application-hosting/compose.application-hosting.yml up -d
```
## Access and data
@@ -43,27 +43,7 @@ Schema validation, relational table construction, registry transactions, data op
## Limits
-Implemented: typed SQL, bounded reads/writes, atomic batches, idempotency, optimistic updates, application isolation, anonymous insert-only forms and declarative MCP hosting. Not implemented: arbitrary-code MCP containers, OAuth authorization server, foreign-key relationship designer, backup/restore console and comprehensive usage billing. The real chat driver is `src/backend/tests/application_hosting_e2e.py`; `application_hosting_e2e_setup.py` prepares its isolated environment. It exercises the production chat API, model, tools, sandbox and publishing. Acceptance status comes from the actual run report.
-
-## Reproduce local end-to-end acceptance
-
-Use a local Docker environment with a real model, industry source and OpenSandbox configured. Build the current frontend first. These drivers use production APIs and actual tools, with no file or publication adapters. Test services bind loopback ports and use separate platform_e2e_verified / applications_e2e_verified databases and storage.
-
-```bash
-npm --prefix src/frontend run build
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_e2e_setup.py
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_e2e.py
-```
-
-Submit test data at `http://127.0.0.1:18533/site/e2e-form-1007/` using email `browser-e2e@example.test`. Then run:
-
-```bash
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_lifecycle_e2e.py
-```
-
-The lifecycle driver restarts only the isolated test services and rotates, rolls back, revokes and republishes the test MCP credential. Credentials stay in `/tmp/application-hosting-full-e2e-credentials.json` (0600), outside pages and acceptance reports. Existing batches are not automatically reset; use a clean isolated database batch for a full chat rerun. This workflow does not upgrade the primary installation or deploy remotely.
-
-For a fresh chat rerun, stop and remove only `hugagent-application-e2e-backend` and `hugagent-application-e2e-mcp`, then run setup with `APPLICATION_E2E_BATCH=run2`. Previous batch databases remain intact and drivers keep the same loopback URLs. Do not remove primary containers or databases.
+Implemented: typed SQL, bounded reads/writes, atomic batches, idempotency, optimistic updates, application isolation, anonymous insert-only forms and declarative MCP hosting. Not implemented: arbitrary-code MCP containers, OAuth authorization server, foreign-key relationship designer, backup/restore console and comprehensive usage billing.
## Desktop integration
@@ -135,9 +115,9 @@ and project files so subsequent upgrades reuse bindings; do not delete applicati
Management and rollback publication reject an existing unpublished project draft instead of overwriting it. Project publication waits for file-tool registration and refreshes the mounted source to its new version before reporting synchronization complete.
-## External review fixes and upgrades
+## Deployment requirements and data safety
-Application hosting remains optional. Set different APPLICATION_DB_PASSWORD and APPLICATION_OWNER_DB_PASSWORD values. Use both docker-compose.yml and docker-compose.application-hosting.yml.
+Application hosting remains optional. Set different APPLICATION_DB_PASSWORD and APPLICATION_OWNER_DB_PASSWORD values. Use both docker-compose.yml and docker/application-hosting/compose.application-hosting.yml.
Run application-database-init for new deployments and existing volumes. It creates recovery history and transfers application schema and table ownership to application_owner. It also grants access to existing application roles. Initialization failures must block backend startup.
diff --git a/document/en/modules/auth.md b/document/en/modules/auth.md
index 5689136e..13a8ca15 100644
--- a/document/en/modules/auth.md
+++ b/document/en/modules/auth.md
@@ -120,7 +120,7 @@ User-facing endpoints:
### Interface layer (CE/EE split seam)
-`core/auth/permissions_iface.py` is the **single import point** for permission symbols: mainline code (deps / files / chats / projects / kb, etc.) imports only from here, never from the concrete implementations. In the EE main repo it purely re-exports three real implementations; in the CE-derived tree the whole file is replaced by a single-tenant stub and the three implementation files do not physically exist:
+`core/auth/personal_permissions_iface.py` is the **single import point** for permission symbols: mainline code (deps / files / chats / projects / kb, etc.) imports only from here, never from the concrete implementations. In the EE main repo it purely re-exports three real implementations; in the CE-derived tree the whole file is replaced by a single-tenant stub and the three implementation files do not physically exist:
| Implementation file | Responsibility |
|---|---|
@@ -185,7 +185,7 @@ The actual gating, verified against each route file's dependencies:
| `PATCH /v1/me/api-keys/{key_id}` | Enable / disable |
| `DELETE /v1/me/api-keys/{key_id}` | Revoke |
-Keys look like `sk-jx-...` and can be used as a Bearer token to call business APIs **in every AUTH_MODE** (external programmatic callers usually have no cookies). Validation (enabled / not revoked / not expired / user flag still on) is centralized in `core/services/api_key_service.py::resolve_api_key`.
+Keys look like `sk-jx-...` and can be used as a Bearer token to call business APIs **in every AUTH_MODE** (external programmatic callers usually have no cookies). Validation (enabled / not revoked / not expired / user flag still on) is centralized in `core/auth/api_keys.py::resolve_api_key`.
Disabling or revoking an API key rejects new requests and cancels its admitted chat tasks that are pending, running, or awaiting attention. Cancellation stops background execution and terminates the SSE stream; completed external tool effects cannot be undone. Re-enabling the key does not resume cancelled tasks. Call duration stays fixed after execution stops, including the needs-attention state.
@@ -210,13 +210,13 @@ Key authentication events all land in the audit table (`audit_logs`): login succ
| SSO ticket exchange | `src/backend/edition_ee/auth/sso.py`, `src/backend/edition_ee/routes/auth.py` |
| Mock SSO / local login & registration page | `src/backend/api/routes/v1/mock_sso.py`, `src/backend/core/auth/mock_ticket_store.py` |
| Password hashing | `src/backend/core/auth/password.py` |
-| Permission interface layer (CE/EE seam) | `src/backend/core/auth/permissions_iface.py` |
+| Permission interface layer (CE/EE seam) | `src/backend/core/auth/personal_permissions_iface.py` |
| Team roles / file permissions (EE) | `src/backend/edition_ee/auth/roles.py`, `src/backend/edition_ee/auth/team_permissions.py` |
| Project / chat-share permissions (EE) | `src/backend/edition_ee/auth/project_permissions.py`, `src/backend/edition_ee/auth/chat_share_permissions.py` |
| Administrative credential dependencies | `src/backend/api/deps.py` |
| Profile / preferences | `src/backend/api/routes/v1/users.py` |
| Per-user permission flags | `src/backend/edition_ee/routes/config_users.py` |
-| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `src/backend/core/services/api_key_service.py` |
+| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `src/backend/core/auth/api_keys.py` |
| Capability-center self-service (owner isolation) | `src/backend/api/routes/v1/me_capabilities.py` |
| Invite codes | `src/backend/edition_ee/auth/invite.py`, `src/backend/edition_ee/routes/config_invites.py` |
| Team management | `src/backend/edition_ee/routes/config_teams.py`, `src/backend/edition_ee/routes/me_teams.py` |
diff --git a/document/en/modules/automation.md b/document/en/modules/automation.md
index 4fb0380c..d2c375f6 100644
--- a/document/en/modules/automation.md
+++ b/document/en/modules/automation.md
@@ -6,8 +6,8 @@ HugAgentOS ships two built-in mechanisms for turning a single instruction into r
| Capability | One-liner | Backend entry |
|---|---|---|
-| Scheduled automation | Run a prompt or a plan automatically on a cron schedule (or once / manually) | `api/routes/v1/automations.py` + `orchestration/schedulers/automation_scheduler.py` |
-| Plan mode | AI first decomposes a complex task into structured steps, then executes them after confirmation | `api/routes/v1/plans.py` + `orchestration/subagents/plan_mode.py` |
+| Scheduled automation | Run a prompt or a plan automatically on a cron schedule (or once / manually) | `features/automation/api/automations.py` + `orchestration/automations/automation_scheduler.py` |
+| Plan mode | AI first decomposes a complex task into structured steps, then executes them after confirmation | `api/routes/v1/plans.py` + `orchestration/plans/` |
## Scheduled automation tasks
@@ -17,7 +17,7 @@ Local scheduled tasks belong to the signed-in cloud account. The account mapping
### Task model
-Tasks live in the `scheduled_tasks` table (`ScheduledTask` in `core/db/models/automation.py`). Key fields:
+Tasks live in the `scheduled_tasks` table (`ScheduledTask` in `edition_ee/db/models/automation.py`). Key fields:
| Field | Description |
|---|---|
@@ -40,7 +40,7 @@ Tasks live in the `scheduled_tasks` table (`ScheduledTask` in `core/db/models/au
### Scheduling mechanics (automation_scheduler.py)
-`orchestration/schedulers/automation_scheduler.py` is an asyncio polling scheduler started with the backend:
+`orchestration/automations/automation_scheduler.py` is an asyncio polling scheduler started with the backend:
- **Polling**: every 15 seconds (plus 0–5 s random jitter) it queries the DB for tasks whose `next_run_at` is due.
- **Distributed lock**: before firing, it acquires the Redis lock `jx:auto:lock:{task_id}` with continuous renewal throughout execution and token-checked release on completion so multiple instances never double-fire.
@@ -48,6 +48,7 @@ Tasks live in the `scheduled_tasks` table (`ScheduledTask` in `core/db/models/au
- **Execution duration**: there is no fixed total runtime limit for scheduled or manually triggered Prompt, Plan, or Loop tasks. Their conversations and successors are also exempt from the ordinary chat absolute-age cap.
- **Failure governance**: when consecutive failures reach `max_failures` (default 3), the task is automatically set to `disabled`.
- **Orphan recovery**: at startup and on every poll, a worker must acquire execution ownership before resuming observation of the linked durable conversation and saving its final outcome. Only orphan runs without a linked conversation are marked `failed`, with overdue schedules advanced and one-shot tasks finalized. Runtime never determines recovery. The lock renews every 15 seconds; its 60-second lease detects a disconnected worker and does not limit total runtime. Missed one-shot tasks also acquire ownership before firing.
+- **Execution lease shutdown**: shutdown signals stop, cancels renewal, waits at most 2 seconds, and always attempts token-checked deletion. Local ownership markers are removed even on cleanup errors. Renewal also stops when the execution Task finishes. Result delivery has a 60-second total wait budget; failures do not overwrite committed success and do not retry automatically. Running records carry an internal ownership token; completion requires both running status and a matching token. Recovery claims the record under its new token.
Each execution creates and immediately links a **real conversation** through the shared ChatRun executor. Prompt runs stream text, thinking and tools; plan runs stream step progress. Opening a running conversation or refreshing the page follows and replays the same event stream as ordinary chat. Users can stop execution from the conversation; prompt runs also accept additional instructions through the durable steering queue at the next safe model boundary. Stopping does not increase consecutive failures, and loss of execution ownership or scheduler shutdown cancels the child run. Final messages, artifacts, citations and delivery remain owned by the shared executor and scheduler outcome handler. Conversation titles use the task name without an automation prefix. Older executions started before live-stream support cannot replay events that were never recorded.
@@ -63,7 +64,7 @@ Opening the right-side Canvas in an execution conversation temporarily hides the
## Plan mode
-Plan mode decomposes a complex task into structured steps before executing them, in two phases (`orchestration/subagents/plan_mode.py`):
+Plan mode decomposes a complex task into structured steps before executing them, in two phases (`orchestration/plans/`):
1. **Generate (Phase 1)**: `POST /v1/plans/generate` (SSE streaming) — the AI analyzes the task description and produces a plan draft with a title, description, and step list; each step can declare `expected_tools` / `expected_skills` / `expected_agents`.
2. **Execute (Phase 2)**: `POST /v1/plans/{plan_id}/execute` (SSE streaming) — steps run sequentially, each with its own agent; step status, tool-call logs, and AI output are persisted incrementally.
@@ -98,14 +99,41 @@ Every morning an Industry morning brief execution conversation is created. The s
| Topic | Path |
|---|---|
-| Automation REST API | `src/backend/api/routes/v1/automations.py` |
-| Scheduler (polling / locks / recovery) | `src/backend/orchestration/schedulers/automation_scheduler.py` |
-| Automation service (cron computation / state machine) | `src/backend/core/services/automation_service.py` |
-| Task / run-record models | `src/backend/core/db/models/automation.py` |
+| Automation REST API | `src/backend/features/automation/api/automations.py` |
+| Scheduler (polling / locks / recovery) | `src/backend/orchestration/automations/automation_scheduler.py` |
+| Automation service (cron computation / state machine) | `src/backend/features/automation/service.py` |
+| Task / run-record models | `src/backend/features/automation/models.py` |
| Plan REST API | `src/backend/api/routes/v1/plans.py` |
-| Plan generation / execution orchestration | `src/backend/orchestration/subagents/plan_mode.py`, `src/backend/core/services/plan_service.py` |
+| Plan generation / execution orchestration | `src/backend/orchestration/plans/`, `src/backend/core/chat/plans.py` |
| Plan / step models | `src/backend/core/db/models/agent.py` |
-| Frontend automation UI | `src/frontend/src/components/lab/AutomationPanel.tsx` etc., `src/frontend/src/components/automation/RunTimelinePanel.tsx` |
+| Frontend automation UI | `src/frontend/src/components/automation/AutomationPanel.tsx` etc., `src/frontend/src/components/automation/RunTimelinePanel.tsx` |
| Frontend state | `src/frontend/src/stores/automationStore.ts`, `automationChatStore.ts` |
Further reading: [Chat System](chat.md) · [MCP Tools](mcp-tools.md) · [Canvas & Artifacts](canvas-artifacts.md)
+
+## Multi-process execution and recovery
+
+Scheduled and manual triggers share one loop, scanned every 3 seconds with at most
+8 observed executions. Trigger consumption, schedule advancement and insertion of
+a pending run commit in one transaction. Conditional database updates fence
+concurrent admission. A manual trigger for a busy task remains queued.
+
+Pending means accepted and awaiting execution. Scheduler shutdown or lease loss
+transfers observation without cancelling the accepted conversation. A successor
+observes the same durable execution and its handoffs. User cancellation still
+stops execution. Child linkage and recovery arguments commit with acceptance;
+recovery never restarts a command to obtain its result. Existing notification
+delivery semantics remain; this change does not promise exactly-once delivery
+to external channels.
+
+Apply FULL migration automationqueue01 or CE migration ce_0032 before rollout.
+Local SQLite uses the installed automation module's schema initialization.
+Downgrade refuses pending records rather than discarding accepted work. Stop old
+schedulers, migrate, then start the new version; do not mix scheduling protocols.
+
+Plan/Loop persist all initial arguments with child acceptance. After a crash before
+worker registration, the successor dispatches only pending children. It never
+replays an already-running child from the beginning. Drain old in-flight work
+before upgrading; missing legacy arguments are not reconstructed by a compatibility path.
+
+Task delivery uses delivery_targets exclusively. The upgrade migrates flat channel_id/conversation_id fields into destination lists, removes the old fields, and preserves explicit destinations and unrelated metadata. Request handling no longer reads the old format.
diff --git a/document/en/modules/autonomous-loop.md b/document/en/modules/autonomous-loop.md
index 0b844bb4..346c5fdc 100644
--- a/document/en/modules/autonomous-loop.md
+++ b/document/en/modules/autonomous-loop.md
@@ -71,7 +71,7 @@ Frontend entry: the **"Autonomous loop" toggle in the chat input** (`components/
## Related code
-- Driver `orchestration/autonomous_loop.py`, planner `orchestration/loop_planner.py`, reviewer `orchestration/subagents/loop_reviewer.py`
-- ChatRun integration `orchestration/chat_run_executor.py` (`start_autonomous_loop_run` / startup reconciliation `resume_running_loops`)
-- Service `core/services/loop_service.py` (ledger mirror / start params / steering queue), API `api/routes/v1/loops.py`
+- Driver `orchestration/loops/driver.py`, planner `orchestration/loops/planner.py`, reviewer `orchestration/loops/reviewer.py`
+- ChatRun integration `core/chat/runtime/` (`start_autonomous_loop_run` / startup reconciliation `resume_running_loops`)
+- Service `orchestration/loops/run.py` (ledger mirror / start params / steering queue), API `api/routes/v1/loops.py`
- Tables `agent_loops` / `loop_iterations`
diff --git a/document/en/modules/browser-automation.md b/document/en/modules/browser-automation.md
index 56149118..883b37c2 100644
--- a/document/en/modules/browser-automation.md
+++ b/document/en/modules/browser-automation.md
@@ -107,7 +107,7 @@ Startup probes the actual Chromium sandbox. Known inner-sandbox failures retry w
After OpenSandbox updates, explicitly build the runtime image from the repository root:
```bash
-docker build -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+docker build -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
```
Ordinary docker compose up --build does not rebuild this on-demand image. Check OPENSANDBOX_IMAGE and create a fresh browser_open session after updating. Resource readiness requires Chromium initialization and real browser verification. Rebuild the corresponding images for other providers too.
diff --git a/document/en/modules/canvas-artifacts.md b/document/en/modules/canvas-artifacts.md
index 707ad1a8..dca1517f 100644
--- a/document/en/modules/canvas-artifacts.md
+++ b/document/en/modules/canvas-artifacts.md
@@ -8,7 +8,7 @@ Personal canvas editing, the artifact center, and chat sharing are all **Communi
## Data canvas
-The data canvas is a slide-out panel on the right of the chat UI (`src/frontend/src/components/canvas/CanvasPanel.tsx`), opened by clicking an attachment card in chat (`components/chat/ArtifactCardList.tsx`) or a file in My Space (`components/myspace/MySpacePanel.tsx`). The panel picks a renderer per file type:
+The data canvas is a slide-out panel on the right of the chat UI (`src/frontend/src/components/canvas/CanvasPanel.tsx`), opened by clicking an attachment card in chat (`components/chat/MessageArtifacts.tsx`) or a file in My Space (`components/myspace/MySpacePanel.tsx`). The panel picks a renderer per file type:
| File type | Rendering | Editable |
|---|---|---|
@@ -87,14 +87,14 @@ limits show a local-download prompt while the main page remains responsive.
### How artifacts are produced and persisted
-During a streamed turn, files produced by tools first enter the per-conversation **workspace** (`core/llm/workspace`); only files that are explicitly **pinned** are written to the `artifacts` table at end of turn by `core/services/artifact_service.py::persist_artifacts` — this strict workspace gate keeps temporary intermediates from polluting My Space.
+During a streamed turn, files produced by tools first enter the per-conversation **workspace** (`core/sandbox/workspace`); only files that are explicitly **pinned** are written to the `artifacts` table at end of turn by `core/files/artifacts/service.py::persist_artifacts` — this strict workspace gate keeps temporary intermediates from polluting My Space.
-Physical storage is managed by `core/artifacts/store.py`, dual-mode by `STORAGE_TYPE`:
+Physical storage is managed by `core/files/artifacts/store.py`, dual-mode by `STORAGE_TYPE`:
- `local` (default, Community Edition): files are written under `${STORAGE_PATH:-result}/artifacts/` with a local JSON index (`index.json`), served via FileResponse.
-- `oss` (Enterprise Edition, `cloud_storage`): files are uploaded to Aliyun OSS; the local index is also backed up to OSS so it survives container restarts. The S3 backend lives in `core/storage/`.
+- `oss` (Enterprise Edition, `cloud_storage`): files are uploaded to Aliyun OSS; the local index is also backed up to OSS so it survives container restarts. The S3 backend lives in `core/files/storage/`.
-Downloads all go through `GET /files/{file_id}` (`api/routes/files.py`), with an owner ∪ team permission check via `core/auth/permissions_iface.py::resolve_artifact_access`; Office files support PDF-converted online preview.
+Downloads all go through `GET /files/{file_id}` (`api/routes/files.py`), with an owner ∪ team permission check via `core/auth/personal_permissions_iface.py::resolve_artifact_access`; Office files support PDF-converted online preview.
### REST API (`/v1/artifacts`, `api/routes/v1/artifacts.py`)
@@ -153,8 +153,8 @@ attachments and My Space, and xlsx files remain editable in the data canvas.
|---|---|
| Canvas panel / Univer spreadsheet | `src/frontend/src/components/canvas/CanvasPanel.tsx`, `UniverSpreadsheet.tsx`, `src/frontend/src/stores/canvasStore.ts` |
| Artifact REST API | `src/backend/api/routes/v1/artifacts.py` |
-| Artifact store (local/OSS) | `src/backend/core/artifacts/store.py` |
-| Artifact persistence (workspace gate) | `src/backend/core/services/artifact_service.py` |
+| Artifact store (local/OSS) | `src/backend/core/files/artifacts/store.py` |
+| Artifact persistence (workspace gate) | `src/backend/core/files/artifacts/service.py` |
| File download / preview | `src/backend/api/routes/files.py` |
| Chat share API | `src/backend/api/routes/v1/chat_shares.py` |
| Share frontend | `src/frontend/src/SharePreviewApp.tsx`, `src/frontend/src/components/share/ShareRecordsPage.tsx` |
diff --git a/document/en/modules/catalog.md b/document/en/modules/catalog.md
index 610135a0..83e1ff2b 100644
--- a/document/en/modules/catalog.md
+++ b/document/en/modules/catalog.md
@@ -13,7 +13,7 @@ core/config/catalog.json system-level single source of truth (admin t
core/config/catalog.py public API: get_catalog / is_enabled / get_enabled_ids
│ set_enabled / reorder_items
▼
-core/config/catalog_resolver.py merge layer: catalog.json defaults ∩ per-user DB overrides
+core/capabilities/catalog/resolver.py merge layer: catalog.json defaults ∩ per-user DB overrides
│ resolve_all_runtime_enabled(db, user_id) → (skills, agents, mcps)
│ resolve_explicit_runtime_capabilities(...) → per-turn invocation (ignores personal switches, keeps hard gates)
▼
@@ -39,7 +39,7 @@ The file lives at `src/backend/core/config/catalog.json` (path overridable via t
}
```
-`catalog_loader.py` performs **dynamic-source synchronization** at load time: new ids discovered from the skill loader (`core/agent_skills/`) and the MCP config service (`core/services/mcp_service.py`, DB table `admin_mcp_servers`) are added automatically; names/descriptions/versions refresh from the source; stale entries whose id no longer exists in any source are removed. Display details (Capability Center intro markdown, icons) are runtime-only fields and never persisted. Results are held in a short-TTL in-memory cache; writes call `invalidate_catalog_cache()`.
+`catalog_loader.py` performs **dynamic-source synchronization** at load time: new ids discovered from the skill loader (`core/capabilities/skills/`) and the MCP config service (`core/capabilities/mcp/service.py`, DB table `admin_mcp_servers`) are added automatically; names/descriptions/versions refresh from the source; stale entries whose id no longer exists in any source are removed. Display details (Capability Center intro markdown, icons) are runtime-only fields and never persisted. Results are held in a short-TTL in-memory cache; writes call `invalidate_catalog_cache()`.
### Gating API
@@ -54,7 +54,7 @@ These two functions are the system-wide read path for capability switches: `fact
## Per-user overrides (catalog_overrides)
-When a user toggles a capability in the Capability Center, the change is written to the `catalog_overrides` DB table (`core/services/catalog_service.py`), per-user and isolated:
+When a user toggles a capability in the Capability Center, the change is written to the `catalog_overrides` DB table (`core/capabilities/catalog/overrides.py`), per-user and isolated:
- **Merge algorithm** (`catalog_resolver.py::_merge_kind`): overrides can only flip the enabled flag of items that *exist* in the base catalog; they can never resurrect deleted items.
- **Admin lock**: items disabled in the base catalog (`enabled=false`) **cannot** be re-enabled by user overrides, and are completely hidden from the user-facing `/v1/catalog` response.
@@ -66,7 +66,7 @@ When a user toggles a capability in the Capability Center, the change is written
- **Explicit plugin guidance**: selecting a plugin loads its skills and MCP tools and recommends using them when relevant, without restricting the first tool call or requiring a call before the answer.
- **Chat-sticky capability restoration**: later turns restore activated skills, connectors, and plugin components before personal-switch filtering, keeping the skill list and tool names/prefixes stable in that chat. Installation, global admin state, dependency readiness, and ownership are revalidated each turn, so hard-gate changes take effect immediately.
-On every chat, `core/chat/context.py::resolve_enabled_capabilities()` writes the default merged result into the workflow context. `api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` then verifies plugin installations and capability ownership, resolves components from the server-side installation record, and merges hard-gate-approved explicit selections into that turn. On later turns, `core/llm/session_capabilities.py` and `core/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` restore chat-activated capabilities.
+On every chat, `core/chat/context.py::resolve_enabled_capabilities()` writes the default merged result into the workflow context. `api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` then verifies plugin installations and capability ownership, resolves components from the server-side installation record, and merges hard-gate-approved explicit selections into that turn. On later turns, `core/llm/runtime/session_capabilities.py` and `core/capabilities/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` restore chat-activated capabilities.
## The /v1/catalog route and KB injection
@@ -116,20 +116,19 @@ State is centralized in `src/frontend/src/stores/catalogStore.ts`; local default
## Admin-side capability management
- **`/admin` content console**: skill upload/toggle/reorder (`api/routes/v1/admin_skills.py`, which calls `catalog.set_enabled` / `reorder_items` to write back to catalog.json), MCP server management (`admin_mcp_servers.py`, DB table + connectivity probe + cache refresh), sub-agent management (`admin_agents.py`), and skill marketplace listing review (`admin_marketplace.py`, **Enterprise Edition (EE)**).
-- **Catalog snapshot migration**: `scripts/export_content.py --only catalog` exports catalog.json + `catalog_overrides`; `scripts/import_content.py --catalog ` imports them.
## Source map
| Topic | Path |
|---|---|
-| Public API (is_enabled, etc.) | `src/backend/core/config/catalog.py` |
+| Public API (is_enabled, etc.) | `src/backend/core/capabilities/catalog/` |
| Single-source file | `src/backend/core/config/catalog.json` (overridable via `CATALOG_PATH`) |
-| Loading / dynamic sync / caching | `src/backend/core/config/catalog_loader.py`, `catalog_common.py`, `catalog_migration.py` |
-| User-override merging | `src/backend/core/config/catalog_resolver.py`, `core/services/catalog_service.py` |
+| Loading / dynamic sync / caching | `src/backend/core/capabilities/catalog/loader.py`, `catalog_common.py`, `catalog_migration.py` |
+| User-override merging | `src/backend/core/capabilities/catalog/resolver.py`, `core/capabilities/catalog/overrides.py` |
| Catalog route | `src/backend/api/routes/v1/catalog.py` |
| User self-service capabilities | `src/backend/api/routes/v1/me_capabilities.py` |
-| MCP server config (DB) | `src/backend/core/services/mcp_service.py`, `api/routes/v1/admin_mcp_servers.py` |
-| Skill management | `src/backend/api/routes/v1/admin_skills.py`, `core/agent_skills/` |
-| Dify KB injection (EE) | `src/backend/edition_ee/kb/dify.py`, via the shared `core/kb/external_provider.py` seam |
+| MCP server config (DB) | `src/backend/core/capabilities/mcp/service.py`, `api/routes/v1/admin_mcp_servers.py` |
+| Skill management | `src/backend/api/routes/v1/admin_skills.py`, `core/capabilities/skills/` |
+| Dify KB injection (EE) | `src/backend/edition_ee/kb/dify.py`, via the shared `features/knowledge/extension.py` seam |
| Frontend Capability Center | `src/frontend/src/components/catalog/`, `stores/catalogStore.ts` |
| Factory consumption | `src/backend/core/llm/factory/build.py::_effective_mcp_server_keys` |
diff --git a/document/en/modules/chat.md b/document/en/modules/chat.md
index a5f2fc76..2fc804de 100644
--- a/document/en/modules/chat.md
+++ b/document/en/modules/chat.md
@@ -71,10 +71,10 @@ orchestration/chat_run_executor.py::start_run()
▼
orchestration/workflow.py::astream_chat_workflow()
│ ├─ orchestration/memory_integration.py non-blocking memory retrieval (bg task + budget)
- │ ├─ core/config/catalog_resolver.py resolve enabled skills/mcp/kb for this request
+ │ ├─ core/capabilities/catalog/resolver.py resolve enabled skills/mcp/kb for this request
│ ├─ core/llm/factory/build.py::create_agent_executor()
│ │ MCP pool + skill registration + file tools + system prompt + middlewares → Agent
- │ ├─ core/llm/context_manager.py trim history to the token budget
+ │ ├─ core/llm/context/window_policy.py trim history to the token budget
│ └─ orchestration/streaming.py::StreamingAgent.stream()
│ consumes agent.reply_stream(), coalesces 25 event kinds while preserving tool-argument deltas
▼
@@ -85,7 +85,7 @@ SSE follower: chat_run_executor.follow_run_as_sse()
### Run decoupling and reconnect/resume
-Every sent message creates a `ChatRun` and a background task (`orchestration/chat_run_executor.py`); events are written to a Redis Stream (`maxlen=5000`; the default TTL is the 7,200-second maximum human wait plus a 1,800-second recovery grace period). The HTTP connection is merely a *follower*, which enables:
+Every sent message creates a `ChatRun` and a background task (`core/chat/runtime/`); events are written to a Redis Stream (`maxlen=5000`; the default TTL is the 7,200-second maximum human wait plus a 1,800-second recovery grace period). The HTTP connection is merely a *follower*, which enables:
| Capability | Endpoint |
|---|---|
@@ -150,7 +150,7 @@ A permission chip sits next to the project selector in the composer toolbar. It
| Approve for me | `auto` | Ordinary writes and edits pass silently; **destructive operations** (`DESTRUCTIVE_OPS`) and commands the local policy tagged `danger:` still stop and ask |
| Full access | `full` | No tool confirmation is ever raised |
-The vocabulary is declared once in `core/llm/tool_permissions.py` (`APPROVAL_ASK` / `APPROVAL_AUTO` / `APPROVAL_FULL`), travels with the chat context into `PermissionRuntime`, and is the single criterion every tool confirmation gate consults. No preset does more than skip the *asking* — a hard deny from the local security policy still denies. Whether a given call counts as dangerous is recovered by `core/sandbox/permissions/models.py::danger_categories()` from the `danger:` prefix in the verdict reasons, never by matching prose. Legacy stored values (`standard`, `readonly`) fall back to the most conservative preset, `ask`.
+The vocabulary is declared once in `core/llm/tool_permissions/` (`APPROVAL_ASK` / `APPROVAL_AUTO` / `APPROVAL_FULL`), travels with the chat context into `PermissionRuntime`, and is the single criterion every tool confirmation gate consults. No preset does more than skip the *asking* — a hard deny from the local security policy still denies. Whether a given call counts as dangerous is recovered by `core/sandbox/permissions/models.py::danger_categories()` from the `danger:` prefix in the verdict reasons, never by matching prose. Legacy stored values (`standard`, `readonly`) fall back to the most conservative preset, `ask`.
Trusted unattended entry points (channel bots, scheduled automation) take the `default_allow` branch and are unaffected by the preset.
@@ -160,17 +160,17 @@ Trusted unattended entry points (channel bots, scheduled automation) take the `d
`create_agent_executor()` is the shared factory for every mode (main chat, plan, batch, sub-agents, automation):
-- **MCP tools**: after the three-layer filter of catalog + per-user overrides + request context (see [Capability Center](catalog.md)), stable servers reuse the process-level connection pool (`core/llm/mcp_pool.py`); per-request servers (e.g. `retrieve_dataset_content`, which needs per-request HTTP headers) are spawned fresh; the user's self-added private MCP servers are merged in with owner isolation.
-- **Skills**: registered as AgentScope Agent Skills via `core/agent_skills/loader.py`, with `view_text_file` allow-listed to read SKILL.md files (see [Agent Skills](agent-skills.md)).
+- **MCP tools**: after the three-layer filter of catalog + per-user overrides + request context (see [Capability Center](catalog.md)), stable servers reuse the process-level connection pool (`core/llm/mcp/`); per-request servers (e.g. `retrieve_dataset_content`, which needs per-request HTTP headers) are spawned fresh; the user's self-added private MCP servers are merged in with owner isolation.
+- **Skills**: registered as AgentScope Agent Skills via `core/capabilities/skills/loader.py`, with `view_text_file` allow-listed to read SKILL.md files (see [Agent Skills](agent-skills.md)).
- **File / sandbox tools**: `bash`, `sandbox_put_artifact`, `sandbox_get_artifact` are always registered; Read/Edit/Write/Glob/Grep/Delete/Move/mkdir plus the MySpace tools are gated by `CODE_CAPABILITY_ENABLED` and share one `ReadStateTracker` to keep the "must Read before Edit" invariant.
-- **Middlewares** (onion model, `core/llm/middlewares.py`): `DynamicModelMiddleware` (switches the model per chat_mode, see [Model Providers](model-providers.md)), `FileContextMiddleware` (injects uploaded/historical file context), `SteerMiddleware` (injects follow-ups after tool results and before the next reasoning round), `WorkspacePinHintMiddleware`, `FinishPinGuardMiddleware`.
+- **Middlewares** (onion model, `core/llm/runtime/`): `DynamicModelMiddleware` (switches the model per chat_mode, see [Model Providers](model-providers.md)), `FileContextMiddleware` (injects uploaded/historical file context), `SteerMiddleware` (injects follow-ups after tool results and before the next reasoning round), `WorkspacePinHintMiddleware`, `FinishPinGuardMiddleware`.
- **Context compression**: `CompactingAgent` routes the pre-turn, mid-turn, and post-turn trigger points through one persistent checkpoint engine and one Codex-style handoff prompt. `ContextConfig` remains only for tool-result limiting. Compaction uses the unified coordinator without a second framework summary fallback. Failure preserves the original context. If the final request still exceeds its budget, the run saves a checkpoint and pauses before calling the reply model.
- **Permissions**: every registered tool gets a native `PermissionRule(ALLOW)` seed, preserving AgentScope's built-in dangerous-operation checks (no blanket BYPASS).
- **Iteration caps**: main agent defaults to `max_iters=50`, isolated sub-agents to 10.
## SSE event types and payloads
-`orchestration/streaming.py::StreamingAgent` coalesces AgentScope 2.0 `reply_stream` events into internal events. Tiny tool-argument fragments are batched at 256 characters or 50ms, keeping them visible without recreating an SSE event storm. `workflow.py` and `chats.py::_stream_sse_response` enrich them with chat-level fields before they hit the wire. Events as the frontend sees them:
+`core/llm/runtime/streaming.py::StreamingAgent` coalesces AgentScope 2.0 `reply_stream` events into internal events. Tiny tool-argument fragments are batched at 256 characters or 50ms, keeping them visible without recreating an SSE event storm. `workflow.py` and `chats.py::_stream_sse_response` enrich them with chat-level fields before they hit the wire. Events as the frontend sees them:
| `type` | Meaning | Key fields |
|---|---|---|
@@ -219,7 +219,7 @@ data: [DONE]
After `meta`, `chat_run_executor.py` persists the assistant message, backfills artifacts,
and launches a background follow-up-question generator
-(`orchestration/followups.py`; results land in the message's
+(`core/chat/followups.py`; results land in the message's
`extra_data.follow_up_questions` and are fetched through
`GET /v1/chats/{chat_id}/messages/{message_id}/followups`). The frontend
collects ontology events in a standalone **Domain Ontology Governance** module
@@ -272,9 +272,9 @@ Each canonical tool-result step is persisted before its display event, independe
Citations make every fact in the answer traceable back to a specific tool result. Numbering authority belongs to a single backend source of truth — the model only **copies** ids, never computes them. The chain has four segments:
-1. **Anchor allocation & injection (backend middleware)**: `core/llm/middlewares.py::CitationAnchorMiddleware` hooks AgentScope 2.0's `on_acting` and, before a tool result reaches the model, calls `orchestration/citation_anchor.py` to extract → allocate → inject: each citable item gets a session-monotonic anchor id (`e1`, `e2`, … — unique across tools, calls, and turns; a new turn continues from the max anchor found in the chat's persisted messages), and `"cite_id": "e7"` is written into the result JSON in place (plain-text results get a trailing `[cite_id: e7]` line). **The allocator is bound to the agent instance** (`attach_allocator()` / `resolve_allocator()`), which is how the orchestrator and the middleware share one counter; the ContextVar is only a fallback for sub-agent chains, because `astream_chat_workflow` is an async generator whose context does not reach the task the agent actually runs in. Extraction degrades through four layers: tool-declared `__citations__` → the tool spec registry (`TOOL_SPECS` config: list paths + CN/EN field aliases) → a generic heuristic (unique dict-array field) → the whole result as one anchor. Operational tools (file writes, pin, etc. — `SKIP_TOOLS`) pass through untouched. Any exception passes the original result through — citations degrade, the conversation never breaks.
+1. **Anchor allocation & injection (backend middleware)**: `core/llm/runtime/` hooks AgentScope 2.0's `on_acting` and, before a tool result reaches the model, calls `core/llm/citations/hook.py` to extract → allocate → inject: each citable item gets a session-monotonic anchor id (`e1`, `e2`, … — unique across tools, calls, and turns; a new turn continues from the max anchor found in the chat's persisted messages), and `"cite_id": "e7"` is written into the result JSON in place (plain-text results get a trailing `[cite_id: e7]` line). **The allocator is bound to the agent instance** (`attach_allocator()` / `resolve_allocator()`), which is how the orchestrator and the middleware share one counter; the ContextVar is only a fallback for sub-agent chains, because `astream_chat_workflow` is an async generator whose context does not reach the task the agent actually runs in. Extraction degrades through four layers: tool-declared `__citations__` → the tool spec registry (`TOOL_SPECS` config: list paths + CN/EN field aliases) → a generic heuristic (unique dict-array field) → the whole result as one anchor. Operational tools (file writes, pin, etc. — `SKIP_TOOLS`) pass through untouched. Any exception passes the original result through — citations degrade, the conversation never breaks.
2. **Prompt contract**: the system prompt (fallback file `prompts/prompt_text/default/system/40_format.system.md`; the active DB version is authoritative at runtime) needs only one tool-count-independent rule: copy the `cite_id` annotated in the result verbatim into `[anchor text](cite:e7)` (or `[来源](cite:e7)` at sentence end); never self-number.
-3. **Orchestration consumption**: each `tool_result` event calls `collect_citation_dicts()`, which fetches `CitationItem`s (`id` / `tool_name` / `tool_id` / `title` / `url` / `snippet` / `source_type` / `item_index`) from the allocator registry keyed by `tool_id`; when no allocator is installed (legacy replay paths) it falls back to the old offset extraction in `orchestration/citations.py`. `source_type` values: CE ships `internet`, `knowledge_base`, and `database`; industry citation types such as `industry_news`, `ai_news`, `chain_info`, and `company_profile` are added by the Enterprise Edition (EE).
+3. **Orchestration consumption**: each `tool_result` event calls `collect_citation_dicts()`, which fetches `CitationItem`s (`id` / `tool_name` / `tool_id` / `title` / `url` / `snippet` / `source_type` / `item_index`) from the allocator registry keyed by `tool_id`; when no allocator is installed (legacy replay paths) it falls back to the old offset extraction in `core/llm/citations/`. `source_type` values: CE ships `internet`, `knowledge_base`, and `database`; industry citation types such as `industry_news`, `ai_news`, `chain_info`, and `company_profile` are added by the Enterprise Edition (EE).
4. **Frontend rendering**: citations ride on `tool_result` and `meta` events and are persisted with the message (what is persisted is the annotated result, so replay/share shows the same numbering as generation). `components/citation/CitationMarkdownBlock.tsx` recognizes three marker forms in parallel — `[anchor text](cite:eN)` (rendered as a text link with a hover source card), `[[eN]]` (obsidian-style tolerance), and the legacy `[ref:tool_name-N]` (historical messages, rendered as a superscript badge). Tool cards show a matching `cite_id` chip on each item (`jx-tr-citeTag`), and tools without a dedicated renderer fall back to a generic list-card renderer.
**Tool development convention**: a tool (in-house or MCP) that wants precise citation granularity should return a `__citations__` field in its JSON — `[{"title": "...", "url": "...", "snippet": "...", "source_type": "..."}, …]`, entries ordered to match the result body; the middleware adopts it verbatim and injects `cite_id` in place. Tools without the field fall back to registry config or heuristics — at worst the whole result becomes one anchor, so **every tool is citable by default**. See the citation-declaration section in [MCP tools](mcp-tools.md).
@@ -297,7 +297,7 @@ Cards are persisted with the message (`metadata.referenced_chats`) and history r
## Plan Mode
-Plan Mode splits complex tasks into "generate plan → user reviews/edits → execute step by step", implemented in `orchestration/subagents/plan_mode.py`:
+Plan Mode splits complex tasks into "generate plan → user reviews/edits → execute step by step", implemented in `orchestration/plans/`:
- **Generate** (`astream_generate_plan` / `POST /v1/plans/generate`): a "bare LLM" agent (`disable_tools=True`) produces a structured JSON plan. System-prompt resolution: active `plan_mode` version in the prompt pool → legacy `system/90_plan_mode` part → fallback file `prompts/prompt_text/plan_mode/plan_mode.system.md` → hardcoded minimal prompt.
- **Execute** (`astream_execute_plan` / `POST /v1/plans/{plan_id}/execute`): each step gets its own agent, executed sequentially, with step-level MCP/skill/sub-agent bindings and cancellation (`is_run_cancelled` polling); execution also goes through ChatRun + Redis Stream, so it survives disconnects.
@@ -351,7 +351,7 @@ the `subagents` prompt-pool kind. That kind stays in backend storage and
migration snapshots for compatibility;
`extensions/agents/default//system.md` is only the seed and failure fallback.
-In addition, user-created sub-agents (`api/routes/v1/agents.py`, DB table `UserAgent`)
+In addition, user-created sub-agents (`edition_ee/routes/agents.py`, DB table `UserAgent`)
can carry their own system prompt, MCP, skill, plugin, and knowledge base
bindings, plus model parameters such as provider, temperature, `max_tokens`,
and `max_iters`. When you create or edit a sub-agent, the resource picker
@@ -428,7 +428,7 @@ or no-longer-owned components are not restored.
- **Autonomous main-agent dispatch**: when neither a structured `@` selection
nor the strict natural-language command matches, the main agent can use the
`call_subagent` tool registered by
- `core/llm/subagent_tool.py`. Each child runs in its own thread and event loop,
+ `core/llm/subagents/tool.py`. Each child runs in its own thread and event loop,
then returns text for the main agent to integrate. This path supports parallel
sub-agents, task decomposition, and cross-domain synthesis.
@@ -438,8 +438,8 @@ Three complementary layers:
| Layer | Implementation | Trigger |
|---|---|---|
-| Chat title summary | `core/llm/summarizer.py::ConversationSummarizer` (`summarizer` model role, `ENABLE_SUMMARY` flag), `POST /v1/summary` | Auto-titling new chats |
-| Unified context checkpoint | `core/services/compaction_service.py::run_compaction()`; pre-turn, mid-turn, and post-turn share one trigger ratio, handoff prompt, replacement shape, and persistent checkpoint | Context reaches the configured fraction of the model window |
+| Chat title summary | `core/models/completions/summarizer.py::ConversationSummarizer` (`summarizer` model role, `ENABLE_SUMMARY` flag), `POST /v1/summary` | Auto-titling new chats |
+| Unified context checkpoint | `core/chat/compaction/service.py::run_compaction()`; pre-turn, mid-turn, and post-turn share one trigger ratio, handoff prompt, replacement shape, and persistent checkpoint | Context reaches the configured fraction of the model window |
| Deterministic overflow protection | `ContextConfig.tool_result_limit` bounds individual tool results first; failed compaction preserves the original context, and an over-budget final request saves a checkpoint and pauses | A tool result is oversized or unified compaction is temporarily unavailable |
Compaction cuts only at legal boundaries: history is first split into steps,
@@ -468,7 +468,7 @@ recovery stay consistent.
## Oversized tool-result offloading
-`core/llm/offloader.py::SandboxOffloader` implements the AgentScope 2.0 `Offloader` protocol: when context compression or tool-result truncation happens, the overflow is no longer silently discarded — it is written into the sandbox at `/workspace/.offload/` (`tool_.txt` / `context_.txt`), the framework appends the path to the model-facing ``, and the model can read it back on demand via `Read` / `bash`. Mounted only when sandbox tools are enabled (`SANDBOX_TOOLS_ENABLED=true`, default on); write failures never raise and degrade to an explanatory message.
+`core/llm/context/offloader.py::SandboxOffloader` implements the AgentScope 2.0 `Offloader` protocol: when context compression or tool-result truncation happens, the overflow is no longer silently discarded — it is written into the sandbox at `/workspace/.offload/` (`tool_.txt` / `context_.txt`), the framework appends the path to the model-facing ``, and the model can read it back on demand via `Read` / `bash`. Mounted only when sandbox tools are enabled (`SANDBOX_TOOLS_ENABLED=true`, default on); write failures never raise and degrade to an explanatory message.
## Chat sharing
@@ -486,29 +486,29 @@ Storage is Redis (`chat_share:*` key groups + TTL) with an in-process memory fal
## Other entry points
-The same orchestration foundation also powers: response regeneration (`POST /v1/chats/{chat_id}/regenerate`), edit-and-resend (`POST /v1/chats/{chat_id}/edit`), non-streaming `POST /v1/agents/responses` with `stream: false` (see [Automation](automation.md)), and scheduled automation (`orchestration/schedulers/`).
+The same orchestration foundation also powers: response regeneration (`POST /v1/chats/{chat_id}/regenerate`), edit-and-resend (`POST /v1/chats/{chat_id}/edit`), non-streaming `POST /v1/agents/responses` with `stream: false` (see [Automation](automation.md)), and scheduled automation (`orchestration/automations/`).
## Source map
| Topic | Path |
|---|---|
| Chat routes / SSE egress | `src/backend/api/routes/v1/chats/__init__.py` |
-| Run decoupling / Redis Stream / resume | `src/backend/orchestration/chat_run_executor.py`, `api/routes/v1/chat_runs.py` |
-| Streaming orchestration | `src/backend/orchestration/workflow.py` |
-| Event mapping (reply_stream → SSE) | `src/backend/orchestration/streaming.py` |
+| Run decoupling / Redis Stream / resume | `src/backend/core/chat/runtime/`, `api/routes/v1/chat_runs.py` |
+| Streaming orchestration | `src/backend/core/agent_runtime/runner.py` |
+| Event mapping (reply_stream → SSE) | `src/backend/core/llm/runtime/streaming.py` |
| Runtime context assembly | `src/backend/core/chat/context.py` |
| Agent factory | `src/backend/core/llm/factory/build.py` |
-| Middlewares | `src/backend/core/llm/middlewares.py` (pure-function helpers in `core/llm/hooks.py`) |
-| Citation extraction | `src/backend/orchestration/citations.py` |
-| Cross-chat references (cards / digests / paging) | `src/backend/core/services/chat_reference_service.py` |
+| Middlewares | `src/backend/core/llm/runtime/` (pure-function helpers in `core/harness/hooks.py`) |
+| Citation extraction | `src/backend/core/llm/citations/` |
+| Cross-chat references (cards / digests / paging) | `src/backend/core/chat/references.py` |
| Cross-chat history tools | `src/backend/core/llm/tools/chat_history_tool.py` |
| Citation rendering | `src/frontend/src/utils/citations.ts`, `src/frontend/src/components/citation/` |
-| Plan mode | `src/backend/orchestration/subagents/plan_mode.py`, `api/routes/v1/plans.py` |
-| Sub-agent tool | `src/backend/core/llm/subagent_tool.py`, `api/routes/v1/agents.py` |
-| Title summarization / context compaction / window protection | `src/backend/core/llm/summarizer.py`, `compaction.py`, `core/services/compaction_service.py`, `context_manager.py` |
-| Oversized-result offloading | `src/backend/core/llm/offloader.py` |
+| Plan mode | `src/backend/orchestration/plans/`, `api/routes/v1/plans.py` |
+| Sub-agent tool | `src/backend/core/llm/subagents/tool.py`, `edition_ee/routes/agents.py` |
+| Title summarization / context compaction / window protection | `src/backend/core/models/completions/summarizer.py`, `compaction.py`, `core/chat/compaction/service.py`, `context_manager.py` |
+| Oversized-result offloading | `src/backend/core/llm/context/offloader.py` |
| Chat sharing | `src/backend/api/routes/v1/chat_shares.py` |
-| Follow-up generation | `src/backend/orchestration/followups.py` |
+| Follow-up generation | `src/backend/core/chat/followups.py` |
| Frontend stream parsing / follow-up queue | `src/frontend/src/hooks/chatStream.ts`, `useStreaming.ts`, `components/chat/QueuedMessageCard.tsx` |
### Desktop tool permissions and operation confirmation
diff --git a/document/en/modules/knowledge-base-wiki.md b/document/en/modules/knowledge-base-wiki.md
index db1ad00b..a25a7c8c 100644
--- a/document/en/modules/knowledge-base-wiki.md
+++ b/document/en/modules/knowledge-base-wiki.md
@@ -140,12 +140,12 @@ The tools work for both self-hosted and external knowledge bases; just pass the
| Layer | Location |
|---|---|
-| Data model | `src/backend/core/db/models/kb_wiki.py` |
-| Pipeline | `src/backend/core/kb/wiki/` (`extract` / `cite` / `dedup` / `taxonomy` / `reduce` / `finalize` / `pipeline`) |
-| Prompts | `src/backend/core/kb/wiki/prompts.py` |
-| Jobs and worker | `src/backend/core/kb/wiki/jobs.py`, `worker.py` |
-| Read surface | `src/backend/core/kb/wiki/local_provider.py` |
-| Per-base dispatch | `src/backend/core/kb/wiki_router.py` |
-| Routes | `src/backend/api/routes/v1/kb_wiki.py` |
+| Data model | `src/backend/features/knowledge/models.py` |
+| Pipeline | `src/backend/features/knowledge/wiki/` (`extract` / `cite` / `dedup` / `taxonomy` / `reduce` / `finalize` / `pipeline`) |
+| Prompts | `src/backend/features/knowledge/wiki/prompts.py` |
+| Jobs and worker | `src/backend/features/knowledge/wiki/jobs.py`, `worker.py` |
+| Read surface | `src/backend/features/knowledge/wiki/local_provider.py` |
+| Per-base dispatch | `src/backend/features/knowledge/wiki_router.py` |
+| Routes | `src/backend/features/knowledge/api/kb_wiki.py` |
| Agent tools | `src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py` |
| Frontend | `src/frontend/src/components/kb/` (`WikiPanel` / `WikiTree` / `ConceptGraph` / `IndexModePicker`) |
diff --git a/document/en/modules/knowledge-base.md b/document/en/modules/knowledge-base.md
index 1e84aa15..32edc8ed 100644
--- a/document/en/modules/knowledge-base.md
+++ b/document/en/modules/knowledge-base.md
@@ -29,7 +29,7 @@ In CE, the frontend shows only **Private Knowledge Base**, and `/v1/catalog` ret
Object storage write (storage_key) │ · embed_text(query)
│ │ · Milvus hybrid_search:
▼ BackgroundTask │ dense (IP) + sparse (BoW) → RRF(k=60)
- core/content/kb_processing.py │ · child hit → fetch parent chunk
+ core/files/content/kb_processing.py │ · child hit → fetch parent chunk
vectorise_document_background() │ text from kb_chunks (PostgreSQL)
· kb_parser.parse_and_chunk() │ · optional reranker pass
· (optional) LLM keywords / questions │ · user_id isolation + global public KBs
@@ -68,9 +68,9 @@ ORM definitions live in `src/backend/core/db/models/knowledge.py`:
| `kb_chunks` | **Parent chunk** text (returned to the LLM on retrieval), with `tags` and related `questions` |
| `kb_assets` | **Media assets** carried by a document (figures cut out by layout parsing, standalone image uploads): bytes live in object storage; this table holds the owning chunk, the source position `locator`, the caption/OCR text `text_content`, and the model-generated description `caption` |
-The three Wiki tables (`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs`) are defined in `core/db/models/kb_wiki.py`; see [Knowledge Base Wiki](./knowledge-base-wiki.md).
+The three Wiki tables (`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs`) are defined in `features/knowledge/models.py`; see [Knowledge Base Wiki](./knowledge-base-wiki.md).
-Child chunks never touch the relational DB — they are vectorized into the Milvus collection `hugagent_kb_private` (`core/kb/kb_vector.py`); every row carries `user_id` / `kb_id` for ownership isolation, and `row_type` distinguishes chunk rows, question rows, and image rows.
+Child chunks never touch the relational DB — they are vectorized into the Milvus collection `hugagent_kb_private` (`features/knowledge/kb_vector.py`); every row carries `user_id` / `kb_id` for ownership isolation, and `row_type` distinguishes chunk rows, question rows, and image rows.
### Chunking and indexing
@@ -84,12 +84,12 @@ Parsing and chunking happen in `core/kb/kb_parser.py::parse_and_chunk()`, suppor
| `recursive` | Recursive fixed-size splitting |
| `embedding_semantic` | Semantic boundary detection via embedding similarity |
-Parent-child parameters are tunable per upload via `indexing_config`: `parent_chunk_size` (default 1024 tokens), `child_chunk_size` (128), `overlap_tokens` (20), `parent_child_indexing` (default true). LLM enrichment is opt-in: `auto_keywords_count` (keywords per parent chunk, stored as tags and fed into sparse retrieval) and `auto_questions_count` (generated questions per parent chunk, indexed as separate question rows in Milvus to boost question-style recall). The background vectorization task is `core/content/kb_processing.py::vectorise_document_background()`.
+Parent-child parameters are tunable per upload via `indexing_config`: `parent_chunk_size` (default 1024 tokens), `child_chunk_size` (128), `overlap_tokens` (20), `parent_child_indexing` (default true). LLM enrichment is opt-in: `auto_keywords_count` (keywords per parent chunk, stored as tags and fed into sparse retrieval) and `auto_questions_count` (generated questions per parent chunk, indexed as separate question rows in Milvus to boost question-style recall). The background vectorization task is `core/files/content/kb_processing.py::vectorise_document_background()`.
### Multimodal: image retrieval
Figures inside documents are no longer discarded. Indexing runs through
-`core/kb/kb_assets.py`:
+`features/knowledge/kb_assets.py`:
1. **Extraction** — for PDFs, the external parse service's layout pass returns the figure
bytes along with its caption, page index and bbox; a standalone image upload is itself
@@ -98,7 +98,7 @@ Figures inside documents are no longer discarded. Indexing runs through
service's temp paths are rewritten to real asset URLs.
2. **Image understanding** — reuses the
[vision bridge](./model-providers.md#vision-bridge-giving-text-only-models-eyes)
- (`core/vision`): the same path and the same `vision` model role that lets a text-only
+ (`core/models/vision`): the same path and the same `vision` model role that lets a text-only
chat model read images. Three things come for free with it — an **evidence cache keyed
by image content** (re-indexing the same document costs nothing), three provider
protocols with structured-output degradation, and the hard constraint that only a
@@ -149,7 +149,7 @@ Returned content follows the `[ref:retrieve_local_kb-N]` citation-marker convent
### API routes
-User-facing routes are prefixed `/v1/catalog/kb` (`src/backend/api/routes/v1/kb.py`, request models in `kb_models.py`):
+User-facing routes are prefixed `/v1/catalog/kb` (`src/backend/features/knowledge/api/kb.py`, request models in `kb_models.py`):
| Method | Path | Description |
|---|---|---|
@@ -165,7 +165,7 @@ User-facing routes are prefixed `/v1/catalog/kb` (`src/backend/api/routes/v1/kb.
| GET / PATCH | `/v1/catalog/kb/{kb_id}/chunks[/{chunk_id}]` | Chunk list / edit tags and questions |
| GET | `/v1/catalog/kb/assets/{asset_id}` | Read a media asset's raw bytes (visibility follows the owning space) |
-Business logic is centralized in `core/services/kb_service.py::KBService`.
+Business logic is centralized in `features/knowledge/service.py::KBService`.
### System-managed space: MySpace sync
@@ -173,13 +173,13 @@ Business logic is centralized in `core/services/kb_service.py::KBService`.
### Admin public knowledge bases (Enterprise Edition, EE)
-The `/v1/admin/kb/*` admin routes live in `src/backend/api/routes/v1/admin_kb.py`, gated by the `content_admin` feature flag (EE router table in `api/routes/v1/__init__.py`). Public spaces are owned by the synthetic system account `system_public_kb` (`kb_service.py::SYSTEM_KB_OWNER_ID`), have `visibility=public`, and are visible and searchable by all users. The admin side additionally offers raw file download, Office-to-PDF online preview, and chunk content editing / deletion.
+The `/v1/admin/kb/*` admin routes live in `src/backend/edition_ee/routes/admin_kb.py`, gated by the `content_admin` feature flag (EE router table in `api/routes/v1/__init__.py`). Public spaces are owned by the synthetic system account `system_public_kb` (`kb_service.py::SYSTEM_KB_OWNER_ID`), have `visibility=public`, and are visible and searchable by all users. The admin side additionally offers raw file download, Office-to-PDF online preview, and chunk content editing / deletion.
Public retrieval through `retrieve_dataset_content` selects exactly one source according to platform configuration. Platform-local mode (`knowledge_base.provider=custom`) searches only local public/shared spaces accessible to the current user, excluding private spaces. External mode searches only the selected Dify, FastGPT or WeKnora backend using that backend's own connection settings. Results are never merged across sources, and empty results or errors never trigger a cross-source fallback. The public knowledge-base list uses the same source selection; private spaces remain accessible through the separate `retrieve_local_kb` tool.
## External Dify knowledge bases (Enterprise Edition, EE)
-The EE-only client is `src/backend/edition_ee/kb/dify.py`; shared routes call it through `core/kb/external_provider.py`. The derived CE tree replaces that seam with a disabled implementation and contains no Dify client. The `is_dify_enabled()` decision has three priority levels:
+The EE-only client is `src/backend/edition_ee/kb/dify.py`; shared routes call it through `features/knowledge/extension.py`. The derived CE tree replaces that seam with a disabled implementation and contains no Dify client. The `is_dify_enabled()` decision has three priority levels:
1. DB system config `knowledge_base.provider == "dify"` (editable in the Config console);
2. Environment variable `KNOWLEDGE_BASE=dify`;
@@ -195,9 +195,9 @@ DIFY_API_KEY=dataset-... # alias: DIFY_AUTH_TOKEN
## Supported file formats
-KB upload validation is in `core/content/file_validation.py::validate_kb_file` (extension + magic-bytes double check), allowing: `.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json`, plus images (`.png` `.jpg` `.jpeg` `.webp` `.gif`).
+KB upload validation is in `core/files/content/file_validation.py::validate_kb_file` (extension + magic-bytes double check), allowing: `.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json`, plus images (`.png` `.jpg` `.jpeg` `.webp` `.gif`).
-The general-purpose parser `core/content/file_parser.py::parse_file()` (shared by chat attachments and MySpace files) covers a wider range: PDF, DOCX, DOC/WPS (via LibreOffice conversion), TXT, XLSX/XLS, CSV, PPTX, plus plain-text formats (HTML / Markdown / JSON / YAML / source code, etc.) decoded directly as UTF-8.
+The general-purpose parser `core/files/content/file_parser.py::parse_file()` (shared by chat attachments and MySpace files) covers a wider range: PDF, DOCX, DOC/WPS (via LibreOffice conversion), TXT, XLSX/XLS, CSV, PPTX, plus plain-text formats (HTML / Markdown / JSON / YAML / source code, etc.) decoded directly as UTF-8.
## Frontend
@@ -213,17 +213,17 @@ The general-purpose parser `core/content/file_parser.py::parse_file()` (shared b
| Path | Responsibility |
|---|---|
-| `src/backend/core/kb/kb_parser.py` | Document parsing + parent-child chunking (5 chunk methods) |
-| `src/backend/core/kb/kb_vector.py` | Milvus collection, embedding, hybrid search, reranking |
-| `src/backend/core/kb/kb_assets.py` | Media assets: storage, image understanding (via the vision bridge), retrieval rows, result attachment |
+| `src/backend/features/knowledge/kb_parser.py` | Document parsing + parent-child chunking (5 chunk methods) |
+| `src/backend/features/knowledge/kb_vector.py` | Milvus collection, embedding, hybrid search, reranking |
+| `src/backend/features/knowledge/kb_assets.py` | Media assets: storage, image understanding (via the vision bridge), retrieval rows, result attachment |
| `src/backend/edition_ee/kb/dify.py` | Dify datasets client and enablement logic (EE only) |
-| `src/backend/core/kb/external_provider.py` | Edition-neutral external-provider seam; disabled by the CE overlay |
-| `src/backend/core/content/kb_processing.py` | Background vectorization, LLM keyword / question enrichment |
-| `src/backend/core/content/file_validation.py` | Upload validation (extension + magic bytes) |
-| `src/backend/core/content/file_parser.py` | General-purpose file parser |
-| `src/backend/core/services/kb_service.py` | KB business logic (incl. the system-managed sync space) |
-| `src/backend/api/routes/v1/kb.py` + `kb_models.py` | User-facing `/v1/catalog/kb` routes |
-| `src/backend/api/routes/v1/admin_kb.py` | Admin public KB routes (Enterprise Edition, EE) |
+| `src/backend/features/knowledge/extension.py` | Edition-neutral external-provider seam; disabled by the CE overlay |
+| `src/backend/features/knowledge/processing.py` | Background vectorization, LLM keyword / question enrichment |
+| `src/backend/core/files/content/file_validation.py` | Upload validation (extension + magic bytes) |
+| `src/backend/core/files/content/file_parser.py` | General-purpose file parser |
+| `src/backend/features/knowledge/service.py` | KB business logic (incl. the system-managed sync space) |
+| `src/backend/features/knowledge/api/kb.py` + `kb_models.py` | User-facing `/v1/catalog/kb` routes |
+| `src/backend/edition_ee/routes/admin_kb.py` | Admin public KB routes (Enterprise Edition, EE) |
| `src/backend/api/routes/v1/catalog.py` | Catalog aggregation (private only in CE; Dify / public spaces added in EE) |
| `src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/` | Retrieval MCP server (both tools) |
| `src/backend/core/db/models/knowledge.py` | `KBSpace` / `KBDocument` / `KBChunk` / `KBAsset` ORM |
diff --git a/document/en/modules/mcp-tools.md b/document/en/modules/mcp-tools.md
index 6f5bad94..404705c8 100644
--- a/document/en/modules/mcp-tools.md
+++ b/document/en/modules/mcp-tools.md
@@ -29,7 +29,7 @@ must execute in separate rounds.
│ agent) │ │ │ :9103 unified Industry Knowledge Center (EE)│
└─────────┘ │ │ :9104 reserved (retired chart MCP) │
│ │ │ :9105 reserved (former report export MCP) │
- MCPConnectionPool │ │ :9106 web_fetch │
+ Native MCP Client │ │ :9106 web_fetch │
mcp_pool.py) └───┬────┘ :9108 automation_task (automation plugin) │
│ 9109–9111 reserved (former office MCPs) │
│ :9112 skill_manager │
@@ -223,7 +223,7 @@ priority order:
injects `"cite_id": "eN"` into each entry in place; the model copies the id as-is.
2. **Standard list shape → one registry line**: list-style tools returning
`{"items": [{"title": …, "content": …}]}` just need an `items_paths` + field-alias
- entry in `orchestration/citation_anchor.py::TOOL_SPECS` — per-item numbering with no
+ entry in `core/llm/citations/hook.py::TOOL_SPECS` — per-item numbering with no
tool code change.
3. **Do nothing → automatic fallback**: a generic heuristic finds the unique dict-array
field at the top level (or one level under `result`) and numbers items; when nothing is
@@ -237,21 +237,18 @@ before the result enters the model context and before persistence; the frontend
render `cite_id` as per-item chips that match the `[anchor text](cite:eN)` references in
the answer body.
-## Backend client: connection pool & bare-name restoration
+## Backend client: native MCP lifetimes
-The backend connects through AgentScope 2.0's `MCPClient`, centred on two files:
+The backend uses MCP SDK 2.3 `Client(mode="auto")` and AgentScope `ToolBase`. `client.py` owns transport authorization; `lifetime.py` owns SDK scopes and complete paginated discovery. HTTP calls reuse the same client throughout a run. STDIO sessions use exclusive leases scoped to the executing user, chat and configuration; successful sessions remain idle for 60 seconds, while failed or cancelled calls destroy them. There is no stable-connection switch or startup connection pool.
-- **`core/llm/mcp_pool.py` — `MCPConnectionPool`** (process-level singleton): at startup `warmup_mcp_tools()` reads all enabled server configs from the DB and pre-connects. Pooling semantics under 2.0:
- - **stdio servers marked `is_stable=true`** keep their connection across requests (saving the 1–7 s subprocess cold start);
- - **HTTP servers are never pooled** — 2.0's stateful HTTP client is bound to its asyncio task, and reuse across requests triggers cancel-scope crashes, so each request gets a fresh `is_stateful=False` connection;
- - per-request servers (KB retrieval with user headers) connect on demand and are closed via `close_transient()` when the request ends.
-- **`core/llm/mcp_manager.py` — `BareNameMCPClient`**: AgentScope 2.0 rewrites tool names to `mcp____`; this subclass restores the server-side bare name (`internet_search`, not `mcp__internet_search__internet_search`) so the display-name mapping (`core/config/display_names.py`), [citation extraction](chat.md) keyed by tool name, and frontend icon rendering all keep working as in 1.x.
+`callbacks.py` binds isolated text sampling and user elicitation to the active run. Resources, URI templates and prompts become permission-governed tools. Progress reaches the existing tool card. Structured results and resource links remain in tool metadata and model-facing output. SDK Tasks extensions are not available in 2.3.
+
+MCP execution budgets exclude time waiting for user answers. Valid questions pause the current invocation; answering or cancelling a question resumes its remaining budget. Repeated or overlapping questions do not reset that budget. Other invocations keep independent timers. Existing human-interaction deadlines still apply, and cancelling a run clears pending questions. Invocation budgets bound SDK requests and HTTP reads; connection, write and discovery timeouts remain active.
-The 2.0 `Toolkit` is constructed once, in `core/llm/factory/build.py`: `Toolkit(tools=[...], mcps=clients)`.
## Unified tool-permission gateway
-`core/llm/tool_permissions.py` provides the run-scoped
+`core/llm/tool_permissions/` provides the run-scoped
`ToolPermissionRegistry` and `ToolPermissionMiddleware`. The registry is a
**governed-tool list**, not a mandatory declaration table:
@@ -270,7 +267,7 @@ The 2.0 `Toolkit` is constructed once, in `core/llm/factory/build.py`: `Toolkit(
tools become dispatchable.
Native tools may opt in with
-`ToolCollector.register_tool_function(..., permission=spec)`. The platform's
+`ToolAssembly.append(function_tool(..., dispatch_spec=spec))`. The platform's
local-file, Bash, and My Space rules live in `builtin_tool_permission()`.
Native tools with neither an explicit spec nor a built-in entry pass through.
@@ -297,7 +294,7 @@ tools that should be governed need a registry rule.
## Registration: DB-driven config + catalog gating
-The source of truth for MCP server configuration is the `admin_mcp_servers` table (ORM: `core/db/models.py::AdminMcpServer`), read through `core/services/mcp_service.py::McpServerConfigService` with a 30-second TTL cache, in a dict format compatible with the legacy `MCP_SERVERS` (`transport / command / args / env / url / headers / is_stable`). `core/config/mcp_config.py` remains as the URL builder for the built-in servers (first-deployment seeds).
+The source of truth for MCP server configuration is the `admin_mcp_servers` table (ORM: `core/db/models/`), read through `core/capabilities/mcp/service.py::McpServerConfigService` with a 30-second TTL cache, using native client configuration (`transport / command / args / env / url / headers`). `core/config/mcp_config.py` remains as the URL builder for the built-in servers (first-deployment seeds).
Whether a server is **visible to the model** additionally passes through [catalog](catalog.md) gating: each entry in the `mcp` section of `core/config/catalog.json` corresponds to a server_id, and a server whose `is_enabled(id, "mcp_server")` is false will not be registered with the agent even if connected.
@@ -309,7 +306,7 @@ The `/config` system console's **MCP Tools → MCP Server Management** view maps
- **Probe-on-create**: `_probe_connectivity` performs a real connection; failures are rejected before persisting;
- **Toggle & ordering**: `POST /{id}/toggle` switches a server instantly (refreshing the catalog and the connection pool);
- **Secret protection**: HTTP header values are encrypted at rest and returned as `***`; secret-looking `env_vars` values are masked as well;
-- **Test & reload**: `POST /{id}/test` re-probes a single server; `POST /reload-pool` hot-rebuilds the connection pool.
+- **Test & reload**: `POST /{id}/test` re-probes a single server; `automatic cache invalidation` refreshes discovery on the next run.
- **Move to MCP Marketplace**: creates a credential-free marketplace snapshot
from an existing remote HTTP/SSE MCP and disables the original global
instance. The service then leaves the MCP server list and can be reviewed,
@@ -329,7 +326,7 @@ Regular users can add remote MCP servers visible **only to themselves** (`api/ro
- `POST /v1/me/mcp-servers`: add a private remote MCP over public HTTP or HTTPS using HTTP/SSE transports; the user entry point deliberately forbids stdio (no arbitrary command execution on the server), and DNS/IP validation blocks localhost, private, link-local, and reserved addresses; probe-on-create means unreachable endpoints are never persisted. HTTP is plaintext, so HTTPS remains recommended for production;
- `DELETE /v1/me/mcp-servers/{id}`: remove one's own private MCP.
-Implementation reuses the same `admin_mcp_servers` table: `owner_user_id` = current user for owner isolation, auto-generated `umcp_` server IDs to avoid clashes, and `is_stable=False` to keep them out of the warmup pool. HTTP header values are encrypted with Fernet at rest and decrypted only at runtime. The feature is gated by the per-user `can_add_mcp` permission flag (open by default in the single-tenant Community Edition; granted per user by organisation admins in the Enterprise Edition — see [editions](../editions/overview.md)).
+Implementation reuses the same `admin_mcp_servers` table: `owner_user_id` = current user for owner isolation, auto-generated `umcp_` server IDs to avoid clashes. HTTP header values are encrypted with Fernet at rest and decrypted only at runtime. The feature is gated by the per-user `can_add_mcp` permission flag (open by default in the single-tenant Community Edition; granted per user by organisation admins in the Enterprise Edition — see [editions](../editions/overview.md)).
## MCP marketplace
@@ -409,19 +406,19 @@ docker-compose up -d --build mcp
| `src/backend/extensions/mcp_servers/_serve.py` | Unified stdio / streamable-http entry point |
| `src/backend/extensions/mcp_servers/registry.py` | Installation placement, exposure, packages, and base ports |
| `src/backend/extensions/mcp_servers/_ports.py` | Deployment port mapping |
-| `src/backend/core/llm/mcp_pool.py` | MCP connection pool (stdio pooled / HTTP per-request) |
-| `src/backend/core/llm/mcp_manager.py` | MCPClient construction + bare tool-name restoration |
-| `src/backend/core/llm/tool_permissions.py` | Governed-tool registry, decisions, approval routing, and execution tickets |
-| `src/backend/core/services/mcp_service.py` | DB-driven server config service (30 s cache) |
-| `src/backend/core/services/mcp_marketplace_service.py` | Marketplace publishing, review, installation, visibility, and security controls |
-| `src/backend/core/services/mcp_oauth_service.py` | OAuth 2.1 login, SDK metadata discovery, encrypted token storage, and refresh |
-| `src/backend/core/services/mcp_marketplace_monitor.py` | Periodic remote tool-snapshot revalidation and drift monitoring |
+| `src/backend/core/llm/mcp/lifetime.py` | Native MCP client ownership and discovery |
+| `src/backend/core/llm/mcp/manager.py` | MCPClient construction + bare tool-name restoration |
+| `src/backend/core/llm/tool_permissions/` | Governed-tool registry, decisions, approval routing, and execution tickets |
+| `src/backend/core/capabilities/mcp/service.py` | DB-driven server config service (30 s cache) |
+| `src/backend/features/marketplace/` | Marketplace publishing, review, installation, visibility, and security controls |
+| `src/backend/core/capabilities/mcp/oauth.py` | OAuth 2.1 login, SDK metadata discovery, encrypted token storage, and refresh |
+| `src/backend/features/marketplace/` | Periodic remote tool-snapshot revalidation and drift monitoring |
| `src/backend/core/config/mcp_config.py` | Built-in server URL builder (http://mcp:NNNN/mcp/) |
| `src/backend/core/config/catalog.json` | Capability catalog: MCP enable/disable seeds |
| `src/backend/api/routes/v1/admin_mcp_servers.py` | Admin custom-MCP API |
| `src/backend/api/routes/v1/me_capabilities.py` | User self-service private MCP / skill API |
-| `src/backend/api/routes/v1/mcp_marketplace.py` | User-facing MCP marketplace API |
-| `src/backend/api/routes/v1/admin_mcp_marketplace.py` | Admin MCP marketplace and review API |
+| `src/backend/features/marketplace/api/mcp_marketplace.py` | User-facing MCP marketplace API |
+| `src/backend/features/marketplace/api/admin_mcp_marketplace.py` | Admin MCP marketplace and review API |
| `docker/Dockerfile.mcp` | mcp container image (MCP runtime, plotting dependencies, and CJK fonts) |
Related docs: [Capability catalog](catalog.md) · [Agent skills](agent-skills.md) · [Knowledge base](knowledge-base.md) · [Editions & licensing](../editions/overview.md)
diff --git a/document/en/modules/memory.md b/document/en/modules/memory.md
index 328e9364..77acffa8 100644
--- a/document/en/modules/memory.md
+++ b/document/en/modules/memory.md
@@ -68,7 +68,7 @@ save_memories_background()
· sanitizer gate → write L1/L2/L3/session → audit side channel
```
-The integration layer for retrieval and injection lives in `src/backend/orchestration/memory_integration.py`. mem0's LLM / embedder / Milvus / reranker configuration is assembled in `src/backend/core/memory/service.py`, while Neo4j graph reads and writes live in `src/backend/core/memory/graph.py`. Model configs are resolved from the DB roles `memory` / `embedding` first, falling back to environment variables.
+The integration layer for retrieval and injection lives in `src/backend/core/agent_runtime/integrations/memory.py`. mem0's LLM / embedder / Milvus / reranker configuration is assembled in `src/backend/core/memory/service.py`, while Neo4j graph reads and writes live in `src/backend/core/memory/graph.py`. Model configs are resolved from the DB roles `memory` / `embedding` first, falling back to environment variables.
## Write pipeline and extractors
@@ -140,7 +140,7 @@ Two independent toggles, both stored in `users_shadow.metadata` (ORM column `ext
| `memory_enabled` | Persistent memory **read**: inject the frozen block at session start | `false` |
| `memory_write_enabled` | **Write**: extract and save memories after each conversation | `false` |
-Project chats get their own scope: personal projects and the default workspace use the real `user_id`, while team projects use `scope_user_id = "team:"` — all team members write into the same mem0 bucket for shared recall, with the real author preserved in `metadata.author_user_id` (see `orchestration/memory_integration.py::save_memories_background` and `api/routes/v1/memories.py::list_memories`). Project-level toggles live in `projects.metadata` (`memory_enabled` / `memory_write_enabled`, defaulting to `true` inside a project); see [Projects & MySpace](./projects-myspace.md).
+Project chats get their own scope: personal projects and the default workspace use the real `user_id`, while team projects use `scope_user_id = "team:"` — all team members write into the same mem0 bucket for shared recall, with the real author preserved in `metadata.author_user_id` (see `core/agent_runtime/integrations/memory.py::save_memories_background` and `api/routes/v1/memories.py::list_memories`). Project-level toggles live in `projects.metadata` (`memory_enabled` / `memory_write_enabled`, defaulting to `true` inside a project); see [Projects & MySpace](./projects-myspace.md).
## Frontend memory center
@@ -233,8 +233,8 @@ See the [environment variable reference](../deployment/environment-variables.md)
| `src/backend/core/memory/sanitizer.py` | Sanitizer gate (hardcoded + DB-managed rules) |
| `src/backend/core/memory/audit.py` | Audit side channel (Enterprise Edition, EE) |
| `src/backend/core/memory/context.py` | `MemoryContext` and workspace / layer resolution |
-| `src/backend/orchestration/memory_integration.py` | Retrieval launch, frozen-block assembly and injection, save delegation |
-| `src/backend/orchestration/workflow.py` | Main orchestration: memory hook wiring |
+| `src/backend/core/agent_runtime/integrations/memory.py` | Retrieval launch, frozen-block assembly and injection, save delegation |
+| `src/backend/core/agent_runtime/runner.py` | Main orchestration: memory hook wiring |
| `src/backend/api/routes/v1/memories.py` | `/v1/memories` management API |
| `src/backend/core/db/models/memory.py` | `ProfileMemory`, `MemoryOutbox`, and `MemorySanitizerRule` ORM |
| `src/backend/edition_ee/db/models/memory.py` | `MemoryAudit` ORM (EE only) |
diff --git a/document/en/modules/model-providers.md b/document/en/modules/model-providers.md
index 969c7049..bea68107 100644
--- a/document/en/modules/model-providers.md
+++ b/document/en/modules/model-providers.md
@@ -9,7 +9,7 @@ In Config → Model Management (EE), use the switch in the provider list's Statu
## Configuring models: providers + roles
-Two tables (`core/db/models.py`): `model_providers` (base_url / api_key / model_name / extra_config / is_active) and `model_role_assignments` (role_key → provider_id). Roles are defined in `core/db/model_repository.py::ROLE_DEFINITIONS`:
+Two tables (`core/db/models/`): `model_providers` (base_url / api_key / model_name / extra_config / is_active) and `model_role_assignments` (role_key → provider_id). Roles are defined in `core/models/repository.py::ROLE_DEFINITIONS`:
| role_key | Purpose | Type |
|---|---|---|
@@ -36,6 +36,8 @@ Precedence is **shared subagent model → an agent's existing explicit model →
### Wire protocol: Responses by default, chat completions only as a fallback
+The pinned AgentScope version natively parses raw reasoning and summary events from Responses. The platform preserves each event's `item_id` and part index to prevent streaming failures and keep distinct reasoning items and parts separate.
+
OpenAI-compatible endpoints come in two wire protocols: the older `POST /chat/completions` and the newer `POST /responses`. The platform **defaults to Responses**, because it carries the model's reasoning as a first-class `reasoning` item that can be handed back verbatim on the next turn; `/chat/completions` has nowhere to put reasoning, so it has to be smuggled through a `reasoning_content` field. Chat completions is used only for endpoints that genuinely do not route `/responses` — commonly thin relay gateways.
The choice lives in `extra_config.api_protocol` (`responses` / `chat_completions`). It is never inferred from the vendor name: saving a provider probes the endpoint, and only an endpoint that clears both stages is recorded as Responses-capable.
@@ -124,7 +126,7 @@ Retry policy: the model layer runs with `max_retries=0`; the agent layer's `Mode
## Dynamic model switching (chat_mode)
-Each frontend message may carry `chat_mode` (`turbo / fast / medium / high / max`); before the reply starts, `core/llm/middlewares.py::DynamicModelMiddleware` (`on_reply`) hot-swaps `agent.model`:
+Each frontend message may carry `chat_mode` (`turbo / fast / medium / high / max`); before the reply starts, `core/llm/runtime/` (`on_reply`) hot-swaps `agent.model`:
```
chat_mode → hooks._resolve_chat_mode(agent.state)
@@ -134,7 +136,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state)
high/max → reasoning_effort=high/max (endpoint must declare support)
```
-Legacy clients that only send `enable_thinking` are mapped to medium / fast. This is the AgentScope 2.0 incarnation of the old 1.x "pre_reply dynamic-model hook" — the hook factories were refactored into middlewares, and `core/llm/hooks.py` now only hosts pure-function helpers.
+Legacy clients that only send `enable_thinking` are mapped to medium / fast. This is the AgentScope 2.0 incarnation of the old 1.x "pre_reply dynamic-model hook" — the hook factories were refactored into middlewares, and `core/harness/hooks.py` now only hosts pure-function helpers.
## Environment variables and MCP subprocess injection
@@ -147,7 +149,7 @@ Full variable list: [Environment Variables](../deployment/environment-variables.
## Personal API keys
-Users can issue personal keys for programmatic access to the platform API (`api/routes/v1/api_keys.py` + `core/services/api_key_service.py`):
+Users can issue personal keys for programmatic access to the platform API (`api/routes/v1/api_keys.py` + `core/auth/api_keys.py`):
- Plaintext keys look like `sk-jx-` and are **returned exactly once at creation**; the DB stores only a SHA256 hash + prefix. Expiry options 7/30/90/180/365 days or never; enable/disable and revoke supported.
- Usage: `Authorization: Bearer sk-jx-...`; the auth layer (`core/auth/backend.py`) recognizes the prefix and resolves the key to a user context via `resolve_api_key`, checking enabled/unrevoked/unexpired.
@@ -157,7 +159,7 @@ Endpoints: `GET/POST /v1/me/api-keys`, `PATCH /v1/me/api-keys/{id}` (toggle), `D
## Billing and usage
-Token usage is accumulated into the end-of-stream `meta.usage` (`orchestration/streaming.py` sums `ModelCallEndEvent`s) and persisted with the message. Two console report groups sit on top (`CONFIG_TOKEN`):
+Token usage is accumulated into the end-of-stream `meta.usage` (`core/llm/runtime/streaming.py` sums `ModelCallEndEvent`s) and persisted with the message. Two console report groups sit on top (`CONFIG_TOKEN`):
- **Usage logs** `api/routes/v1/admin_usage_logs.py`: `GET /v1/admin/usage-logs` (detail), `/summary` (aggregate), `/models` (distinct model names).
- **Billing reports** `api/routes/v1/admin_billing.py` (**Enterprise Edition (EE)** — part of the full admin console): `GET /v1/admin/billing/summary` (cost aggregated by user/model), model pricing CRUD (`/pricing`, input/output unit prices, currency), `GET /v1/admin/billing/export` (CSV cost export).
@@ -166,21 +168,21 @@ Community Edition users can see their own token usage; organization-level billin
## Routing strategy (ROUTER_STRATEGY)
-`orchestration/strategy.py`: `ROUTER_STRATEGY=main_only` (default) always routes to the main agent; `llm_router` is a reserved placeholder that currently also falls back to `MainOnlyStrategy` (safe by default). Actual multi-agent dispatch happens via `@mentions` + the `call_subagent` tool — see [Chat & Agent Orchestration](chat.md).
+`core/agent_runtime/strategy.py`: `ROUTER_STRATEGY=main_only` (default) always routes to the main agent; `llm_router` is a reserved placeholder that currently also falls back to `MainOnlyStrategy` (safe by default). Actual multi-agent dispatch happens via `@mentions` + the `call_subagent` tool — see [Chat & Agent Orchestration](chat.md).
## Source map
| Topic | Path |
|---|---|
-| Model factory / JxOpenAIChatModel | `src/backend/core/llm/chat_models.py` |
-| Role resolution service (DB + cache) | `src/backend/core/services/model_config.py` |
-| Role definitions / provider repository | `src/backend/core/db/model_repository.py` |
+| Model factory / JxOpenAIChatModel | `src/backend/core/models/chat_models.py` |
+| Role resolution service (DB + cache) | `src/backend/core/models/config.py` |
+| Role definitions / provider repository | `src/backend/core/models/repository.py` |
| Management API | `src/backend/api/routes/v1/models.py` |
-| Dynamic-model middleware | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`, helpers in `core/llm/hooks.py` |
-| MCP env injection | `src/backend/core/services/model_config.py::get_mcp_env_overlay`, `core/config/runtime_env.py` |
-| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `core/services/api_key_service.py`, `core/auth/backend.py` |
+| Dynamic-model middleware | `src/backend/core/llm/runtime/`, helpers in `core/harness/hooks.py` |
+| MCP env injection | `src/backend/core/models/config.py::get_mcp_env_overlay`, `core/config/runtime_env.py` |
+| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `core/auth/api_keys.py`, `core/auth/backend.py` |
| Usage logs / billing | `src/backend/api/routes/v1/admin_usage_logs.py`, `api/routes/v1/admin_billing.py` |
-| Routing strategy | `src/backend/orchestration/strategy.py` |
+| Routing strategy | `src/backend/core/agent_runtime/strategy.py` |
| Fail-fast on missing main model | `src/backend/api/routes/v1/chats/__init__.py::_ensure_main_model_configured` |
### Child cancellation and parameter overrides
diff --git a/document/en/modules/personal-agent.md b/document/en/modules/personal-agent.md
index ae667c61..9bf7a32a 100644
--- a/document/en/modules/personal-agent.md
+++ b/document/en/modules/personal-agent.md
@@ -217,24 +217,18 @@ When a Linux desktop runtime has no input driver, the connection offers session
Prepare Enterprise Edition, PostgreSQL, Redis, and the OpenSandbox images using the existing deployment guide.
-1. Build the computer image: `docker build -f docker/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .`. Do not overwrite a tag that a running computer uses.
+1. Build the computer image: `docker build -f docker/personal-agent/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .`. Do not overwrite a tag that a running computer uses.
2. Set HOST_STORAGE_PATH to an absolute host path.
3. Set PERSONAL_AGENT_OPENSANDBOX_API_KEY to a separate random secret of at least 32 characters.
4. Set PERSONAL_AGENT_OPENSANDBOX_DOMAIN to personal-agent-opensandbox:8080.
5. Set PERSONAL_AGENT_IMAGE to hugagent-personal-agent:v5. Existing computers use the new image when they are recreated.
-6. Run the root Compose file with docker/compose.personal-agent.yml and the personal-agent profile.
+6. Run the root Compose file with docker/personal-agent/compose.personal-agent.yml and the personal-agent profile.
7. Apply the existing database migration procedure. Apply personalagent01 through personalagent05. These migrations create module tables, backfill message links, create task sessions, and add device session operations.
The extra Compose file creates a separate controller, compute network, and private configuration volume. The controller needs Docker socket access. Do not commit credentials. The existing database encryption key protects sandbox and email credentials.
Missing Personal Agent controller settings cause a failure. The service does not fall back to the ordinary controller. The worker checks permanent computers once a minute. Pausing the task queue preserves the computer.
-## Verification
-
-Tests cover identity, concurrency, leases, approvals, revocation, history pagination, and ordinary chat restrictions. Isolated service tests also cover real models, cloud screenshots and input, persistent files, SMTP/IMAP attachments, and the frontend flow.
-
-Ethereal captures test email. It does not prove delivery to normal Internet inboxes. Mac and Windows native desktop tests require their respective systems. See the internal acceptance record for passed and pending checks.
-
Wait until the agent has replied to every message before you downgrade personalagent02. The downgrade archives task links and scoped instructions in message metadata. Upgrading again restores them.
## Status panel
diff --git a/document/en/modules/projects-myspace.md b/document/en/modules/projects-myspace.md
index 3fb51354..2435daf7 100644
--- a/document/en/modules/projects-myspace.md
+++ b/document/en/modules/projects-myspace.md
@@ -35,7 +35,7 @@ Key ORM models (`src/backend/core/db/models/`):
## Projects
-Routes: `src/backend/api/routes/v1/projects.py` (CE router table); business logic in `core/services/project_service.py` and `project_file_service.py`.
+Routes: `src/backend/api/routes/v1/projects.py` (CE router table); business logic in `core/projects/` and `project_file_service.py`.
| Method | Path | Description |
|---|---|---|
@@ -71,16 +71,16 @@ In the project overview, the model and thinking-effort menu always opens downwar
### How project context enters a conversation
-When a conversation is started inside a project (the request carries `project_id`), `core/services/session_operations/request_context.py` assembles the workflow context:
+When a conversation is started inside a project (the request carries `project_id`), `core/chat/operations/request_context.py` assembles the workflow context:
1. Project metadata is loaded — `project_name`, `project_instructions`, the linked folder name and working directory. Instructions still come from `AGENTS.md`; cloud and local project files are not recursively enumerated;
2. `core/llm/factory/build.py` injects these into the system prompt via `_build_project_section` (`build_system_prompt(cfg, ctx=...)`);
3. The project-level memory scope takes effect: the workspace becomes `project:`; team projects use the mem0 bucket `team:` (members share memories), and the project's own `metadata.memory_enabled` / `memory_write_enabled` override the user-level toggles (defaulting to on inside projects) — see [Memory System](./memory.md);
-4. Sandbox path scoping: in project conversations, the agent's `/myspace/...` file operations are redirected under the linked folder (the explicit `ProjectScope` parameter mechanism in `core/llm/tools/myspace_vfs/__init__.py`).
+4. Sandbox path scoping: in project conversations, the agent's `/myspace/...` file operations are redirected under the linked folder (the explicit `ProjectScope` parameter mechanism in `core/files/spaces/`).
Custom project-mode prompt templates use project names, directory scope and instructions. Before upgrading an existing deployment, remove obsolete file-inventory and file-count placeholders from saved templates; the runtime no longer supplies those variables.
-Team project permissions follow team roles: owner/admin always have admin rights; members are governed by `file_permission` (editor/viewer) (`core/auth/permissions_iface.py::require_project_access`).
+Team project permissions follow team roles: owner/admin always have admin rights; members are governed by `file_permission` (editor/viewer) (`core/auth/personal_permissions_iface.py::require_project_access`).
### Project sidebar controls
@@ -167,7 +167,7 @@ Permission model: `TeamMember.role` (owner/admin/member) + `file_permission` (vi
Three complementary paths:
-1. **Attachment injection (hooks)**: frontend requests send only `file_id`, `name`, and `mime_type` in each `attachments[]` item, never the file body. `core/llm/hooks.py` lazily downloads and parses by `file_id`, caching `parsed_text` and its summary; the first turn injects only metadata and a bounded preview of at most 5,000 characters, while xlsx previews also include total dimensions, headers, and a few data rows. The model reads content beyond the preview on demand through `read_artifact`; images still use multimodal injection. Every read validates `user_id` ownership to block forged cross-user access;
+1. **Attachment injection (hooks)**: frontend requests send only `file_id`, `name`, and `mime_type` in each `attachments[]` item, never the file body. `core/harness/hooks.py` lazily downloads and parses by `file_id`, caching `parsed_text` and its summary; the first turn injects only metadata and a bounded preview of at most 5,000 characters, while xlsx previews also include total dimensions, headers, and a few data rows. The model reads content beyond the preview on demand through `read_artifact`; images still use multimodal injection. Every read validates `user_id` ownership to block forged cross-user access;
2. **Project file access**: a project conversation's system prompt supplies project instructions and directory scope without an automatic file inventory or count. The agent discovers files on demand through directory, search and read tools;
3. **Sandbox virtual filesystem**: with code execution enabled, the `/myspace/...` path maps MySpace into the sandbox (lazy loading + reverse sync); team projects map the team folder — see [Sandbox](./sandbox.md).
@@ -176,8 +176,8 @@ Three complementary paths:
| Path | Responsibility |
|---|---|
| `src/backend/api/routes/v1/projects.py` | Projects API |
-| `src/backend/core/services/project_service.py` / `project_file_service.py` | Project business logic |
-| `src/backend/core/services/project_scope.py` | `ProjectScope` (sandbox path scoping) |
+| `src/backend/core/projects/` / `project_file_service.py` | Project business logic |
+| `src/backend/core/projects/personal_scope.py` | `ProjectScope` (sandbox path scoping) |
| `src/backend/api/routes/v1/myspace_folders.py` | Personal folders API |
| `src/backend/api/routes/v1/artifacts.py` | Asset list / chat favorites / add-to-KB |
| `src/backend/edition_ee/routes/team_files.py` | Team folders & files API (Enterprise Edition, EE) |
@@ -186,9 +186,9 @@ Three complementary paths:
| `src/backend/core/db/models/identity.py` | Shared identity ORM such as `UserFolder` |
| `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM (EE only) |
| `src/backend/core/db/models/artifact.py` | `Artifact` ORM |
-| `src/backend/core/llm/hooks.py` | Attachment context injection (`_build_file_context`, etc.) |
+| `src/backend/core/harness/hooks.py` | Attachment context injection (`_build_file_context`, etc.) |
| `src/backend/core/llm/factory/build.py` | Project section injection into the system prompt |
-| `src/backend/core/llm/tools/myspace_vfs/__init__.py` | MySpace ↔ sandbox mapping layer |
+| `src/backend/core/files/spaces/` | MySpace ↔ sandbox mapping layer |
| `src/frontend/src/components/projects/` | Project frontend components |
| `src/frontend/src/components/sidebar/Sidebar.tsx` | Sidebar project groups, quick chat creation, and project actions |
| `src/frontend/src/components/myspace/` | MySpace frontend components |
@@ -262,11 +262,8 @@ staleness is allowed. Failed registrations, missing files and failed downloads a
revisited at the next normal synchronization, without continuous background retries.
Before upgrading, stop writers and back up the database and myspace_cache.
-Preview with `python src/backend/scripts/clean_myspace_duplicates.py --all`.
-Apply only with `--apply --backup-dir `.
-Cleanup deletes every live record in each duplicate group and its mirror path;
-it does not retain the newest record. Tombstones prevent historical backfill from
-resurrecting deleted identities. Original objects and recovery journals are retained.
+Inspect duplicate names and review a cleanup and recovery plan before upgrading.
+There is no standalone deduplication script.
Then upgrade Alembic (EE: personalname01; CE: ce_0018), resume services and run
reconciliation and end-to-end checks. Migration refuses remaining duplicates
instead of silently deleting data in another environment.
@@ -275,13 +272,13 @@ never restore duplicate records into the running service.
### Space file tools
-Space file tools share the `core/llm/tools/space_tools.py` registration entry point: `space_list_myspace_files`, `space_stage_myspace_file`, `space_create_folder`, `space_move`, and `space_delete`. EE also provides `space_list_team_files`. Generic Read/Write/Edit/Glob/Grep names remain unchanged.
+Space file tools share the `core/files/spaces/` registration entry point: `space_list_myspace_files`, `space_stage_myspace_file`, `space_create_folder`, `space_move`, and `space_delete`. EE also provides `space_list_team_files`. Generic Read/Write/Edit/Glob/Grep names remain unchanged.
Favorite-chat tools have been removed; historical chats remain accessible through `list_related_chats` and `read_chat`.
### Unified mounted-space synchronization
-Personal listeners live in `core/space_sync/`; team adapters live in `edition_ee/services/space_sync/`. Callbacks enqueue in memory, workers batch a durable SQLite journal outside writable mounts, then update database metadata and object storage with authorization and version checks. Native moves retain file/folder IDs. Empty directories, deletion, restart recovery and parsed-cache invalidation share this pipeline. Committed database changes refresh registered mounts; rollback publishes nothing. Conflicts preserve local bytes and fail synchronization barriers. Network transfers and human approval are outside callback latency; measure complete synchronization in the actual deployment.
+Personal listeners live in `core/files/spaces/`; team adapters live in `edition_ee/services/workspace_adapter.py`. Callbacks enqueue in memory, workers batch a durable SQLite journal outside writable mounts, then update database metadata and object storage with authorization and version checks. Native moves retain file/folder IDs. Empty directories, deletion, restart recovery and parsed-cache invalidation share this pipeline. Committed database changes refresh registered mounts; rollback publishes nothing. Conflicts preserve local bytes and fail synchronization barriers. Network transfers and human approval are outside callback latency; measure complete synchronization in the actual deployment.
### Project environment
@@ -291,7 +288,7 @@ Custom environments bind subagents, plugins, skills and connectors. The main age
The backend limits capability registration, explicit invocation, restored session capabilities, subagents and dynamic plugin loading. An empty custom environment never falls back to all resources. Changes apply from the next turn; running responses and their derived jobs keep the original configuration. Conversations outside the project are unaffected. Basic file and conversation tools retain their existing permission rules.
-Implementation follows separate responsibilities: `core/project_environment/` validates configuration and resolves dependencies without catalog access. The resource service reads authorized inventory, the environment service saves and previews configuration, and the runtime policy freezes and narrows registration and invocation. The frontend editor composes the interface; `useProjectEnvironment` loads resources, previews dependencies and handles marketplace acquisition. Resource pickers display explicit bindings and inferred dependencies.
+Implementation follows separate responsibilities: `core/projects/environment/` validates configuration and resolves dependencies without catalog access. The resource service reads authorized inventory, the environment service saves and previews configuration, and the runtime policy freezes and narrows registration and invocation. The frontend editor composes the interface; `useProjectEnvironment` loads resources, previews dependencies and handles marketplace acquisition. Resource pickers display explicit bindings and inferred dependencies.
### Online cloud document editing
diff --git a/document/en/modules/prompts.md b/document/en/modules/prompts.md
index 6277e574..752eb6da 100644
--- a/document/en/modules/prompts.md
+++ b/document/en/modules/prompts.md
@@ -30,7 +30,7 @@ src/backend/prompts/prompt_text/default/system/
└── 40_format.system.md # output format + [ref:tool-N] citation contract
```
-After the base prompt, runtime appends dynamic sections per context: the tools & skills notice (`_TOOLS_AND_SKILLS_NOTICE`), the lightweight KB catalog (`prompts/kb_lite_section.py`), the project-mode section (`prompts/project_section.py`, injected only for project chats), the code-execution segment and batch-mode hint (appended by `factory/build.py`), and the sub-agent routing table (`core/llm/subagent_tool.py::build_subagent_prompt_section`).
+After the base prompt, runtime appends dynamic sections per context: the tools & skills notice (`_TOOLS_AND_SKILLS_NOTICE`), the lightweight KB catalog (`prompts/kb_lite_section.py`), the project-mode section (`prompts/project_section.py`, injected only for project chats), the code-execution segment and batch-mode hint (appended by `factory/build.py`), and the sub-agent routing table (`core/llm/subagents/tool.py::build_subagent_prompt_section`).
### Caching
@@ -49,7 +49,7 @@ Pool management and runtime assembly share this read path. After an active versi
### Execution manifest and runtime binding
-Before an Agent can execute, `core/llm/execution_manifest.py::PromptManifestBuilder` records the exact final request surface. Every ordered prompt section records `id`, origin, trust level, priority, cache class, budget, token estimate, content hash and version. Every final tool definition similarly hashes its name, complete description, canonical parameter schema, permission policy and recovery policy. Tool definitions are sorted by stable id; prompt sections retain their deliberate prefix order.
+Before an Agent can execute, `core/harness/execution_manifest.py::PromptManifestBuilder` records the exact final request surface. Every ordered prompt section records `id`, origin, trust level, priority, cache class, budget, token estimate, content hash and version. Every final tool definition similarly hashes its name, complete description, canonical parameter schema, permission policy and recovery policy. Tool definitions are sorted by stable id; prompt sections retain their deliberate prefix order.
The builder emits `prompt_hash`, `prompt_manifest_hash`, `tool_manifest_hash`, `context_hash` and one `aggregate_hash`. Manifest construction and AgentScope model input consume the same run-scoped frozen tool/skill snapshot, so the recorded surface cannot differ from the request. Progressive plugin activation invalidates that snapshot and publishes an explicit next `surface_generation` before the next model call.
@@ -59,15 +59,15 @@ Prompt text, project instructions, file names, tool descriptions and schemas are
### Canonical Context IR
-Immediately before every model call, `core/llm/context_ir.py` converts the complete request context into immutable `ContextItem` records. User input, assistant history, account identity, frozen memory, project/file material, attachments, tool calls/results, compaction summaries, steer messages and harness reminders therefore carry machine-readable `kind`, `origin`, `trust`, visibility, priority, item budget, truncation policy, content reference/hash, cache class and creation sequence. Compatibility rendering may still use a `user` role for memory and mid-turn reminders, but their provenance is no longer inferred from XML tags or from that role.
+Immediately before every model call, `core/llm/context/ir.py` converts the complete request context into immutable `ContextItem` records. User input, assistant history, account identity, frozen memory, project/file material, attachments, tool calls/results, compaction summaries, steer messages and harness reminders therefore carry machine-readable `kind`, `origin`, `trust`, visibility, priority, item budget, truncation policy, content reference/hash, cache class and creation sequence. Compatibility rendering may still use a `user` role for memory and mid-turn reminders, but their provenance is no longer inferred from XML tags or from that role.
`ContextAssembler` applies a deterministic order and an explicit message budget: the model window minus a 15% response reserve and the canonical final-tool-schema reserve. Critical system/current-user items cannot be silently displaced; optional items are selected by priority; text and tool outputs use declared head/tail policies; and every tool call/result pair is kept, shortened or excluded as one unit. The sanitized context manifest explains every inclusion, truncation and exclusion without retaining plaintext. Its hash is folded into the request's `context_hash` and `aggregate_hash`, then rebound to the run's Asset Bundle.
-`core/llm/context_adapter.py::AgentScopeContextAdapter` is the sole Context IR ↔ AgentScope message conversion seam. Persisted session metadata survives history replay, while final model roles and SSE behavior remain compatible. `ManifestBoundAgent` performs final assembly through the public `Agent.reply` path after the run-scoped tool/skill surface has been frozen.
+`core/llm/context/adapter.py::AgentScopeContextAdapter` is the sole Context IR ↔ AgentScope message conversion seam. Persisted session metadata survives history replay, while final model roles and SSE behavior remain compatible. `ManifestBoundAgent` performs final assembly through the public `Agent.reply` path after the run-scoped tool/skill surface has been frozen.
## Prompt version pool (prompt_versions)
-The pool stores multiple prompt sets in a single `ContentBlock(id="prompt_versions")` row with payload `{active: {kind: version_id}, versions: [...]}`; the service layer is `core/services/prompt_version_service.py`:
+The pool stores multiple prompt sets in a single `ContentBlock(id="prompt_versions")` row with payload `{active: {kind: version_id}, versions: [...]}`; the service layer is `core/llm/prompt_versions.py`:
- **Six kinds** (`VALID_KINDS`): `system` (main agent), `code_exec` (code-execution segment), `distillation` (skill distillation), `plan_mode` (plan mode), `subagents` (independent prompts for the platform-default Explorer, Worker, and Reviewer), and `turbo` (the standalone turbo-mode prompt).
- Each version carries `(kind, id, name, description, parts[])`, a part being `{part_id, display_name, content, sort_order, is_enabled}`.
@@ -100,8 +100,8 @@ their own enablement state.
| kind | Runtime consumer | Resolution order |
|---|---|---|
| `code_exec` | `factory/build.py` appends this segment (the code-capability prompt) to the system prompt when `CODE_CAPABILITY_ENABLED=true`; the single source of truth is `prompt_version_service.render_code_capability_segment()`, also used by the console preview | active DB version → `prompts/prompt_text/code_exec/system/*.system.md` |
-| `distillation` | Skill distillation (`core/llm/skill_distiller.py`, distilling conversation trajectories into reusable skills) | active DB version → `prompts/prompt_text/distillation/skill_distiller.system.md` |
-| `plan_mode` | The plan-generation sub-agent (`orchestration/subagents/plan_mode.py::_load_plan_prompt`) | active `plan_mode` DB version → legacy `system/90_plan_mode` part → `prompts/prompt_text/plan_mode/plan_mode.system.md` → hardcoded fallback |
+| `distillation` | Skill distillation (`features/distillation/`, distilling conversation trajectories into reusable skills) | active DB version → `prompts/prompt_text/distillation/skill_distiller.system.md` |
+| `plan_mode` | The plan-generation sub-agent (`orchestration/plans/`) | active `plan_mode` DB version → legacy `system/90_plan_mode` part → `prompts/prompt_text/plan_mode/plan_mode.system.md` → hardcoded fallback |
| `turbo` | Turbo mode (when the chat-mode picker is set to Turbo, `factory/build.py` **replaces** the assembled main system prompt with this one). The tool set is admin-configurable via Config "System Config → Turbo" (`turbo.mcp_server_ids`, defaulting to internet search / web fetch / KB retrieval, **independent of catalog enable/disable state**); explicitly summoned skills / @sub-agents / plugins may join per turn (`turbo.manual_invoke_enable`); the react-iteration cap is configurable (`turbo.max_iters`, default 4). The single source of truth is `prompt_version_service.render_turbo_system_prompt()` | active `turbo` DB version → `prompts/prompt_text/turbo/turbo.system.md` → hardcoded fallback |
Sub-agents do not use the full pool assembly: `prompt_runtime.py::build_subagent_system_prompt()` builds around the user-defined `system_prompt`, reusing the `20_tools_policy` / `65_citations` / `60_format` parts from the active version (or files). See [Chat & Agent Orchestration](chat.md).
@@ -126,18 +126,6 @@ Prompts live only in the database and never ship with code; migration across env
Both accept `ADMIN_TOKEN` or `CONFIG_TOKEN`. Snapshots are validated against `PROMPT_BLOCK_MAP`, so docs snapshots and prompt snapshots cannot be imported through the wrong endpoint.
-### Scripts (src/backend/scripts/)
-
-```bash
-# Export (via a running backend API; --database-url for direct DB access also works)
-python scripts/export_content.py --api-url http://localhost:3000/api --only prompts
-# → scripts/exported/prompts_snapshot_.json
-
-# Import into the target environment (use the target machine's ADMIN_TOKEN)
-python scripts/import_content.py --api-url http:///api --prompts prompts_snapshot_.json
-# supports --no-overwrite / --dry-run
-```
-
The same flow works for offline production (image-pack delivery, persistent DB volume): copy the snapshot in alongside the image pack and `curl -X POST .../v1/content/prompts/import` inside the backend container — no restart required. When migrating to a differently-branded environment, review the snapshot manually and rewrite brand-specific wording in context before importing (never do mechanical find-and-replace).
## Source map
@@ -147,10 +135,9 @@ The same flow works for offline production (image-pack delivery, persistent DB v
| Runtime assembly + caching | `src/backend/prompts/prompt_runtime.py` |
| Providers (filesystem/inline/minimal) | `src/backend/prompts/provider.py` |
| Config (provider/parts/PROMPT_DIR) | `src/backend/prompts/prompt_config.py`, `prompts/config/default.json` |
-| Version pool service | `src/backend/core/services/prompt_version_service.py` |
+| Version pool service | `src/backend/core/llm/prompt_versions.py` |
| Console routes | `src/backend/api/routes/v1/admin_prompts.py` |
-| Migration endpoints (export/import) | `src/backend/api/routes/v1/content.py`, `core/content/content_blocks.py` |
-| Migration scripts | `src/backend/scripts/export_content.py`, `scripts/import_content.py` |
+| Migration endpoints (export/import) | `src/backend/api/routes/v1/content.py`, `core/files/content/content_blocks.py` |
| System prompt fallback files | `src/backend/prompts/prompt_text/default/system/` |
| Scenario prompt fallbacks | `src/backend/prompts/prompt_text/{code_exec,distillation,plan_mode,turbo}/` |
| Dynamic sections | `src/backend/prompts/kb_lite_section.py`, `prompts/project_section.py` |
diff --git a/document/en/modules/sandbox.md b/document/en/modules/sandbox.md
index c0921ff9..640e625f 100644
--- a/document/en/modules/sandbox.md
+++ b/document/en/modules/sandbox.md
@@ -65,7 +65,7 @@ The runtime retains at most 64 process records and only evicts completed records
The sandbox identity is resolved by `resolve_sandbox_session(sandbox_session_id, chat_id)`: a non-empty explicit ID wins, while a missing or empty value falls back to `chat_id`. The main agent, plan execution, batch items, and every subagent therefore use one conversation sandbox, and a child run never destroys it when it finishes.
-**MySpace real-time registration**: a file written under `/myspace` **is** a file in the user's My Space, and both sides must show the same state at any moment. Registration is driven by **the filesystem itself** (`core/myspace/watcher.py`): any write or delete under the mirror directory `myspace_cache/{uid}/` is registered back into the artifact ledger, **regardless of who wrote it**.
+**MySpace real-time registration**: a file written under `/myspace` **is** a file in the user's My Space, and both sides must show the same state at any moment. Registration is driven by **the filesystem itself** (`core/files/spaces/`): any write or delete under the mirror directory `myspace_cache/{uid}/` is registered back into the artifact ledger, **regardless of who wrote it**.
The earlier design had each writing tool register its own change (write / edit / file delete-move-mkdir / a before-and-after directory snapshot around `Bash`). That assumes changes can only arrive through those entry points, and they cannot: a `nohup`-ed background process writes after the command returns, sub-agents and batch runs write from another coroutine, skill CLIs write directly, and so do MCP servers. Every entry point left out shows up the same way — the file sits on the user's disk, is invisible and undeletable in the UI, and every new sandbox mounts that directory again, which is where "leftover intermediate files from the previous session" came from.
@@ -82,13 +82,11 @@ Registration splits by the nature of the file — it is **not** a blanket push:
Other points:
-- **Registration attaches no chat, and pins no workspace card**: MySpace is one per-user directory that every session's sandbox mounts, so a filesystem event carries a path and no chat identity — who wrote it is simply not knowable here. Registration therefore records only the owning user (`Artifact.chat_id` stays null), and the registry never pins: a card can only be pinned by the run that wrote the file (the model calling `pin_to_workspace`, or that run's own Write / delivery guard — all through the `core.llm.workspace` ContextVar), where both "which chat" and "which file" are certain. Picking "whichever run of this user happens to be live" instead used to file one chat's output under another chat and surface it in that chat's workspace.
+- **Registration attaches no chat, and pins no workspace card**: MySpace is one per-user directory that every session's sandbox mounts, so a filesystem event carries a path and no chat identity — who wrote it is simply not knowable here. Registration therefore records only the owning user (`Artifact.chat_id` stays null), and the registry never pins: a card can only be pinned by the run that wrote the file (the model calling `pin_to_workspace`, or that run's own Write / delivery guard — all through the `core.llm.runtime.workspace` ContextVar), where both "which chat" and "which file" are certain. Picking "whichever run of this user happens to be live" instead used to file one chat's output under another chat and surface it in that chat's workspace.
- **The reference point is the artifact record, not the mirror cache**: with the bind mount on, the sandbox's `/workspace/myspace/{uid}` *is* the backend's `myspace_cache/{uid}`, so "sandbox file matches mirror cache" is trivially true and can no longer mean "already registered". That mismatch used to make write-back a permanent no-op.
- **"Newer on disk than in the ledger" is the only signal**: when the backend writes into the mirror itself (materialising a UI upload, restoring a declined change) it aligns the file's mtime to the registration time, so the watcher never mistakes the backend's own write for a fresh sandbox change — no separate "I wrote this" bookkeeping is needed.
- **Every worker watches; a claim deduplicates**: the confirmation bar lives in the process running that chat, so the watcher has to run in every worker (the cost is inotify watches scaling as workers × directories against `fs.inotify.max_user_watches`; raise it on deployments with very many folders — exceeding it drops events **silently**); one change is still handled once, via the claim in `core.infra.ephemeral`, and preferably by the worker hosting that user's chat (only it can raise the bar).
-- **When the sandbox directory is not on this host** (`script_runner` / `cube`: the session workspace lives inside a container, or the whole sandbox is remote), `core/myspace/sandbox_sync.py` copies the sandbox's current state into the mirror directory after the command, and everything downstream is identical.
-
-The backlog that accumulated before this shipped is a **one-off data migration**: run `scripts/reconcile_myspace_mirror.py` by hand once (`--dry-run` first; add `--prune-stale` to clear leftover copies). It is deliberately not a scan that runs on every startup — that would be wasted work, and it would paper over real gaps in the live path.
+- **When the sandbox directory is not on this host** (`script_runner` / `cube`: the session workspace lives inside a container, or the whole sandbox is remote), `core/files/spaces/sandbox.py` copies the sandbox's current state into the mirror directory after the command, and everything downstream is identical.
### Large artifact delivery
@@ -185,8 +183,8 @@ fix on both execution surfaces. Updating only the cloud does not fix old clients
## Administrator sandbox management
-- **Read-only monitoring (security console)**: `api/routes/v1/config_security.py` exposes `/v1/config/security/sandbox/*` — overview, instance list, per-instance detail, snapshot list, rebuild history and effective configuration; everything goes through the providers' `admin_*` interfaces, with the UI trimmed by each provider's `admin_capabilities()` declaration (script_runner can't enumerate instances, so those columns are greyed out).
-- **Dependency rebuild (Enterprise EE)**: `api/routes/v1/admin_sandbox.py` (`/v1/admin/sandbox/*`) aggregates the pip/apt dependencies declared by all skills (`core/services/skill_deps_aggregator.py`) and lets an admin trigger a sandbox image rebuild with one click: the `script-runner` / `opensandbox` targets run a local `docker compose build`, while the `cube` target rebuilds the template on the remote node over SSH and hot-swaps it (`core/services/sandbox_rebuild_service.py` + `cube_template_builder`), with per-run status and logs. New skill dependencies get baked into the images without hand-editing Dockerfiles.
+- **Read-only monitoring (security console)**: `features/security_console/api/config_security.py` exposes `/v1/config/security/sandbox/*` — overview, instance list, per-instance detail, snapshot list, rebuild history and effective configuration; everything goes through the providers' `admin_*` interfaces, with the UI trimmed by each provider's `admin_capabilities()` declaration (script_runner can't enumerate instances, so those columns are greyed out).
+- **Dependency rebuild (Enterprise EE)**: `features/sandbox_ops/api/admin_sandbox.py` (`/v1/admin/sandbox/*`) aggregates the pip/apt dependencies declared by all skills (`core/capabilities/skills/deps/aggregator.py`) and lets an admin trigger a sandbox image rebuild with one click: the `script-runner` / `opensandbox` targets run a local `docker compose build`, while the `cube` target rebuilds the template on the remote node over SSH and hot-swaps it (`features/sandbox_ops/rebuild.py` + `cube_template_builder`), with per-run status and logs. New skill dependencies get baked into the images without hand-editing Dockerfiles.
## Key environment variables
@@ -220,17 +218,17 @@ Full list in the [environment variable reference](../deployment/environment-vari
| `src/backend/core/sandbox/factory.py` | Explicit provider selection; initialization errors propagate |
| `src/backend/core/sandbox/providers/script_runner/provider.py` | Lightweight sandbox (CE) |
| `src/backend/core/sandbox/providers/opensandbox/provider.py` | Shared OpenSandbox provider |
-| `src/backend/core/sandbox/providers/opensandbox/session.py` | Sessions / snapshots / park-restore workers (EE) |
+| `src/backend/core/sandbox/providers/opensandbox/` | Sessions / snapshots / park-restore workers (EE) |
| `src/backend/core/sandbox/providers/opensandbox/execution.py` | Execution path + idle reaper (EE) |
-| `src/backend/core/sandbox/providers/opensandbox/internals.py` | Volume builders, metadata, user pool (EE) |
+| `src/backend/core/sandbox/providers/opensandbox/` | Volume builders, metadata, user pool (EE) |
| `src/backend/core/sandbox/providers/opensandbox/pool.py` | Two-bucket warm pool |
| `src/backend/core/sandbox/providers/cube/provider.py` | Cube remote-MicroVM provider (EE) |
| `src/backend/core/llm/tools/sandbox_tool.py` | Bash / write_stdin / sandbox_put_artifact / sandbox_get_artifact |
-| `src/backend/core/llm/offloader.py` | Overflow offloading to the session workspace |
-| `src/backend/api/routes/v1/admin_sandbox.py` | Dependency-rebuild admin API (EE) |
-| `src/backend/api/routes/v1/config_security.py` | Security-console read-only sandbox views |
-| `src/backend/core/services/sandbox_rebuild_service.py` | Image/template rebuild orchestration (EE) |
-| `docker/Dockerfile.script-runner` / `docker/Dockerfile.opensandbox` / `docker/Dockerfile.cube-sandbox` | The three sandbox images |
+| `src/backend/core/llm/context/offloader.py` | Overflow offloading to the session workspace |
+| `src/backend/features/sandbox_ops/api/admin_sandbox.py` | Dependency-rebuild admin API (EE) |
+| `src/backend/features/security_console/api/config_security.py` | Security-console read-only sandbox views |
+| `src/backend/features/sandbox_ops/rebuild.py` | Image/template rebuild orchestration (EE) |
+| `docker/sandbox/Dockerfile.script-runner` / `docker/sandbox/Dockerfile.opensandbox` / `docker/sandbox/Dockerfile.cube-sandbox` | The three sandbox images |
Related docs: [Agent skills](agent-skills.md) · [MCP tool system](mcp-tools.md) · [Projects & My Space](projects-myspace.md) · [Editions & licensing](../editions/overview.md)
@@ -258,7 +256,7 @@ The backend collects status and logs and keeps the sandbox active. The model can
The command ledger stores bounded logs, execution leases and completion delivery state. OpenSandbox recovery uses the original execution identity, sandbox binding and committed log cursor. It never restarts the command. Other providers stop owned processes on orderly shutdown. After an unexpected restart, an unrecoverable execution is reported as `unknown`.
-Migration `bashbackground01` creates the command ledger. Apply it through the normal release procedure before using the feature. Tests use isolated databases and disposable sandbox sessions. See the [architecture decision](../../internal/adr/20261018-background-bash-completion.md) and [validation record](../../internal/background-bash-validation-20261018.md).
+Migration `bashbackground01` creates the command ledger. Apply it through the normal release procedure before using the feature. Tests use isolated databases and disposable sandbox sessions.
### Reclaim accounting and multi-worker administration
@@ -273,5 +271,31 @@ OpenSandbox instance lists and counts use the controller's current running
inventory instead of cached process handles. Controller query failures mean
the count is unavailable, never a successful zero. Pool diagnostics include
scope=worker and worker_pid. They describe one backend process and are not a
-global per-user quota. This fix does not convert local pool limits into
-cross-process admission limits.
+global per-user quota. Controller admission now enforces the shared ceiling described below.
+
+## Shared capacity and maintenance ownership
+
+The controller now enforces OPENSANDBOX_POOL_MAX_TOTAL against Docker inventory
+across backend workers, warm instances, restored sessions and computer instances.
+Worker pool counts remain local diagnostics. Container creation holds a file lock
+on the controller data volume; mount initialization stays outside that critical
+section. Capacity exhaustion or a competing admission returns 429 for later retry.
+Processes of the same controller must share its opensandbox_data volume.
+
+The leader discovers idle sessions from shared bindings, including sessions
+created by other workers or surviving a restart. Identity-checked cleanup of a
+missing binding does not block other sessions. Snapshot work stops with leadership;
+GC deletes metadata only for the snapshot_id it processed.
+
+Output reads no longer prepare mounts or write object directory markers.
+Directory initialization happens during creation, while execution and file access
+still check authorization. An uncertain or timed-out health probe preserves the
+FUSE helper; only confirmed missing or disconnected mounts permit replacement.
+Command observation retries are bounded before durable recovery takes over.
+Processes without a recoverable reference require confirmed termination, not loss
+of their control handle merely because observation failed.
+
+Roll out both backend and OpenSandbox controller images. Updating only the backend
+does not activate the shared capacity limit.
+
+Skill mounts require HOST_STORAGE_PATH to identify the host directory corresponding to backend STORAGE_PATH. Missing configuration fails explicitly; source-tree fallback is removed. Restart adoption requires explicit ownership metadata and never infers ownership from old entrypoints.
diff --git a/document/en/modules/sites.md b/document/en/modules/sites.md
index 92ae7e65..6cf07e80 100644
--- a/document/en/modules/sites.md
+++ b/document/en/modules/sites.md
@@ -66,13 +66,13 @@ The platform counts HTML page views per site (asset files excluded), shown on th
| Part | Location |
|---|---|
| Publish and lookup tools | `src/backend/extensions/mcp_servers/site_publish_mcp/` (`publish_site` / `list_sites`, shipped by the Sites plugin) |
-| Internal endpoints | `src/backend/api/routes/v1/internal_sites.py` (`/v1/internal/sites/publish`, `/v1/internal/sites/list`) |
-| Site lookup service | `src/backend/core/services/site_listing.py` (one record shape for local and cloud) |
-| Business service | `src/backend/core/services/site_service.py` |
-| Public hosting route | `src/backend/api/routes/sites_serve.py` (`GET /site/{slug}/{path}`) |
-| Management API | `src/backend/api/routes/v1/sites.py` (`/v1/sites`) |
-| Access password | `src/backend/core/services/site_password.py` (hash + credential), `src/backend/api/routes/site_gate.py` (unified password page) |
-| Database table | `sites` (`core/db/models/site.py`) |
+| Internal endpoints | `src/backend/features/sites/api/internal_sites.py` (`/v1/internal/sites/publish`, `/v1/internal/sites/list`) |
+| Site lookup service | `src/backend/features/sites/listing.py` (one record shape for local and cloud) |
+| Business service | `src/backend/features/sites/management.py` |
+| Public hosting route | `src/backend/features/sites/api/sites_serve.py` (`GET /site/{slug}/{path}`) |
+| Management API | `src/backend/features/sites/api/sites.py` (`/v1/sites`) |
+| Access password | `src/backend/features/sites/` (hash + credential), `src/backend/features/sites/api/site_gate.py` (unified password page) |
+| Database table | `sites` (`features/sites/models.py`) |
| Frontend management panel | `src/frontend/src/components/sites/SitesPanel.tsx` (Lab → Sites), `SitePasswordField.tsx` (password management) |
| nginx forwarding | `location /site/` in `src/frontend/default.conf.template` |
diff --git a/document/en/modules/storage.md b/document/en/modules/storage.md
index ab669079..947ea06c 100644
--- a/document/en/modules/storage.md
+++ b/document/en/modules/storage.md
@@ -4,7 +4,7 @@ Cloud Docker deployments in CE and EE default to S3-compatible storage. Desktop
Set STORAGE_TYPE=s3 and configure S3_ENDPOINT, S3_BUCKET, S3_REGION, S3_ACCESS_KEY and S3_SECRET_KEY. The backend and sandbox controller must reach the endpoint.
-The optional docker/compose.object-storage.yml starts an independent S3 service. It uses its own volume and does not publish host ports. It does not share Milvus storage.
+The optional docker/object-storage/compose.object-storage.yml starts an independent S3 service. It uses its own volume and does not publish host ports. It does not share Milvus storage.
CE initialization generates separate random access and secret keys. Run python3 scripts/configure-ce-docker.py in a new CE checkout. It refuses to overwrite an existing .env.
@@ -12,7 +12,7 @@ Build the backend image before starting the bucket initializer. Object mounts us
Existing installations must back up configuration, databases and file references before changing providers. Copy required objects and verify their SHA-256 values. Preserve the source objects.
-The storage factory is core/storage/factory.py. S3StorageBackend uses boto3 and includes directory operations. LocalStorageBackend serves desktop local files. OSS and OBS adapters remain EE providers.
+The storage factory is core/files/storage/factory.py. S3StorageBackend uses boto3 and includes directory operations. LocalStorageBackend serves desktop local files. OSS and OBS adapters remain EE providers.
The artifact store supports S3, OSS and OBS. It keeps local metadata records and mirrors them into object storage. Historical download references remain valid when migrated objects keep their storage keys.
diff --git a/document/en/modules/workflow-jobs.md b/document/en/modules/workflow-jobs.md
index 40d7f5e9..69f867b6 100644
--- a/document/en/modules/workflow-jobs.md
+++ b/document/en/modules/workflow-jobs.md
@@ -23,11 +23,3 @@ Omitting `tools` inherits the parent's MCPs. A non-empty explicit list selects i
Derived instances use the main-agent iteration policy without lightweight or subagent turn caps. Each complete task executes once. Invalid output, execution failure or an uncertain callback result never automatically repeats potentially completed side effects. Inspect existing actions before explicitly resuming.
Windows local jobs use `.job//` to reduce directory depth and keep packaged Python job SDK imports below the traditional path-length limit. Each execution retains its complete unique attempt ID; resume and cancellation use the persisted actual directory and exact process reference.
-
-The native desktop end-to-end acceptance script is `src/backend/tests/integration/job_agent_local_smoke.py`. Run it with the packaged private Python and a new isolated output directory. It exercises real HTTP callbacks, managed job processes, concurrent derived main agents, child agents, Bash and file results. Only model decisions are supplied by a local scripted service. It uses no real accounts or data and does not replace installer or WebView2 window acceptance.
-
-## Validation
-
-Unit and regression tests live under src/backend/tests/orchestration/test_job*.py. The opt-in real-sandbox test, src/backend/tests/integration/job_managed_smoke.py, requires an existing job for sandbox context. It creates synthetic jobs and a temporary callback endpoint to test concurrent ledger writes, thread exit, isolated cancellation, cross-process adoption, resume, and startup failure. It does not call a model or rerun the original business script.
-
-Before deployment, verify the actual source directory mounted into the container. The working tree and running service may use different checkouts; restarting a container with the wrong mount will not apply these changes.
diff --git a/document/zh-CN/README.md b/document/zh-CN/README.md
index 4d737be3..dd1765e7 100644
--- a/document/zh-CN/README.md
+++ b/document/zh-CN/README.md
@@ -1,60 +1,95 @@
# HugAgentOS 中文文档
-> 最后更新:2026-07-21 | [English](../en/README.md)
+> 最后更新:2026-10-23 | [English](../en/README.md)
-HugAgentOS 是一个企业级 AI Agent 平台:FastAPI 后端 + React 前端 + AgentScope 2.0 智能体底座,全栈 Docker 化部署,以 open-core 模式提供**开源社区版(CE)**与**商业版(EE)**。本目录是面向部署者、使用者与二次开发者的完整产品文档。
+HugAgentOS 是一个开源、可自托管的 Agent Harness:你提出的问题在云端持续执行,关掉电脑也不会中断;
+在此之上提供 7×24 小时在线的 Personal Agent,并以 HugAgentOS Cloud(服务器)和 HugAgentOS Local
+(桌面本机)两种形态运行。平台以开源社区版(CE)与商业版(EE)双形态发行。本目录是面向部署者、
+使用者与二次开发者的完整产品文档。
## 快速上手
| 文档 | 说明 |
|------|------|
-| [产品简介](getting-started/introduction.md) | 是什么、核心特性总览、CE/EE 一览、技术栈与整体架构 |
-| [快速开始](getting-started/quick-start.md) | 选择一键安装或 Docker Compose,快速运行 HugAgentOS |
+| [产品简介](getting-started/introduction.md) | Durable Agent、运行形态、能力总览、技术栈与整体架构 |
+| [快速开始](getting-started/quick-start.md) | 选择 HugAgentOS Cloud、桌面客户端或一键安装,并启用 Personal Agent |
| [快速构建领域本体](getting-started/domain-ontology-quickstart.md) | 用最小 Domain Pack 跑通校验、发布、测试和工具绑定 |
## 部署
-先看 [部署指南(选择部署方式)](deployment/README.md),再按方式查阅:
+先看[部署指南(选择部署方式)](deployment/README.md),再按方式查阅:
| 文档 | 说明 |
|------|------|
| [部署指南 · 总览](deployment/README.md) | 各部署方式对比与选型、部署后验证 |
-| [无 Docker 一键安装](deployment/quick-install.md) | 个人单机零依赖:一条命令装好,SQLite + 进程内事件流/短时状态 + 子进程 MCP/沙箱 |
-| [Docker Compose 部署](deployment/docker-compose.md) | 团队/生产标准形态:全部服务拓扑、profiles、rebuild 流程、数据库迁移 |
-| [离线生产部署](deployment/offline-production.md) | 隔离环境:镜像 tarball 打包 / 生产侧加载 / 提示词快照迁移(商业版 EE) |
-| [Windows 部署](deployment/windows-deployment.md) | Docker Desktop / WSL2 部署差异、换行符、路径与沙箱限制 |
+| [Docker Compose 部署](deployment/docker-compose.md) | HugAgentOS Cloud 标准形态:服务拓扑、profiles、重建流程、数据库迁移 |
+| [桌面离线本机模式](deployment/desktop-local-mode.md) | HugAgentOS Local:私有运行时、仅本机 / 仅云端 / 本机 + 云端、升级与回滚 |
+| [本机项目工作目录](deployment/local-project-workspace.md) | 桌面端绑定本地项目后的执行目录与权限规则 |
+| [本机与云端技能、插件管理](deployment/local-capability-management.md) | 混合模式下技能与插件的来源和管理 |
+| [无 Docker 一键安装](deployment/quick-install.md) | 个人单机:SQLite + 进程内状态 + 子进程 MCP / 沙箱 |
+| [Windows 安装与部署](deployment/windows-deployment.md) | 桌面端单机服务或 Docker Desktop + WSL2 |
+| [macOS 签名与升级](deployment/macos-signing.md) | 桌面端签名、公证与升级 |
+| [我的空间文件挂载](deployment/space-files.md) | 网页与沙箱共用的文件目录 |
| [环境变量参考](deployment/environment-variables.md) | 全量环境变量逐组说明(默认值 / 作用 / CE·EE 相关性) |
## 架构
| 文档 | 说明 |
|------|------|
-| [总体架构](architecture/overview.md) | 分层架构图、一次对话的完整生命周期、容器拓扑、关键设计决策 |
-| [后端架构](architecture/backend.md) | `src/backend/` 全目录详解:api / orchestration / core 15 子模块 / 路由注册表 |
-| [前端架构](architecture/frontend.md) | 五入口分发、组件组、Zustand store、hooks、构建链 |
-| [数据模型](architecture/data-model.md) | 76 张表分组速览、alembic 双迁移链、CE/EE 表边界 |
+| [总体架构](architecture/overview.md) | 分层架构、本体控制平面、一次对话的完整生命周期、容器拓扑 |
+| [后端架构](architecture/backend.md) | `src/backend/` 目录详解与路由注册表 |
+| [前端架构](architecture/frontend.md) | 入口分发、组件组、Zustand store、模块页面与应用中心 |
+| [数据模型](architecture/data-model.md) | 表分组速览、迁移链、CE/EE 表边界 |
## 功能模块
+**Harness 与长时运行**
+
| 文档 | 说明 |
|------|------|
-| [对话与编排](modules/chat.md) | 端到端对话流、SSE 事件、引用系统、计划模式、子智能体、断线续播 |
-| [提示词系统](modules/prompts.md) | 装配优先级、版本池、提示词广场、跨环境迁移 |
-| [能力目录](modules/catalog.md) | catalog 单一真源、能力门控、用户自助能力 |
-| [模型接入](modules/model-providers.md) | 模型供应商与角色体系、动态切换、个人 API-Key、计费 |
-| [MCP 工具系统](modules/mcp-tools.md) | 内置及插件 Server、连接池、管理员/用户自定义 MCP |
-| [技能系统](modules/agent-skills.md) | SKILL.md 机制、技能包分层、技能市场、技能蒸馏 |
-| [沙箱执行](modules/sandbox.md) | 三种 provider、bash 工具、快照持久化、MySpace 直挂 |
-| [记忆系统](modules/memory.md) | L1/L2/L3 三层记忆、脱敏、审计、mem0 基础设施 |
-| [领域本体治理与校验](modules/ontology-harness.md) | CE 设置页治理、Domain Pack、工具门禁、交付前评审与人审演进 |
-| [知识库](modules/knowledge-base.md) | 自建知识库(混合检索)与 Dify 外接、公共库管理 |
-| [知识库 Wiki](modules/knowledge-base-wiki.md) | LLM 抽取实体与概念、生成互链条目页与概念图谱 |
-| [对象存储](modules/storage.md) | local / s3 / oss 三后端、产物仓、文件链路 |
-| [项目空间与我的空间](modules/projects-myspace.md) | 项目工作区、个人/团队文件夹、文件入上下文 |
-| [认证与权限](modules/auth.md) | AUTH_MODE 三模式、SSO、权限位、管理凭证 |
-| [管理台](modules/admin-console.md) | /admin 与 /config 双管理台、19 组管理路由 |
+| [对话与智能体编排](modules/chat.md) | Run 与连接解耦、断线续播、运行中转向、上下文按步恢复与压缩、计划模式、子智能体 |
+| [Personal Agent](modules/personal-agent.md) | 7×24 个人智能体:后台任务、永久云电脑、定时与心跳、IM 渠道、审批、文件记忆与 Dreaming |
+| [自主循环](modules/autonomous-loop.md) | 目标跨轮存活、maker≠checker、停滞护栏、断点续跑 |
+| [工作流后台作业](modules/workflow-jobs.md) | 沙箱受管进程、租约接管、派生主智能体 |
| [自动化与计划模式](modules/automation.md) | 定时任务调度、计划模式 |
-| [数据画布与产物](modules/canvas-artifacts.md) | Univer 画布、代码产物、Artifact 中心、会话分享 |
+
+**能力与扩展**
+
+| 文档 | 说明 |
+|------|------|
+| [MCP 工具系统](modules/mcp-tools.md) | 内置及插件 Server、连接池、自定义 MCP |
+| [技能系统](modules/agent-skills.md) | SKILL.md 机制、技能包分层、技能市场 |
+| [浏览器自动化插件](modules/browser-automation.md) | 浏览网页、操作表单与网页测试 |
+| [能力目录](modules/catalog.md) | catalog 单一真源、能力门控、用户自助能力 |
+| [沙箱执行](modules/sandbox.md) | provider、Bash 工具、快照持久化、我的空间直挂 |
+| [模型接入](modules/model-providers.md) | 模型供应商与角色体系、动态切换、个人 API-Key |
+| [提示词系统](modules/prompts.md) | 装配优先级、版本池、跨环境迁移 |
+
+**知识、记忆与治理**
+
+| 文档 | 说明 |
+|------|------|
+| [记忆系统](modules/memory.md) | L1/L2/L3 三层记忆、持久 Outbox、做法沉淀、脱敏与审计 |
+| [领域本体治理与校验](modules/ontology-harness.md) | Domain Pack、工具门禁、交付前评审、演进草案人工审核 |
+| [知识库](modules/knowledge-base.md) | 自建知识库(混合检索)与外接知识库 |
+| [知识库 Wiki](modules/knowledge-base-wiki.md) | 抽取实体与概念,生成互链条目页与概念图谱 |
+
+**工作区与产物**
+
+| 文档 | 说明 |
+|------|------|
+| [项目空间与我的空间](modules/projects-myspace.md) | 项目工作区、个人文件夹、云文档在线编辑、文件入上下文 |
+| [数据画布与产物](modules/canvas-artifacts.md) | Univer 画布、代码产物、产物中心、会话分享 |
+| [站点](modules/sites.md) | 对话建站与平台托管 |
+| [应用数据库与 MCP 托管](modules/application-hosting.md) | 应用独立数据库、表单数据与 MCP 发布 |
+| [对象存储](modules/storage.md) | 本地与 S3 兼容存储、产物仓、文件链路 |
+
+**平台管理**
+
+| 文档 | 说明 |
+|------|------|
+| [认证与权限](modules/auth.md) | AUTH_MODE、SSO、权限位、管理凭证 |
+| [管理台](modules/admin-console.md) | /admin 与 /config 管理台 |
## API 参考
@@ -67,21 +102,40 @@ HugAgentOS 是一个企业级 AI Agent 平台:FastAPI 后端 + React 前端 +
| 文档 | 说明 |
|------|------|
-| [社区版 vs 商业版](editions/overview.md) | open-core 模式、功能对比矩阵、三种运行形态、升级路径 |
-| [License 机制](editions/license.md) | 离线签名验签、状态机、能力位执法、签发工具(商业版 EE) |
+| [社区版 vs 商业版](editions/overview.md) | open-core 模式、功能对比矩阵、运行形态、升级路径 |
+| [License 机制](editions/license.md) | 离线签名验签、状态机、能力位执法(商业版 EE) |
| [CE 构建管线](editions/build-ce.md) | manifest、build_ce.py 流水、overlay、验收闸门 |
## 开发指南
| 文档 | 说明 |
|------|------|
-| [后端开发](development/backend.md) | Docker 内开发模式、测试、迁移、分层规范、新增路由/MCP/技能 |
-| [前端开发](development/frontend.md) | 构建热替、目录规范、API 调用约定、edition 门控 |
-
----
-
-内部设计与方案文档(CE/EE 拆分施工图、沙箱快照设计等)见 [docs 根目录](../README.md)。
+| [后端开发](development/backend.md) | 开发模式、测试、迁移、分层规范、新增路由 / MCP / 技能 |
+| [前端开发](development/frontend.md) | 构建、目录规范、API 调用约定、edition 门控 |
+| [客户对话界面定制](development/frontend-customization.md) | 页面配置、客户组件包与独立前端 |
-- [浏览器自动化插件](modules/browser-automation.md)
+**工作台应用(Workbench)**
-- [Personal Agent](modules/personal-agent.md)
+| 文档 | 说明 |
+|------|------|
+| [工作台协议](development/workbench-protocol.md) | JSON Schema、协议版本与接入方式 |
+| [Workbench SDK 与插件页面](development/workbench-sdk.md) | SDK 1.0.0、UI v3 插件页面 |
+| [开发预览](development/workbench-preview.md) | 独立工程与权限预览 |
+| [构建与离线制品](development/workbench-build.md) | 协议、SDK 与宿主前端的构建交付 |
+| [应用地址与插件导航](development/workbench-routing.md) | `/apps/` 地址与 applicationKey |
+| [应用资源](development/workbench-resources.md) | 包内 resource runtime |
+| [界面状态与有限保活](development/workbench-state.md) | 状态格式版本与保存恢复 |
+| [角标、通知与后台运行](development/workbench-activity.md) | badge、notification、background 能力 |
+| [平台适配](development/workbench-platform-adapters.md) | Web、Tauri 与 Electron 适配接口 |
+| [来源与离线安装](development/workbench-origins.md) | 本机与云端同名插件的入口 |
+| [插件升级与权限审批](development/workbench-plugin-upgrades.md) | 升级、重装与授权继承 |
+| [应用目录与布局策略](development/workbench-access-policies.md) | 目录接口、可用性与默认入口 |
+| [模块导航与应用目录](development/workbench-navigation-preferences.md) | 侧栏模块与应用中心 |
+| [目录通知与撤销](development/workbench-invalidation.md) | ETag 与目录失效通知 |
+| [普通外部应用](development/workbench-external-apps.md) | 外链应用的管理与使用 |
+| [外部应用认证](development/workbench-external-auth.md) | 外部应用认证配置(商业版 EE 管理台) |
+| [外部页面嵌入](development/workbench-external-embedding.md) | 嵌入工作区的 origin 与就绪确认 |
+| [应用诊断](development/workbench-diagnostics.md) | 诊断编号与重试 |
+| [错误码](development/workbench-errors.md) | 工作台错误码 |
+| [管理员指南](development/workbench-administration.md) | 工作台管理 |
+| [兼容与迁移](development/workbench-migration.md) | 版本兼容与迁移 |
diff --git a/document/zh-CN/api/overview.md b/document/zh-CN/api/overview.md
index 3906b512..8583aa0c 100644
--- a/document/zh-CN/api/overview.md
+++ b/document/zh-CN/api/overview.md
@@ -59,13 +59,13 @@ HugAgentOS 后端是一个 FastAPI 应用,所有业务接口挂在 `/v1/*` 前
| 方式 | 凭据携带方法 | 适用范围 | 实现 |
|---|---|---|---|
| **会话 Cookie** | `Cookie: jx_session=`(登录后自动种下) | 所有面向最终用户的 `/v1/*` 接口(chats、projects、kb、memories…) | `get_current_user`:Cookie → Redis 会话查找 |
-| **个人 API-Key** | `Authorization: Bearer sk-jx-...` | 与会话 Cookie 等价的用户身份,适合脚本/第三方集成调用 | `sk-jx-` 前缀识别;明文仅创建时返回一次,DB 只存 SHA256(`core/services/api_key_service.py`);在 `/v1/me/api-keys` 自助管理 |
+| **个人 API-Key** | `Authorization: Bearer sk-jx-...` | 与会话 Cookie 等价的用户身份,适合脚本/第三方集成调用 | `sk-jx-` 前缀识别;明文仅创建时返回一次,DB 只存 SHA256(`core/auth/api_keys.py`);在 `/v1/me/api-keys` 自助管理 |
| **Bearer token(mock/remote 模式)** | `Authorization: Bearer ` | `AUTH_MODE=mock`(开发:任意 token / 缺省 token 均放行为 mock 用户)或 `remote`(对接用户中心校验) | `get_current_user` 的兜底分支;`AUTH_MODE=session` 下不接受 |
| **ADMIN_TOKEN** | `Authorization: Bearer ` | `/admin` 管理台后端(`/v1/admin/skills`、`/v1/admin/kb` 等)及内容块写接口 | `require_admin`(环境变量 `ADMIN_TOKEN`) |
| **CONFIG_TOKEN** | `Authorization: Bearer ` | `/config` 系统配置台后端(`/v1/config/*`、`/v1/models`、`/v1/service-configs`、部分 `/v1/admin/*` 观测接口) | `require_config`(环境变量 `CONFIG_TOKEN`) |
| **ADMIN 或 CONFIG 任一** | 同上 | 提示词快照导入/导出(`/v1/content/prompts/export|import`) | `require_admin_or_config` |
| **super_admin 会话** | 会话 Cookie(用户 `extra_data.role == "super_admin"`),合法 ADMIN_TOKEN 可兜底 | 个别跨用户管理操作 | `require_super_admin` |
-| **BACKEND_INTERNAL_TOKEN** | `Authorization: Bearer ` | 站点、浏览器与安全工具的内部服务调用;未配置时拒绝请求 | `api/routes/v1/internal_site_auth.py`、`internal_security.py` |
+| **BACKEND_INTERNAL_TOKEN** | `Authorization: Bearer ` | 站点、浏览器与安全工具的内部服务调用;未配置时拒绝请求 | `features/sites/api/internal_site_auth.py`、`internal_security.py` |
鉴权优先级(`get_current_user`):**Cookie 会话 → API-Key Bearer(`sk-jx-` 前缀)→ mock/remote Bearer**。携带了形似 API-Key 的 Bearer 但校验失败时直接 401,绝不静默降级为匿名。
@@ -120,7 +120,7 @@ curl -X POST 'https:///api/v1/agents/responses' \
## SSE 流式协议
-流式聊天端点 `POST /v1/agents/responses`(`src/backend/api/routes/v1/agent_responses.py`,`stream: true`)校验通过后启动一个后台 run,再以 SSE 跟随该 run 实时下发事件;断线后可用 `GET /v1/chats/stream/{run_id}` 从任意偏移续播。事件由 `src/backend/orchestration/workflow.py` 产出、`src/backend/orchestration/chat_run_executor.py` 序列化上 wire。
+流式聊天端点 `POST /v1/agents/responses`(`src/backend/api/routes/v1/agent_responses.py`,`stream: true`)校验通过后启动一个后台 run,再以 SSE 跟随该 run 实时下发事件;断线后可用 `GET /v1/chats/stream/{run_id}` 从任意偏移续播。事件由 `src/backend/core/agent_runtime/runner.py` 产出、`src/backend/core/chat/runtime/` 序列化上 wire。
**Wire 格式**:`Content-Type: text/event-stream`,每个事件一行 `data: {JSON}`,事件类型放在 JSON 的 `type` 字段里(不使用 SSE 的 `event:` 行);流以 `data: [DONE]` 结束。静默超过 15 秒时发送 SSE 注释行 `: heartbeat` 维持反代连接。
@@ -197,7 +197,7 @@ data: [DONE]
答案必须与原问题数量、ID 和顺序完全一致;私有 option ID 必须来自 pending 响应,单选题不能提交多个选项,且每题必须作答或明确 `skipped=true`。后端会先把这些私有 ID 映射回原始选项 label,再向模型返回 DSH 兼容结果 `{"answers":[{"id":"scope","selected":["最小范围"]}]}`。第一个有效回答/取消会原子取得请求,之后的并发提交返回 `stale=true`,不会重复唤醒工具。成功 POST 已代表服务端原子取得请求,当前标签页可立即按 `request_id` 移除;`user_question_resolved` 和周期 pending 权威快照负责同步其他标签页与断线客户端。
-正文中的 `[ref:tool_name-N]` 引用标记由 `orchestration/citations.py` 解析为 `citations` 项,前端据此渲染角标(见 [对话模块](../modules/chat.md))。
+正文中的 `[ref:tool_name-N]` 引用标记由 `core/llm/citations/` 解析为 `citations` 项,前端据此渲染角标(见 [对话模块](../modules/chat.md))。
### 续播、Steer 与取消
diff --git a/document/zh-CN/architecture/backend.md b/document/zh-CN/architecture/backend.md
index ab242a68..beb8c6bb 100644
--- a/document/zh-CN/architecture/backend.md
+++ b/document/zh-CN/architecture/backend.md
@@ -128,7 +128,7 @@ src/backend/
`runner` 仅依赖不加载后端配置或数据库的进程和路径基础模块。OpenSandbox 专属组件留在自己的驱动目录;Cube 与 OpenSandbox 共用的进程、路径和账号环境能力由共享包提供。
-### core/agent_skills — 技能引擎
+### core/capabilities/skills — 技能引擎
| 文件 | 职责 |
|---|---|
@@ -144,12 +144,12 @@ src/backend/
| 子模块 | 职责与关键文件 |
|---|---|
| `core/chat` | workflow 上下文组装(`context.py`)、SSE 工具日志事件构造(`tool_log.py`) |
-| `core/content` | 附件解析(`file_parser.py`)、KB 文档分块/关键词/向量化(`kb_processing.py`)、上传校验(`file_validation.py`)、产物读取与摘要(`artifact_reader/refs/summary.py`)、内容块导入导出(`content_blocks.py`)、`svg_fit.py` |
-| `core/kb` / `edition_ee/kb` | 自建知识库解析与 Milvus 向量库保留在 `core/kb`;Dify 客户端与外部检索适配器只存在于 `edition_ee/kb`(商业版 EE) |
-| `core/artifacts` | 产物存储 `store.py`:本地 / OSS 双模式 |
-| `core/infra` | 统一响应(`responses.py`)、异常(`exceptions.py`)、结构化日志(`logging.py`)、限流(`rate_limit.py`)、Redis 单例(`redis.py`)、指标(`metrics.py`)、后台任务注册表(`runtime_state.py`)、脱敏(`data_masking.py`)、蒸馏预算闸门(`distillation_budget.py`,商业版 EE) |
+| `core/files/content` | 附件解析(`file_parser.py`)、KB 文档分块/关键词/向量化(`kb_processing.py`)、上传校验(`file_validation.py`)、产物读取与摘要(`artifact_reader/refs/summary.py`)、内容块导入导出(`content_blocks.py`)、`svg_fit.py` |
+| `features/knowledge` / `edition_ee/kb` | 自建知识库解析与 Milvus 向量库保留在 `features/knowledge`;Dify 客户端与外部检索适配器只存在于 `edition_ee/kb`(商业版 EE) |
+| `core/files/artifacts` | 产物存储 `store.py`:本地 / OSS 双模式 |
+| `core/infra` | 统一响应(`responses.py`)、异常(`exceptions.py`)、结构化日志(`logging.py`)、Redis 单例(`redis.py`)、指标(`metrics.py`)、后台任务注册表(`runtime_state.py`)、脱敏(`data_masking.py`)、蒸馏预算闸门(`distillation_budget.py`,商业版 EE) |
| `edition_ee/licensing` | 能力枚举、验签、时钟回拨防护、席位策略、中间件和 manager 全部只存在于商业版 EE;CE 树物理不含 License 包,由 `api/middleware/edition.py` 的 CE overlay 提供固定版本探针 |
-| `core/storage` | 存储协议 `protocol.py` + 工厂 `factory.py`;`local.py`(CE)、`s3.py` / `oss.py`(商业版 EE) |
+| `core/files/storage` | 存储协议 `protocol.py` + 工厂 `factory.py`;`local.py`(CE)、`s3.py` / `oss.py`(商业版 EE) |
## orchestration/ — 编排层
@@ -219,12 +219,12 @@ CE 路由位于 `api/routes/v1/`;已物理拆分的商业路由位于 `edition
| 应用入口与启动钩子 | `src/backend/api/app.py` |
| 路由注册表 | `src/backend/api/routes/v1/__init__.py` |
| 智能体工厂 | `src/backend/core/llm/factory/build.py` |
-| Run 执行器 / 工作流 | `src/backend/orchestration/chat_run_executor.py`、`workflow.py` |
-| 能力目录 | `src/backend/core/config/catalog.py` |
+| Run 执行器 / 工作流 | `src/backend/core/chat/runtime/`、`workflow.py` |
+| 能力目录 | `src/backend/core/capabilities/catalog/` |
| 沙箱协议 | `src/backend/core/sandbox/protocol.py` |
| 记忆流水线 | `src/backend/core/memory/pipeline.py` |
| EE License 实现 / CE 版本中间件 | `src/backend/edition_ee/licensing/`、`src/backend/api/middleware/edition.py` |
-| 技能引擎 | `src/backend/core/agent_skills/loader.py` |
+| 技能引擎 | `src/backend/core/capabilities/skills/loader.py` |
| MCP 注册表与端口 | `src/backend/extensions/mcp_servers/registry.py`、`_ports.py` |
空间 MCP 接口位于 `extensions/mcp_servers/space_files_mcp/`。注册表分别管理安装位置和暴露方式;混合模式通过桌面能力网关访问空间,修改操作接入统一审批。详见[空间文件挂载](../deployment/space-files.md)。
diff --git a/document/zh-CN/architecture/data-model.md b/document/zh-CN/architecture/data-model.md
index 8a99e9f7..126931cd 100644
--- a/document/zh-CN/architecture/data-model.md
+++ b/document/zh-CN/architecture/data-model.md
@@ -179,11 +179,11 @@ user_feedback # 问题反馈审核 /
|---|---|
| 共享 ORM / EE ORM | `src/backend/core/db/models/`、`src/backend/edition_ee/db/models/` |
| 本体仓储 | `src/backend/core/db/repository/ontology.py` |
-| 问题反馈模型 / 仓储 | `src/backend/edition_ee/db/models/feedback.py`、`db/repository/feedback.py` |
+| 问题反馈模型 / 仓储 | `src/backend/edition_ee/db/models/feedback.py`、`edition_ee/db/repository/feedback.py` |
| 引擎与启动建表 | `src/backend/core/db/engine.py` |
| 仓储层 | `src/backend/core/db/repository/` |
-| CE/EE 建表边界 | `src/backend/core/db/edition_tables.py` |
+| CE/EE 建表边界 | `src/backend/edition_ee/db/edition_tables.py` |
| EE 迁移主链 | `src/backend/alembic/versions/` |
| CE 迁移基线 | `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` |
-| 提示词版本池服务 | `src/backend/core/services/prompt_version_service.py` |
+| 提示词版本池服务 | `src/backend/core/llm/prompt_versions.py` |
| 版本边界权威方案 | [版本与授权](../editions/overview.md) |
diff --git a/document/zh-CN/architecture/overview.md b/document/zh-CN/architecture/overview.md
index 9219e0e8..06677e4c 100644
--- a/document/zh-CN/architecture/overview.md
+++ b/document/zh-CN/architecture/overview.md
@@ -1,8 +1,8 @@
# 系统整体架构
-> 最后更新:2026-07-19
+> 最后更新:2026-10-23
-HugAgentOS 是一个面向企业的 AgentOS,同时提供轻量本地运行形态和全容器化生产形态。前端为 React 单页应用,后端为 FastAPI 异步服务,智能体底座为 AgentScope 2.0 的 ReActAgent,工具生态通过 MCP(Model Context Protocol)以独立进程提供,代码执行落在可插拔的沙箱驱动上。平台以「开源社区版(CE)+ 商业版(EE)」双形态发行,两版共用一套代码,通过路由注册表、建表边界与 license 能力位三道接缝裁剪(详见 [版本与授权](../editions/overview.md))。
+HugAgentOS 是一个开源、可自托管的 Agent Harness,提供 HugAgentOS Cloud(全容器化服务端)与 HugAgentOS Local(桌面本机)两种运行形态。前端为 React 单页应用,后端为 FastAPI 异步服务,智能体底座为 AgentScope 2.0 的 ReActAgent,工具生态通过 MCP(Model Context Protocol)以独立进程提供,代码执行落在可插拔的沙箱驱动上。平台以「开源社区版(CE)+ 商业版(EE)」双形态发行,两版共用一套代码,通过路由注册表、建表边界与 license 能力位三道接缝裁剪(详见 [版本与授权](../editions/overview.md))。
本文从五个视角描述系统:分层架构、本体驱动的企业可信控制平面、一次对话请求的完整生命周期、容器拓扑、关键设计决策。更深入的模块拆解见 [后端架构](./backend.md)、[前端架构](./frontend.md) 与 [数据模型](./data-model.md)。
@@ -23,48 +23,38 @@ HugAgentOS 是一个面向企业的 AgentOS,同时提供轻量本地运行形
```
┌────────────────────────────────────────────────────────────────────┐
-│ 浏览器(React SPA) │
-│ App.tsx 主应用 · AdminApp 内容台(EE) · ConfigApp 系统台(EE) │
-│ api.ts 类型化客户端 + useStreaming SSE 解析 │
+│ 入口:Web(React SPA)· 桌面客户端(Tauri)· IM 渠道 · API · 定时 · 心跳 │
└──────────────────────────────┬─────────────────────────────────────┘
│ HTTP / SSE(/api 前缀)
┌──────────────────────────────▼─────────────────────────────────────┐
-│ Nginx(frontend 容器,:80) │
-│ 静态资源托管 · /api → backend 反向代理 │
+│ FastAPI(src/backend/api/) │
+│ 中间件:CORS · 结构化日志 · 统一错误处理 · 版本路由注册表 │
└──────────────────────────────┬─────────────────────────────────────┘
│
┌──────────────────────────────▼─────────────────────────────────────┐
-│ FastAPI(backend 容器,src/backend/api/) │
-│ 中间件:CORS · 结构化日志 · 统一错误处理 │
-│ 路由注册表:CE_ROUTERS + EE_ROUTERS(license 能力位守卫) │
+│ Run 层 │
+│ core/runs:持久 Run、数据库租约、启动对账、工具副作用意图与回执 │
+│ core/chat/runtime:Run 准入、聊天 worker、续播、steer、压缩 │
└──────────────────────────────┬─────────────────────────────────────┘
│
┌──────────────────────────────▼─────────────────────────────────────┐
-│ 编排层(src/backend/orchestration/) │
-│ chat_run_executor:Run 与 HTTP 连接解耦、断线续播 │
-│ workflow:流式编排 · strategy:路由策略 · citations:引用提取 │
-│ memory_integration:记忆读写 · subagents/plan_mode:计划模式 │
-│ schedulers:自动化定时任务 · 技能蒸馏 cron(EE) │
-└──────────────────────────────┬─────────────────────────────────────┘
- │
-┌──────────────────────────────▼─────────────────────────────────────┐
-│ 智能体层(src/backend/core/llm/) │
-│ factory/build → AgentScope 2.0 ReActAgent │
-│ ToolCollector → Toolkit(自研工具 + 技能 + MCP 工具) │
-│ middlewares:动态模型/文件上下文/系统提醒 · offloader:超长结果落盘 │
-│ prompts/prompt_runtime:系统提示词装配(DB 优先,文件兜底) │
+│ 智能体执行 │
+│ core/agent_runtime:runner.astream_chat_workflow、路由策略、 │
+│ 记忆 / 本体 / 进化集成 │
+│ core/llm:factory/build → AgentScope ReActAgent、工具、上下文、引用 │
+│ orchestration/:plans 计划模式 · loops 自主循环 · automations 自动化 │
+│ personal_agent/:身份、任务队列、调度、渠道、设备连接、工作区 │
└───────┬──────────────────────┬──────────────────────┬──────────────┘
│ streamable-http │ 沙箱驱动协议 │ SQL / 向量
┌───────▼───────────┐ ┌────────▼──────────┐ ┌─────────▼─────────────┐
-│ MCP 工具池 │ │ 沙箱 │ │ 数据层 │
-│ mcp 工具进程 │ │ script-runner(CE) │ │ PostgreSQL · Redis │
-│ 端口见 _ports.py │ │ OpenSandbox(CE/EE) │ │ Milvus · Neo4j(可选) │
-│ 搜索/抓取/插件工具 │ │ CubeSandbox(EE) │ │ 本地 / S3 / OSS 存储 │
-│ 数仓/产业链(EE) │ │ │ │ │
+│ MCP 工具池 │ │ 沙箱与作业 │ │ 数据层 │
+│ mcp 工具进程 │ │ OpenSandbox │ │ PostgreSQL / SQLite │
+│ 端口见 _ports.py │ │ script-runner │ │ Redis · Milvus · Neo4j│
+│ 插件工具 │ │ core/jobs 后台作业 │ │ S3 兼容 / 本地存储 │
└───────────────────┘ └───────────────────┘ └───────────────────────┘
```
-依赖方向自上而下单向:`api → orchestration → core`,`core` 内部 `services → db`。MCP 服务器是独立进程,与后端只通过 HTTP 通信,崩溃互不影响。
+依赖方向自上而下单向:`api → features / orchestration / personal_agent → core`;`core` 不依赖上层,`integrations/` 承载钉钉、飞书、邮箱等外部系统接入。MCP 服务器是独立进程,与后端只通过 HTTP 通信,崩溃互不影响。
## 领域本体驱动的企业可信控制平面
@@ -139,21 +129,22 @@ RAG 则提供支撑决策所需的文档与证据。
### 2. 启动 Run(与 HTTP 连接解耦)
-`orchestration/chat_run_executor.py::start_run` 把 AI 工作流注册为后台 Run(落 `chat_runs` 表),HTTP 响应只是「跟随」这个 Run:
+`core/chat/runtime/admission.py::start_run` 把本轮注册为后台 Run(落 `chat_runs` 表),由持有数据库租约的聊天 worker 执行;HTTP 响应只是「跟随」这个 Run:
-- 浏览器断线、刷新页面后可调 `GET /v1/chats/stream/{run_id}` 从指定偏移续播;
-- 后端重启时 lifespan 钩子 `_startup_recover_chat_runs` 恢复/收割未完成的 Run。
+- 浏览器断线、刷新页面后可调 `GET /v1/chats/stream/{run_id}?from=N` 从指定偏移续播;
+- 运行中可通过 `POST /v1/chat-runs/{run_id}/steer` 追加指令,在下一个安全 ReAct 边界生效;
+- 后端启动时 `core/chat/runtime/recovery.py::recover_orphan_runs` 依据最后提交的安全点对账租约过期的 Run:可恢复的继续,不可恢复的明确收尾。
### 3. 编排与智能体执行
-`orchestration/workflow.py::astream_chat_workflow` 驱动单轮执行:
+`core/agent_runtime/runner.py::astream_chat_workflow` 驱动单轮执行:
-1. **记忆检索**——`orchestration/memory_integration.py` 读取 L1 画像 / L2 向量记忆(开启时),注入系统消息;
-2. **路由决策**——`orchestration/strategy.py`,`ROUTER_STRATEGY=main_only`(默认)恒路由到主智能体;
+1. **记忆检索**——`core/agent_runtime/integrations/memory.py` 读取 L1 画像 / L2 向量记忆(开启时),注入系统消息;
+2. **路由决策**——`core/agent_runtime/strategy.py`,`ROUTER_STRATEGY=main_only`(默认)恒路由到主智能体;
3. **构建智能体**——`core/llm/factory/build.py::create_agent_executor`:装配提示词(`prompts/prompt_runtime.py`,DB 版本池优先)、按 catalog 与用户能力位筛选 MCP 服务器、注册自研工具(Read/Write/Edit/Glob/Grep/bash/技能加载等,见 `core/llm/tools/`)、最终一次性构造 AgentScope Toolkit;
-4. **流式执行**——`orchestration/streaming.py::StreamingAgent` 包装 ReActAgent,把推理增量、工具调用、工具结果逐条产出;
-5. **引用提取**——`orchestration/citations.py` 从工具结果中解析 `[ref:tool_name-N]` 标记生成引用项;
-6. **收尾**——落库助手消息与工具调用日志,`core/memory/pipeline.py` 在后台异步抽取记忆(不阻塞 SSE 主链路),独立接口生成追问问题(`orchestration/followups.py`)。
+4. **流式执行**——`core/llm/runtime/streaming.py::StreamingAgent` 包装 ReActAgent,把推理增量、工具调用、工具结果逐条产出;
+5. **引用提取**——`core/llm/citations/` 从工具结果中解析 `[ref:tool_name-N]` 标记生成引用项;
+6. **收尾**——落库助手消息与工具调用日志,`core/memory/pipeline.py` 在后台异步抽取记忆(不阻塞 SSE 主链路),独立接口生成追问问题(`core/chat/followups.py`)。
### 4. SSE 事件流
@@ -166,9 +157,9 @@ RAG 则提供支撑决策所需的文档与证据。
| `tool_call_start` | 工具调用开始(稳定 ID + 名称) | `core/chat/tool_log.py` |
| `tool_call_delta` | 合批后的工具参数 JSON 增量 | `core/chat/tool_log.py` |
| `tool_call` | 工具参数完整、即将执行 | `core/chat/tool_log.py` |
-| `tool_result` | 工具执行结果(含引用、产物卡片载荷) | `core/chat/tool_log.py` + `orchestration/tool_payloads.py` |
+| `tool_result` | 工具执行结果(含引用、产物卡片载荷) | `core/chat/tool_log.py` + `core/chat/tool_payloads.py` |
| `tool_progress` | 长工具的进度上报 | `chats.py` |
-| `meta` | 末尾元信息:路由、引用源、产物列表等 | `orchestration/workflow.py` |
+| `meta` | 末尾元信息:路由、引用源、产物列表等 | `core/agent_runtime/runner.py` |
| `error` | 错误事件(随后立即 `[DONE]`) | `chats.py` |
前端 `src/frontend/src/hooks/useStreaming.ts` 按 `type` 分发,并按 `tool_id` 把 start、delta、最终调用与结果原位合并为一张工具卡片。
@@ -181,13 +172,14 @@ RAG 则提供支撑决策所需的文档与证据。
|---|---|---|---|
| `frontend` | hugagent-frontend | `3002:80` | Nginx:静态资源 + `/api` 反代 |
| `backend` | hugagent-backend | `3001` | FastAPI 主服务 |
-| `mcp` | hugagent-mcp | 内部独立端口 | 9 个 MCP 服务器常驻进程(streamable-http;CE 7 个通用工具,EE 另含 2 个行业工具) |
+| `mcp` | hugagent-mcp | 内部独立端口 | 内置 MCP 服务器常驻进程(streamable-http;数仓查询与行业工具为 EE) |
| `postgres` | hugagent-postgres | `5432` | 主数据库 |
| `redis` | hugagent-redis | `6379` | 会话 / 缓存 |
-| `script-runner`(profile `script_runner`) | hugagent-script-runner | 内部 | 轻量脚本执行 sidecar(无 DB/密钥访问权) |
-| `opensandbox` + `opensandbox-config-init`(profile `opensandbox`) | hugagent-opensandbox | 内部 | 持久沙箱(商业版 EE) |
+| `script-runner`(profile `script_runner`) | hugagent-script-runner | 内部 | 可选的轻量脚本执行 sidecar(无 DB/密钥访问权) |
+| `opensandbox` + `opensandbox-config-init`(profile `opensandbox`) | hugagent-opensandbox | 内部 | 默认沙箱:持久会话、快照恢复(与 script-runner 互斥) |
| `etcd` / `minio` / `milvus`(profile `mem0`) | hugagent-etcd 等 | 内部 | L2 向量记忆基础设施 |
| `neo4j`(profile `mem0`) | hugagent-neo4j | `7474/7687` | L3 图谱记忆(可选) |
+| `personal-agent-opensandbox`(profile `personal-agent`,`docker/personal-agent/compose.personal-agent.yml`) | — | 内部 | Personal Agent 永久云电脑的专用控制器 |
所有服务挂在同一 `hugagent-network` 网络;`postgres_data` 等具名卷跨重建持久化。部署细节见 [Docker Compose 部署](../deployment/docker-compose.md)。
@@ -195,7 +187,7 @@ RAG 则提供支撑决策所需的文档与证据。
### Catalog 单一真源
-`src/backend/core/config/catalog.json` 是能力开关的唯一权威,四类条目:`skills` / `agents` / `mcp` / `kb`。`core/config/catalog.py` 暴露 `is_enabled(kind, id)` / `get_enabled_ids(kind)`,从 MCP 工具装载、技能注册到路由决策全部经此门控;管理台的启停操作写 `catalog_overrides` 表叠加在文件之上(`core/config/catalog_loader.py`)。新能力只要进 catalog 即可被前端能力中心与智能体工厂同时感知,无需改编排代码。详见 [能力目录](../modules/catalog.md)。
+`src/backend/core/config/catalog.json` 是能力开关的唯一权威,四类条目:`skills` / `agents` / `mcp` / `kb`。`core/capabilities/catalog/` 暴露 `is_enabled(kind, id)` / `get_enabled_ids(kind)`,从 MCP 工具装载、技能注册到路由决策全部经此门控;管理台的启停操作写 `catalog_overrides` 表叠加在文件之上(`core/capabilities/catalog/loader.py`)。新能力只要进 catalog 即可被前端能力中心与智能体工厂同时感知,无需改编排代码。详见 [能力目录](../modules/catalog.md)。
### 统一响应信封
@@ -203,19 +195,19 @@ RAG 则提供支撑决策所需的文档与证据。
### 提示词 DB 化
-系统提示词不再硬编码:版本池存于 `content_blocks` 表(`id=prompt_versions`),由 `core/services/prompt_version_service.py` 管理 CRUD、激活与首启 seed;`prompts/prompt_runtime.py` 装配时按「DB 激活版本 → `prompts/prompt_text/default/system/` 文件兜底 → 最小硬编码兜底」三级降级,保证永不为空。运营可在管理台灰度切换提示词版本而无需发版。详见 [提示词系统](../modules/prompts.md)。
+系统提示词不再硬编码:版本池存于 `content_blocks` 表(`id=prompt_versions`),由 `core/llm/prompt_versions.py` 管理 CRUD、激活与首启 seed;`prompts/prompt_runtime.py` 装配时按「DB 激活版本 → `prompts/prompt_text/default/system/` 文件兜底 → 最小硬编码兜底」三级降级,保证永不为空。运营可在管理台灰度切换提示词版本而无需发版。详见 [提示词系统](../modules/prompts.md)。
### MCP 工具独立进程
-每个工具是 `src/backend/extensions/mcp_servers//` 下的独立 MCP 服务器,统一运行在专用 `mcp` 容器内,以 streamable-http 常驻监听(端口表 `extensions/mcp_servers/_ports.py` 为单一真源),后端经 `core/llm/mcp_manager.py` 的 HTTP 客户端连接池复用连接。好处:工具崩溃不影响主进程、依赖隔离(文档编辑与导出已迁入沙箱技能)、可独立扩缩。CE 保留 7 个通用 Server;`query_database` 与 `ai_chain_information_mcp` 两个行业工具依赖内网数据源,归商业版 EE。详见 [MCP 工具](../modules/mcp-tools.md)。
+每个工具是 `src/backend/extensions/mcp_servers//` 下的独立 MCP 服务器,统一运行在专用 `mcp` 容器内,以 streamable-http 常驻监听(端口表 `extensions/mcp_servers/_ports.py` 为单一真源),后端经 `core/llm/mcp/manager.py` 的 HTTP 客户端连接池复用连接。好处:工具崩溃不影响主进程、依赖隔离(文档编辑与导出已迁入沙箱技能)、可独立扩缩。`query_database` 与 `ai_chain_information_mcp` 两个行业工具依赖内网数据源,归商业版 EE。详见 [MCP 工具](../modules/mcp-tools.md)。
-### Run 与连接解耦
+### Run 与连接、进程解耦
-流式对话不绑定 HTTP 连接生命周期:`chat_run_executor` 把执行状态写入 `chat_runs` 表并支持按偏移续播,彻底解决「刷新页面丢回复」「弱网断流」问题,也让自动化调度器能复用同一执行通道。
+流式对话不绑定 HTTP 连接,也不绑定某个进程:Run 状态与按步上下文落库,worker 通过数据库租约领取执行,过期执行者不能再提交结果或调用工具;进程中断后由其他 worker 接管。工具调用先记录意图、有回执才算完成(`core/runs/tool_effects/`),恢复时只重放能证明尚未生效的调用,结果未知的外部操作交给用户确认。自动化、计划模式、自主循环与 Personal Agent 复用同一套 Run 机制。
### CE/EE 同源双形态
-商业版主仓即全量代码;社区版由 `scripts/build_ce.py` 按 `ce/manifest.yaml` 派生(排除 `edition_ee` 实现 + 文本变换 + CE overlay)。运行时边界包括路由注册表 `api/routes/v1/__init__.py`、建表边界 `core/db/edition_tables.py`(CE 不注册 20 张 EE 专属表)、EE 的 `edition_ee/licensing/` 与 CE 固定版本中间件,以及站点可见性等版本策略缝隙。详见 [版本与授权](../editions/overview.md)。
+商业版主仓即全量代码;社区版由 `scripts/build_ce.py` 按 `ce/manifest.yaml` 派生(排除 `edition_ee` 实现 + 文本变换 + CE overlay)。运行时边界包括路由注册表 `api/routes/v1/__init__.py`、建表边界 `edition_ee/db/edition_tables.py`(CE 不注册 20 张 EE 专属表)、EE 的 `edition_ee/licensing/` 与 CE 固定版本中间件,以及站点可见性等版本策略缝隙。详见 [版本与授权](../editions/overview.md)。
## 相关源码
@@ -223,12 +215,13 @@ RAG 则提供支撑决策所需的文档与证据。
|---|---|
| FastAPI 应用与启动钩子 | `src/backend/api/app.py` |
| 流式对话入口 | `src/backend/api/routes/v1/chats/__init__.py` |
-| Run 执行器 | `src/backend/orchestration/chat_run_executor.py` |
-| 流式编排 | `src/backend/orchestration/workflow.py` |
+| Run 执行器 | `src/backend/core/chat/runtime/` |
+| 流式编排 | `src/backend/core/agent_runtime/runner.py` |
| 智能体工厂 | `src/backend/core/llm/factory/build.py` |
| MCP 配置与端口 | `src/backend/core/config/mcp_config.py`、`src/backend/extensions/mcp_servers/_ports.py` |
-| 能力目录 | `src/backend/core/config/catalog.json`、`catalog.py` |
+| 能力目录 | `src/backend/core/config/catalog.json`、`src/backend/core/capabilities/catalog/` |
+| Run、租约与恢复 | `src/backend/core/runs/`、`src/backend/core/chat/runtime/recovery.py` |
| 响应信封 | `src/backend/core/infra/responses.py` |
| 提示词装配 | `src/backend/prompts/prompt_runtime.py` |
-| CE/EE 接缝 | `src/backend/api/routes/v1/__init__.py`、`src/backend/core/db/edition_tables.py`、`src/backend/api/middleware/edition.py`、`src/backend/edition_ee/licensing/` |
+| CE/EE 接缝 | `src/backend/api/routes/v1/__init__.py`、`src/backend/edition_ee/db/edition_tables.py`、`src/backend/api/middleware/edition.py`、`src/backend/edition_ee/licensing/` |
| 容器编排 | `docker-compose.yml` |
diff --git a/document/zh-CN/deployment/README.md b/document/zh-CN/deployment/README.md
index 5a64dba2..cb1ce781 100644
--- a/document/zh-CN/deployment/README.md
+++ b/document/zh-CN/deployment/README.md
@@ -9,10 +9,8 @@ HugAgentOS 支持多种部署方式,从「个人单机零依赖尝鲜」到「
| 部署方式 | 适用场景 | Docker | 数据库 | 多用户 | 文档 |
|---|---|---|---|---|---|
| **桌面端离线本机模式** | Windows / macOS / Linux 个人用户;安装后直接使用 | 不需要 | SQLite | 否(单用户) | [desktop-local-mode.md](desktop-local-mode.md) |
-| **UOS 1070 ARM64 桌面端** | 统信 UOS 1070 aarch64;Electron 43 `.deb` | 不需要 | SQLite / 远程 | 本机单用户 / 云端多用户 | [uos1070-desktop.md](uos1070-desktop.md) |
| **无 Docker 一键安装** | 个人单机尝鲜、二次开发体验;一条命令装好即用 | 不需要 | SQLite | 否(单用户) | [quick-install.md](quick-install.md) |
| **Docker Compose** | 团队 / 生产的标准形态,多用户、全功能 | 需要 | PostgreSQL | 是 | [docker-compose.md](docker-compose.md) |
-| **离线生产部署(商业版 EE)** | 政务 / 内网等隔离环境,镜像 tarball 离线交付 | 需要 | PostgreSQL | 是 | [offline-production.md](offline-production.md) |
跨平台与参考:
@@ -20,7 +18,6 @@ HugAgentOS 支持多种部署方式,从「个人单机零依赖尝鲜」到「
|---|---|
| [Windows 部署](windows-deployment.md) | 桌面端本机服务一键安装,或以 Docker Desktop + WSL2 跑 Compose 版 |
| [桌面端离线本机模式](desktop-local-mode.md) | 三平台随包私有 Python、原子升级回滚与完整/精简包 |
-| [UOS 1070 ARM64 桌面端](uos1070-desktop.md) | Electron 43、glibc 2.28、ARM64 `.deb` 构建与验收 |
| [环境变量参考](environment-variables.md) | 全量环境变量逐组说明(默认值 / 作用 / CE·EE 相关性) |
## 一句话对比
@@ -30,7 +27,6 @@ HugAgentOS 支持多种部署方式,从「个人单机零依赖尝鲜」到「
联网构建。定位是单进程单用户。
- **无 Docker 一键安装**:Linux/macOS 上最快的命令行路径。运行 `curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh | bash`,安装器会创建管理员、配置模型、启动服务并打开浏览器。数据保存在 `~/.hugagent/`。**单进程单用户**,适合个人试用与开发,不适合多人或生产环境。
- **Docker Compose**:推荐的标准部署。全部服务由一个 `docker-compose.yml` 编排(PostgreSQL + Redis + 后端 + MCP + 前端 + 沙箱),支持多用户、持久沙箱、分层记忆等全部能力。
-- **离线生产部署(EE)**:面向无法在线拉镜像的隔离环境,在联网侧打镜像 tarball,拷到生产侧 `docker load` + `compose up`。属商业版交付范畴。
> 能力对比、社区版 / 商业版差异见 [版本对比](../editions/overview.md)。
diff --git a/document/zh-CN/deployment/artifact-storage-key-repair.md b/document/zh-CN/deployment/artifact-storage-key-repair.md
deleted file mode 100644
index 0a63ffd0..00000000
--- a/document/zh-CN/deployment/artifact-storage-key-repair.md
+++ /dev/null
@@ -1,48 +0,0 @@
-# 历史产物存储地址修复
-
-适用于旧版本把 `artifacts/.docx` 等对象登记为 `artifacts/` 的问题。
-当前生成流程要求真实存储地址,个人空间文件列表只等待本地改动登记,不再等待所有云端文件下载。
-导入窗口把请求失败显示为错误并提供重试,失败不会再显示为“暂无文件”。
-
-## 安全边界
-
-修复只匹配 `storage_key == artifacts/`。必须确认旧对象不存在、
-对应文件名后缀的对象存在、下载字节数与原登记大小一致。服务超时、鉴权失败、
-大小不符或找不到替代对象时,保留原记录。真实存在的无后缀对象保持不变。
-
-写入采用带原地址、文件名、大小、更新时间条件的原子更新;不修改内容版本。
-原地址、新地址、SHA-256 和原内容版本与新地址一起存入
-`metadata.storage_key_repair`,支持审计及有条件回滚。
-个人空间投影下载失败时会用同样的校验自动修复;未保存的本地内容仍按原有冲突保护处理。
-
-## 执行
-
-遵守仓库发布授权:预发验证后,生产代码与数据变更必须另行取得用户确认。
-在已授权环境的项目根目录执行,使用该环境已有配置与 Python 运行时。
-不要复制其他环境的 `.env` 或凭据。
-
-```bash
-# 默认只验证、输出候选地址,不改数据库。
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id <用户ID>
-
-# 确认候选结果后执行;重复运行不会重复修改已修复项。
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id <用户ID> --apply
-
-# 仅回滚内容版本、当前地址都未变化的已审计修复。
-# 回滚会恢复旧地址;若后台投影已升级,仍可能再次进行自动修复。
-PYTHONPATH=src/backend python scripts/repair_artifact_storage_keys.py --user-id <用户ID> --rollback
-```
-
-工具一次只处理明确指定的账号,按文件 ID 分批读取。错误退出非零,避免把部分失败当作全部成功。
-修复既有记录包含回收站中的历史项;不会恢复已删除文件或变更所属目录。
-
-## 验收
-
-1. 地址修复候选、对象大小和审计记录正确;再次运行没有重复改写。
-2. 个人空间根目录和原文件目录的列表 API 返回成功;有效文件能下载。
-3. Windows 桌面本机模式分别用 `@` 和 `+ → 从我的空间导入` 选择文件,实际发送后可读取正文。
-4. 注入单个不存在的对象时,其他文件列表可用;文件获取仍报告真实失败。
-5. 请求失败显示错误和重试;切换筛选或目录后旧响应不能覆盖当前结果。
-
-前端源码变化必须随桌面客户端发行才对已安装客户端生效。完成源码构建不代表已经发布,
-也不代表现有生产数据已修改;发布记录应注明预发、生产和客户端各自的验证结果。
diff --git a/document/zh-CN/deployment/desktop-local-mode.md b/document/zh-CN/deployment/desktop-local-mode.md
index 9add6669..2459a1ff 100644
--- a/document/zh-CN/deployment/desktop-local-mode.md
+++ b/document/zh-CN/deployment/desktop-local-mode.md
@@ -1,7 +1,6 @@
# 桌面端离线本机模式
-> 适用于 Windows x86_64、macOS Apple Silicon / Intel、Linux x86_64;UOS 1070
-> aarch64 使用独立的 [Electron 43 桌面发行线](uos1070-desktop.md)。
+> 适用于 Windows x86_64、macOS Apple Silicon / Intel、Linux x86_64。
桌面端默认发行“完整离线包”。安装包同时携带当前平台的 CE 服务源码和私有 CPython 3.11
运行时,最终用户不需要安装 Python、uv、pip、编译器或 Docker,也不需要在首次启动时访问
diff --git a/document/zh-CN/deployment/docker-compose.md b/document/zh-CN/deployment/docker-compose.md
index d16978e4..8ede9286 100644
--- a/document/zh-CN/deployment/docker-compose.md
+++ b/document/zh-CN/deployment/docker-compose.md
@@ -14,7 +14,7 @@ CE 的 Docker 部署默认使用 OpenSandbox;无 Docker 安装及桌面本机
```bash
python3 scripts/configure-ce-docker.py
-docker build -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+docker build -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
docker compose build backend
docker compose up -d --build
```
@@ -65,8 +65,8 @@ CE 在空数据库首次启动时会全局安装并启用 `automation`、
| 服务 | profile | 容器名 | 镜像 / 构建 | 作用 |
|---|---|---|---|---|
-| `script-runner` | `script_runner` | hugagent-script-runner | `docker/Dockerfile.script-runner` | 轻量沙箱:1 GB 内存 / 1 CPU / read-only rootfs + tmpfs,技能目录只读挂载到 `/workspace/skills` |
-| `opensandbox-config-init` | `opensandbox` | hugagent-opensandbox-config-init | `python:3.12-alpine` | 一次性 init:把 `docker/opensandbox-config.toml.tpl` 中的 `@@HOST_REPO_PATH@@` / `@@HOST_STORAGE_PATH@@` 渲染进 named volume `opensandbox_config` |
+| `script-runner` | `script_runner` | hugagent-script-runner | `docker/sandbox/Dockerfile.script-runner` | 轻量沙箱:1 GB 内存 / 1 CPU / read-only rootfs + tmpfs,技能目录只读挂载到 `/workspace/skills` |
+| `opensandbox-config-init` | `opensandbox` | hugagent-opensandbox-config-init | `python:3.12-alpine` | 一次性 init:把 `docker/opensandbox-server/opensandbox-config.toml.tpl` 中的 `@@HOST_REPO_PATH@@` / `@@HOST_STORAGE_PATH@@` 渲染进 named volume `opensandbox_config` |
| `opensandbox` | `opensandbox` | hugagent-opensandbox | `hugagent-opensandbox-server:v0.1.13-3` | CE / EE 共享控制器:经 Docker socket 管理沙盒;execd 执行命令并保留会话文件。CE 不映射宿主端口;全量仓库可配置调试端口 |
### mem0 记忆基础设施(profile `mem0`,可选)
@@ -214,19 +214,19 @@ make migrate-new msg="describe change"
| 功能 | 文件 |
|---|---|
-| 服务编排 | `docker-compose.yml`(CubeSandbox 叠加:`docker-compose.cube.yml`) |
+| 服务编排 | `docker-compose.yml`(CubeSandbox 叠加:`docker/sandbox/compose.cube.yml`) |
| 后端镜像 | `docker/Dockerfile`(multi-stage,target `production`) |
| MCP 镜像 | `docker/Dockerfile.mcp` |
-| 沙箱镜像 | `docker/Dockerfile.script-runner`、`docker/Dockerfile.opensandbox` |
+| 沙箱镜像 | `docker/sandbox/Dockerfile.script-runner`、`docker/sandbox/Dockerfile.opensandbox` |
| 前端镜像 + nginx 反代 | `src/frontend/Dockerfile`、`src/frontend/nginx.conf`、`src/frontend/default.conf.template` |
| 后端入口(自动迁移) | `src/backend/scripts/backend_entrypoint.sh` |
-| OpenSandbox 配置模板 | `docker/opensandbox-config.toml.tpl` |
+| OpenSandbox 配置模板 | `docker/opensandbox-server/opensandbox-config.toml.tpl` |
| 迁移脚本 | `alembic.ini`、`src/backend/alembic/` |
| MCP 端口单一真源 | `src/backend/extensions/mcp_servers/_ports.py` |
### OpenSandbox 命令延迟与控制器镜像
-控制器使用 `docker/Dockerfile.opensandbox-server` 构建
+控制器使用 `docker/opensandbox-server/Dockerfile.opensandbox-server` 构建
`hugagent-opensandbox-server:v0.1.13-3`。它基于固定的上游版本,对已知沙盒使用
按名称 inspect 并校验归属标签,且在线程池中执行生命周期与代理的同步 Docker 查询,
避免阻塞其它请求。构建时先校验上游源文件摘要并运行回归测试;升级上游版本时须重新审查补丁。
@@ -239,7 +239,7 @@ make migrate-new msg="describe change"
## 站点数据与浏览器部署检查
-普通 Compose 启动不启用应用数据托管,也不重建 OpenSandbox 按需运行镜像。需要这两项能力时,部署前分别完成[应用数据升级](../modules/application-hosting.md#外部检验修复与升级)和[浏览器运行镜像检查](../modules/browser-automation.md)。配置不同的 APPLICATION_DB_PASSWORD 与 APPLICATION_OWNER_DB_PASSWORD、共享 Redis 和适合网络环境的 BROWSER_DNS_RESOLVER_URL;使用 application_owner 作为应用数据运行账号。已有数据库卷也必须重新执行 application-database-init,然后更新后端、MCP、前端及沙箱运行镜像,并创建新浏览器会话进行验证。
+普通 Compose 启动不启用应用数据托管,也不重建 OpenSandbox 按需运行镜像。需要这两项能力时,部署前分别完成[应用数据升级](../modules/application-hosting.md#部署要求与数据安全)和[浏览器运行镜像检查](../modules/browser-automation.md)。配置不同的 APPLICATION_DB_PASSWORD 与 APPLICATION_OWNER_DB_PASSWORD、共享 Redis 和适合网络环境的 BROWSER_DNS_RESOLVER_URL;使用 application_owner 作为应用数据运行账号。已有数据库卷也必须重新执行 application-database-init,然后更新后端、MCP、前端及沙箱运行镜像,并创建新浏览器会话进行验证。
### WSL 内存紧张时限制构建资源
@@ -254,11 +254,11 @@ make migrate-new msg="describe change"
BUILDX_BUILDER=hugagent-limited COMPOSE_PARALLEL_LIMIT=1 docker compose build mcp
BUILDX_BUILDER=hugagent-limited COMPOSE_PARALLEL_LIMIT=1 docker compose build frontend
docker buildx build --builder hugagent-limited --load \
- -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
+ -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .
docker buildx stop hugagent-limited
builder 已存在时直接复用。构建结束后停止专用 builder,保留缓存供下次使用。
-应用托管部署仍需叠加 docker-compose.application-hosting.yml。
+应用托管部署仍需叠加 docker/application-hosting/compose.application-hosting.yml。
不要同时运行完整构建和多组浏览器测试。WSL 上限应结合其他容器及编辑器的实际占用设置。
## 模型网关启动与排障(商业版 EE)
diff --git a/document/zh-CN/deployment/environment-variables.md b/document/zh-CN/deployment/environment-variables.md
index e83e6ab5..3361ef4b 100644
--- a/document/zh-CN/deployment/environment-variables.md
+++ b/document/zh-CN/deployment/environment-variables.md
@@ -23,7 +23,7 @@
| `HOST_REPO_PATH` | 仓库绝对路径 | 宿主机仓库根路径;backend 经 docker.sock 调宿主 daemon 重建沙盒镜像时解析 compose 相对路径用 | CE |
| `DOCKER_GID` | `999` | 宿主 docker 组 GID,授予 backend 容器读写 `/var/run/docker.sock`(`stat -c '%g' /var/run/docker.sock` 查询) | CE |
| `COMPOSE_PROFILES` | CE Docker: `opensandbox` | compose profile 选择(`script_runner` / `opensandbox`,可叠加 `,mem0`),须与 `SANDBOX_PROVIDER` 一致 | CE |
-| `COMPOSE_FILE` | (未设) | 叠加 compose 文件;CubeSandbox 时设 `docker-compose.yml:docker-compose.cube.yml` | CE |
+| `COMPOSE_FILE` | (未设) | 叠加 compose 文件;CubeSandbox 时设 `docker-compose.yml:docker/sandbox/compose.cube.yml` | CE |
| `MAX_REQUEST_SIZE` | `52428800`(代码默认 50 MB) | 后端单请求体上限(字节) | CE |
| `DB_MIGRATION_RETRIES` / `DB_MIGRATION_RETRY_INTERVAL` | `20` / `2`(入口脚本默认) | 启动迁移重试次数 / 间隔秒 | CE |
@@ -57,7 +57,7 @@
| `ENABLE_SUMMARY` | `true` | 关闭则标题回退为消息截断 | CE |
| `SUMMARY_MAX_ROUNDS` | `3` | 超过该轮数后标题不再更新 | CE |
| `OPENAI_API_KEY` / `OPENAI_API_BASE` | (空)/ `https://api.openai.com/v1` | 备用 OpenAI 直连配置(compose 透传) | CE |
-| `ROUTER_STRATEGY` | `main_only`(代码默认) | 路由策略(`orchestration/strategy.py`) | CE |
+| `ROUTER_STRATEGY` | `main_only`(代码默认) | 路由策略(`core/agent_runtime/strategy.py`) | CE |
| `FOLLOWUP_ENABLED` | `true`(代码默认) | 追问建议生成开关 | CE |
## 鉴权与登录
@@ -217,13 +217,13 @@ Config 管理台会分别保存 Dify、FastGPT、WeKnora 的 URL、API Key 与
|---|---|---|
| `OPENSANDBOX_DOMAIN` | `http://opensandbox:8080` | OpenSandbox server 地址 |
| `OPENSANDBOX_API_KEY` | CE 初始化生成 | 控制器与后端使用相同密钥;不在日志中输出 |
-| `OPENSANDBOX_IMAGE` | `hugagent-opensandbox-custom:latest`(compose 默认;代码兜底 `opensandbox/code-interpreter:v1.0.2`) | 沙盒运行时镜像;自建镜像预装项目全部依赖(`docker/Dockerfile.opensandbox`) |
+| `OPENSANDBOX_IMAGE` | `hugagent-opensandbox-custom:latest`(compose 默认;代码兜底 `opensandbox/code-interpreter:v1.0.2`) | 沙盒运行时镜像;自建镜像预装项目全部依赖(`docker/sandbox/Dockerfile.opensandbox`) |
| `OPENSANDBOX_READY_TIMEOUT_S` | `90` | 等沙盒就绪上限(秒) |
| `OPENSANDBOX_REQUEST_TIMEOUT_S` | `120` | 单次 HTTP 调用超时(秒) |
| `OPENSANDBOX_PORT` | 全量仓库 `8910`;CE 不映射 | 控制器容器内固定 8080 |
| `OPENSANDBOX_POOL_JUPYTER_MIN_IDLE` / `MAX_IDLE` | `.env.example` 1/3;compose 默认 2/3 | 用户会话池(历史配置名);CE 最小预热数为 0 |
| `OPENSANDBOX_POOL_LIGHT_MIN_IDLE` / `MAX_IDLE` | `2` / `5` | 轻量桶(一次性执行) |
-| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | 全池上限(含使用中) |
+| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | 控制器全局上限(所有 worker、预热与使用中实例);backend 和控制器使用同一值 |
| `OPENSANDBOX_SNAPSHOT_ENABLED` | `true` | 快照持久化总开关(idle 时 snapshot+kill,重连恢复) |
| `OPENSANDBOX_SNAPSHOT_RETENTION_DAYS` | `7` | 快照保留天数(GC 周期清理) |
| `OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S` | `120` | 等快照 Ready 的轮询上限(秒) |
@@ -317,16 +317,10 @@ Config 管理台会分别保存 Dify、FastGPT、WeKnora 的 URL、API Key 与
License 细节见 [License 机制](../editions/license.md)。
-## 其他(限流、熔断、安全、告警)
+## 其他(安全、告警)
| 变量 | 默认值 | 说明 | 版本 |
|---|---|---|---|
-| `RATE_LIMIT_ENABLED` | `true` | 接口限流开关 | CE |
-| `RATE_LIMIT_STORAGE` | `memory://` | 限流计数存储(可 `redis://...`) | CE |
-| `RATE_LIMIT_GLOBAL` / `RATE_LIMIT_PER_USER` | `500/minute` / `50/minute` | 全局 / 单用户限流 | CE |
-| `CB_USER_CENTER_THRESHOLD` / `CB_USER_CENTER_TIMEOUT` | `5` / `60` | 用户中心熔断器 | EE |
-| `CB_MODEL_API_THRESHOLD` / `CB_MODEL_API_TIMEOUT` | `10` / `30` | 模型 API 熔断器 | CE |
-| `CB_STORAGE_THRESHOLD` / `CB_STORAGE_TIMEOUT` | `5` / `60` | 存储熔断器 | CE |
| `CORS_ORIGINS` | compose 默认 `http://localhost:3000,http://localhost:5173` | 允许跨域来源 | CE |
| `ENABLE_SECURITY_HEADERS` | (注释,建议生产 `true`) | 安全响应头 | CE |
| `AUDIT_LOG_RETENTION_DAYS` / `AUDIT_LOG_EXPORT_ENABLED` | `90` / `true`(注释示例) | 审计日志保留 / 导出 | EE |
@@ -340,7 +334,7 @@ License 细节见 [License 机制](../editions/license.md)。
|---|---|
| 变量样例(部署起点) | `.env.example` |
| 集中式设置读取(唯一入口) | `src/backend/core/config/settings.py` |
-| compose 层注入与默认值 | `docker-compose.yml`、`docker-compose.cube.yml` |
+| compose 层注入与默认值 | `docker-compose.yml`、`docker/sandbox/compose.cube.yml` |
| 沙箱 provider 选择 | `src/backend/core/sandbox/`、`settings.py::SandboxSettings` |
| 记忆设置 | `settings.py::MemorySettings`、`src/backend/core/memory/`(service.py / pipeline.py) |
| 问题反馈 GitHub 配置 | `src/backend/edition_ee/services/system_config.py`、`feedback_service.py` |
@@ -389,7 +383,7 @@ License 细节见 [License 机制](../editions/license.md)。
模块开启后,社区版已存在的登录用户即可使用。企业版额外检查 Config 后台的用户、角色或团队 **Personal Agent** 权限(默认关闭);收回权限会隐藏入口,并停止排队、定时触发和渠道转交任务。不需要 persistent_sandbox License。两个版本都保留对象归属检查和操作批准。
-请阅读 [Personal Agent](../modules/personal-agent.md)。部署时使用根 Compose 文件、docker/compose.personal-agent.yml 和 personal-agent profile。
+请阅读 [Personal Agent](../modules/personal-agent.md)。部署时使用根 Compose 文件、docker/personal-agent/compose.personal-agent.yml 和 personal-agent profile。
diff --git a/document/zh-CN/deployment/offline-production.md b/document/zh-CN/deployment/offline-production.md
deleted file mode 100644
index db13e3e6..00000000
--- a/document/zh-CN/deployment/offline-production.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# 离线生产部署(商业版 EE)
-
-> 最后更新:2026-10-16 | [English](../../en/deployment/offline-production.md) | 返回 [部署指南](README.md)
-
-离线交付使用 `deploy` 分支的编译加固镜像及配套部署 kit。
-构建和打包在供应方的独立 `deploy` 工作区执行;目标机加载镜像并使用同批 kit 启动。
-构建、分支同步、上传和生产发布分别遵守项目授权规则。
-
-## 联网侧:准备完整交付包
-
-1. 核对已验证的 `deploy` 提交、干净工作区、产品身份及目标 CPU 架构。
-2. 完整读取该提交的 `deploy_kit/save_pack_ee.sh` 和配套 Compose 文件。
-3. 构建并打包:
-
- ```bash
- # 在 deploy 分支的工作区根目录执行
- bash deploy_kit/save_pack_ee.sh --build
- # 编译镜像已存在且已核对本次提交时,可直接打包
- bash deploy_kit/save_pack_ee.sh
- ```
-
-脚本校验编译后端、MCP、前端及必备基础设施镜像,并按本机存在情况加入可选镜像。
-交付方还须核对目标 profiles 的全部依赖,不能把“可选镜像未安装”当作目标机不需要它。
-镜像名、版本、输出目录与文件名以该提交的脚本和 manifest 为准,不手工改名。
-交付包同时包含镜像归档、manifest 和匹配的部署 kit;License 单独安全交付。
-不得用普通源码镜像或只包含后端和前端的增量包替代完整 EE 交付包。
-
-## 目标侧:加载与上线
-
-按交付 kit 的说明配置独立项目名、数据目录、License 和必要的受保护环境文件。
-升级前备份数据库与文件,确认镜像 manifest、目标架构和所选 profiles 一致。
-授权后加载指定镜像归档,并使用同批 kit 的启动入口及完整 Compose 覆盖文件。
-目标机不能联网拉镜像时,必须先确认所需镜像已全部加载,启动不得触发现场构建。
-不使用旧源码部署脚本,也不自动选取目录里时间最新但未经核验的归档。
-
-验收至少包括 `/health`、`/ready`、登录、附件、模型调用及实际启用的沙箱能力。
-数据库迁移会改变 schema;数据卷持久化不代表升级不会修改业务数据。
-失败时按交付记录中的镜像、配置和数据备份恢复,不删除数据卷。
-
-## 提示词快照随包带入
-
-系统提示词存在数据库 `content_blocks` 表(`prompt_versions` 版本池 + `prompt_hub` 提示词广场),**不进仓库、不在镜像里**。要把新提示词推上生产,需作为单独数据文件随镜像包带入并在生产侧导入:
-
-### 1. 联网侧导出
-
-```bash
-python src/backend/scripts/export_content.py \
- --api-url http://localhost:3001 --only prompts
-# 产出 src/backend/scripts/exported/prompts_snapshot_.json
-```
-
-(也支持 `--database-url` 直连库导出;`ADMIN_TOKEN` 自动从 `.env` 读取。)
-
-### 2. 拷贝
-
-把 `prompts_snapshot_.json` 与镜像 tarball 一起拷到生产机。
-
-### 3. 生产侧导入
-
-前提:生产 backend 镜像须已包含 `POST /v1/content/prompts/import` 接口(`src/backend/api/routes/v1/content.py`)。在新 backend 起好后:
-
-```bash
-BACKEND_CONTAINER=your-backend-container
-docker cp prompts_snapshot_.json "${BACKEND_CONTAINER}:/tmp/"
-docker exec "$BACKEND_CONTAINER" curl -sS -X POST \
- 'http://localhost:3001/v1/content/prompts/import?overwrite=true' \
- -H "Authorization: Bearer <生产ADMIN_TOKEN>" \
- -H 'Content-Type: application/json' \
- -d @/tmp/prompts_snapshot_.json
-```
-
-API 导入会自动失效提示词缓存,**无需重启 backend**。
-
-> 这是按需的一次性数据操作:DB 卷持久化,不必每次发版都导提示词——只在要把新提示词推上生产时执行。
-
-## 相关源码
-
-| 功能 | 文件 |
-|---|---|
-| EE 完整镜像包(deploy 分支) | `deploy_kit/save_pack_ee.sh` |
-| EE 编译镜像(deploy 分支) | `docker/Dockerfile.backend-compiled`、`docker/Dockerfile.mcp-compiled` |
-| Compose 覆盖(deploy 分支) | `docker-compose.prod.yml`、`docker-compose.deploy.yml`;运行时以交付 kit 为准 |
-| 提示词导出 / 导入 | `src/backend/scripts/export_content.py`、`src/backend/scripts/import_content.py` |
-| 提示词版本池 | `src/backend/core/services/prompt_version_service.py` |
diff --git a/document/zh-CN/deployment/quick-install.md b/document/zh-CN/deployment/quick-install.md
index 961e6099..d24ab26b 100644
--- a/document/zh-CN/deployment/quick-install.md
+++ b/document/zh-CN/deployment/quick-install.md
@@ -187,7 +187,7 @@ hugagent doctor # 环境自检(Python 版本、端口占用、数据目录
| CLI(onboard / serve / doctor) | `src/backend/cli.py` |
| 本地模式开关 | `src/backend/core/config/settings.py`(`DeploySettings`,`DEPLOY_PROFILE=local`) |
| 前端静态托管 + `/api` 桥接 | `src/backend/api/local_hosting.py` |
-| MCP / 沙箱子进程督管 | `src/backend/orchestration/local_subprocess.py` |
-| 进程内事件流 / 短时状态 | `src/backend/orchestration/run_event_stream.py`、`src/backend/core/infra/ephemeral.py`(`REDIS_URL` 留空即启用) |
-| 内置 MCP 目录种子 | `src/backend/core/services/mcp_service.py`(`seed_builtin_mcp_servers_if_empty`) |
+| MCP / 沙箱子进程督管 | `src/backend/core/infra/local_runtime/` |
+| 进程内事件流 / 短时状态 | `src/backend/core/runs/events/`、`src/backend/core/infra/ephemeral.py`(`REDIS_URL` 留空即启用) |
+| 内置 MCP 目录种子 | `src/backend/core/capabilities/mcp/service.py`(`seed_builtin_mcp_servers_if_empty`) |
| 环境变量参考 | [environment-variables.md](environment-variables.md)(「无 Docker 本地模式」段) |
diff --git a/document/zh-CN/deployment/space-files.md b/document/zh-CN/deployment/space-files.md
index ea24a7cc..571ec0cd 100644
--- a/document/zh-CN/deployment/space-files.md
+++ b/document/zh-CN/deployment/space-files.md
@@ -31,19 +31,11 @@ S3 服务部署在本机时,请使用容器可达的服务地址。
控制器不再持有 FUSE 连接,无需共享传播目录或宿主挂载准备脚本。
沙盒文件操作仍使用普通用户。
-## 文件导入
+## 存量文件迁移
-先备份数据库和对象数据。暂停对目标目录的写入。
-对每个用户或团队生成导入清单:
-
-```bash
-PYTHONPATH=src/backend python -m scripts.import_space_files --user USER_ID
-PYTHONPATH=src/backend python -m scripts.import_space_files --team TEAM_ID
-```
-
-确认目标目录和文件清单后,追加 `--apply` 执行导入。
-工具拒绝同名冲突,并校验复制后的内容。
-工具保留源对象和文件引用。导入中断后可重新运行。
+独立文件导入命令已移除。迁移存量文件前,先备份数据库和对象数据,
+暂停目标目录写入,制定并审核迁移方案。核对目标路径、同名冲突和文件内容,
+保留源对象及文件引用。
## 验证
@@ -52,20 +44,16 @@ PYTHONPATH=src/backend python -m scripts.import_space_files --team TEAM_ID
撤回团队权限。确认该用户无法访问该团队目录。
验证成功后恢复用户操作。
+## 限制
-## 验证范围
-
-S3 挂载通过本地独立 MinIO、控制器、沙盒和浏览器验证。
-测试覆盖上传、下载、目录列表、只读写入拒绝、目录隔离和挂载撤销。
-OSS 和 OBS 适配尚需在各自的测试桶中验证。
-对象存储目录移动包含多个对象操作。它不是原子的文件系统重命名。
+对象存储目录移动包含多个对象操作,不是原子的文件系统重命名。
OBS 的 V4 配置依据[华为云说明](https://support.huaweicloud.com/obs_faq/obs_faq_0274.html)。
OSS 的 S3 V2 签名依据[阿里云说明](https://www.alibabacloud.com/help/tc/oss/user-guide/0002-00000307)。
## 本机 S3 服务(CE / EE)
-将 `docker/compose.object-storage.yml` 加入 `COMPOSE_FILE`。
+将 `docker/object-storage/compose.object-storage.yml` 加入 `COMPOSE_FILE`。
该服务使用独立数据卷。它不发布宿主端口,也不使用 Milvus 的数据卷。
配置 `S3_ENDPOINT=http://object-storage:9000` 和 `S3_BUCKET=hugagent-files`。
分别生成随机 `S3_ACCESS_KEY` 与 `S3_SECRET_KEY`,写入权限为 600 的环境文件。
@@ -122,4 +110,4 @@ MCP 接口位于 `src/backend/extensions/mcp_servers/space_files_mcp/`。
缺失、过期或参数变更的票据不能发出网关请求。
桌面升级需同时交付新版内置后端和云端后端,并重新同步能力清单。
-对象存储镜像使用 MinIO 官方 GitHub Release 中固定 SHA-256 的二进制文件。Docker 通过 docker/Dockerfile.object-storage 构建此镜像。已有 Milvus 镜像和数据卷保持独立。
+对象存储镜像使用 MinIO 官方 GitHub Release 中固定 SHA-256 的二进制文件。Docker 通过 docker/object-storage/Dockerfile.object-storage 构建此镜像。已有 Milvus 镜像和数据卷保持独立。
diff --git a/document/zh-CN/deployment/uos1070-desktop.md b/document/zh-CN/deployment/uos1070-desktop.md
deleted file mode 100644
index ea0c741e..00000000
--- a/document/zh-CN/deployment/uos1070-desktop.md
+++ /dev/null
@@ -1,87 +0,0 @@
-# UOS 统信 1070 aarch64 桌面端
-
-> [English](../../en/deployment/uos1070-desktop.md)
-
-UOS 1070 aarch64 使用仓库中的 `desktop-uos/` Electron 43 发行线。交付格式固定为
-Debian ARM64 `.deb`;它不依赖系统 WebKitGTK 4.1,因此不受 Tauri 2 在 Debian 10 / UOS
-1070 上的 WebView 依赖阻断。
-
-## 支持范围
-
-| 项目 | 生产基线 |
-|---|---|
-| 操作系统 | UOS 1070 桌面版,aarch64 |
-| libc | glibc 2.28 |
-| 桌面壳 | Electron 43.4.1 |
-| 安装包 | Debian `arm64` `.deb` |
-| 本机运行时 | 私有 CPython 3.11,`aarch64-manylinux_2_28` 锁 |
-| 数据目录 | `~/.hugagent` |
-| 配置目录 | `~/.config/com.hugagent.desktop` |
-
-完整包支持本机、云端和双模式,能力与现有桌面端一致:deep-link 登录、回环反代、流式
-对话、托盘/快捷问答、原生通知、本地文件夹项目、双上游路由、身份与模型配置桥接、离线
-本机服务安装/升级/回滚、SHA-256 验证更新。
-
-UOS 默认隐藏窗口内菜单栏,保留系统标题栏与窗口控制。托盘的“文件、编辑、视图、帮助”
-提供完整桌面命令;Ctrl+Shift+N 新建窗口、Ctrl+N 新建对话继续可用。工作区与 macOS
-共用模块导航渐变、内层侧栏与内容圆角、浅深主题和加载布局;快速问答保持原有紧凑布局。
-UOS 的原生标题栏位于网页区域外,因此不再额外占用网页顶部安全区。
-
-发行品牌可以为完整包预设首启模式;固定双模式的发行版会直接进入一键初始化。初始化与
-安装页使用与现有桌面端一致的光晕、轨道和进度高光动画,只显示阶段、百分比与可操作错误,
-不在页面底部展示原始日志。日志仍保存在本机服务日志目录,便于排障。
-
-## 构建与验证
-
-完整包只能在原生 Linux aarch64 构建机生成,推荐直接使用 UOS 1070 构建机。x86_64
-机器只能交叉生成不含 Python 运行时的精简包。
-
-```bash
-cd desktop-uos
-npm install
-npm test
-npm run build
-npm run verify:deb -- "dist/HugAgentOS UOS_0.5.15_uos1070_arm64.deb"
-```
-
-验收时至少检查:
-
-1. `dpkg-deb --field <包> Architecture` 返回 `arm64`;
-2. Electron 主 ELF 的最高 GLIBC 引用不超过 2.28;
-3. 断网冷启动完整包,本机服务安装阶段没有 `pip install`、uv 或 Python 下载;
-4. 本机、云端和双模式分别完成登录、SSE 对话、附件上传下载、文件夹选择和通知;
-5. 双模式下云端项目进云端,本地文件夹项目进本机;
-6. 使用测试版更新清单完成 SHA-256 校验、系统授权安装和重启。
-
-UOS 与 Windows、macOS 共用同一个发布索引:客户端按自己的平台取更新,某个平台还没发包
-时明确提示「当前平台暂无可用更新」,而不带平台参数的旧客户端读到的共同版本要等该版本
-三个平台都发齐才前进。桌面端版本号三平台保持一致,构建时会校验。
-
-具体命令、产物和发布步骤见 [`desktop-uos/README.md`](../../../desktop-uos/README.md)。
-
-> Electron 43 官方支持计划在 2027 年 1 月结束。它是 UOS 1070 的当前兼容基线,生产
-> 维护仍需跟进最新 43.x 补丁,并持续验证 Electron 44+ 后续迁移。
-
-## 与当前 Tauri 功能同步
-
-UOS 复用当前源码中的 React 前端与 CE 本机后端。重新构建完整包会带入最新能力来源、
-本机启停、技能同步和工作目录逻辑;只更新云端不会替换已安装客户端的这些资源。
-
-- “文件”菜单支持打开文件夹并进入本机项目对话;普通包可重新选择运行模式。
-- “视图”提供放大、缩小、实际大小(Ctrl + +/-/0,或 Ctrl+滚轮)。缩放档位与 Tauri
- 一致,保存在 `prefs.json`,新窗口和重启沿用;系统 DPI 保持原生处理。
-- “帮助”、托盘和侧栏共用更新流程。后台订阅发布通知,仅提示新版;确认后显示下载进度,
- 下载阶段可取消。安装仍校验 Ed25519 签名和 SHA-256,并由 UOS 系统授权。
- 旧服务器使用低频检查,断线退避重连;并发检查不会重复安装。
-- 回环反代在注入会话和桥接凭据前检查 Origin/Fetch Metadata;不带来源的直接请求被拒绝。
-- 服务日志运行中轮转,每份最大 32 MiB,只保留当前和上一份。能力未就绪时的检查间隔
- 从 500 ms 逐步增加至 15 秒。升级保留原 UOS 数据及能力目录,不自动迁移已有配置。
-
-完整包还支持与 Tauri 相同的仅混合模式构建入口:
-
-```bash
-JX_DEFAULT_SERVER_BASE=https://agent.example.com npm run build:hybrid-only
-```
-
-该入口固定云端地址与双模式,必须提供云端地址;精简包不能用于仅混合模式。
-完整 ARM64 离线包仍须在原生 aarch64 构建机完成构建和安装验收。
diff --git a/document/zh-CN/development/backend.md b/document/zh-CN/development/backend.md
index d8c53c55..8e8c4b21 100644
--- a/document/zh-CN/development/backend.md
+++ b/document/zh-CN/development/backend.md
@@ -41,7 +41,7 @@ docker-compose logs -f backend
| `make migrate` | `alembic upgrade head` |
| `make migrate-new msg="..."` | `alembic revision --autogenerate -m "..."` |
| `make migrate-down` / `migrate-history` | 回滚一步 / 查看历史 |
-| `make db-reset` / `db-seed` | 重置(破坏性)/ 种子数据 |
+| `make db-reset` | 重置数据库(破坏性) |
| `make build` / `up` / `down` / `logs` / `ps` 等 | docker-compose 包装 |
> ⚠️ 已提交代码并非 formatter-clean:**不要对未改动的文件整体跑 black/isort**(会把真实 diff 淹没在格式化噪音里),只格式化自己改的范围。
@@ -68,10 +68,10 @@ PYTHONPATH=src/backend python -m pytest src/backend/tests/memory -q
## Alembic 迁移流程
-1. 在 `src/backend/core/db/models.py` 改 ORM 模型;
+1. 在 `src/backend/core/db/models/` 改 ORM 模型;
2. `make migrate-new msg="add xxx table"` 自动生成迁移(**人工审阅生成结果**,autogenerate 不可全信);
3. `make migrate`(或容器内 `alembic upgrade head`)应用;
-4. 若是 **EE 专属表**:把表名加进 `core/db/edition_tables.py::EE_ONLY_TABLES`(CE 不建 EE 空表,集合内表名有存在性断言)。
+4. 若是 **EE 专属表**:把表名加进 `edition_ee/db/edition_tables.py::EE_ONLY_TABLES`(CE 不建 EE 空表,集合内表名有存在性断言)。
注意:主仓 alembic 链是 EE 链;CE 派生树用独立的 `ce_0001` 基线链(见 [CE 构建管线](../editions/build-ce.md#ce-数据库差异)),主仓新增迁移不会进入 CE。
@@ -175,7 +175,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest
## 新增技能(Agent Skill)步骤
-技能加载是多源架构(`core/agent_skills/config.py`):内置(`extensions/skills/default/`,always-on)、admin(DB / `/app/storage/admin_skills/`)、用户、项目四个加载源按优先级合并;`extensions/skills/marketplace/` 是安装制技能市场种子,由 marketplace 服务单独扫描,**不在默认加载源内**。
+技能加载是多源架构(`core/capabilities/skills/config.py`):内置(`extensions/skills/default/`,always-on)、admin(DB / `/app/storage/admin_skills/`)、用户、项目四个加载源按优先级合并;`extensions/skills/marketplace/` 是安装制技能市场种子,由 marketplace 服务单独扫描,**不在默认加载源内**。
新增**内置技能**:
@@ -188,7 +188,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest
## 新功能检查清单
-- [ ] ORM 模型 + 索引 + 时间戳(`core/db/models.py`),EE 表已加 `EE_ONLY_TABLES`
+- [ ] ORM 模型 + 索引 + 时间戳(`core/db/models/`),EE 表已加 `EE_ONLY_TABLES`
- [ ] Repository 过滤软删除;Service 承载业务与权限
- [ ] 路由用信封响应、异常走 `core/infra/exceptions`
- [ ] 路由已登记 `CE_ROUTERS` / `EE_ROUTERS`(EE 配能力位 + manifest 排除)
@@ -205,10 +205,10 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest
| 响应信封 | `src/backend/core/infra/responses.py` |
| 异常体系 | `src/backend/core/infra/exceptions.py` |
| 依赖注入 | `src/backend/api/deps.py` |
-| ORM / 建表边界 | `src/backend/core/db/models.py`、`src/backend/core/db/edition_tables.py` |
+| ORM / 建表边界 | `src/backend/core/db/models/`、`src/backend/edition_ee/db/edition_tables.py` |
| MCP 端口表 / 连接配置 | `src/backend/extensions/mcp_servers/_ports.py`、`src/backend/core/config/mcp_config.py` |
| 能力目录 | `src/backend/core/config/catalog.json` |
-| 技能加载 | `src/backend/core/agent_skills/`(`config.py` / `loader.py` / `registry.py`) |
-| 工作流编排 | `src/backend/orchestration/workflow.py` |
+| 技能加载 | `src/backend/core/capabilities/skills/`(`config.py` / `loader.py` / `registry.py`) |
+| 工作流编排 | `src/backend/core/agent_runtime/runner.py` |
相关阅读:[前端开发指南](frontend.md) · [API 总览](../api/overview.md) · [MCP 工具](../modules/mcp-tools.md) · [技能系统](../modules/agent-skills.md)
diff --git a/document/zh-CN/development/container-layout.md b/document/zh-CN/development/container-layout.md
deleted file mode 100644
index 3eb59e90..00000000
--- a/document/zh-CN/development/container-layout.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# 容器响应式布局
-
-聊天工具栏跟随所在面板的可用宽度调整,首页、对话页、项目页和分屏共用同一套布局。
-
-- 工具区与模型/发送区分组,空间不足时自然换行。
-- 标签逐级收起,保留图标、可访问名称和模式退出按钮;模型名称保留并省略长文本。
-- 触屏使用真实的 44px 点击区域,参与排版,不叠加透明命中层。
-- 项目、能力目录和知识库卡片按可用宽度自动排列,长名称不会撑破网格。
-- 系统安全区、页面高度与弹出菜单仍按其实际视口处理。
-
-实现入口:前端的 components/chat/ComposerToolbar.tsx、styles/chat/toolbar-layout.css、
-styles/projects.css 和 styles/catalog/。容器边界负责提供宽度,后代负责响应;
-不要让元素查询自身,也不要通过 JavaScript 重复维护 CSS 断点。
-
-在 WSL 项目目录进入 src/frontend,使用项目支持的 Node.js 版本执行:
-
- npm run test:container-layout-browser
- npm run test:canvas-layout-browser
- npm run test:chat-width-browser
- npm run lint
- npm run build
-
-新增用例覆盖固定大窗口中的窄容器、手机窗口、分数像素边界、深浅主题、
-长名称、全部模式、触屏目标、键盘退出及发送/中止。
diff --git a/document/zh-CN/development/frontend.md b/document/zh-CN/development/frontend.md
index 93e5c6cf..9c2a8d93 100644
--- a/document/zh-CN/development/frontend.md
+++ b/document/zh-CN/development/frontend.md
@@ -132,6 +132,13 @@ const multiTenancy = useEditionStore((s) => (s.loaded ? !!s.features.multi_tenan
- 新增 EE 界面区块时:① 用上面的选择器按对应能力位门控;② 若整个页面属 EE(如 /admin、/config),无需 runtime 门控——CE 派生树直接物理排除组件目录并用 overlay 的 `main.tsx` 摘掉路由(见 [CE 构建管线](../editions/build-ce.md));
- 处理接口报错时记得识别 `LicenseError`(402),给出「联系管理员更新 license」而非笼统失败提示。
+## 容器响应式布局
+
+聊天工具栏、项目 / 能力目录 / 知识库卡片按所在面板的可用宽度排版,首页、对话页、项目页和分屏共用同一套规则:
+容器提供宽度,后代按容器响应;不要让元素查询自身,也不要用 JavaScript 重复维护 CSS 断点。触屏点击区域为真实的 44px。
+入口:`components/chat/ComposerToolbar.tsx`、`styles/chat/toolbar-layout.css`、`styles/projects.css`、`styles/catalog/`。
+修改后运行 `npm run test:container-layout-browser`、`npm run test:chat-width-browser`。
+
## 新功能检查清单
- [ ] 组件 PascalCase + named export + Props 接口
diff --git a/document/zh-CN/development/workbench-migration.md b/document/zh-CN/development/workbench-migration.md
index f1593a59..5585d389 100644
--- a/document/zh-CN/development/workbench-migration.md
+++ b/document/zh-CN/development/workbench-migration.md
@@ -44,24 +44,7 @@ SDK CLI 只校验声明,不迁移数据库或业务数据。
# 工作台配置迁移与恢复
-迁移是显式运维操作,不会随启动自动执行。先在隔离数据库副本完成演练;真实配置迁移需要单独授权。当前命令只接受本机隔离副本。
-
-## 盘点与切换
-
-1. 停止副本的所有写入,包括内容导入、用户偏好、插件安装和后台写入。--writes-frozen 只是操作员确认,不会自动停止服务。
-2. 配置 WB20_COPY_DATABASE_URL。SQLite 文件须在当前源码目录的 .scratch/workbench-20 下。PostgreSQL 须为回环主机且数据库名以 workbench_20 开头。不要将真实库设置为演练目标。
-3. 创建权限为 0700 的快照目录。运行如下命令:
-
-```bash
-PYTHONPATH=src/backend python -m scripts.workbench_migration inventory \
- --snapshot /absolute/private/path/before.json
-PYTHONPATH=src/backend python -m scripts.workbench_migration apply \
- --snapshot /absolute/private/path/before.json --digest --writes-frozen
-```
-
-盘点不写数据库。报告包含数量、来源版本、脱敏位置和错误码;不打印 URL、用户标识、票据或秘密。完整迁移范围快照保存在 0600 文件中,包含原配置、偏好、角色/团队关系、插件声明和工作台数据。它不是业务文件或完整数据库备份,须与完整数据库/文件备份共同保管。
-
-摘要匹配且源未变化时,切换在单一事务中执行。失败回滚事务。切换前后快照的摘要记录在 workbench_migration:v1 receipt 中。原始 app_config 行和旧导航列表移除;新目录只读取规范来源。重复外部 ID 的全部副本、未知 ID、危险 URL 和未知字段隔离,不产生入口或授权。原始副本保留在受保护快照。
+独立迁移命令已移除。以下内容记录内部迁移与恢复约束,不构成可直接执行的运维流程。现有部署需要迁移时,应先备份并制定针对目标版本的方案,在隔离副本验证后单独授权执行。
## 保留的语义
@@ -69,7 +52,7 @@ PYTHONPATH=src/backend python -m scripts.workbench_migration apply \
- 外部条目顺序和启停保留。默认侧栏转换为策略。已有现代偏好优先;已确认旧 preferences.pinned_apps/app_order 转换为 web 偏好。null 继承,空列表明确取消固定。聊天排序、主题和其他偏好保持不变。
- 无法映射的权限引用保留原值并报告,不成为新授权。旧插件 UI v1/v2 不改写为 v3。不会按名称或 slug 绑定新安装。
- 旧认证配置 workbench_external_auth:legacy: 保持原 key。缺少目标侧适配时报告 auth_configuration_required。管理员须显式审核登录方式;真实旧目标尚未验证时,不得宣称外部登录无回归。不得恢复通用 SSO token 拼接。
-- 同服务恢复保留 workbench_origin_identity。克隆到另一服务须先在克隆副本重新盘点,再执行 clone-origin;旧插件 key 不自动映射为新来源授权。跨来源偏好和数据须另行核对。
+- 同服务恢复保留 workbench_origin_identity。克隆到另一服务须先在克隆副本重新盘点,再重建服务来源标识;旧插件 key 不自动映射为新来源授权。跨来源偏好和数据须另行核对。
## 对外版本窗口
@@ -77,13 +60,7 @@ content-v1 接口在工作台协议 1.x 内保留,协议 2.0 迁移前另行
## 恢复与新增数据
-先冻结全部写入,再执行:
-
-```bash
-PYTHONPATH=src/backend python -m scripts.workbench_migration rollback \
- --snapshot /absolute/private/path/before.json --digest \
- --delta /absolute/private/path/delta.json --writes-frozen
-```
+恢复前冻结全部写入,并按已审核的恢复方案操作。
恢复前导出受保护增量。已映射普通旧外链的修改可回放到旧格式。新身份、应用状态、资源、权限、插件、认证或策略变化不能安全表达时,返回 delta_not_representable 并保留新数据。不要仅恢复旧快照。继续运行已验证的新版本,核对增量并另行制定迁移方案。恢复 receipt 幂等;已恢复批次不能直接重复 apply,应创建新的审核批次。
diff --git a/document/zh-CN/development/workbench-routing.md b/document/zh-CN/development/workbench-routing.md
index 2c558825..40ab74c9 100644
--- a/document/zh-CN/development/workbench-routing.md
+++ b/document/zh-CN/development/workbench-routing.md
@@ -10,6 +10,6 @@
一个应用可以在二级视图中引用自己的 workspace 模块。启动接口只接受主模块及该应用声明的二级模块,不接受安装中其他未声明模块。
-此功能没有数据库结构迁移。旧 UI v1/v2、托管应用接口和旧业务地址保留原有契约。完整多平台端到端验证由集成验收报告记录;专项夹具通过不表示原生客户端已通过。
+此功能没有数据库结构迁移。旧 UI v1/v2、托管应用接口和旧业务地址保留原有契约。
目录为空时保留深链接并显示空状态。准确应用查询的网络或服务器故障显示重试,不误报为无权限或不存在。
diff --git a/document/zh-CN/editions/build-ce.md b/document/zh-CN/editions/build-ce.md
index ecd3efbb..966063fa 100644
--- a/document/zh-CN/editions/build-ce.md
+++ b/document/zh-CN/editions/build-ce.md
@@ -26,7 +26,7 @@ manifest 按处理顺序分为以下几段:
glob 模式(相对仓库根),命中即不拷贝。覆盖:
-- **后端 EE 模块**:完整的 `edition_ee/**` 实现根(团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成),以及云存储(`core/storage/s3.py`、`oss.py`)、Cube 沙箱 provider、记忆审计、技能蒸馏等 EE 服务;
+- **后端 EE 模块**:完整的 `edition_ee/**` 实现根(团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成),以及云存储(`core/files/storage/s3.py`、`oss.py`)、Cube 沙箱 provider、记忆审计、技能蒸馏等 EE 服务;
- **EE 路由**:`api/routes/v1/admin_*.py`、`config_*.py`、`audit.py`、`auth.py`、`team_files.py`、`service_configs.py`、`data_sources.py`、`db_metadata.py`、`gateway_*.py`;
- **行业 MCP**:`extensions/mcp_servers/query_database_mcp/**`、`ai_chain_information_mcp/**`;
- **行业专属前端与种子**:产业链 Canvas、资讯 / 企业画像渲染器、行业 API 客户端、图标资源,
@@ -81,13 +81,13 @@ CE 前端不按工具名猜测远程 MCP 的业务结构。用户接入的远程
| `.hugagent-edition` | 仅在派生后出现的机器可读 `ce` 标识;让发布工具区分派生 CE checkout 与生成器异常缺失的源代码 checkout |
| `.github/workflows/desktop-release.yml` | 公开 CE 桌面发版 workflow,包含 release tag / 版本前置门禁 |
| `src/backend/api/routes/v1/__init__.py` | CE 路由注册表;`EE_ROUTERS` 恒为空 |
-| `src/backend/core/auth/permissions_iface.py` | 单租户 owner-only 权限接口;不导出团队权限函数 |
-| `src/backend/core/services/artifact_edition.py` | 个人空间 artifact 作用域接口;不暴露团队字段、权限或仓储方法 |
+| `src/backend/core/auth/personal_permissions_iface.py` | 单租户 owner-only 权限接口;不导出团队权限函数 |
+| `src/backend/core/files/edition.py` | 个人空间 artifact 作用域接口;不暴露团队字段、权限或仓储方法 |
| `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | 个人空间工具、VFS 与恢复接口;组织空间实现不进入 CE |
-| `src/backend/core/config/edition_display_names.py` | CE 工具展示名;不包含团队工具名称 |
+| `src/backend/core/config/display_names.py` | CE 工具展示名;不包含团队工具名称 |
| `src/backend/core/memory/audit.py` | 记忆审计 no-op stub(同名接口、不落数据) |
| `src/backend/alembic/versions/ce_000{1,2}_*.py` | CE 独立迁移链基线与幂等 Schema 升级(见下节) |
-| `src/backend/core/services/edition_startup.py` | CE 启动接缝;为 EE 数据源恢复、蒸馏调度提供 no-op 实现,避免导入已裁除模块 |
+| `src/backend/orchestration/workers/edition_startup.py` | CE 启动接缝;为 EE 数据源恢复、蒸馏调度提供 no-op 实现,避免导入已裁除模块 |
| `src/backend/tests/ce_release/` | 可执行 CE 发布回归套件,覆盖登录、私有技能/MCP、数据库升级及启动/Compose 契约 |
| `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | 去除管理端点与组织字段后的 CE API 契约 |
| `src/backend/extensions/mcp_servers/_ports.py` | 7 个通用工具的端口表(EE 行业工具与已退役端口标注 reserved) |
@@ -183,7 +183,7 @@ EE 始终全量建表。维护规则:**新增 EE 专属模型必须放进 `edi
| overlay 目录 | `ce/overlay/` |
| 派生树版本标识 | `ce/overlay/.hugagent-edition` |
| 公开桌面发版 workflow | `ce/overlay/.github/workflows/desktop-release.yml` |
-| CE/EE 建表边界 | `src/backend/core/db/edition_tables.py` |
+| CE/EE 建表边界 | `src/backend/edition_ee/db/edition_tables.py` |
| 幂等 Schema 升级器 | `src/backend/core/db/schema_reconcile.py` |
| 启动建表 CE 分支 | `src/backend/core/db/engine.py::init_db` |
| CE 独立迁移链 | `ce/overlay/src/backend/alembic/versions/ce_000{1,2}_*.py` |
diff --git a/document/zh-CN/getting-started/introduction.md b/document/zh-CN/getting-started/introduction.md
index c937aa53..8193115f 100644
--- a/document/zh-CN/getting-started/introduction.md
+++ b/document/zh-CN/getting-started/introduction.md
@@ -1,93 +1,118 @@
# 产品简介
-> 最后更新:2026-07-02
+> 最后更新:2026-10-23 | [English](../../en/getting-started/introduction.md)
-HugAgentOS 是一个企业级 AI Agent 平台:以 ReAct 智能体为内核,把对话、工具调用(MCP)、技能、沙箱代码执行、长期记忆、知识库 RAG、自动化与批量处理整合在一套全容器化(Docker Compose)的系统里。后端基于 FastAPI + AgentScope 2.0,前端基于 React 19 + Vite + Zustand,所有能力通过统一的 catalog 注册表按需启停,开箱即是一个可私有化部署的完整产品,而非框架半成品。
+HugAgentOS 是一个开源、可自托管的 **Agent Harness**。模型负责思考,Harness 负责模型之外的一切:
+任务在哪里运行、断了怎么接上、工具和沙箱怎么用、什么操作需要人批准、经验怎么沉淀。
-平台分为**社区版(CE,开源)**与**商业版(EE)**两个形态:社区版面向个人与小团队,包含完整的对话、工具、技能、沙箱、三层记忆、自动化、批量、数据画布等能力;商业版在此之上叠加团队协作、SSO、审计合规、行业数据工具、持久沙箱、云存储与完整管理台等组织级能力。详细边界见 [版本对比](../editions/overview.md)。
+它的核心是 **Durable Agent**:你提出的问题在云端执行,而不是依赖你的浏览器或电脑。关掉网页、
+合上笔记本,任务继续推进;服务器重启或宕机,其他实例从上次保存的进度接着做;不确定是否已经
+执行的外部操作不会被自动重做。在此之上,HugAgentOS 提供一个 7×24 小时在线的
+**Personal Agent**,并以 **HugAgentOS Cloud** 与 **HugAgentOS Local** 两种形态运行。
+
+平台分为**社区版(CE,开源)**与**商业版(EE)**:社区版让个人完整运行 Harness 与
+Personal Agent;商业版补充团队协作、SSO、审计合规、行业数据工具等组织级能力。
+详细边界见[版本对比](../editions/overview.md)。
+
+## 运行形态
+
+| 形态 | 运行位置 | 数据与状态 | 适合 | 部署文档 |
+|---|---|---|---|---|
+| **HugAgentOS Cloud** | 你的服务器或私有云 | PostgreSQL、Redis、S3 兼容存储、OpenSandbox | 7×24 Personal Agent、长任务、IM 渠道、多设备访问 | [Docker Compose](../deployment/docker-compose.md) |
+| **HugAgentOS Local** | 你的电脑(Windows / macOS / Linux) | 本地 SQLite、Milvus Lite、本地文件 | 离线、敏感数据、直接处理本地项目 | [桌面离线本机模式](../deployment/desktop-local-mode.md) |
+| **本机 + 云端** | 云端为控制面,本机为执行面 | 云端项目存云端,本地项目留本机 | 云端常驻,同时操作本机电脑与项目 | [桌面离线本机模式](../deployment/desktop-local-mode.md) |
+| 一键安装 | 开发机 | SQLite、进程内状态、子进程沙箱 | 试用与二次开发 | [无 Docker 一键安装](../deployment/quick-install.md) |
+
+7×24 持久运行需要 HugAgentOS Cloud。一键安装不提供跨进程的租约接管,只适合试用。
## 核心特性总览
| 能力 | 说明 | 版本 |
|---|---|---|
-| 对话智能体 | SSE 流式对话、AgentScope 2.0 ReActAgent、计划模式(Plan Mode)子智能体、深度思考、`[ref:tool-N]` 引用溯源、会话标题自动摘要 | CE |
-| 子智能体 | 用户自建子智能体、@提及协作、路由策略(`ROUTER_STRATEGY`) | CE(版本管理/组织级智能体库为 EE) |
-| MCP 工具生态 | 9 个内置 MCP server(CE 7 个通用工具:联网搜索、网页抓取、图表生成、批量计划、自动化任务管理、技能管理、知识检索;EE 2 个行业工具:数仓查询、产业链信息),独立 `mcp` 容器以 streamable-http 提供;用户可自助接入远程 HTTP/SSE MCP | CE(产业链/企业画像/数仓等行业工具为 EE) |
-| 技能系统 | Agent Skills(SKILL.md + 脚本):内置技能包、管理台上传、技能市场浏览安装、技能蒸馏 | CE(技能审核/组织治理为 EE) |
-| 沙箱执行 | `bash` / `sandbox_put_artifact` / `sandbox_get_artifact` 工具,三种 provider 可切换:script_runner(轻量内置)、OpenSandbox(持久会话 + Jupyter 上下文 + 快照)、CubeSandbox(E2B 兼容 MicroVM) | CE 轻量沙箱;持久沙箱(会话保持/快照)为 EE |
-| 记忆系统 | mem0 三层记忆:L1 个人画像、L2 向量记忆(Milvus)、L3 知识图谱(Neo4j),跨会话注入与后台抽取 | CE(记忆审计为 EE) |
-| 知识库 RAG | 文档上传、分块、向量 + 关键词混合检索、私有知识库;EE 可追加公共库与 Dify 外部知识库 | CE(公共库与 Dify 对接为 EE) |
-| 自动化 | 定时任务 / Cron 调度 / Prompt 与计划自动化 / 失败重试(`orchestration/schedulers/`) | CE |
-| 数据画布 | 基于 Univer 的在线表格编辑 | CE 个人编辑;多人实时协同为 EE |
-| 项目空间 / 我的空间 | 个人项目(文件容量配额)、个人网盘(myspace)、收藏、会话分享 | CE(团队文件夹/权限矩阵为 EE) |
-| 管理台 | `/admin` 内容管理(技能、能力中心、版本说明)+ `/config` 系统配置台(提示词版本池、模型、用户/团队、计费、审计、安全) | CE 基础管理;完整管理台(用户/团队/计费/审计/安全)为 EE |
-
-## CE 与 EE
-
-一句话区分:**社区版让一个人把平台用到极致,商业版让一个组织规模化地用起来**——团队协作、SSO、RBAC、审计合规、行业数据工具、持久沙箱、云存储、white-label 属于商业版。完整功能矩阵与授权方式见 [版本对比](../editions/overview.md) 与 [License 机制](../editions/license.md)。
+| Durable Run | 每次请求是一条持久化的 Run,后台执行;断线续播、运行中追加指令;租约接管与启动对账;工具副作用记录意图与回执,结果未知不自动重放 | CE |
+| Personal Agent | 单一对话入口、并行后台任务、永久云电脑、定时任务与心跳、IM 渠道、操作审批、文件化记忆与 Dreaming、连接本机电脑 | CE(授权治理与跨设备会话管理为 EE) |
+| 自主循环 | 目标跨轮存活、做完为止;worker 干活、只读评审员验收,命令可判定的需求由 driver 机检;中断后可续跑 | CE |
+| 对话与计划模式 | SSE 流式、ReAct 工具编排、深度思考、计划模式、引用溯源、上下文压缩检查点、会话分叉 | CE |
+| 子智能体 | 内置 explorer / worker / reviewer,用户自建子智能体,自动路由或 `@` 提及协作 | CE(组织级智能体库为 EE) |
+| MCP 工具与插件 | 内置联网搜索、网页抓取、知识检索、图表、自动化、技能与插件管理等工具;MCP 市场、自建私有 MCP、插件市场、浏览器自动化插件 | CE(数仓查询与行业工具为 EE) |
+| Agent Skills | SKILL.md + 脚本,内置技能、技能市场与个人技能 | CE(技能审核与组织治理为 EE) |
+| 沙箱执行 | OpenSandbox 会话、环境复用与快照恢复,或本地子进程沙箱;工作流后台作业以受管进程运行,可按租约接管 | CE(CubeSandbox 与团队挂载为 EE) |
+| 记忆与个人进化 | L1 画像、L2 做法记忆(Milvus,可选)、L3 关系图谱(Neo4j,可选);持久 Outbox 写入;进化建议逐条审批 | CE(记忆审计为 EE) |
+| 知识库 | 文档分块、向量 + 关键词混合检索、私有知识库、知识库 Wiki | CE(公共库管理与 Dify 对接为 EE) |
+| 领域本体治理 | Domain Pack 导入与版本发布、工具执行前规则检查、高风险答案独立评审、演进草案人工审核 | CE |
+| 自动化 | 自然语言创建定时任务、Cron 调度、失败重试 | CE |
+| 站点与应用托管 | 对话建站并一键发布;应用独立数据库与 MCP 托管 | CE(项目转换后的站点协作为 EE) |
+| 数据画布与产物 | Univer 在线表格、图表、报告、Office 文件、网页等产物与产物中心 | CE 个人编辑;多人协同为 EE |
+| 项目空间 / 我的空间 | 个人项目、文件夹、收藏、会话分享;桌面端可绑定本地项目目录 | CE(团队文件夹与权限矩阵为 EE) |
+| 桌面客户端 | Tauri 2 客户端,自带私有运行时,支持仅本机、仅云端、本机 + 云端三种模式 | CE |
## 技术栈
-| 层 | 技术 | 备注 |
-|---|---|---|
-| 后端 | FastAPI + Uvicorn(Python 3.11) | `src/backend/api/app.py`,统一响应 envelope |
-| 智能体框架 | AgentScope 2.0(`agentscope==2.0.0`) | ReActAgent + 工具注册,`core/llm/factory/build.py` |
-| 前端 | React 19 + Vite 7 + Zustand 5 + Ant Design 6 | `src/frontend/`,nginx 容器托管并反代 `/api` |
-| 数据库 | PostgreSQL 15(生产)/ SQLite(本地调试兜底) | SQLAlchemy 2 + Alembic 迁移 |
-| 缓存 / 会话 | Redis 7 | 会话存储、流式 follower(Redis Streams) |
-| 向量库 | Milvus 2.4(mem0 profile) | L2 向量记忆与自建知识库检索 |
-| 图数据库 | Neo4j 5 Community(mem0 profile,可选) | L3 知识图谱记忆 |
-| 沙箱 | script_runner sidecar / 阿里 OpenSandbox / 腾讯 CubeSandbox | `core/sandbox/` provider 协议,env 切换 |
-| 部署 | Docker Compose(profiles:`script_runner` / `opensandbox` / `mem0`) | 全部服务容器化,无本地 dev server |
+| 层 | 技术 |
+|---|---|
+| 智能体运行时 | AgentScope 2.0(ReActAgent)、Model Context Protocol |
+| 后端 | Python 3.11、FastAPI、SQLAlchemy 2、Alembic |
+| 前端 | React 19、TypeScript、Vite、Zustand、Ant Design |
+| 桌面端 | Tauri 2、私有 CPython 3.11 运行时 |
+| 数据与状态 | PostgreSQL 15 或 SQLite;Redis 7 或进程内状态 |
+| 存储 | S3 兼容存储(云端)、本地文件(桌面与一键安装) |
+| 沙箱 | OpenSandbox、本地子进程(CubeSandbox 为 EE) |
+| 可选记忆 | Milvus 2.4、Neo4j 5 Community、mem0 |
+| 部署 | Docker Compose、桌面安装包、一键安装脚本、Nginx |
## 整体架构
```
- ┌──────────────────────────────────────────────┐
- Browser ──► Nginx ────►│ FastAPI backend (src/backend/api/app.py) │
- (frontend 容器, │ api/routes/v1/* · 50+ 路由 · 统一 envelope │
- /api 反向代理) └───────────────────┬──────────────────────────┘
- │
- ┌──────────────────────┼─────────────────────────┐
- ▼ ▼ ▼
- orchestration/workflow.py core/services/* core/auth/*
- (SSE 流式编排:text / (业务服务层) (local / mock /
- tool_call / tool_result remote+SSO)
- / meta / done)
- │
- ┌─────────────┼───────────────────┬─────────────────────┐
- ▼ ▼ ▼ ▼
- core/llm/ orchestration/ orchestration/ core/memory/ (svc)
- factory/build strategy.py citations.py + memory_integration
- (AgentScope (路由策略) (引用解析) (mem0 检索/保存)
- 2.0 ReAct) │
- │ ▼
- ├──► core/llm/mcp_manager ──► mcp 容器 Milvus / Neo4j
- │ (9 个 MCP server,独立 streamable-http 端口) (mem0 profile)
- ├──► core/sandbox/* ──► script-runner / OpenSandbox / CubeSandbox
- └──► PostgreSQL · Redis · 存储 (local / S3 / OSS)
+ 入口:Web · 桌面客户端 · IM 渠道 · API · 定时任务 · 心跳
+ │
+ ▼
+ FastAPI(api/) 鉴权 · 版本路由 · 消息先落库再返回
+ │
+ ▼
+ Run 层(core/runs · core/chat/runtime)
+ 持久化 Run · 数据库租约 · 启动对账 · 事件流续播 · steer · 工具副作用回执
+ │
+ ▼
+ 智能体执行(core/agent_runtime · core/llm)
+ ReActAgent · 上下文按步装配与压缩 · 记忆 / 本体 / 进化集成
+ 外层编排(orchestration/):计划模式 · 自主循环 · 自动化
+ │
+ ┌────────────────┼──────────────────┐
+ ▼ ▼ ▼
+ MCP · 技能 · 插件 沙箱 · 后台作业 知识库 · 记忆
+ │ │ │
+ └────────────────┴──────────────────┘
+ ▼
+ PostgreSQL / SQLite · Redis · S3 / 本地存储 · Milvus · Neo4j
```
-请求主链路:浏览器 → frontend 容器内 nginx(`/api` 反代)→ FastAPI → `orchestration/workflow.py` 编排流式输出 → `core/llm/factory/build.py` 构建 ReActAgent → MCP 工具 / 沙箱 / 记忆 → SSE 事件(`text` / `tool_call` / `tool_result` / `meta` / `done`)回推前端。
+Personal Agent(`personal_agent/`)复用同一套 Run、租约与沙箱机制,增加身份、任务队列、
+调度、渠道、设备连接和工作区文件。更完整的请求生命周期见[架构总览](../architecture/overview.md)。
## 下一步
-- [10 分钟快速开始](quick-start.md)
-- [Docker Compose 完整部署](../deployment/docker-compose.md)
-- [环境变量参考](../deployment/environment-variables.md)
-- [架构总览](../architecture/overview.md)
+- [快速开始](quick-start.md)
+- [Personal Agent](../modules/personal-agent.md)
+- [对话与智能体编排](../modules/chat.md)
+- [自主循环](../modules/autonomous-loop.md)
+- [部署指南](../deployment/README.md)
## 相关源码
-| 功能 | 文件 |
+| 功能 | 路径 |
|---|---|
-| FastAPI 应用与路由注册 | `src/backend/api/app.py`、`src/backend/api/routes/v1/` |
-| 流式编排(SSE) | `src/backend/orchestration/workflow.py`、`orchestration/streaming.py` |
+| FastAPI 应用与路由 | `src/backend/api/app.py`、`src/backend/api/routes/v1/` |
+| Run 生命周期、租约与恢复 | `src/backend/core/runs/` |
+| 对话 worker、续播与转向 | `src/backend/core/chat/runtime/` |
+| 智能体执行入口 | `src/backend/core/agent_runtime/`(`runner.astream_chat_workflow`) |
| 智能体构建 | `src/backend/core/llm/factory/build.py` |
-| MCP server 与端口映射 | `src/backend/extensions/mcp_servers/`、`src/backend/extensions/mcp_servers/_ports.py`、`src/backend/core/config/mcp_config.py` |
-| 能力注册表 | `src/backend/core/config/catalog.json`、`core/config/catalog.py` |
-| 沙箱 provider | `src/backend/core/sandbox/`、`src/backend/core/config/settings.py::SandboxSettings` |
-| 记忆系统 | `src/backend/core/memory/`(service.py / pipeline.py)、`src/backend/orchestration/memory_integration.py` |
-| 自动化调度 | `src/backend/orchestration/schedulers/automation_scheduler.py` |
-| 前端入口(App / Admin / Config) | `src/frontend/src/main.tsx`、`App.tsx`、`AdminApp.tsx`、`ConfigApp.tsx` |
-| 版本与 License 设置 | `src/backend/core/config/settings.py::EditionSettings / LicenseSettings` |
+| Harness 契约与执行台账 | `src/backend/core/harness/` |
+| 工作流后台作业 | `src/backend/core/jobs/` |
+| 计划模式、自主循环、自动化 | `src/backend/orchestration/plans/`、`orchestration/loops/`、`orchestration/automations/` |
+| Personal Agent | `src/backend/personal_agent/` |
+| 记忆与进化 | `src/backend/core/memory/`、`src/backend/core/evolution/` |
+| 领域本体 | `src/backend/core/ontology/` |
+| 沙箱 | `src/backend/core/sandbox/` |
+| 内置 MCP、技能、插件与智能体 | `src/backend/extensions/` |
+| 前端入口 | `src/frontend/src/main.tsx`、`App.tsx` |
+| 桌面客户端 | `desktop/src-tauri/` |
diff --git a/document/zh-CN/getting-started/quick-start.md b/document/zh-CN/getting-started/quick-start.md
index 840f1976..e9127b62 100644
--- a/document/zh-CN/getting-started/quick-start.md
+++ b/document/zh-CN/getting-started/quick-start.md
@@ -1,21 +1,85 @@
# 快速开始
-> 最后更新:2026-07-21 |
+> 最后更新:2026-10-23 |
> [English](../../en/getting-started/quick-start.md)
-个人试用可以选择一键安装,长期服务或服务隔离场景可以选择 Docker Compose。
-两种方式都需要准备 OpenAI 兼容 API 或本地模型。
+HugAgentOS 有三种上手方式。需要智能体 7×24 小时在云端持续运行时选择 HugAgentOS Cloud;
+想在自己的电脑上离线使用时选择 HugAgentOS Local 桌面客户端;做二次开发或快速试用时选择
+一键安装。所有方式都需要准备 OpenAI 兼容 API 或本地模型。
## 选择部署方式
-两种部署方式使用不同的运行与持久化模型。
-
| 方式 | 适用场景 | 运行形态 |
|---|---|---|
-| 一键安装 | 个人试用与开发 | SQLite、进程内状态、本地子进程沙箱 |
-| Docker Compose | 长期运行服务器与服务隔离 | PostgreSQL、Redis、容器沙箱、持久化服务卷 |
+| HugAgentOS Cloud(Docker Compose) | 7×24 运行、Personal Agent、IM 渠道、多设备访问 | PostgreSQL、Redis、OpenSandbox、持久化卷 |
+| HugAgentOS Local(桌面客户端) | 离线使用、敏感数据、处理本地项目 | 自带私有运行时、本地 SQLite 与文件 |
+| 一键安装 | 试用与二次开发 | SQLite、进程内状态、本地子进程沙箱 |
+
+**初始账号与密码均为 `admin`,首次登录必须修改;社区版不提供自助注册。**
+
+## 方式一:HugAgentOS Cloud(Docker Compose)
+
+安装 Git、Docker Engine 或 Docker Desktop 和 Docker Compose v2,然后运行:
+
+```bash
+git clone https://github.com/ZJU-REAL/HugAgentOS.git
+cd HugAgentOS
+cp .env.example .env
+mkdir -p data/storage
+docker compose build backend
+docker compose up -d --build
+```
+
+打开 [http://localhost:3002](http://localhost:3002),使用 `admin` / `admin`
+登录并修改密码,然后进入「设置 → 系统管理 → 模型服务」接入模型。使用
+`docker compose ps` 检查服务,使用 `docker compose down` 停止服务且保留数据。
+
+Profiles、数据持久化、生产配置和重建流程见
+[Docker Compose 部署指南](../deployment/docker-compose.md)。
+
+### 启用 Personal Agent(可选)
+
+Personal Agent 默认关闭。它使用独立的 OpenSandbox 控制器为每个用户维护一台永久云电脑。
+
+1. 构建专用电脑镜像:
+
+ ```bash
+ docker build -f docker/personal-agent/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .
+ ```
+
+2. 在 `.env` 中设置:
+
+ | 变量 | 值 |
+ |---|---|
+ | `PERSONAL_AGENT_ENABLED` | `true` |
+ | `HOST_STORAGE_PATH` | 宿主机上的绝对路径 |
+ | `PERSONAL_AGENT_OPENSANDBOX_API_KEY` | 至少 32 个字符的独立随机密钥,例如 `openssl rand -hex 32` 的输出 |
-## 方式一:一键安装
+3. 使用附加 Compose 文件启动:
+
+ ```bash
+ docker compose -f docker-compose.yml -f docker/personal-agent/compose.personal-agent.yml --profile personal-agent up -d
+ ```
+
+启动后,在「新建对话」下方点击「Personal Agent」即可使用。数据库迁移、镜像升级和渠道绑定见
+[Personal Agent](../modules/personal-agent.md)。密钥不要提交到 Git。
+
+## 方式二:HugAgentOS Local(桌面客户端)
+
+1. 从 [Releases](https://github.com/ZJU-REAL/HugAgentOS/releases) 下载对应平台
+ (Windows x86_64、macOS Apple Silicon / Intel、Linux x86_64)的桌面安装包。
+2. 首次启动时选择运行模式:
+ - **仅本机**:服务、数据和执行都在这台电脑上,可完全离线运行;
+ - **仅云端**:连接已部署的 HugAgentOS Cloud;
+ - **本机 + 云端**:使用云端账号,云端项目存云端,本地项目在本机执行;
+ Personal Agent 可以借此连接这台电脑。
+3. 选择本机模式时,客户端会校验并解压自带的运行时,通常数十秒内完成,不需要安装 Python、
+ Docker 或数据库。
+
+安装、升级回滚和数据目录见[桌面离线本机模式](../deployment/desktop-local-mode.md);
+绑定本地项目后的执行目录规则见[本机项目工作目录](../deployment/local-project-workspace.md)。
+
+## 方式三:一键安装
在 Linux、macOS 或通过 WSL2 使用 Windows 时选择此方式。开始前安装下列工具。
@@ -61,33 +125,17 @@ curl -fsSL https://raw.githubusercontent.com/ZJU-REAL/HugAgentOS/main/install.sh
export PATH="$HOME/.hugagent/venv/bin:$PATH"
```
-## 方式二:Docker Compose
-
-需要持久化服务卷和容器隔离时选择此方式。安装 Git、Docker Engine 或
-Docker Desktop 和 Docker Compose v2,然后运行:
-
-```bash
-git clone https://github.com/ZJU-REAL/HugAgentOS.git
-cd HugAgentOS
-cp .env.example .env
-mkdir -p data/storage
-docker compose up -d --build
-```
-
-打开 [http://localhost:3002](http://localhost:3002),使用 `admin` / `admin`
-登录并修改密码,然后进入「设置 → 系统管理 → 模型服务」接入模型。使用
-`docker compose ps` 检查服务,使用 `docker compose down` 停止服务且保留数据。
-
-Profiles、数据持久化、生产配置和重建流程见
-[Docker Compose 部署指南](../deployment/docker-compose.md)。
+一键安装不提供跨进程的租约接管,不适合 7×24 运行。
## 下一步
首次登录成功后,可以继续阅读:
-- [完整无 Docker 安装指南](../deployment/quick-install.md):安装选项、能力边界和故障排查;
+- [产品简介](introduction.md):Durable Agent、运行形态与能力总览;
- [模型接入](../modules/model-providers.md);
-- [领域本体快速入门](domain-ontology-quickstart.md):用最小 Domain Pack 构建受治理工作流;
+- [Personal Agent](../modules/personal-agent.md)与[自主循环](../modules/autonomous-loop.md);
- [MCP 工具](../modules/mcp-tools.md)、
[Agent Skills](../modules/agent-skills.md)和
- [私有知识库](../modules/knowledge-base.md)。
+ [私有知识库](../modules/knowledge-base.md);
+- [领域本体快速入门](domain-ontology-quickstart.md):用最小 Domain Pack 构建受治理工作流;
+- [完整无 Docker 安装指南](../deployment/quick-install.md):安装选项、能力边界和故障排查。
diff --git a/document/zh-CN/modules/admin-console.md b/document/zh-CN/modules/admin-console.md
index 122ad6e4..a90ed117 100644
--- a/document/zh-CN/modules/admin-console.md
+++ b/document/zh-CN/modules/admin-console.md
@@ -53,11 +53,11 @@ HugAgentOS 提供**两个相互独立的管理入口**,分别面向内容运
`/v1/admin/skills`(`api/routes/v1/admin_skills.py`,`ADMIN_TOKEN`):技能全生命周期管理——CRUD、zip 上传、启停、排序、图标、依赖扫描与编辑、技能内单文件读写删、复刻内置技能、导入 / 导出。技能存于 `admin_skills` 表,技能体系详见 [Agent 技能](agent-skills.md)。
-`/v1/admin/skill-drafts`(`api/routes/v1/admin_skill_drafts.py`,`ADMIN_TOKEN`):自动蒸馏出的技能草稿审核——列表 / 计数 / 详情、先编辑后通过(approve)、驳回(reject)、删除,以及手动触发每日蒸馏扫描。
+`/v1/admin/skill-drafts`(`features/distillation/api/admin_skill_drafts.py`,`ADMIN_TOKEN`):自动蒸馏出的技能草稿审核——列表 / 计数 / 详情、先编辑后通过(approve)、驳回(reject)、删除,以及手动触发每日蒸馏扫描。
### 市场审核(admin_marketplace)(商业版 EE:content_admin)
-`/v1/admin/marketplace`(`api/routes/v1/admin_marketplace.py`,`ADMIN_TOKEN`):两件事——
+`/v1/admin/marketplace`(`features/marketplace/api/admin_marketplace.py`,`ADMIN_TOKEN`):两件事——
1. **安装市场技能为全局技能**:浏览市场列表(标注「是否已全局安装」)、安装后 owner 为空、全员可用,可继续在「技能管理」编辑。
2. **审核用户上架申请**:申请列表(按 status 过滤)、详情(含 SKILL.md 预览)、通过(上架,全员可装)、驳回 / 下架。
@@ -91,14 +91,14 @@ GitHub 绑定默认在此面板维护,也可用 `FEEDBACK_GITHUB_*` 环境变
### 知识库管理(admin_kb)(商业版 EE:content_admin)
-`/v1/admin/kb`(`api/routes/v1/admin_kb.py`,`ADMIN_TOKEN`):自建**公共知识库**管理(`KBSpace.visibility == "public"`,归属系统属主),能力镜像用户态 `kb.py`——库 CRUD、AI 生成简介、文档上传 / 列表 / 原文预览(Office 转 PDF)/ 删除 / 重新索引、分块预览与逐块编辑(内容 / 标签 / 问题)。公共库对全部用户在能力目录可见、可被检索。Dify 模式下仅只读展示 Dify 数据集,写操作返回 409。详见 [知识库](knowledge-base.md)。
+`/v1/admin/kb`(`edition_ee/routes/admin_kb.py`,`ADMIN_TOKEN`):自建**公共知识库**管理(`KBSpace.visibility == "public"`,归属系统属主),能力镜像用户态 `kb.py`——库 CRUD、AI 生成简介、文档上传 / 列表 / 原文预览(Office 转 PDF)/ 删除 / 重新索引、分块预览与逐块编辑(内容 / 标签 / 问题)。公共库对全部用户在能力目录可见、可被检索。Dify 模式下仅只读展示 Dify 数据集,写操作返回 409。详见 [知识库](knowledge-base.md)。
### 沙箱管理(admin_sandbox + config_security/sandbox)
两个互补面板:
-- `/v1/admin/sandbox`(`api/routes/v1/admin_sandbox.py`,`ADMIN_TOKEN`,**商业版 EE:content_admin**):聚合技能依赖清单预览、触发沙盒镜像重建 + 容器热切、重建历史与日志查询。
-- `/v1/config/security/sandbox/*`(`api/routes/v1/config_security.py`,`CONFIG_TOKEN`,**商业版 EE:system_config**):只读安全视图——沙盒概览、运行实例列表 / 详情、快照、重建历史、脱敏配置。Provider 不支持的能力(如 ScriptRunner 无法枚举实例)返回 `code=42210`,前端置灰。
+- `/v1/admin/sandbox`(`features/sandbox_ops/api/admin_sandbox.py`,`ADMIN_TOKEN`,**商业版 EE:content_admin**):聚合技能依赖清单预览、触发沙盒镜像重建 + 容器热切、重建历史与日志查询。
+- `/v1/config/security/sandbox/*`(`features/security_console/api/config_security.py`,`CONFIG_TOKEN`,**商业版 EE:system_config**):只读安全视图——沙盒概览、运行实例列表 / 详情、快照、重建历史、脱敏配置。Provider 不支持的能力(如 ScriptRunner 无法枚举实例)返回 `code=42210`,前端置灰。
详见 [沙箱](sandbox.md)。
@@ -166,13 +166,13 @@ GitHub 绑定默认在此面板维护,也可用 `FEEDBACK_GITHUB_*` 环境变
| 管理凭证依赖 | `src/backend/api/deps.py` |
| License 能力位 | `src/backend/edition_ee/licensing/features.py`、`src/backend/edition_ee/licensing/deps.py` |
| 技能 / 草稿 / 市场 | `src/backend/api/routes/v1/admin_skills.py`、`admin_skill_drafts.py`、`admin_marketplace.py` |
-| 问题反馈与 GitHub 发布 | `src/backend/edition_ee/routes/feedbacks.py`、`admin_feedbacks.py`、`services/feedback_service.py` |
+| 问题反馈与 GitHub 发布 | `src/backend/edition_ee/routes/feedbacks.py`、`admin_feedbacks.py`、`edition_ee/services/feedback_service.py` |
| 提示词 / MCP / 子智能体 | `src/backend/api/routes/v1/admin_prompts.py`、`admin_mcp_servers.py`、`admin_agents.py` |
-| 知识库 / 沙盒 | `src/backend/edition_ee/routes/admin_kb.py`、`src/backend/api/routes/v1/admin_sandbox.py` |
+| 知识库 / 沙盒 | `src/backend/edition_ee/routes/admin_kb.py`、`src/backend/features/sandbox_ops/api/admin_sandbox.py` |
| 计费 / 用量 / 日志 / 会话审查 | `src/backend/api/routes/v1/admin_billing.py`、`admin_usage_logs.py`、`admin_logs.py`、`admin_chat_history.py` |
| 内容管理 | `src/backend/api/routes/v1/content.py` |
| 用户 / 团队 / 邀请 / License(EE) | `src/backend/edition_ee/routes/config_users.py`、`config_teams.py`、`config_invites.py`、`config_license.py` |
-| 安全 | `src/backend/api/routes/v1/config_security.py` |
+| 安全 | `src/backend/features/security_console/api/config_security.py` |
| 服务配置 | `src/backend/api/routes/v1/service_configs.py` |
延伸阅读:[认证与权限](auth.md) · [提示词系统](prompts.md) · [版本与授权](../editions/overview.md)
diff --git a/document/zh-CN/modules/agent-skills.md b/document/zh-CN/modules/agent-skills.md
index a1927829..c3d857dd 100644
--- a/document/zh-CN/modules/agent-skills.md
+++ b/document/zh-CN/modules/agent-skills.md
@@ -29,11 +29,11 @@
└── _scripts.json # 可选:脚本白名单声明(缺省时按扩展名自动探测)
```
-解析与数据结构在 `core/agent_skills/registry.py`:`AgentSkillMetadata`(轻量元数据,列表场景用)与 `AgentSkillSpec`(含完整指令,执行场景用)。`_scripts.json` 缺失时 `loader._auto_detect_scripts` 按扩展名自动生成白名单,用户无需手写。
+解析与数据结构在 `core/capabilities/skills/registry.py`:`AgentSkillMetadata`(轻量元数据,列表场景用)与 `AgentSkillSpec`(含完整指令,执行场景用)。`_scripts.json` 缺失时 `loader._auto_detect_scripts` 按扩展名自动生成白名单,用户无需手写。
## 多源加载与优先级
-`core/agent_skills/loader.py::MultiSourceSkillLoader` 通过后端抽象层(`backends/`:filesystem / database / composite)从多个来源加载技能,同 ID 冲突时高优先级覆盖(`core/agent_skills/config.py`):
+`core/capabilities/skills/loader.py::MultiSourceSkillLoader` 通过后端抽象层(`backends/`:filesystem / database / composite)从多个来源加载技能,同 ID 冲突时高优先级覆盖(`core/capabilities/skills/config.py`):
| 来源 | 目录 | 优先级 | 说明 |
|---|---|---|---|
@@ -142,11 +142,11 @@
- **范围**:主对话链路 + 子智能体。子智能体绑定的插件同样延迟加载,但激活只在该次运行内生效、不落库(子智能体运行短暂且相互隔离,落到父会话的键上会把激活泄漏进主智能体装配);收窄模式(对话模式)圈定的插件保持全量装配(那是管理员的显式圈定);组件含 stdio 形态 MCP 的插件不延迟。
- **开关**:环境变量 `PLUGIN_PROGRESSIVE_LOADING=false` 可整体回退到全量装配。
-实现:`core/plugins/runtime/__init__.py`(延迟解析 / 目录渲染 / 激活工具)+ `core/llm/factory/build.py` 装配接入。
+实现:`core/capabilities/plugins/runtime/__init__.py`(延迟解析 / 目录渲染 / 激活工具)+ `core/llm/factory/build.py` 装配接入。
## 技能市场(Skill Marketplace)
-技能市场是"预置 + 社区"双来源的可安装技能库,核心服务在 `core/services/marketplace_service.py`。
+技能市场是"预置 + 社区"双来源的可安装技能库,核心服务在 `features/marketplace/`。
### 浏览与安装
@@ -197,7 +197,7 @@
GET /v1/admin/visibility/principals ← 用户/团队/角色简表(选择器数据源)
```
-存储:`marketplace_listing_states.visibility`(缺行 = public)+ `marketplace_visibility_grants` 白名单表;用户侧解析收口在 `core/auth/marketplace_visibility.py`(角色含个人直配与经团队获得的部门默认角色)。
+存储:`marketplace_listing_states.visibility`(缺行 = public)+ `marketplace_visibility_grants` 白名单表;用户侧解析收口在 `edition_ee/auth/marketplace_visibility.py`(角色含个人直配与经团队获得的部门默认角色)。
## 管理员技能管理
@@ -206,7 +206,7 @@
- **创建/编辑**:手写 SKILL.md(`POST /`、`PUT /{id}`),或 **zip 上传**整个技能包(`POST /upload`,自动识别目录前缀、二进制文件 base64 入库,上限 200MB);
- **文件级管理**:`GET/PUT/DELETE /{id}/files/{filename}` 在线编辑技能附属文件;
- **运维**:启停(`/{id}/toggle`)、排序、图标、fork 复制、批量导出/导入(跨环境迁移);
-- **依赖管理**:`POST /{id}/rescan-deps` 用 `core/agent_skills/deps_detector.py` 静态扫描脚本的 pip/apt 依赖,`PUT /{id}/dependencies` 人工修正;聚合后的依赖清单用于沙箱镜像重建(见[沙箱模块](sandbox.md)的管理员依赖重建)。
+- **依赖管理**:`POST /{id}/rescan-deps` 用 `core/capabilities/skills/deps/detector.py` 静态扫描脚本的 pip/apt 依赖,`PUT /{id}/dependencies` 人工修正;聚合后的依赖清单用于沙箱镜像重建(见[沙箱模块](sandbox.md)的管理员依赖重建)。
## 技能蒸馏(Distillation)(商业版 EE)
@@ -223,7 +223,7 @@
approve → 转正为 AdminSkill;reject / delete;可手动触发 trigger-daily-scan
```
-阈值、关键词、预算等全部可由 `DISTILL_*` 环境变量覆盖(`core/config/distillation.py`)。蒸馏管线(`skill_distiller.py`、调度器)属**商业版 EE**,社区版派生树剔除;技能草稿审核台随之仅在商业版可见。
+阈值、关键词、预算等全部可由 `DISTILL_*` 环境变量覆盖(`features/distillation/config.py`)。蒸馏管线(`skill_distiller.py`、调度器)属**商业版 EE**,社区版派生树剔除;技能草稿审核台随之仅在商业版可见。
## 用户自助技能(能力中心)
@@ -265,22 +265,22 @@
| 路径 | 说明 |
|---|---|
-| `src/backend/core/agent_skills/registry.py` | SKILL.md 解析、元数据/完整 Spec 数据结构 |
-| `src/backend/core/agent_skills/loader.py` | 多源加载、DB 技能物化、沙箱路径重定向 |
-| `src/backend/core/agent_skills/config.py` | 加载源优先级、统一沙箱技能目录、内置技能同步 |
-| `src/backend/core/agent_skills/selector.py` | 按用户意图 LLM 选技能(渐进式披露) |
-| `src/backend/core/agent_skills/deps_detector.py` | 脚本 pip/apt 依赖静态探测 |
-| `src/backend/core/agent_skills/backends/` | filesystem / database / composite 加载后端 |
+| `src/backend/core/capabilities/skills/registry.py` | SKILL.md 解析、元数据/完整 Spec 数据结构 |
+| `src/backend/core/capabilities/skills/loader.py` | 多源加载、DB 技能物化、沙箱路径重定向 |
+| `src/backend/core/capabilities/skills/config.py` | 加载源优先级、统一沙箱技能目录、内置技能同步 |
+| `src/backend/core/llm/skill_selection.py` | 按用户意图 LLM 选技能(渐进式披露) |
+| `src/backend/core/capabilities/skills/deps/detector.py` | 脚本 pip/apt 依赖静态探测 |
+| `src/backend/core/capabilities/skills/backends/` | filesystem / database / composite 加载后端 |
| `src/backend/core/llm/tools/skill_tool.py` | 受限 view_text_file + {baseDir} 替换 + Runtime Hint |
| `src/backend/extensions/skills/default/` | 12 个内置技能(含 10 个 OfficeCLI 系列) |
| `src/backend/extensions/skills/marketplace/` | 45 个可安装市场技能包 |
-| `src/backend/core/services/marketplace_service.py` | 市场列表/安装/凭据注入/上架审核 |
-| `src/backend/api/routes/v1/marketplace.py` | 用户侧市场 API(浏览/安装/提交/撤回) |
-| `src/backend/api/routes/v1/admin_marketplace.py` | 管理员市场 API(全局安装/审核上架) |
+| `src/backend/features/marketplace/` | 市场列表/安装/凭据注入/上架审核 |
+| `src/backend/features/marketplace/api/marketplace.py` | 用户侧市场 API(浏览/安装/提交/撤回) |
+| `src/backend/features/marketplace/api/admin_marketplace.py` | 管理员市场 API(全局安装/审核上架) |
| `src/backend/api/routes/v1/admin_skills.py` | 管理员技能 CRUD / zip / 依赖管理 |
-| `src/backend/api/routes/v1/admin_skill_drafts.py` | 蒸馏草稿审核(EE) |
-| `src/backend/core/llm/skill_distiller.py` | 技能蒸馏 LLM 管线(EE) |
-| `src/backend/orchestration/schedulers/distillation_cron_scheduler.py` | 蒸馏每日调度(EE) |
+| `src/backend/features/distillation/api/admin_skill_drafts.py` | 蒸馏草稿审核(EE) |
+| `src/backend/features/distillation/` | 技能蒸馏 LLM 管线(EE) |
+| `src/backend/orchestration/workers/distillation_cron_scheduler.py` | 蒸馏每日调度(EE) |
| `src/backend/api/routes/v1/me_capabilities.py` | 用户自助技能/私有 MCP API |
相关文档:[沙箱执行系统](sandbox.md) · [MCP 工具系统](mcp-tools.md) · [能力目录](catalog.md) · [管理台](admin-console.md) · [版本与许可](../editions/overview.md)
diff --git a/document/zh-CN/modules/application-hosting.md b/document/zh-CN/modules/application-hosting.md
index 20a707ee..a3891425 100644
--- a/document/zh-CN/modules/application-hosting.md
+++ b/document/zh-CN/modules/application-hosting.md
@@ -6,11 +6,11 @@
配置 `APPLICATION_DATABASE_URL` 指向独立数据库。不能指向平台 `DATABASE_URL` 的同一数据库。云部署未配置时,管理界面提示未启用;本地模式可使用 SQLite,但生产使用 PostgreSQL。首次使用先执行 `python -m core.services.application_hosting_setup` 初始化注册表。
-可选 Compose 覆盖文件 `docker-compose.application-hosting.yml` 包含独立 PostgreSQL、持久卷和一次性初始化任务。分别提供不同的 `APPLICATION_DB_PASSWORD`(初始化)与 `APPLICATION_OWNER_DB_PASSWORD`(运行账号),使用 URL 安全的密码或正确编码连接串;初始化任务使用与后端相同的 Dockerfile 构建目标,并只读挂载当前后端源码。部署时后端也必须使用本次源码构建的镜像,并按仓库部署流程重建前端,使 nginx 的 `/applications-mcp/` 转发配置生效。
+可选 Compose 覆盖文件 `docker/application-hosting/compose.application-hosting.yml` 包含独立 PostgreSQL、持久卷和一次性初始化任务。分别提供不同的 `APPLICATION_DB_PASSWORD`(初始化)与 `APPLICATION_OWNER_DB_PASSWORD`(运行账号),使用 URL 安全的密码或正确编码连接串;初始化任务使用与后端相同的 Dockerfile 构建目标,并只读挂载当前后端源码。部署时后端也必须使用本次源码构建的镜像,并按仓库部署流程重建前端,使 nginx 的 `/applications-mcp/` 转发配置生效。
```bash
-docker compose -f docker-compose.yml -f docker-compose.application-hosting.yml config --quiet
-docker compose -f docker-compose.yml -f docker-compose.application-hosting.yml up -d
+docker compose -f docker-compose.yml -f docker/application-hosting/compose.application-hosting.yml config --quiet
+docker compose -f docker-compose.yml -f docker/application-hosting/compose.application-hosting.yml up -d
```
实际启动、重建、迁移和部署仍遵守项目授权流程。存量迁移使用独立应用数据库。本地验证不代表远程环境已经升级。端到端验证使用独立数据库及平台的本地 mock 认证;该认证模式仅供本地测试。
@@ -47,27 +47,7 @@ PostgreSQL 每个应用使用独立 schema 与受限 NOLOGIN 角色;查询和
已实现结构化 SQL 表、约束与索引、原子批量写入、幂等重试、版本更新、受限查询、应用隔离、匿名表单和声明式 MCP 托管。请求有行大小、批量大小、分页和记录配额限制。CSV 流式导出全表。
-未实现任意用户代码 MCP 容器、OAuth 授权服务器、外键关系设计器、全量备份恢复控制台和完整用量计费。当前固定 Bearer 方案不可宣称已完成 OAuth。真实聊天端到端驱动位于 `src/backend/tests/application_hosting_e2e.py`,独立测试环境由同目录的 `application_hosting_e2e_setup.py` 准备。通过生产聊天 API 验证模型、工具、沙箱与发布,验收结果以实际运行记录为准。
-
-## 本地端到端复现
-
-使用已配置真实模型、产业来源和 OpenSandbox 的本地 Docker 环境;先构建当前前端。以下驱动使用生产 API 与真实工具,不替换文件工具或发布接口。测试服务仅绑定回环端口,使用独立 platform_e2e_verified / applications_e2e_verified 数据库和独立存储目录。
-
-```bash
-npm --prefix src/frontend run build
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_e2e_setup.py
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_e2e.py
-```
-
-在 `http://127.0.0.1:18533/site/e2e-form-1007/` 用测试信息提交一次,邮箱使用 `browser-e2e@example.test`。再运行:
-
-```bash
-PYTHONPATH=src/backend .venv/bin/python src/backend/tests/application_hosting_lifecycle_e2e.py
-```
-
-生命周期验证会重启上述独立测试服务,并轮换、回滚、撤销和重新发布测试 MCP 凭据。凭据只保存在 `/tmp/application-hosting-full-e2e-credentials.json`(0600),不放入网页或验收报告。已有批次不自动清库;完整聊天重跑应使用干净的独立数据库批次。该流程不升级主环境,也不执行远程部署。
-
-重跑完整聊天时,先停止并移除 `hugagent-application-e2e-backend` 和 `hugagent-application-e2e-mcp` 这两个测试容器,再以 `APPLICATION_E2E_BATCH=run2` 执行 setup。旧批次数据库保留,驱动仍使用相同的回环访问地址。不要删除主环境容器或数据库。
+未实现任意用户代码 MCP 容器、OAuth 授权服务器、外键关系设计器、全量备份恢复控制台和完整用量计费。当前固定 Bearer 方案不可宣称已完成 OAuth。
## 桌面端
@@ -128,9 +108,9 @@ project_synced=false;保留新草稿并报告同步失败,核对线上版本
管理表单和回滚发布遇到尚未发布的项目草稿时返回冲突,不覆盖草稿。项目发布会等待文件工具登记,发布后刷新挂载文件的新版本;只有这些同步步骤完成才报告 project_synced=true。
-## 外部检验修复与升级
+## 部署要求与数据安全
-应用数据仍为可选模块。云端部署必须配置不同的 APPLICATION_DB_PASSWORD 与 APPLICATION_OWNER_DB_PASSWORD,并同时使用 docker-compose.yml 与 docker-compose.application-hosting.yml。首次部署及已有卷升级都执行 application-database-init;该步骤创建恢复历史表,把专用库对象交给 application_owner,并为已有应用角色补齐授权。后端仅使用 NOSUPERUSER、NOCREATEDB、CREATEROLE 的 application_owner;application_admin 只用于专用数据库初始化。CREATEROLE 是创建和回收应用隔离角色所需权限,因此必须使用独立 PostgreSQL 实例,不能复用业务平台数据库。初始化失败时不启动后端,不手工跳过依赖检查。
+应用数据仍为可选模块。云端部署必须配置不同的 APPLICATION_DB_PASSWORD 与 APPLICATION_OWNER_DB_PASSWORD,并同时使用 docker-compose.yml 与 docker/application-hosting/compose.application-hosting.yml。首次部署及已有卷升级都执行 application-database-init;该步骤创建恢复历史表,把专用库对象交给 application_owner,并为已有应用角色补齐授权。后端仅使用 NOSUPERUSER、NOCREATEDB、CREATEROLE 的 application_owner;application_admin 只用于专用数据库初始化。CREATEROLE 是创建和回收应用隔离角色所需权限,因此必须使用独立 PostgreSQL 实例,不能复用业务平台数据库。初始化失败时不启动后端,不手工跳过依赖检查。
访客 API 的成功及错误响应均带 CORS。云端访客写入与密码尝试采用 Redis 原子共享计数;所有后端进程必须使用同一 Redis。Redis 未配置或不可用时返回 503。仅未配置 Redis 的本机单进程模式使用内存计数。
diff --git a/document/zh-CN/modules/auth.md b/document/zh-CN/modules/auth.md
index e62c68cd..0f0c65df 100644
--- a/document/zh-CN/modules/auth.md
+++ b/document/zh-CN/modules/auth.md
@@ -111,7 +111,7 @@ SSO 提供方响应必须是 JSON 对象,且成功码必须为有限数值 0
### 接口层(CE/EE 拆分接缝)
-`core/auth/permissions_iface.py` 是权限符号的**唯一导入入口**:主干代码(deps / files / chats / projects / kb 等)一律从这里导入,不直接引用具体实现。EE 主仓中它纯转发三个真实现;CE 派生树用单租户 stub 整体替换本文件,三个真实现物理不存在:
+`core/auth/personal_permissions_iface.py` 是权限符号的**唯一导入入口**:主干代码(deps / files / chats / projects / kb 等)一律从这里导入,不直接引用具体实现。EE 主仓中它纯转发三个真实现;CE 派生树用单租户 stub 整体替换本文件,三个真实现物理不存在:
| 实现文件 | 职责 |
|---|---|
@@ -176,7 +176,7 @@ SSO 提供方响应必须是 JSON 对象,且成功码必须为有限数值 0
| `PATCH /v1/me/api-keys/{key_id}` | 启用 / 禁用 |
| `DELETE /v1/me/api-keys/{key_id}` | 撤销 |
-Key 形如 `sk-jx-...`,在**所有 AUTH_MODE 下**都可作为 Bearer 调用业务 API(外部程序化调用通常没有 Cookie)。校验逻辑(启用 / 未撤销 / 未过期 / 用户权限位仍开启)集中在 `core/services/api_key_service.py::resolve_api_key`。
+Key 形如 `sk-jx-...`,在**所有 AUTH_MODE 下**都可作为 Bearer 调用业务 API(外部程序化调用通常没有 Cookie)。校验逻辑(启用 / 未撤销 / 未过期 / 用户权限位仍开启)集中在 `core/auth/api_keys.py::resolve_api_key`。
禁用或撤销 API Key 会拒绝新请求,并取消该 Key 已接收且仍在等待、运行或暂停待处理的对话任务。取消会停止后台执行并结束 SSE 流;已完成的外部工具操作不会撤回。重新启用 Key 不会恢复已取消任务。调用记录进入“需要处理”等停止状态后,耗时固定在停止时间。
@@ -201,13 +201,13 @@ Key 形如 `sk-jx-...`,在**所有 AUTH_MODE 下**都可作为 Bearer 调用
| SSO 票据交换 | `src/backend/edition_ee/auth/sso.py`、`src/backend/edition_ee/routes/auth.py` |
| Mock SSO / 本地登录注册页 | `src/backend/api/routes/v1/mock_sso.py`、`src/backend/core/auth/mock_ticket_store.py` |
| 密码哈希 | `src/backend/core/auth/password.py` |
-| 权限接口层(CE/EE 接缝) | `src/backend/core/auth/permissions_iface.py` |
+| 权限接口层(CE/EE 接缝) | `src/backend/core/auth/personal_permissions_iface.py` |
| 团队角色 / 文件权限(EE) | `src/backend/edition_ee/auth/roles.py`、`src/backend/edition_ee/auth/team_permissions.py` |
| 项目 / 会话分享权限(EE) | `src/backend/edition_ee/auth/project_permissions.py`、`src/backend/edition_ee/auth/chat_share_permissions.py` |
| 管理凭证依赖 | `src/backend/api/deps.py` |
| 用户资料 / 偏好 | `src/backend/api/routes/v1/users.py` |
| 用户权限位管理 | `src/backend/edition_ee/routes/config_users.py` |
-| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`、`src/backend/core/services/api_key_service.py` |
+| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`、`src/backend/core/auth/api_keys.py` |
| 能力中心自助(owner 隔离) | `src/backend/api/routes/v1/me_capabilities.py` |
| 邀请码 | `src/backend/edition_ee/auth/invite.py`、`src/backend/edition_ee/routes/config_invites.py` |
| 团队管理 | `src/backend/edition_ee/routes/config_teams.py`、`src/backend/edition_ee/routes/me_teams.py` |
diff --git a/document/zh-CN/modules/automation.md b/document/zh-CN/modules/automation.md
index cd888e35..2c53f631 100644
--- a/document/zh-CN/modules/automation.md
+++ b/document/zh-CN/modules/automation.md
@@ -6,8 +6,8 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制,两
| 能力 | 一句话定位 | 后端入口 |
|---|---|---|
-| 定时自动化 | 按 cron 周期(或一次性 / 手动)自动执行一段 Prompt 或一个计划 | `api/routes/v1/automations.py` + `orchestration/schedulers/automation_scheduler.py` |
-| 计划模式 | AI 先把复杂任务拆成结构化步骤,确认后逐步执行 | `api/routes/v1/plans.py` + `orchestration/subagents/plan_mode.py` |
+| 定时自动化 | 按 cron 周期(或一次性 / 手动)自动执行一段 Prompt 或一个计划 | `features/automation/api/automations.py` + `orchestration/automations/automation_scheduler.py` |
+| 计划模式 | AI 先把复杂任务拆成结构化步骤,确认后逐步执行 | `api/routes/v1/plans.py` + `orchestration/plans/` |
## 定时自动化任务
@@ -17,7 +17,7 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制,两
### 任务模型
-任务存于 `scheduled_tasks` 表(`core/db/models/automation.py` 的 `ScheduledTask`),核心字段:
+任务存于 `scheduled_tasks` 表(`edition_ee/db/models/automation.py` 的 `ScheduledTask`),核心字段:
| 字段 | 说明 |
|---|---|
@@ -40,7 +40,7 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制,两
### 调度机制(automation_scheduler.py)
-`orchestration/schedulers/automation_scheduler.py` 是一个随后端启动的 asyncio 轮询调度器:
+`orchestration/automations/automation_scheduler.py` 是一个随后端启动的 asyncio 轮询调度器:
- **轮询**:每 15 秒(外加 0–5 秒随机抖动)查询 DB 中 `next_run_at` 到期的任务。
- **分布式锁**:触发前先抢 Redis 锁 `jx:auto:lock:{task_id}`,执行期间持续续期,完成后按持有者令牌释放,防止多实例重复触发。
@@ -48,6 +48,7 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制,两
- **执行时长**:单次执行没有固定总时长上限;定时触发与手动执行的 Prompt、Plan、Loop 任务均可持续运行,关联会话及其接续运行也不受普通会话的总时长硬限制。
- **失败治理**:连续失败计数 ≥ `max_failures`(默认 3)时任务自动置为 `disabled`。
- **失联恢复**:启动时及每次轮询检查 `running` 记录。取得该任务的执行锁后,继续跟随已关联的耐久会话并保存最终结果;只有未关联会话的孤儿记录才改为 `failed`,并按需推进过期调度、收尾一次性任务。任务运行多久不影响恢复判断。执行锁每 15 秒续期,60 秒租约用于识别执行者失联,不限制任务总时长。错过的一次性任务同样先取得执行锁。
+- **执行锁关闭**:设置停止信号并取消续期,最多等待 2 秒后始终尝试按 token 删除锁;清理失败也移除本地占位并记录错误。执行 Task 结束后续期自动停止。结果投递总等待上限为 60 秒,投递失败不覆盖已提交的成功,也不自动重试。运行记录保存内部所有权 token,完成更新同时校验运行中状态与 token;恢复执行先认领该记录。
每次执行通过统一的 ChatRun 会话执行器创建**真实会话**,并在开跑时立即关联执行记录。指令任务实时展示正文、思考和工具调用;计划任务展示步骤进度。点击运行中的会话或刷新页面,可以跟随和重播与普通会话相同的事件流。用户可在会话中停止执行,指令任务还可通过耐久指令队列追加要求,在下一个安全的模型执行边界生效。主动停止不计入连续失败次数;执行锁失效或调度器关闭会取消后台会话。最终消息、文件、引用和结果投递继续由统一执行器与调度收尾模块负责。会话标题直接使用任务名称,不添加自动化前缀。更新前启动的执行无法补回未记录的实时事件。
@@ -63,7 +64,7 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制,两
## 计划模式
-计划模式把复杂任务拆成结构化步骤再执行,分两阶段(`orchestration/subagents/plan_mode.py`):
+计划模式把复杂任务拆成结构化步骤再执行,分两阶段(`orchestration/plans/`):
1. **生成(Phase 1)**:`POST /v1/plans/generate`(SSE 流式)——AI 分析任务描述,产出含标题、描述、步骤列表的计划草稿;每步可声明 `expected_tools` / `expected_skills` / `expected_agents`(期望用到的工具 / 技能 / 子智能体)。
2. **执行(Phase 2)**:`POST /v1/plans/{plan_id}/execute`(SSE 流式)——逐步顺序执行,每步用独立的 agent;步骤状态、工具调用日志、AI 输出逐步落库。
@@ -98,14 +99,37 @@ POST /v1/automations
| 主题 | 路径 |
|---|---|
-| 自动化 REST API | `src/backend/api/routes/v1/automations.py` |
-| 调度器(轮询 / 锁 / 恢复) | `src/backend/orchestration/schedulers/automation_scheduler.py` |
-| 自动化服务(cron 计算 / 状态机) | `src/backend/core/services/automation_service.py` |
-| 任务 / 运行记录模型 | `src/backend/core/db/models/automation.py` |
+| 自动化 REST API | `src/backend/features/automation/api/automations.py` |
+| 调度器(轮询 / 锁 / 恢复) | `src/backend/orchestration/automations/automation_scheduler.py` |
+| 自动化服务(cron 计算 / 状态机) | `src/backend/features/automation/service.py` |
+| 任务 / 运行记录模型 | `src/backend/features/automation/models.py` |
| 计划 REST API | `src/backend/api/routes/v1/plans.py` |
-| 计划生成 / 执行编排 | `src/backend/orchestration/subagents/plan_mode.py`、`src/backend/core/services/plan_service.py` |
+| 计划生成 / 执行编排 | `src/backend/orchestration/plans/`、`src/backend/core/chat/plans.py` |
| 计划 / 步骤模型 | `src/backend/core/db/models/agent.py` |
-| 前端自动化 UI | `src/frontend/src/components/lab/AutomationPanel.tsx` 等、`src/frontend/src/components/automation/RunTimelinePanel.tsx` |
+| 前端自动化 UI | `src/frontend/src/components/automation/AutomationPanel.tsx` 等、`src/frontend/src/components/automation/RunTimelinePanel.tsx` |
| 前端状态 | `src/frontend/src/stores/automationStore.ts`、`automationChatStore.ts` |
延伸阅读:[对话系统](chat.md) · [MCP 工具](mcp-tools.md) · [数据画布与产物](canvas-artifacts.md)
+
+## 多进程运行与恢复
+
+定时和手动触发共用一个调度循环,每 3 秒扫描一次,最多同时观察 8 个执行。
+消费触发信号、推进排期和插入 pending 执行记录在一个数据库事务内完成;
+数据库条件更新保护并发接纳。忙碌任务的新手动触发保留至当前执行结束。
+
+pending 表示已接纳、等待执行。调度器停止或租约失效只转交观察权,
+不会取消已接纳的会话;后续实例继续观察同一个持久执行及其后继会话。
+用户在会话中主动停止仍会停止实际执行。子执行关联和恢复参数与接纳一起提交,
+不通过重跑已启动命令恢复结果。运行中的通知投递仍沿用原有投递语义,
+此变更不承诺外部渠道的恰好一次投递。
+
+升级数据库须应用 FULL 的 automationqueue01 或 CE 的 ce_0032;
+本机 SQLite 使用已安装 automation 模块的 schema 初始化完成约束升级。
+存在 pending 记录时拒绝降级,以免丢弃已接纳工作。先停止旧版本调度器、
+迁移后启动新版本,不混用两套调度协议。
+
+Plan/Loop 将完整启动参数与子任务接纳一同持久化。接纳后、启动前崩溃时,
+后继调度器只领取 pending 子任务;已开始的 running 子任务不会从头重放。
+升级前应排空旧版在途任务,旧版未保存的参数不会由兼容路径猜测补齐。
+
+任务投递统一存储为 delivery_targets。升级迁移将旧 channel_id/conversation_id 转换为目标列表并删除旧字段,保留已明确配置的目标和其他元数据;请求处理不再读取旧格式。
diff --git a/document/zh-CN/modules/autonomous-loop.md b/document/zh-CN/modules/autonomous-loop.md
index d1c5d60a..5ba8e397 100644
--- a/document/zh-CN/modules/autonomous-loop.md
+++ b/document/zh-CN/modules/autonomous-loop.md
@@ -71,7 +71,7 @@
## 相关代码
-- 驱动器 `orchestration/autonomous_loop.py`、规划器 `orchestration/loop_planner.py`、评审员 `orchestration/subagents/loop_reviewer.py`
-- ChatRun 接入 `orchestration/chat_run_executor.py`(`start_autonomous_loop_run` / 启动对账 `resume_running_loops`)
-- 服务 `core/services/loop_service.py`(账本镜像 / 启动参数 / steering 队列)、API `api/routes/v1/loops.py`
+- 驱动器 `orchestration/loops/driver.py`、规划器 `orchestration/loops/planner.py`、评审员 `orchestration/loops/reviewer.py`
+- ChatRun 接入 `core/chat/runtime/`(`start_autonomous_loop_run` / 启动对账 `resume_running_loops`)
+- 服务 `orchestration/loops/run.py`(账本镜像 / 启动参数 / steering 队列)、API `api/routes/v1/loops.py`
- 数据表 `agent_loops` / `loop_iterations`
diff --git a/document/zh-CN/modules/browser-automation.md b/document/zh-CN/modules/browser-automation.md
index 9e674501..b0856011 100644
--- a/document/zh-CN/modules/browser-automation.md
+++ b/document/zh-CN/modules/browser-automation.md
@@ -105,7 +105,7 @@ Docker 使用沙箱镜像内的 Python Playwright 与 Chromium;桌面完整包
启动时实际尝试 Chromium 沙箱。仅经过宿主验证的外层隔离容器在出现明确的内部沙箱错误时允许关闭 Chromium 内层沙箱后重试;本机环境不采用此降级。启动失败显示 chromium_sandbox_unavailable、chromium_runtime_missing 或 chromium_launch_failed,而不是仅显示异常类型。
-更新 OpenSandbox 时需显式从仓库根目录构建运行镜像:docker build -f docker/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .。普通 docker compose up --build 不会重建这一按需启动的镜像。核对 OPENSANDBOX_IMAGE 实际标签,并在更新后用 browser_open 创建新会话;只有 Chromium 初始化及真实浏览器验证通过后才报告资源就绪。其他运行提供方同样需重建各自运行镜像。
+更新 OpenSandbox 时需显式从仓库根目录构建运行镜像:docker build -f docker/sandbox/Dockerfile.opensandbox -t hugagent-opensandbox-custom:latest .。普通 docker compose up --build 不会重建这一按需启动的镜像。核对 OPENSANDBOX_IMAGE 实际标签,并在更新后用 browser_open 创建新会话;只有 Chromium 初始化及真实浏览器验证通过后才报告资源就绪。其他运行提供方同样需重建各自运行镜像。
Fake-IP DNS 返回198.18.0.0/15时仍禁止连接,错误提示 fake_ip_dns,并建议配置 BROWSER_DNS_RESOLVER_URL 为受信任的 HTTPS JSON DNS 服务。解析失败返回 dns_failed;不得为解决公网 DNS 问题打开私网访问。缺失元素和重复定位器分别返回 element_not_found、ambiguous_locator,未执行动作不再标为结果未知。
diff --git a/document/zh-CN/modules/canvas-artifacts.md b/document/zh-CN/modules/canvas-artifacts.md
index b29574e2..6a92fafd 100644
--- a/document/zh-CN/modules/canvas-artifacts.md
+++ b/document/zh-CN/modules/canvas-artifacts.md
@@ -8,7 +8,7 @@
## 数据画布(Canvas)
-数据画布是聊天界面右侧的滑出面板(`src/frontend/src/components/canvas/CanvasPanel.tsx`),点击聊天附件卡片(`components/chat/ArtifactCardList.tsx`)或「我的空间」中的文件(`components/myspace/MySpacePanel.tsx`)即可打开。面板按文件类型选择渲染器:
+数据画布是聊天界面右侧的滑出面板(`src/frontend/src/components/canvas/CanvasPanel.tsx`),点击聊天附件卡片(`components/chat/MessageArtifacts.tsx`)或「我的空间」中的文件(`components/myspace/MySpacePanel.tsx`)即可打开。面板按文件类型选择渲染器:
| 文件类型 | 渲染方式 | 可编辑 |
|---|---|---|
@@ -78,14 +78,14 @@ Web Worker 中执行,并按真实存在的单元格稀疏转换,不遍历工
### 产物的产生与落库
-对话流式过程中,工具产出的文件先进入会话**工作区**(`core/llm/workspace`),只有被「钉住」(pinned)的文件才会在回合结束时由 `core/services/artifact_service.py::persist_artifacts` 写入 `artifacts` 表——这道**严格工作区闸门**保证临时中间文件不会污染我的空间。
+对话流式过程中,工具产出的文件先进入会话**工作区**(`core/sandbox/workspace`),只有被「钉住」(pinned)的文件才会在回合结束时由 `core/files/artifacts/service.py::persist_artifacts` 写入 `artifacts` 表——这道**严格工作区闸门**保证临时中间文件不会污染我的空间。
-物理存储由 `core/artifacts/store.py` 管理,按 `STORAGE_TYPE` 双模式:
+物理存储由 `core/files/artifacts/store.py` 管理,按 `STORAGE_TYPE` 双模式:
- `local`(默认,社区版):文件写在 `${STORAGE_PATH:-result}/artifacts/`,维护本地 JSON 索引(`index.json`),FileResponse 直出。
-- `oss`(商业版 EE,`cloud_storage`):上传到阿里云 OSS,本地索引同时备份到 OSS,容器重启不丢。S3 后端见 `core/storage/`。
+- `oss`(商业版 EE,`cloud_storage`):上传到阿里云 OSS,本地索引同时备份到 OSS,容器重启不丢。S3 后端见 `core/files/storage/`。
-下载统一走 `GET /files/{file_id}`(`api/routes/files.py`),按 `core/auth/permissions_iface.py::resolve_artifact_access` 做 owner ∪ team 权限判定,Office 文件支持转 PDF 在线预览。
+下载统一走 `GET /files/{file_id}`(`api/routes/files.py`),按 `core/auth/personal_permissions_iface.py::resolve_artifact_access` 做 owner ∪ team 权限判定,Office 文件支持转 PDF 在线预览。
### REST 接口(`/v1/artifacts`,`api/routes/v1/artifacts.py`)
@@ -141,8 +141,8 @@ officecli-xlsx、officecli-pptx 三个基座技能(以及它们之上的表单
|---|---|
| 画布面板 / Univer 表格 | `src/frontend/src/components/canvas/CanvasPanel.tsx`、`UniverSpreadsheet.tsx`、`src/frontend/src/stores/canvasStore.ts` |
| Artifact REST API | `src/backend/api/routes/v1/artifacts.py` |
-| Artifact 存储(local/OSS) | `src/backend/core/artifacts/store.py` |
-| 产物落库(工作区闸门) | `src/backend/core/services/artifact_service.py` |
+| Artifact 存储(local/OSS) | `src/backend/core/files/artifacts/store.py` |
+| 产物落库(工作区闸门) | `src/backend/core/files/artifacts/service.py` |
| 文件下载 / 预览 | `src/backend/api/routes/files.py` |
| 会话分享 API | `src/backend/api/routes/v1/chat_shares.py` |
| 分享前端 | `src/frontend/src/SharePreviewApp.tsx`、`src/frontend/src/components/share/ShareRecordsPage.tsx` |
diff --git a/document/zh-CN/modules/catalog.md b/document/zh-CN/modules/catalog.md
index d8d1bd40..7e0529be 100644
--- a/document/zh-CN/modules/catalog.md
+++ b/document/zh-CN/modules/catalog.md
@@ -13,7 +13,7 @@ core/config/catalog.json 系统级单一真源(admin 启停 + 展
core/config/catalog.py 公共 API:get_catalog / is_enabled / get_enabled_ids
│ set_enabled / reorder_items
▼
-core/config/catalog_resolver.py 合并层:catalog.json 默认 ∩ 用户 DB 覆盖
+core/capabilities/catalog/resolver.py 合并层:catalog.json 默认 ∩ 用户 DB 覆盖
│ resolve_all_runtime_enabled(db, user_id) → (skills, agents, mcps)
│ resolve_explicit_runtime_capabilities(...) → 本轮显式调用(忽略个人开关,保留硬门控)
▼
@@ -39,7 +39,7 @@ core/config/catalog_resolver.py 合并层:catalog.json 默认 ∩ 用户 D
}
```
-`catalog_loader.py` 在加载时做**动态源同步**:从技能 loader(`core/agent_skills/`)与 MCP 配置服务(`core/services/mcp_service.py`,DB 表 `admin_mcp_servers`)发现新 id 自动补入、名称/描述/版本随源刷新、源中已消失的陈旧项自动移除;展示详情(能力中心介绍 markdown、图标)属运行时字段,不落盘。结果带短 TTL 内存缓存,写操作调 `invalidate_catalog_cache()` 失效。
+`catalog_loader.py` 在加载时做**动态源同步**:从技能 loader(`core/capabilities/skills/`)与 MCP 配置服务(`core/capabilities/mcp/service.py`,DB 表 `admin_mcp_servers`)发现新 id 自动补入、名称/描述/版本随源刷新、源中已消失的陈旧项自动移除;展示详情(能力中心介绍 markdown、图标)属运行时字段,不落盘。结果带短 TTL 内存缓存,写操作调 `invalidate_catalog_cache()` 失效。
### 门控 API
@@ -54,7 +54,7 @@ get_enabled_ids("mcp") # 某类全部启用 id
## 用户级覆盖(catalog_overrides)
-用户在能力中心里开/关某项能力,写入 DB 表 `catalog_overrides`(`core/services/catalog_service.py`),按用户独立、互不影响:
+用户在能力中心里开/关某项能力,写入 DB 表 `catalog_overrides`(`core/capabilities/catalog/overrides.py`),按用户独立、互不影响:
- **合并算法**(`catalog_resolver.py::_merge_kind`):覆盖只能翻转 base catalog 中**已存在**项的启用位,不能复活已删除项。
- **管理员锁**:base catalog 中 `enabled=false` 的项,用户覆盖**不可**重新启用,且在 `/v1/catalog` 响应中对前台完全隐藏。
@@ -66,7 +66,7 @@ get_enabled_ids("mcp") # 某类全部启用 id
- **显式插件使用引导**:选择插件后加载其技能和 MCP 工具,提示模型优先按需使用;不限制首个工具调用,也不要求必须调用才能完成回答。
- **会话级能力恢复**:后续回合在个人开关筛选前恢复已激活的技能、连接器及插件组件,使技能清单与工具名/前缀在该会话中稳定;恢复时仍重新校验安装、管理员全局状态、依赖与归属,硬门控变化会立即生效。
-每次对话时 `core/chat/context.py::resolve_enabled_capabilities()` 把默认合并结果写入 workflow context;`api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` 再校验插件安装实例与能力归属,从服务端安装记录解析组件,并把通过硬门控的显式选择并入当前回合。`core/llm/session_capabilities.py` 和 `core/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` 在后续回合恢复会话已激活能力。
+每次对话时 `core/chat/context.py::resolve_enabled_capabilities()` 把默认合并结果写入 workflow context;`api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` 再校验插件安装实例与能力归属,从服务端安装记录解析组件,并把通过硬门控的显式选择并入当前回合。`core/llm/runtime/session_capabilities.py` 和 `core/capabilities/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` 在后续回合恢复会话已激活能力。
## /v1/catalog 路由与 KB 注入
@@ -116,20 +116,19 @@ get_enabled_ids("mcp") # 某类全部启用 id
## 管理端能力管理
- **`/admin` 内容管理台**:技能上传/启停/排序(`api/routes/v1/admin_skills.py`,内部调 `catalog.set_enabled` / `reorder_items` 回写 catalog.json)、MCP server 管理(`admin_mcp_servers.py`,DB 表 + 试连 + 缓存刷新)、子智能体管理(`admin_agents.py`)、技能市场上架审核(`admin_marketplace.py`,**商业版 EE**)。
-- **catalog 快照迁移**:`scripts/export_content.py --only catalog` 导出 catalog.json + `catalog_overrides`,`scripts/import_content.py --catalog ` 导入。
## 相关源码
| 主题 | 路径 |
|---|---|
-| 公共 API(is_enabled 等) | `src/backend/core/config/catalog.py` |
+| 公共 API(is_enabled 等) | `src/backend/core/capabilities/catalog/` |
| 单一真源文件 | `src/backend/core/config/catalog.json`(`CATALOG_PATH` 可覆盖) |
-| 加载 / 动态同步 / 缓存 | `src/backend/core/config/catalog_loader.py`,`catalog_common.py`,`catalog_migration.py` |
-| 用户覆盖合并 | `src/backend/core/config/catalog_resolver.py`,`core/services/catalog_service.py` |
+| 加载 / 动态同步 / 缓存 | `src/backend/core/capabilities/catalog/loader.py`,`catalog_common.py`,`catalog_migration.py` |
+| 用户覆盖合并 | `src/backend/core/capabilities/catalog/resolver.py`,`core/capabilities/catalog/overrides.py` |
| 目录路由 | `src/backend/api/routes/v1/catalog.py` |
| 用户自助能力 | `src/backend/api/routes/v1/me_capabilities.py` |
-| MCP 服务配置(DB) | `src/backend/core/services/mcp_service.py`,`api/routes/v1/admin_mcp_servers.py` |
-| 技能管理 | `src/backend/api/routes/v1/admin_skills.py`,`core/agent_skills/` |
-| Dify KB 注入(EE) | `src/backend/edition_ee/kb/dify.py`,经共享接缝 `core/kb/external_provider.py` 调用 |
+| MCP 服务配置(DB) | `src/backend/core/capabilities/mcp/service.py`,`api/routes/v1/admin_mcp_servers.py` |
+| 技能管理 | `src/backend/api/routes/v1/admin_skills.py`,`core/capabilities/skills/` |
+| Dify KB 注入(EE) | `src/backend/edition_ee/kb/dify.py`,经共享接缝 `features/knowledge/extension.py` 调用 |
| 前端能力中心 | `src/frontend/src/components/catalog/`,`stores/catalogStore.ts` |
| 工厂消费侧 | `src/backend/core/llm/factory/build.py::_effective_mcp_server_keys` |
diff --git a/document/zh-CN/modules/chat.md b/document/zh-CN/modules/chat.md
index 0e5f7005..e6540f58 100644
--- a/document/zh-CN/modules/chat.md
+++ b/document/zh-CN/modules/chat.md
@@ -71,10 +71,10 @@ orchestration/chat_run_executor.py::start_run()
▼
orchestration/workflow.py::astream_chat_workflow()
│ ├─ orchestration/memory_integration.py 非阻塞记忆检索(后台 task + 预算超时)
- │ ├─ core/config/catalog_resolver.py 解析本次启用的 skills/mcp/kb
+ │ ├─ core/capabilities/catalog/resolver.py 解析本次启用的 skills/mcp/kb
│ ├─ core/llm/factory/build.py::create_agent_executor()
│ │ MCP 连接池 + 技能注册 + 文件工具 + 系统提示词 + 中间件 → Agent
- │ ├─ core/llm/context_manager.py 历史按 token 预算裁剪
+ │ ├─ core/llm/context/window_policy.py 历史按 token 预算裁剪
│ └─ orchestration/streaming.py::StreamingAgent.stream()
│ 消费 agent.reply_stream(),归并 25 种细粒度事件并保留工具参数增量
▼
@@ -85,7 +85,7 @@ SSE follower:chat_run_executor.follow_run_as_sse()
### Run 解耦与断线续播
-每次发送消息会创建一条 `ChatRun` 并启动后台任务(`orchestration/chat_run_executor.py`),事件写入 Redis Stream(`maxlen=5000`;默认 TTL 为最长人机等待 7200 秒加 1800 秒恢复余量)。HTTP 连接只是"跟随者",因此:
+每次发送消息会创建一条 `ChatRun` 并启动后台任务(`core/chat/runtime/`),事件写入 Redis Stream(`maxlen=5000`;默认 TTL 为最长人机等待 7200 秒加 1800 秒恢复余量)。HTTP 连接只是"跟随者",因此:
| 能力 | 端点 |
|---|---|
@@ -133,7 +133,7 @@ SSE follower:chat_run_executor.follow_run_as_sse()
| 替我批准 | `auto` | 普通写入 / 编辑直接放行;**删除类操作**(`DESTRUCTIVE_OPS`)与本地策略标了 `danger:<类别>` 的命令仍然停下来问 |
| 完全放开 | `full` | 所有工具确认一律不再询问 |
-档位在 `core/llm/tool_permissions.py` 里统一声明(`APPROVAL_ASK` / `APPROVAL_AUTO` / `APPROVAL_FULL`),随对话上下文传到 `PermissionRuntime`,是所有工具确认闸的唯一判据;任何一档都只跳过"问"这一步,本地安全策略判定的硬拒绝仍然拒绝。"这次危不危险"由 `core/sandbox/permissions/models.py::danger_categories()` 从裁决理由里的 `danger:` 前缀还原,不靠匹配提示文案。读到早期版本存下的 `standard` / `readonly` 时按最保守的 `ask` 兜底。
+档位在 `core/llm/tool_permissions/` 里统一声明(`APPROVAL_ASK` / `APPROVAL_AUTO` / `APPROVAL_FULL`),随对话上下文传到 `PermissionRuntime`,是所有工具确认闸的唯一判据;任何一档都只跳过"问"这一步,本地安全策略判定的硬拒绝仍然拒绝。"这次危不危险"由 `core/sandbox/permissions/models.py::danger_categories()` 从裁决理由里的 `danger:` 前缀还原,不靠匹配提示文案。读到早期版本存下的 `standard` / `readonly` 时按最保守的 `ask` 兜底。
渠道机器人、定时任务这类无人值守入口走 `default_allow` 分支,不受该档位影响。
@@ -143,17 +143,17 @@ SSE follower:chat_run_executor.follow_run_as_sse()
`create_agent_executor()` 是所有模式(主对话、计划、批量、子智能体、自动化)共用的工厂:
-- **MCP 工具**:经 catalog + 用户覆盖 + 请求上下文三层过滤后(见 [能力目录](catalog.md)),stable 服务复用进程级连接池(`core/llm/mcp_pool.py`),per-request 服务(如 `retrieve_dataset_content` 需带每请求 HTTP header)每次新建;用户自助添加的私有 MCP 按 owner 现查合入。
-- **技能**:经 `core/agent_skills/loader.py` 注册为 AgentScope Agent Skills,并放行 `view_text_file` 读取 SKILL.md(详见 [技能系统](agent-skills.md))。
+- **MCP 工具**:经 catalog + 用户覆盖 + 请求上下文三层过滤后(见 [能力目录](catalog.md)),stable 服务复用进程级连接池(`core/llm/mcp/`),per-request 服务(如 `retrieve_dataset_content` 需带每请求 HTTP header)每次新建;用户自助添加的私有 MCP 按 owner 现查合入。
+- **技能**:经 `core/capabilities/skills/loader.py` 注册为 AgentScope Agent Skills,并放行 `view_text_file` 读取 SKILL.md(详见 [技能系统](agent-skills.md))。
- **文件/沙箱工具**:`bash`、`sandbox_put_artifact`、`sandbox_get_artifact` 无条件注册;Read/Edit/Write/Glob/Grep/Delete/Move/mkdir + MySpace 工具受 `CODE_CAPABILITY_ENABLED` 门控,共享同一个 `ReadStateTracker` 维持「先 Read 才能 Edit」不变量。
-- **中间件**(洋葱模型,`core/llm/middlewares.py`):`DynamicModelMiddleware`(按 chat_mode 切模型,见 [模型接入](model-providers.md))、`FileContextMiddleware`(注入上传/历史文件上下文)、`SteerMiddleware`(工具结果之后、下一轮推理之前注入追加指令)、`WorkspacePinHintMiddleware`、`FinishPinGuardMiddleware`。
+- **中间件**(洋葱模型,`core/llm/runtime/`):`DynamicModelMiddleware`(按 chat_mode 切模型,见 [模型接入](model-providers.md))、`FileContextMiddleware`(注入上传/历史文件上下文)、`SteerMiddleware`(工具结果之后、下一轮推理之前注入追加指令)、`WorkspacePinHintMiddleware`、`FinishPinGuardMiddleware`。
- **上下文压缩**:`CompactingAgent` 把轮前、轮内和轮末三个触发时机接入同一个持久化 checkpoint 引擎,并共用 Codex 风格交接提示词;`ContextConfig` 只保留工具结果限长。压缩只使用统一协调器,不再运行框架的第二条摘要回退。压缩失败时保留原始上下文;最终请求仍超预算时保存断点并暂停,不继续调用回复模型。
- **权限**:所有已注册工具 seed 原生 `PermissionRule(ALLOW)`,保留 AgentScope 内置工具的危险操作检查(不使用一刀切 BYPASS)。
- **迭代上限**:主智能体默认 `max_iters=50`,隔离子智能体默认 10。
## SSE 事件类型与负载
-`orchestration/streaming.py::StreamingAgent` 把 AgentScope 2.0 `reply_stream` 的细粒度事件归并为内部事件;工具参数小分片按 256 字符或 50ms 合批,兼顾实时可见与 SSE 事件量。`workflow.py` 与 `chats.py::_stream_sse_response` 再补充会话级字段后落到 wire 上。前端实际收到的事件:
+`core/llm/runtime/streaming.py::StreamingAgent` 把 AgentScope 2.0 `reply_stream` 的细粒度事件归并为内部事件;工具参数小分片按 256 字符或 50ms 合批,兼顾实时可见与 SSE 事件量。`workflow.py` 与 `chats.py::_stream_sse_response` 再补充会话级字段后落到 wire 上。前端实际收到的事件:
| `type` | 含义 | 关键字段 |
|---|---|---|
@@ -200,7 +200,7 @@ data: {"type":"meta","route":"main","citations":[...],"usage":{"prompt_tokens":3
data: [DONE]
```
-`meta` 之后,`chat_run_executor.py` 持久化助手消息、回填 artifact,并起后台任务生成追问问题(`orchestration/followups.py`,结果写进消息 `extra_data.follow_up_questions`,前端经 `GET /v1/chats/{chat_id}/messages/{message_id}/followups` 拉取)。本体事件在前端汇总为独立的“领域本体治理”模块,不再写入或显示在“思考过程”中。模型草稿保持逐 token 流式展示;委员会仅在实际修订答案时发送一次 `content_replace`,前端原位替换正文,数据库只保存评审后的最终答案。`ontology_governance` 随助手消息持久化,刷新历史会话后仍可回显。
+`meta` 之后,`chat_run_executor.py` 持久化助手消息、回填 artifact,并起后台任务生成追问问题(`core/chat/followups.py`,结果写进消息 `extra_data.follow_up_questions`,前端经 `GET /v1/chats/{chat_id}/messages/{message_id}/followups` 拉取)。本体事件在前端汇总为独立的“领域本体治理”模块,不再写入或显示在“思考过程”中。模型草稿保持逐 token 流式展示;委员会仅在实际修订答案时发送一次 `content_replace`,前端原位替换正文,数据库只保存评审后的最终答案。`ontology_governance` 随助手消息持久化,刷新历史会话后仍可回显。
`usage` 是本轮所有模型调用的累计计费数据;上下文仪表使用的是
`context_usage`,即最近一次主模型调用的 `prompt_tokens + completion_tokens`,
@@ -233,9 +233,9 @@ data: [DONE]
引用让回答里的每个事实可溯源到具体工具结果。编号权收归后端唯一真源——模型只**复制**编号、不做任何计算,链路分四段:
-1. **发号回注(后端中间件)**:`core/llm/middlewares.py::CitationAnchorMiddleware` 挂在 AgentScope 2.0 的 `on_acting` 钩子上,在工具结果回给模型前调用 `orchestration/citation_anchor.py` 完成 提取 → 发号 → 回注:为每条可引用条目分配会话内单调唯一的锚点 id(`e1`、`e2`、…,跨工具、跨调用、跨轮不重复,新一轮从该会话历史消息的最大锚点续号),并把 `"cite_id": "e7"` 就地写进结果 JSON(纯文本结果在文末追加 `[cite_id: e7]` 行)。**发号器绑在 agent 实例上**(`attach_allocator()` / `resolve_allocator()`)——编排层与中间件由此共享同一个计数器;ContextVar 只作子智能体链路的兜底,因为 `astream_chat_workflow` 是 async generator,其上下文与 agent 实际执行所在的 task 并不互通。提取按四层降级:工具自声明 `__citations__` → 工具规格注册表(`TOOL_SPECS` 配置,列表路径 + 中英字段别名)→ 通用启发式(唯一字典数组字段)→ 整份结果 1 个锚点;操作型工具(写文件、pin 等,`SKIP_TOOLS`)直接放行。任何异常原样放行、绝不阻断对话。
+1. **发号回注(后端中间件)**:`core/llm/runtime/` 挂在 AgentScope 2.0 的 `on_acting` 钩子上,在工具结果回给模型前调用 `core/llm/citations/hook.py` 完成 提取 → 发号 → 回注:为每条可引用条目分配会话内单调唯一的锚点 id(`e1`、`e2`、…,跨工具、跨调用、跨轮不重复,新一轮从该会话历史消息的最大锚点续号),并把 `"cite_id": "e7"` 就地写进结果 JSON(纯文本结果在文末追加 `[cite_id: e7]` 行)。**发号器绑在 agent 实例上**(`attach_allocator()` / `resolve_allocator()`)——编排层与中间件由此共享同一个计数器;ContextVar 只作子智能体链路的兜底,因为 `astream_chat_workflow` 是 async generator,其上下文与 agent 实际执行所在的 task 并不互通。提取按四层降级:工具自声明 `__citations__` → 工具规格注册表(`TOOL_SPECS` 配置,列表路径 + 中英字段别名)→ 通用启发式(唯一字典数组字段)→ 整份结果 1 个锚点;操作型工具(写文件、pin 等,`SKIP_TOOLS`)直接放行。任何异常原样放行、绝不阻断对话。
2. **提示词约定**:系统提示词(`prompts/prompt_text/default/system/40_format.system.md` 的兜底版本,运行时以 DB 激活版本为准)只需一条与工具数量无关的通用规则:把结果里标注的 `cite_id` 原样复制进 `[锚文本](cite:e7)`(或句末 `[来源](cite:e7)`),禁止自行编号。
-3. **编排层消费**:每个 `tool_result` 事件经 `collect_citation_dicts()` 按 `tool_id` 从发号器注册表精确取 `CitationItem`(`id` / `tool_name` / `tool_id` / `title` / `url` / `snippet` / `source_type` / `item_index`);发号器缺位(旧对话回放等)时回退 `orchestration/citations.py` 的旧偏移提取。`source_type` 取值:CE 内置 `internet`、`knowledge_base`、`database`;`industry_news`、`ai_news`、`chain_info`、`company_profile` 等行业引用类型由商业版 EE 扩展。
+3. **编排层消费**:每个 `tool_result` 事件经 `collect_citation_dicts()` 按 `tool_id` 从发号器注册表精确取 `CitationItem`(`id` / `tool_name` / `tool_id` / `title` / `url` / `snippet` / `source_type` / `item_index`);发号器缺位(旧对话回放等)时回退 `core/llm/citations/` 的旧偏移提取。`source_type` 取值:CE 内置 `internet`、`knowledge_base`、`database`;`industry_news`、`ai_news`、`chain_info`、`company_profile` 等行业引用类型由商业版 EE 扩展。
4. **前端渲染**:citations 随 `tool_result` 与 `meta` 事件下发并随消息持久化(落库的就是注号后的结果,回放/分享与生成时编号一致)。`components/citation/CitationMarkdownBlock.tsx` 并行识别三种标记——`[锚文本](cite:eN)`(渲染为带悬浮出处卡片的文字链接)、`[[eN]]`(obsidian 双链容错)、旧格式 `[ref:工具名-序号]`(历史消息,渲染为角标);工具卡片条目上同步显示 `cite_id` 小徽章(`jx-tr-citeTag`),没有专属渲染器的工具由通用列表渲染器兜底成标准卡片。
**工具开发约定**:需要精确控制引用粒度的工具(自研或 MCP),在返回 JSON 里带 `__citations__` 字段——`[{"title": "...", "url": "...", "snippet": "...", "source_type": "..."}, …]`,条目顺序与结果正文对应;中间件优先采用并就地注入 `cite_id`。未自声明的工具按注册表配置或启发式提取,最差整份结果 1 个锚点——**任何工具默认可引用**。详见《[MCP 工具](mcp-tools.md)》的引用声明一节。
@@ -258,7 +258,7 @@ data: [DONE]
## 计划模式(Plan Mode)
-计划模式把复杂任务拆成「生成计划 → 用户确认/编辑 → 逐步执行」两阶段,实现在 `orchestration/subagents/plan_mode.py`:
+计划模式把复杂任务拆成「生成计划 → 用户确认/编辑 → 逐步执行」两阶段,实现在 `orchestration/plans/`:
- **生成**(`astream_generate_plan` / `POST /v1/plans/generate`):以 `disable_tools=True` 的"裸模型"产出结构化 JSON 计划。系统提示词解析顺序:版本池 `plan_mode` 激活版本 → 旧版 `system/90_plan_mode` 分段 → 文件兜底 `prompts/prompt_text/plan_mode/plan_mode.system.md` → 硬编码最小提示。
- **执行**(`astream_execute_plan` / `POST /v1/plans/{plan_id}/execute`):每个步骤独立建 agent 顺序执行,支持步骤级 MCP/技能/子智能体绑定、取消(`is_run_cancelled` 轮询);执行同样走 ChatRun + Redis Stream,可断线续播。
@@ -284,7 +284,7 @@ data: [DONE]
上下文和工作区是两个独立维度:探索员、审查员能检查当前文件,但不会被主对话或执行过程锚定;执行员则需要完整继承用户约束和既有决定。主智能体的动态路由表只展示本轮最终启用且角色策略允许下放的技能、MCP、知识库和基础工具;主智能体已关闭、未授权或被运行时过滤的能力既不会展示,也不会传给默认子智能体。三个角色都不能继续调用子智能体。默认列表不含 planner:普通主对话已用 `update_plan` 维护计划,避免形成“计划里的计划”。角色提示词在运行时仍来自版本池 `subagents` kind 的 `explorer / worker / reviewer` 三个独立 part,随包提示词 `extensions/agents/default//system.md` 只负责种子和故障回退;该 kind 保留在后端与迁移快照中,以兼容既有环境。
-除此之外,用户自建子智能体(`api/routes/v1/agents.py`,DB 表 `UserAgent`)可绑定独立的系统提示词、MCP / 技能 / 插件 / KB 集合与模型参数(provider / temperature / max_tokens / max_iters)。创建或编辑时,资源选择器支持以下来源:
+除此之外,用户自建子智能体(`edition_ee/routes/agents.py`,DB 表 `UserAgent`)可绑定独立的系统提示词、MCP / 技能 / 插件 / KB 集合与模型参数(provider / temperature / max_tokens / max_iters)。创建或编辑时,资源选择器支持以下来源:
- 已安装的技能与插件;
- 技能市场和插件市场,安装完成后自动绑定到当前子智能体;需要凭据的市场资源仍先走原有凭据配置与安装权限校验;
@@ -326,7 +326,7 @@ Enter / Tab 确认和 Escape 返回或关闭。个人在能力中心关闭的技
- **专属会话**:从子智能体详情页进入的会话使用 `agent_id`,后续轮次持续由该子智能体执行。
- **主智能体自主编排**:既没有结构化 `@`,也没有命中严格自然语言调用语法时,主智能体可
按任务需要调用
- `core/llm/subagent_tool.py` 注册的 `call_subagent`。子智能体在独立线程和事件循环中运行,结果
+ `core/llm/subagents/tool.py` 注册的 `call_subagent`。子智能体在独立线程和事件循环中运行,结果
回传给主智能体整合。这一路径适合多子智能体并行、任务拆分和跨领域汇总。
## 会话摘要与上下文压缩
@@ -335,8 +335,8 @@ Enter / Tab 确认和 Escape 返回或关闭。个人在能力中心关闭的技
| 层次 | 实现 | 触发 |
|---|---|---|
-| 会话标题摘要 | `core/llm/summarizer.py::ConversationSummarizer`(`summarizer` 模型角色,`ENABLE_SUMMARY` 开关),`POST /v1/summary` | 新会话标题自动生成 |
-| 统一上下文检查点 | `core/services/compaction_service.py::run_compaction()`;轮前、轮内、轮末共用同一触发比例、交接提示词、替代历史结构和持久化 checkpoint | 上下文达到模型窗口的配置比例时 |
+| 会话标题摘要 | `core/models/completions/summarizer.py::ConversationSummarizer`(`summarizer` 模型角色,`ENABLE_SUMMARY` 开关),`POST /v1/summary` | 新会话标题自动生成 |
+| 统一上下文检查点 | `core/chat/compaction/service.py::run_compaction()`;轮前、轮内、轮末共用同一触发比例、交接提示词、替代历史结构和持久化 checkpoint | 上下文达到模型窗口的配置比例时 |
| 确定性溢出保护 | `ContextConfig.tool_result_limit` 先限制单条工具结果;统一压缩失败时保留原始上下文,最终请求超预算则保存断点并暂停 | 单条工具结果过大,或统一压缩暂时不可用时 |
压缩只在合法边界切分:历史先按步骤展开,从尾部保留装得下
@@ -356,7 +356,7 @@ framing 的后端估算。当前回合结束后若后台压缩已启动,`meta.
## 超长工具结果 offload
-`core/llm/offloader.py::SandboxOffloader` 实现 AgentScope 2.0 `Offloader` 协议:上下文压缩/工具结果截断时,溢出部分不再被静默丢弃,而是落盘到沙箱 `/workspace/.offload/`(`tool_.txt` / `context_.txt`),框架会在给模型的 `` 里附上路径,模型可用 `Read` / `bash` 按需读回。仅在沙箱工具启用(`SANDBOX_TOOLS_ENABLED=true`,默认开)时挂载,写失败永不抛异常、返回降级说明。
+`core/llm/context/offloader.py::SandboxOffloader` 实现 AgentScope 2.0 `Offloader` 协议:上下文压缩/工具结果截断时,溢出部分不再被静默丢弃,而是落盘到沙箱 `/workspace/.offload/`(`tool_.txt` / `context_.txt`),框架会在给模型的 `` 里附上路径,模型可用 `Read` / `bash` 按需读回。仅在沙箱工具启用(`SANDBOX_TOOLS_ENABLED=true`,默认开)时挂载,写失败永不抛异常、返回降级说明。
## 会话分享(Chat Shares)
@@ -374,29 +374,29 @@ framing 的后端估算。当前回合结束后若后台压缩已启动,`meta.
## 其它入口
-同一编排底座还服务:消息重新生成(`POST /v1/chats/{chat_id}/regenerate`)、编辑重发(`POST /v1/chats/{chat_id}/edit`)、非流式 `POST /v1/agents/responses`(`stream: false`)与定时自动化(`orchestration/schedulers/`)。
+同一编排底座还服务:消息重新生成(`POST /v1/chats/{chat_id}/regenerate`)、编辑重发(`POST /v1/chats/{chat_id}/edit`)、非流式 `POST /v1/agents/responses`(`stream: false`)与定时自动化(`orchestration/automations/`)。
## 相关源码
| 主题 | 路径 |
|---|---|
| 聊天路由 / SSE 出口 | `src/backend/api/routes/v1/chats/__init__.py` |
-| Run 解耦 / Redis Stream / 续播 | `src/backend/orchestration/chat_run_executor.py`,`api/routes/v1/chat_runs.py` |
-| 流式编排主流程 | `src/backend/orchestration/workflow.py` |
-| 事件映射(reply_stream → SSE) | `src/backend/orchestration/streaming.py` |
+| Run 解耦 / Redis Stream / 续播 | `src/backend/core/chat/runtime/`,`api/routes/v1/chat_runs.py` |
+| 流式编排主流程 | `src/backend/core/agent_runtime/runner.py` |
+| 事件映射(reply_stream → SSE) | `src/backend/core/llm/runtime/streaming.py` |
| 运行时上下文装配 | `src/backend/core/chat/context.py` |
| Agent 工厂 | `src/backend/core/llm/factory/build.py` |
-| 中间件 | `src/backend/core/llm/middlewares.py`(纯函数 helper 在 `core/llm/hooks.py`) |
-| 引用抽取 | `src/backend/orchestration/citations.py` |
-| 跨会话引用(名片 / 摘要 / 分页) | `src/backend/core/services/chat_reference_service.py` |
+| 中间件 | `src/backend/core/llm/runtime/`(纯函数 helper 在 `core/harness/hooks.py`) |
+| 引用抽取 | `src/backend/core/llm/citations/` |
+| 跨会话引用(名片 / 摘要 / 分页) | `src/backend/core/chat/references.py` |
| 跨会话历史工具 | `src/backend/core/llm/tools/chat_history_tool.py` |
| 引用前端渲染 | `src/frontend/src/utils/citations.ts`,`src/frontend/src/components/citation/` |
-| 计划模式 | `src/backend/orchestration/subagents/plan_mode.py`,`api/routes/v1/plans.py` |
-| 子智能体工具 | `src/backend/core/llm/subagent_tool.py`,`api/routes/v1/agents.py` |
-| 标题摘要 / 上下文压缩 / 窗口保护 | `src/backend/core/llm/summarizer.py`、`compaction.py`、`core/services/compaction_service.py`、`context_manager.py` |
-| 超长结果 offload | `src/backend/core/llm/offloader.py` |
+| 计划模式 | `src/backend/orchestration/plans/`,`api/routes/v1/plans.py` |
+| 子智能体工具 | `src/backend/core/llm/subagents/tool.py`,`edition_ee/routes/agents.py` |
+| 标题摘要 / 上下文压缩 / 窗口保护 | `src/backend/core/models/completions/summarizer.py`、`compaction.py`、`core/chat/compaction/service.py`、`context_manager.py` |
+| 超长结果 offload | `src/backend/core/llm/context/offloader.py` |
| 会话分享 | `src/backend/api/routes/v1/chat_shares.py` |
-| 追问生成 | `src/backend/orchestration/followups.py` |
+| 追问生成 | `src/backend/core/chat/followups.py` |
| 前端流式解析 / 追加消息 | `src/frontend/src/hooks/chatStream.ts`,`useStreaming.ts`,`components/chat/QueuedMessageCard.tsx` |
### 桌面端工具权限与操作确认
diff --git a/document/zh-CN/modules/knowledge-base-wiki.md b/document/zh-CN/modules/knowledge-base-wiki.md
index f936e071..f37a6cda 100644
--- a/document/zh-CN/modules/knowledge-base-wiki.md
+++ b/document/zh-CN/modules/knowledge-base-wiki.md
@@ -138,12 +138,12 @@ Wiki 是纯离线批处理,调用量随文档量线性增长,**建议单独
| 层 | 位置 |
|---|---|
-| 数据模型 | `src/backend/core/db/models/kb_wiki.py` |
-| 生成管线 | `src/backend/core/kb/wiki/`(`extract` / `cite` / `dedup` / `taxonomy` / `reduce` / `finalize` / `pipeline`) |
-| 提示词 | `src/backend/core/kb/wiki/prompts.py` |
-| 作业与 worker | `src/backend/core/kb/wiki/jobs.py`、`worker.py` |
-| 读取面 | `src/backend/core/kb/wiki/local_provider.py` |
-| 按库分发 | `src/backend/core/kb/wiki_router.py` |
-| 路由 | `src/backend/api/routes/v1/kb_wiki.py` |
+| 数据模型 | `src/backend/features/knowledge/models.py` |
+| 生成管线 | `src/backend/features/knowledge/wiki/`(`extract` / `cite` / `dedup` / `taxonomy` / `reduce` / `finalize` / `pipeline`) |
+| 提示词 | `src/backend/features/knowledge/wiki/prompts.py` |
+| 作业与 worker | `src/backend/features/knowledge/wiki/jobs.py`、`worker.py` |
+| 读取面 | `src/backend/features/knowledge/wiki/local_provider.py` |
+| 按库分发 | `src/backend/features/knowledge/wiki_router.py` |
+| 路由 | `src/backend/features/knowledge/api/kb_wiki.py` |
| 智能体工具 | `src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py` |
| 前端 | `src/frontend/src/components/kb/`(`WikiPanel` / `WikiTree` / `ConceptGraph` / `IndexModePicker`) |
diff --git a/document/zh-CN/modules/knowledge-base.md b/document/zh-CN/modules/knowledge-base.md
index 15c3f525..5dd8ba6b 100644
--- a/document/zh-CN/modules/knowledge-base.md
+++ b/document/zh-CN/modules/knowledge-base.md
@@ -29,7 +29,7 @@ HugAgentOS 支持自建与外部知识库。公有检索按平台配置选择一
对象存储落盘(storage_key) │ · embed_text(query)
│ │ · Milvus hybrid_search:
▼ BackgroundTask │ 稠密(IP) + 稀疏(BoW) → RRF(k=60)
- core/content/kb_processing.py │ · 命中子块 → 回表 kb_chunks 取父块原文
+ core/files/content/kb_processing.py │ · 命中子块 → 回表 kb_chunks 取父块原文
vectorise_document_background() │ · 可选 Reranker 重排
· kb_parser.parse_and_chunk() │ · user_id 隔离 + public kb 全局可见
·(可选)LLM 抽关键词/生成问题 │
@@ -67,9 +67,9 @@ ORM 定义在 `src/backend/core/db/models/knowledge.py`:
| `kb_chunks` | **父块**原文(检索命中后返回给 LLM),含标签 `tags` 与关联问题 `questions` |
| `kb_assets` | 文档里的**媒体资产**(版面解析切出的图片、单独上传的图片):字节在对象存储,本表存归属分块、原文位置 `locator`、图注/OCR `text_content`、模型生成的描述 `caption` |
-Wiki 相关的三张表(`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs`)定义在 `core/db/models/kb_wiki.py`,见 [知识库 Wiki](./knowledge-base-wiki.md)。
+Wiki 相关的三张表(`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs`)定义在 `features/knowledge/models.py`,见 [知识库 Wiki](./knowledge-base-wiki.md)。
-子块不入关系库——向量化后写 Milvus collection `hugagent_kb_private`(`core/kb/kb_vector.py`),每行带 `user_id` / `kb_id` 字段做归属隔离,`row_type` 区分 chunk 行、question 行与 image 行。
+子块不入关系库——向量化后写 Milvus collection `hugagent_kb_private`(`features/knowledge/kb_vector.py`),每行带 `user_id` / `kb_id` 字段做归属隔离,`row_type` 区分 chunk 行、question 行与 image 行。
### 分块与索引
@@ -83,16 +83,16 @@ Wiki 相关的三张表(`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs`
| `recursive` | 递归定长切分 |
| `embedding_semantic` | 基于 embedding 相似度的语义边界检测 |
-父子分块参数可在上传时通过 `indexing_config` 调整:`parent_chunk_size`(默认 1024 token)、`child_chunk_size`(128)、`overlap_tokens`(20)、`parent_child_indexing`(默认 true)。还可启用 LLM 增强:`auto_keywords_count`(每父块抽关键词入 tags,参与稀疏检索)与 `auto_questions_count`(每父块生成关联问题,作为独立 question 行入 Milvus,提高问句召回)。后台向量化任务在 `core/content/kb_processing.py::vectorise_document_background()`。
+父子分块参数可在上传时通过 `indexing_config` 调整:`parent_chunk_size`(默认 1024 token)、`child_chunk_size`(128)、`overlap_tokens`(20)、`parent_child_indexing`(默认 true)。还可启用 LLM 增强:`auto_keywords_count`(每父块抽关键词入 tags,参与稀疏检索)与 `auto_questions_count`(每父块生成关联问题,作为独立 question 行入 Milvus,提高问句召回)。后台向量化任务在 `core/files/content/kb_processing.py::vectorise_document_background()`。
### 多模态:图片检索
-文档里的图不再被丢弃。索引时走 `core/kb/kb_assets.py`:
+文档里的图不再被丢弃。索引时走 `features/knowledge/kb_assets.py`:
1. **抽图**——PDF 经外部解析服务的版面切分拿回图片本体、图注、页码与 bbox;单独上传的
图片自身即一个资产(此前会被当作纯文本硬解成乱码入库,现已修复)。字节存对象存储,
正文里原本指向解析服务临时路径的死链被改写成真实资产 URL。
-2. **图像理解**——复用[视觉桥](./model-providers.md#视觉桥让纯文本模型能读图)(`core/vision`),和对话里「纯文本主模型读图」
+2. **图像理解**——复用[视觉桥](./model-providers.md#视觉桥让纯文本模型能读图)(`core/models/vision`),和对话里「纯文本主模型读图」
走的是同一条通路、同一个 `vision` 模型角色,因此白得三样东西:按图片内容哈希的**证据缓存**
(重新索引同一篇文档不用再付一次钱)、三协议与结构化输出降级、以及「该角色只能指派多模态模型」
的硬约束。产出映射:`summary` → `caption`,`ocr.full_text` 并入 `text_content`(与原图注共存),
@@ -134,7 +134,7 @@ ASR 转写写进 `text_content`,其后的索引与检索完全复用。
### API 路由
-用户侧路由前缀 `/v1/catalog/kb`(`src/backend/api/routes/v1/kb.py`,请求模型在 `kb_models.py`):
+用户侧路由前缀 `/v1/catalog/kb`(`src/backend/features/knowledge/api/kb.py`,请求模型在 `kb_models.py`):
| 方法 | 路径 | 说明 |
|---|---|---|
@@ -150,7 +150,7 @@ ASR 转写写进 `text_content`,其后的索引与检索完全复用。
| GET / PATCH | `/v1/catalog/kb/{kb_id}/chunks[/{chunk_id}]` | 分块列表 / 编辑标签与问题 |
| GET | `/v1/catalog/kb/assets/{asset_id}` | 读取媒体资产原始字节(可见性跟随所属空间) |
-业务逻辑集中在 `core/services/kb_service.py::KBService`。
+业务逻辑集中在 `features/knowledge/service.py::KBService`。
### 系统托管知识库:我的空间同步
@@ -158,13 +158,13 @@ ASR 转写写进 `text_content`,其后的索引与检索完全复用。
### 管理员公共知识库(商业版 EE)
-`/v1/admin/kb/*` 管理台路由在 `src/backend/api/routes/v1/admin_kb.py`,挂 `content_admin` 能力位(EE 路由表见 `api/routes/v1/__init__.py`)。公共知识库由合成系统账号 `system_public_kb` 持有(`kb_service.py::SYSTEM_KB_OWNER_ID`),`visibility=public`,对全体用户可见可检索。管理端额外提供原始文件下载、Office 转 PDF 在线预览、分块内容编辑 / 删除等能力。
+`/v1/admin/kb/*` 管理台路由在 `src/backend/edition_ee/routes/admin_kb.py`,挂 `content_admin` 能力位(EE 路由表见 `api/routes/v1/__init__.py`)。公共知识库由合成系统账号 `system_public_kb` 持有(`kb_service.py::SYSTEM_KB_OWNER_ID`),`visibility=public`,对全体用户可见可检索。管理端额外提供原始文件下载、Office 转 PDF 在线预览、分块内容编辑 / 删除等能力。
公有知识库检索 `retrieve_dataset_content` 严格按平台配置选择一种来源:自建模式(`knowledge_base.provider=custom`)只检索当前用户有权访问的本地公有/共享库,排除个人私有库;外部模式只检索当前选择的 Dify、FastGPT 或 WeKnora 数据集,连接配置也按后端隔离。两种来源不合并,空结果或服务错误均不触发跨来源回退。公有知识库列表遵循相同的来源选择;私有知识库仍通过独立的 `retrieve_local_kb` 入口访问。
## Dify 外接知识库(商业版 EE)
-商业版客户端位于 `src/backend/edition_ee/kb/dify.py`,共享路由通过 `core/kb/external_provider.py` 接缝调用。CE 派生树把该接缝替换为禁用实现,且不包含 Dify 客户端。启用判定 `is_dify_enabled()` 的优先级:
+商业版客户端位于 `src/backend/edition_ee/kb/dify.py`,共享路由通过 `features/knowledge/extension.py` 接缝调用。CE 派生树把该接缝替换为禁用实现,且不包含 Dify 客户端。启用判定 `is_dify_enabled()` 的优先级:
1. DB 系统配置 `knowledge_base.provider == "dify"`(Config 管理台可改);
2. 环境变量 `KNOWLEDGE_BASE=dify`;
@@ -180,9 +180,9 @@ DIFY_API_KEY=dataset-... # 兼容别名 DIFY_AUTH_TOKEN
## 文件解析支持
-知识库上传校验在 `core/content/file_validation.py::validate_kb_file`(扩展名 + magic bytes 双重校验),允许:`.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json` 及图片(`.png` `.jpg` `.jpeg` `.webp` `.gif`)。
+知识库上传校验在 `core/files/content/file_validation.py::validate_kb_file`(扩展名 + magic bytes 双重校验),允许:`.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json` 及图片(`.png` `.jpg` `.jpeg` `.webp` `.gif`)。
-通用文件解析器 `core/content/file_parser.py::parse_file()` 覆盖更广(对话附件、我的空间文件共用):PDF、DOCX、DOC/WPS(经 LibreOffice 转换)、TXT、XLSX/XLS、CSV、PPTX,外加 HTML / Markdown / JSON / YAML / 代码等纯文本格式直接 UTF-8 解码。
+通用文件解析器 `core/files/content/file_parser.py::parse_file()` 覆盖更广(对话附件、我的空间文件共用):PDF、DOCX、DOC/WPS(经 LibreOffice 转换)、TXT、XLSX/XLS、CSV、PPTX,外加 HTML / Markdown / JSON / YAML / 代码等纯文本格式直接 UTF-8 解码。
## 前端
@@ -196,17 +196,17 @@ DIFY_API_KEY=dataset-... # 兼容别名 DIFY_AUTH_TOKEN
| 路径 | 职责 |
|---|---|
-| `src/backend/core/kb/kb_parser.py` | 文档解析 + 父子分块(5 种 chunk_method) |
-| `src/backend/core/kb/kb_vector.py` | Milvus collection、embedding、混合检索、重排 |
-| `src/backend/core/kb/kb_assets.py` | 媒体资产:落盘、图像理解(调用视觉桥)、检索行、结果附图 |
+| `src/backend/features/knowledge/kb_parser.py` | 文档解析 + 父子分块(5 种 chunk_method) |
+| `src/backend/features/knowledge/kb_vector.py` | Milvus collection、embedding、混合检索、重排 |
+| `src/backend/features/knowledge/kb_assets.py` | 媒体资产:落盘、图像理解(调用视觉桥)、检索行、结果附图 |
| `src/backend/edition_ee/kb/dify.py` | Dify datasets 客户端与启用判定(仅 EE) |
-| `src/backend/core/kb/external_provider.py` | 版本中立的外部知识库接缝;CE overlay 将其禁用 |
-| `src/backend/core/content/kb_processing.py` | 后台向量化任务、LLM 关键词 / 问题增强 |
-| `src/backend/core/content/file_validation.py` | 上传文件校验(扩展名 + magic bytes) |
-| `src/backend/core/content/file_parser.py` | 通用文件解析器 |
-| `src/backend/core/services/kb_service.py` | 知识库业务逻辑(含系统托管同步库) |
-| `src/backend/api/routes/v1/kb.py` + `kb_models.py` | 用户侧 `/v1/catalog/kb` 路由 |
-| `src/backend/api/routes/v1/admin_kb.py` | 管理台公共知识库路由(商业版 EE) |
+| `src/backend/features/knowledge/extension.py` | 版本中立的外部知识库接缝;CE overlay 将其禁用 |
+| `src/backend/features/knowledge/processing.py` | 后台向量化任务、LLM 关键词 / 问题增强 |
+| `src/backend/core/files/content/file_validation.py` | 上传文件校验(扩展名 + magic bytes) |
+| `src/backend/core/files/content/file_parser.py` | 通用文件解析器 |
+| `src/backend/features/knowledge/service.py` | 知识库业务逻辑(含系统托管同步库) |
+| `src/backend/features/knowledge/api/kb.py` + `kb_models.py` | 用户侧 `/v1/catalog/kb` 路由 |
+| `src/backend/edition_ee/routes/admin_kb.py` | 管理台公共知识库路由(商业版 EE) |
| `src/backend/api/routes/v1/catalog.py` | 能力目录聚合(CE 仅私有库;EE 追加 Dify / 公共库) |
| `src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/` | 检索 MCP server(两个工具) |
| `src/backend/core/db/models/knowledge.py` | `KBSpace` / `KBDocument` / `KBChunk` / `KBAsset` ORM |
diff --git a/document/zh-CN/modules/mcp-tools.md b/document/zh-CN/modules/mcp-tools.md
index ad8af2da..b8706b50 100644
--- a/document/zh-CN/modules/mcp-tools.md
+++ b/document/zh-CN/modules/mcp-tools.md
@@ -26,7 +26,7 @@ HugAgentOS 的工具生态构建在 [MCP(Model Context Protocol)](https://mo
│ agent) │ │ │ :9103 产业知识中心统一 MCP(插件,EE) │
└─────────┘ │ │ :9104 保留(图表 MCP 已退役) │
│ │ │ :9105 保留(原报告导出 MCP) │
- MCPConnectionPool │ │ :9106 web_fetch(网页抓取) │
+ Native MCP Client │ │ :9106 web_fetch(网页抓取) │
mcp_pool.py) └───┬────┘ :9108 automation_task(自动化插件提供) │
│ 9109–9111 保留(原 excel/ppt/pdf MCP) │
│ :9112 skill_manager(技能管理) │
@@ -190,7 +190,7 @@ extensions/mcp_servers/_mcp/
- 中间件(`CitationAnchorMiddleware`)会**优先采用**该声明,就地为每条注入
`"cite_id": "eN"`,模型引用时原样复制。
2. **标准列表结构 → 加一行注册表配置**:结果形如 `{"items": [{"title": …, "content": …}]}`
- 的列表型工具,在 `orchestration/citation_anchor.py::TOOL_SPECS` 里登记
+ 的列表型工具,在 `core/llm/citations/hook.py::TOOL_SPECS` 里登记
`items_paths` + 字段别名即可逐条编号,不改工具代码。
3. **什么都不做 → 自动兜底**:通用启发式能识别顶层(或 `result` 下一层)唯一的
字典数组字段并逐条编号;彻底认不出时整份结果作为 1 个锚点。
@@ -201,21 +201,18 @@ extensions/mcp_servers/_mcp/
Server 同样适用(返回 JSON 即可)。回注发生在结果进模型上下文与落库之前,前端工具卡片
会把 `cite_id` 渲染成条目徽章,与正文 `[锚文本](cite:eN)` 引用一一对应。
-## 后端客户端:连接池与裸名还原
+## 后端客户端:原生 MCP 生命周期
-后端基于 AgentScope 2.0 的 `MCPClient` 连接 MCP Server,核心在两个文件:
+后端使用 MCP SDK 2.3 的 `Client(mode="auto")` 和 AgentScope `ToolBase`。`client.py` 处理传输授权,`lifetime.py` 管理 SDK 上下文并完整遍历分页。HTTP 在一次运行内复用客户端。STDIO 按执行用户、聊天和配置独占租用,成功结束后保留 60 秒;失败或取消则销毁。已删除稳定连接开关和启动连接池。
-- **`core/llm/mcp_pool.py` — `MCPConnectionPool`**(进程级单例):启动时 `warmup_mcp_tools()` 从 DB 读取全部启用的 Server 配置并预连接。注意 2.0 语义下的池化策略:
- - **stdio 且 `is_stable=true`** 的 Server 跨请求保持连接(省掉 1–7 秒子进程冷启动);
- - **HTTP Server 一律不池化**——2.0 的 stateful HTTP 客户端与 asyncio task 绑定,跨请求复用会触发 cancel-scope 崩溃,因此每请求用 `is_stateful=False` 新建连接;
- - per-request Server(知识库检索带用户头)每次现连,请求结束 `close_transient()` 关闭。
-- **`core/llm/mcp_manager.py` — `BareNameMCPClient`**:AgentScope 2.0 默认把工具名改写为 `mcp____`,该子类还原为服务器侧裸名(`internet_search` 而非 `mcp__internet_search__internet_search`),保证展示名映射(`core/config/display_names.py`)、[引用溯源](chat.md)的按工具名分发、前端图标渲染等 1.x 约定继续成立。
+`callbacks.py` 将隔离的文本 Sampling 与用户 Elicitation 绑定到当前运行。资源、URI 模板和提示词转换为受权限控制的工具。进度显示在现有工具卡片。结构化输出和资源链接保留在工具 metadata 与模型输出中。SDK 2.3 尚未提供新版 Tasks 扩展。
+
+MCP 工具的执行预算不包含等待用户回答的时间。有效提问会暂停当前调用的计时;回答或放弃后,从剩余预算继续。多次、重叠提问不会重置预算,其他调用继续独立计时。用户交互仍受现有等待期限约束,取消运行会清理待回答问题。SDK 请求和 HTTP 读取由调用预算约束,连接、写入和发现仍保留超时。
-`Toolkit` 在 2.0 是一次性构造,由 `core/llm/factory/build.py` 统一执行 `Toolkit(tools=[...], mcps=clients)`。
## 统一工具权限网关
-`core/llm/tool_permissions.py` 提供运行级 `ToolPermissionRegistry` 与
+`core/llm/tool_permissions/` 提供运行级 `ToolPermissionRegistry` 与
`ToolPermissionMiddleware`。Registry 是**受控工具清单**,不是全工具声明表:
- 工具名没有进入 Registry 时保持既有行为,直接放行,不要求新工具声明权限;
@@ -228,7 +225,7 @@ Server 同样适用(返回 JSON 即可)。回注发生在结果进模型上
一律拒绝,不会再以“记录审计后继续执行”降级;
- 渐进加载插件把新出现的 MCP 工具加入同一个运行级 Registry,因此热加载不会绕开网关。
-原生工具可以在 `ToolCollector.register_tool_function(..., permission=spec)` 注册规则;平台
+原生工具可以在 `ToolAssembly.append(function_tool(..., dispatch_spec=spec))` 注册规则;平台
内置的本机文件、Bash 与 MySpace 工具集中列在 `builtin_tool_permission()`。没有传
`permission` 且不在该表中的原生工具不受网关管理。
@@ -251,7 +248,7 @@ MCP 规则只接受平台/管理员持有的可信连接配置,不采信远端
## 配置注册:DB 驱动 + catalog 门控
-MCP Server 的配置真源是数据库表 `admin_mcp_servers`(ORM:`core/db/models.py::AdminMcpServer`),由 `core/services/mcp_service.py::McpServerConfigService` 以 30 秒 TTL 缓存读出,格式与旧 `MCP_SERVERS` dict 兼容(`transport / command / args / env / url / headers / is_stable`)。`core/config/mcp_config.py` 保留为内置 Server 的 URL 构造器(首次部署种子)。
+MCP Server 的配置真源是数据库表 `admin_mcp_servers`(ORM:`core/db/models/`),由 `core/capabilities/mcp/service.py::McpServerConfigService` 以 30 秒 TTL 缓存读出,使用原生客户端配置(`transport / command / args / env / url / headers`)。`core/config/mcp_config.py` 保留为内置 Server 的 URL 构造器(首次部署种子)。
是否**对模型可见**还要过一道 [catalog](catalog.md) 门控:`core/config/catalog.json` 中 `mcp` 段的每项对应一个 server_id,`is_enabled(id, "mcp_server")` 为 false 的 Server 即使连接着也不会注册给智能体。
@@ -263,7 +260,7 @@ MCP Server 的配置真源是数据库表 `admin_mcp_servers`(ORM:`core/db/m
- **创建即试连**:`_probe_connectivity` 真实连一次,失败拒绝落库;
- **开关与排序**:`POST /{id}/toggle` 即时启停(联动刷新 catalog 与连接池);
- **密钥保护**:HTTP Header 值加密存储、接口统一以 `***` 返回;`env_vars` 中疑似密钥的值同样脱敏;
-- **测试与重载**:`POST /{id}/test` 单独试连;`POST /reload-pool` 热重建连接池;
+- **测试与重载**:`POST /{id}/test` 单独试连;`配置更新自动失效缓存` 更新后下一次运行重新发现;
- **移至 MCP 市场**:把现有远程 HTTP/SSE MCP 生成无凭据的市场快照,
并停用原全局实例;该服务随后从 MCP 服务管理列表移除,可在市场中复核、
编辑展示信息,再按需全局安装。原服务已配置的 Token 仍加密保留在服务端,
@@ -279,7 +276,7 @@ MCP Server 的配置真源是数据库表 `admin_mcp_servers`(ORM:`core/db/m
- `POST /v1/me/mcp-servers`:添加私有远程 MCP,支持公网 HTTP/HTTPS 上的 HTTP/SSE——用户入口禁止 stdio,避免在服务器执行任意命令;地址会经过 DNS/IP 检查以阻断 localhost、内网、链路本地与保留地址;创建即试连,连不上不落库。HTTP 为明文传输,生产环境仍建议使用 HTTPS;
- `DELETE /v1/me/mcp-servers/{id}`:删除自己的私有 MCP。
-实现上复用同一张 `admin_mcp_servers` 表:`owner_user_id` = 当前用户实现 owner 隔离,server_id 自动生成 `umcp_` 防冲突,`is_stable=False` 不进 warmup 池。HTTP Header 值使用 Fernet 加密后落库,运行时才解密。该功能由管理台的 per-user 权限位 `can_add_mcp` 控制(社区版单租户默认即可放开;商业版由组织管理员按用户授予,见[版本说明](../editions/overview.md))。
+实现上复用同一张 `admin_mcp_servers` 表:`owner_user_id` = 当前用户实现 owner 隔离,server_id 自动生成 `umcp_` 防冲突。HTTP Header 值使用 Fernet 加密后落库,运行时才解密。该功能由管理台的 per-user 权限位 `can_add_mcp` 控制(社区版单租户默认即可放开;商业版由组织管理员按用户授予,见[版本说明](../editions/overview.md))。
## MCP 市场
@@ -359,19 +356,19 @@ docker-compose up -d --build mcp
| `src/backend/extensions/mcp_servers/_serve.py` | stdio / streamable-http 双 transport 统一入口 |
| `src/backend/extensions/mcp_servers/registry.py` | 安装位置、暴露方式、包名和基础端口 |
| `src/backend/extensions/mcp_servers/_ports.py` | 实际部署端口映射 |
-| `src/backend/core/llm/mcp_pool.py` | MCP 连接池(stdio 池化 / HTTP per-request) |
-| `src/backend/core/llm/mcp_manager.py` | MCPClient 构造 + 工具裸名还原 |
-| `src/backend/core/llm/tool_permissions.py` | 受控工具 Registry、权限判定、确认路由与执行票据 |
-| `src/backend/core/services/mcp_service.py` | DB 驱动的 Server 配置服务(30s 缓存) |
-| `src/backend/core/services/mcp_marketplace_service.py` | 市场发布、审核、安装、可见性与安全控制 |
-| `src/backend/core/services/mcp_oauth_service.py` | OAuth 2.1 登录、SDK 元数据发现、加密令牌存储与刷新 |
-| `src/backend/core/services/mcp_marketplace_monitor.py` | 远程工具快照定期复检与漂移监控 |
+| `src/backend/core/llm/mcp/` | MCP 连接池(stdio 池化 / HTTP per-request) |
+| `src/backend/core/llm/mcp/manager.py` | MCPClient 构造 + 工具裸名还原 |
+| `src/backend/core/llm/tool_permissions/` | 受控工具 Registry、权限判定、确认路由与执行票据 |
+| `src/backend/core/capabilities/mcp/service.py` | DB 驱动的 Server 配置服务(30s 缓存) |
+| `src/backend/features/marketplace/` | 市场发布、审核、安装、可见性与安全控制 |
+| `src/backend/core/capabilities/mcp/oauth.py` | OAuth 2.1 登录、SDK 元数据发现、加密令牌存储与刷新 |
+| `src/backend/features/marketplace/` | 远程工具快照定期复检与漂移监控 |
| `src/backend/core/config/mcp_config.py` | 内置 Server URL 构造(http://mcp:NNNN/mcp/) |
| `src/backend/core/config/catalog.json` | 能力目录:MCP 启停门控种子 |
| `src/backend/api/routes/v1/admin_mcp_servers.py` | 管理员自定义 MCP API |
| `src/backend/api/routes/v1/me_capabilities.py` | 用户自助私有 MCP / 技能 API |
-| `src/backend/api/routes/v1/mcp_marketplace.py` | 用户端 MCP 市场 API |
-| `src/backend/api/routes/v1/admin_mcp_marketplace.py` | 管理端 MCP 市场与审核 API |
+| `src/backend/features/marketplace/api/mcp_marketplace.py` | 用户端 MCP 市场 API |
+| `src/backend/features/marketplace/api/admin_mcp_marketplace.py` | 管理端 MCP 市场与审核 API |
| `docker/Dockerfile.mcp` | mcp 容器镜像(MCP 运行时、绘图依赖与中文字体) |
相关文档:[能力目录](catalog.md) · [技能系统](agent-skills.md) · [知识库](knowledge-base.md) · [版本与许可](../editions/overview.md)
diff --git a/document/zh-CN/modules/memory.md b/document/zh-CN/modules/memory.md
index 98d7822f..c596185d 100644
--- a/document/zh-CN/modules/memory.md
+++ b/document/zh-CN/modules/memory.md
@@ -64,7 +64,7 @@ save_memories_background()
· sanitize 脱敏闸门 → 写 L1/L2/L3/Session → audit 旁路
```
-检索与注入的整合层在 `src/backend/orchestration/memory_integration.py`;mem0 的 LLM / Embedder / Milvus / Reranker 配置在 `src/backend/core/memory/service.py`,Neo4j 图谱读写在 `src/backend/core/memory/graph.py`。模型配置优先取 DB 中 `memory` / `embedding` 角色,缺省回落到环境变量。
+检索与注入的整合层在 `src/backend/core/agent_runtime/integrations/memory.py`;mem0 的 LLM / Embedder / Milvus / Reranker 配置在 `src/backend/core/memory/service.py`,Neo4j 图谱读写在 `src/backend/core/memory/graph.py`。模型配置优先取 DB 中 `memory` / `embedding` 角色,缺省回落到环境变量。
## 写入流水线与抽取器
@@ -136,7 +136,7 @@ save_memories_background()
| `memory_enabled` | 永久记忆**读取**:会话启动时是否注入冻结块 | `false` |
| `memory_write_enabled` | **写入**:对话结束后是否抽取并保存记忆 | `false` |
-项目对话有独立作用域:个人项目 / 默认空间用真实 `user_id`,团队项目用 `scope_user_id = "team:"`——同团队成员写入同一个 mem0 桶实现共享,真实作者保留在 `metadata.author_user_id`(见 `orchestration/memory_integration.py::save_memories_background` 与 `api/routes/v1/memories.py::list_memories`)。项目级开关存于 `projects.metadata`(`memory_enabled` / `memory_write_enabled`,项目内缺省 `true`),详见 [项目空间与我的空间](./projects-myspace.md)。
+项目对话有独立作用域:个人项目 / 默认空间用真实 `user_id`,团队项目用 `scope_user_id = "team:"`——同团队成员写入同一个 mem0 桶实现共享,真实作者保留在 `metadata.author_user_id`(见 `core/agent_runtime/integrations/memory.py::save_memories_background` 与 `api/routes/v1/memories.py::list_memories`)。项目级开关存于 `projects.metadata`(`memory_enabled` / `memory_write_enabled`,项目内缺省 `true`),详见 [项目空间与我的空间](./projects-myspace.md)。
## 前端记忆中心
@@ -229,8 +229,8 @@ RERANKER_API_KEY=...
| `src/backend/core/memory/sanitizer.py` | 脱敏闸门(硬编码规则 + DB 动态规则) |
| `src/backend/core/memory/audit.py` | 审计旁路(商业版 EE) |
| `src/backend/core/memory/context.py` | `MemoryContext` 与 workspace / 层级解析 |
-| `src/backend/orchestration/memory_integration.py` | 检索启动、冻结块组装与注入、保存转调 |
-| `src/backend/orchestration/workflow.py` | 主编排:记忆 hook 接线点 |
+| `src/backend/core/agent_runtime/integrations/memory.py` | 检索启动、冻结块组装与注入、保存转调 |
+| `src/backend/core/agent_runtime/runner.py` | 主编排:记忆 hook 接线点 |
| `src/backend/api/routes/v1/memories.py` | `/v1/memories` 管理 API |
| `src/backend/core/db/models/memory.py` | `ProfileMemory`、`MemoryOutbox`、`MemorySanitizerRule` ORM |
| `src/backend/edition_ee/db/models/memory.py` | `MemoryAudit` ORM(仅 EE) |
diff --git a/document/zh-CN/modules/model-providers.md b/document/zh-CN/modules/model-providers.md
index 6a048edf..40e28d2a 100644
--- a/document/zh-CN/modules/model-providers.md
+++ b/document/zh-CN/modules/model-providers.md
@@ -9,7 +9,7 @@ HugAgentOS 通过 **OpenAI 兼容协议**接入任意大模型端点(vLLM、Ol
## 配置模型:供应商 + 角色
-两张表(`core/db/models.py`):`model_providers`(base_url / api_key / model_name / extra_config / is_active)与 `model_role_assignments`(role_key → provider_id)。角色定义在 `core/db/model_repository.py::ROLE_DEFINITIONS`:
+两张表(`core/db/models/`):`model_providers`(base_url / api_key / model_name / extra_config / is_active)与 `model_role_assignments`(role_key → provider_id)。角色定义在 `core/models/repository.py::ROLE_DEFINITIONS`:
| role_key | 用途 | 类型 |
|---|---|---|
@@ -36,6 +36,8 @@ HugAgentOS 通过 **OpenAI 兼容协议**接入任意大模型端点(vLLM、Ol
### 接口协议:默认 Responses,不支持才回退
+Responses 的思考原文和摘要事件由当前锁定的 AgentScope 原生解析。平台原样保留事件的 `item_id` 与分段索引,避免流式发送失败,以及不同思考条目或分段被错误合并。
+
OpenAI 兼容上游有两套接口:较老的 `POST /chat/completions` 和较新的 `POST /responses`。平台**默认走 Responses**,因为它把模型的思考做成独立的 `reasoning` 条目,下一轮可原样回传;`/chat/completions` 没有放思考的位置,只能把思考夹带进 `reasoning_content` 字段。只有确实没有 `/responses` 路由的上游(常见于只做转发的中转网关)才使用 Chat Completions。
走哪套由 `extra_config.api_protocol` 决定,取值 `responses` / `chat_completions`。该值不靠厂商名推断,而是保存供应商时向上游探测后写入。探测分两步,两步都过才判为 Responses 可用:
@@ -124,7 +126,7 @@ DeepSeek、GLM、Qwen 文本版这类主力模型看不见图片。视觉桥的
## 动态模型切换(chat_mode)
-前端每条消息可带 `chat_mode`(`turbo / fast / medium / high / max`),在 reply 开始前由 `core/llm/middlewares.py::DynamicModelMiddleware`(`on_reply`)热切 `agent.model`:
+前端每条消息可带 `chat_mode`(`turbo / fast / medium / high / max`),在 reply 开始前由 `core/llm/runtime/`(`on_reply`)热切 `agent.model`:
```
chat_mode → hooks._resolve_chat_mode(agent.state)
@@ -134,7 +136,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state)
high/max → reasoning_effort=high/max(端点须声明支持)
```
-旧客户端只传 `enable_thinking` 时回退映射为 medium / fast。这就是 1.x「pre_reply 动态模型 hook」在 AgentScope 2.0 下的形态——hook 工厂已重构为中间件,`core/llm/hooks.py` 仅保留纯函数 helper。
+旧客户端只传 `enable_thinking` 时回退映射为 medium / fast。这就是 1.x「pre_reply 动态模型 hook」在 AgentScope 2.0 下的形态——hook 工厂已重构为中间件,`core/harness/hooks.py` 仅保留纯函数 helper。
## 环境变量与 MCP 子进程注入
@@ -147,7 +149,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state)
## 个人 API-Key
-用户可签发个人密钥以编程方式调用平台 API(`api/routes/v1/api_keys.py` + `core/services/api_key_service.py`):
+用户可签发个人密钥以编程方式调用平台 API(`api/routes/v1/api_keys.py` + `core/auth/api_keys.py`):
- 明文形如 `sk-jx-`,**仅创建时返回一次**,DB 只存 SHA256 哈希 + 前缀;支持 7/30/90/180/365 天有效期或永不过期、启停与撤销。
- 调用方式 `Authorization: Bearer sk-jx-...`,鉴权层(`core/auth/backend.py`)识别 key 前缀后经 `resolve_api_key` 解析为用户上下文,校验启用/未撤销/未过期。
@@ -157,7 +159,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state)
## 计费与用量
-token 用量在流式收尾的 `meta.usage` 中统计(`orchestration/streaming.py` 从 `ModelCallEndEvent` 累计)并随消息持久化,之上有两组管理台报表(`CONFIG_TOKEN`):
+token 用量在流式收尾的 `meta.usage` 中统计(`core/llm/runtime/streaming.py` 从 `ModelCallEndEvent` 累计)并随消息持久化,之上有两组管理台报表(`CONFIG_TOKEN`):
- **用量日志** `api/routes/v1/admin_usage_logs.py`:`GET /v1/admin/usage-logs`(明细)、`/summary`(汇总)、`/models`(去重模型名)。
- **计费报表** `api/routes/v1/admin_billing.py`(**商业版 EE**——完整管理台能力):`GET /v1/admin/billing/summary`(按用户/模型聚合成本)、模型定价 CRUD(`/pricing`,输入/输出单价、币种)、`GET /v1/admin/billing/export`(CSV 成本导出)。
@@ -166,21 +168,21 @@ token 用量在流式收尾的 `meta.usage` 中统计(`orchestration/streaming
## 路由策略(ROUTER_STRATEGY)
-`orchestration/strategy.py`:`ROUTER_STRATEGY=main_only`(默认)恒路由到主智能体;`llm_router` 为预留占位,当前实现同样回落 `MainOnlyStrategy`(safe-by-default)。实际的多智能体分流由 `@提及` + `call_subagent` 工具完成,见 [对话与智能体编排](chat.md)。
+`core/agent_runtime/strategy.py`:`ROUTER_STRATEGY=main_only`(默认)恒路由到主智能体;`llm_router` 为预留占位,当前实现同样回落 `MainOnlyStrategy`(safe-by-default)。实际的多智能体分流由 `@提及` + `call_subagent` 工具完成,见 [对话与智能体编排](chat.md)。
## 相关源码
| 主题 | 路径 |
|---|---|
-| 模型工厂 / JxOpenAIChatModel | `src/backend/core/llm/chat_models.py` |
-| 角色解析服务(DB + 缓存) | `src/backend/core/services/model_config.py` |
-| 角色定义 / 供应商仓储 | `src/backend/core/db/model_repository.py` |
+| 模型工厂 / JxOpenAIChatModel | `src/backend/core/models/chat_models.py` |
+| 角色解析服务(DB + 缓存) | `src/backend/core/models/config.py` |
+| 角色定义 / 供应商仓储 | `src/backend/core/models/repository.py` |
| 管理 API | `src/backend/api/routes/v1/models.py` |
-| 动态模型中间件 | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`,helper 在 `core/llm/hooks.py` |
-| MCP env 注入 | `src/backend/core/services/model_config.py::get_mcp_env_overlay`,`core/config/runtime_env.py` |
-| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`,`core/services/api_key_service.py`,`core/auth/backend.py` |
+| 动态模型中间件 | `src/backend/core/llm/runtime/`,helper 在 `core/harness/hooks.py` |
+| MCP env 注入 | `src/backend/core/models/config.py::get_mcp_env_overlay`,`core/config/runtime_env.py` |
+| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`,`core/auth/api_keys.py`,`core/auth/backend.py` |
| 用量日志 / 计费 | `src/backend/api/routes/v1/admin_usage_logs.py`,`api/routes/v1/admin_billing.py` |
-| 路由策略 | `src/backend/orchestration/strategy.py` |
+| 路由策略 | `src/backend/core/agent_runtime/strategy.py` |
| 主模型缺失快速失败 | `src/backend/api/routes/v1/chats/__init__.py::_ensure_main_model_configured` |
## 模型专属思考档位
diff --git a/document/zh-CN/modules/personal-agent.md b/document/zh-CN/modules/personal-agent.md
index d26c3ac1..ec8bce3a 100644
--- a/document/zh-CN/modules/personal-agent.md
+++ b/document/zh-CN/modules/personal-agent.md
@@ -219,24 +219,18 @@ Linux 桌面运行环境未提供输入驱动时,连接只包含会话管理
先按项目部署文档准备企业版、PostgreSQL、Redis 和 OpenSandbox 镜像。
-1. 构建专用电脑镜像:`docker build -f docker/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .`。不要覆盖正在运行的电脑使用的标签。
+1. 构建专用电脑镜像:`docker build -f docker/personal-agent/Dockerfile.personal-agent -t hugagent-personal-agent:v5 .`。不要覆盖正在运行的电脑使用的标签。
2. 设置 `HOST_STORAGE_PATH` 为宿主机绝对路径。
3. 为 `PERSONAL_AGENT_OPENSANDBOX_API_KEY` 设置至少 32 个字符的独立随机密钥。
4. 设置 `PERSONAL_AGENT_OPENSANDBOX_DOMAIN=personal-agent-opensandbox:8080`。
5. 设置 `PERSONAL_AGENT_IMAGE=hugagent-personal-agent:v5`。已有电脑在下次重建时使用新镜像。
-6. 使用根 Compose 文件和附加文件:`docker compose -f docker-compose.yml -f docker/compose.personal-agent.yml --profile personal-agent up -d`。
+6. 使用根 Compose 文件和附加文件:`docker compose -f docker-compose.yml -f docker/personal-agent/compose.personal-agent.yml --profile personal-agent up -d`。
7. 按项目现有迁移流程升级数据库。依次应用 `personalagent01` 至 `personalagent05`。这些迁移创建模块表、回填消息关联、建立独立任务会话,并增加跨设备会话操作表。
附加文件创建独立控制器、专用网络和私有配置卷。控制器需要访问 Docker socket。密钥不得提交到 Git。现有数据库密钥用于加密沙盒和邮箱凭据。
配置缺失时,Personal Agent 不能连接普通控制器作为替代。后台服务每分钟检查永久电脑。用户暂停任务队列时,电脑仍保留。
-## 验证范围
-
-自动化测试覆盖单一身份、并发任务、租约、审批、设备撤销、历史分页和普通聊天入口限制。隔离环境还验证了实际模型、真实云端截图和输入、文件持久化、SMTP/IMAP 附件和前后端流程。
-
-Ethereal 测试邮箱捕获邮件。它不能证明向普通互联网邮箱投递成功。Mac 和 Windows 原生桌面测试必须在对应系统完成。具体已通过项和待验项见内部验收记录。
-
回滚 personalagent02 前,必须等待所有消息回复完成。回滚把任务关联及独立指令归档到原消息元数据中。重新升级会恢复这些关联和指令。
## 状态栏
diff --git a/document/zh-CN/modules/projects-myspace.md b/document/zh-CN/modules/projects-myspace.md
index 3e42a848..d79a3b0e 100644
--- a/document/zh-CN/modules/projects-myspace.md
+++ b/document/zh-CN/modules/projects-myspace.md
@@ -35,7 +35,7 @@ teams ─────────┬── team_members(role: owner/admin/memb
## 项目(Projects)
-路由:`src/backend/api/routes/v1/projects.py`(CE 路由表),业务在 `core/services/project_service.py` 与 `project_file_service.py`。
+路由:`src/backend/api/routes/v1/projects.py`(CE 路由表),业务在 `core/projects/` 与 `project_file_service.py`。
| 方法 | 路径 | 说明 |
|---|---|---|
@@ -71,16 +71,16 @@ teams ─────────┬── team_members(role: owner/admin/memb
### 项目上下文如何进入对话
-在项目内发起对话时(请求携带 `project_id`),`core/services/session_operations/request_context.py` 组装 workflow context:
+在项目内发起对话时(请求携带 `project_id`),`core/chat/operations/request_context.py` 组装 workflow context:
1. 读取项目元信息——`project_name`、`project_instructions`、挂钩文件夹名和工作目录;项目指令仍读取 `AGENTS.md`,不递归枚举云端或本机项目文件;
2. `core/llm/factory/build.py` 把这些经 `_build_project_section` 注入 system prompt(`build_system_prompt(cfg, ctx=...)`);
3. 项目级记忆作用域随之生效:workspace 为 `project:`,团队项目的 mem0 桶为 `team:`(成员共享记忆),项目自身的 `metadata.memory_enabled` / `memory_write_enabled` 覆盖用户级开关(项目内缺省开启)——详见 [记忆系统](./memory.md);
-4. 沙箱侧路径作用域:项目对话中 agent 的 `/myspace/...` 文件操作被重定向到挂钩文件夹之下(`core/llm/tools/myspace_vfs/__init__.py` 的 `ProjectScope` 显式传参机制)。
+4. 沙箱侧路径作用域:项目对话中 agent 的 `/myspace/...` 文件操作被重定向到挂钩文件夹之下(`core/files/spaces/` 的 `ProjectScope` 显式传参机制)。
项目模式自定义提示词模板使用项目名称、目录范围和项目指令变量。升级已有部署前,须从已保存的模板中移除旧文件清单及文件总数占位符;运行时不再提供这些变量。
-团队项目权限沿用团队角色:owner/admin 恒为管理权限,member 按 `file_permission`(editor/viewer)二级控制(`core/auth/permissions_iface.py::require_project_access`)。
+团队项目权限沿用团队角色:owner/admin 恒为管理权限,member 按 `file_permission`(editor/viewer)二级控制(`core/auth/personal_permissions_iface.py::require_project_access`)。
### 项目侧栏交互
@@ -158,7 +158,7 @@ teams ─────────┬── team_members(role: owner/admin/memb
三条路径,互为补充:
-1. **附件注入(hooks)**:前端请求的 `attachments[]` 只携带 `file_id`、`name`、`mime_type`,不携带文件正文。`core/llm/hooks.py` 按 `file_id` 延迟下载、解析并缓存 `parsed_text` 与摘要;首轮只注入文件元数据和最多 5,000 字符的有界预览,xlsx 额外提供总行列数、表头和少量数据行。模型需要预览外的正文时通过 `read_artifact` 按需读取;图片仍走多模态注入。所有读取都校验 `user_id` 归属,防止伪造 `file_id` 跨用户访问;
+1. **附件注入(hooks)**:前端请求的 `attachments[]` 只携带 `file_id`、`name`、`mime_type`,不携带文件正文。`core/harness/hooks.py` 按 `file_id` 延迟下载、解析并缓存 `parsed_text` 与摘要;首轮只注入文件元数据和最多 5,000 字符的有界预览,xlsx 额外提供总行列数、表头和少量数据行。模型需要预览外的正文时通过 `read_artifact` 按需读取;图片仍走多模态注入。所有读取都校验 `user_id` 归属,防止伪造 `file_id` 跨用户访问;
2. **项目文件访问**:项目对话的 system prompt 提供项目指令和目录范围,不自动携带文件列表或文件总数。agent 按需使用目录、搜索和读取工具获取文件信息;
3. **沙箱虚拟文件系统**:开启代码执行时,`/myspace/...` 路径把我的空间映射进沙箱(懒加载 + 文件系统实时登记),团队项目映射团队文件夹——见 [沙箱](./sandbox.md)。
@@ -167,8 +167,8 @@ teams ─────────┬── team_members(role: owner/admin/memb
| 路径 | 职责 |
|---|---|
| `src/backend/api/routes/v1/projects.py` | 项目 API |
-| `src/backend/core/services/project_service.py` / `project_file_service.py` | 项目业务逻辑 |
-| `src/backend/core/services/project_scope.py` | `ProjectScope`(沙箱路径作用域) |
+| `src/backend/core/projects/` / `project_file_service.py` | 项目业务逻辑 |
+| `src/backend/core/projects/personal_scope.py` | `ProjectScope`(沙箱路径作用域) |
| `src/backend/api/routes/v1/myspace_folders.py` | 个人文件夹 API |
| `src/backend/api/routes/v1/artifacts.py` | 资产列表 / 会话收藏 / 加入知识库 |
| `src/backend/edition_ee/routes/team_files.py` | 团队文件夹与文件 API(商业版 EE) |
@@ -177,9 +177,9 @@ teams ─────────┬── team_members(role: owner/admin/memb
| `src/backend/core/db/models/identity.py` | `UserFolder` 等共享身份 ORM |
| `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM(仅 EE) |
| `src/backend/core/db/models/artifact.py` | `Artifact` ORM |
-| `src/backend/core/llm/hooks.py` | 附件上下文注入(`_build_file_context` 等) |
+| `src/backend/core/harness/hooks.py` | 附件上下文注入(`_build_file_context` 等) |
| `src/backend/core/llm/factory/build.py` | 项目 section 注入 system prompt |
-| `src/backend/core/llm/tools/myspace_vfs/__init__.py` | 我的空间 ↔ 沙箱映射层 |
+| `src/backend/core/files/spaces/` | 我的空间 ↔ 沙箱映射层 |
| `src/frontend/src/components/projects/` | 项目前端组件 |
| `src/frontend/src/components/sidebar/Sidebar.tsx` | 侧栏项目分组、快捷新建与项目菜单 |
| `src/frontend/src/components/myspace/` | 我的空间前端组件 |
@@ -249,23 +249,20 @@ Config「团队管理」新增成员或调整成员权限时,可选「所有
缺失文件或失败下载也在下次正常同步时补齐。
升级前先停止本地写入并备份数据库及 myspace_cache。
-运行 `python src/backend/scripts/clean_myspace_duplicates.py --all` 预览;
-加 `--apply --backup-dir <新的备份目录>` 才执行。
-清理删除每个重名组的全部有效记录及镜像路径,不保留“最新一条”。
-采用删除标记避免历史记录回填复活,原对象和恢复日志保留在维护备份中。
+升级前检查同名记录,并制定经审核的数据清理与恢复方案。当前不提供独立去重脚本。
然后执行 Alembic 升级(EE: personalname01;CE: ce_0018),恢复服务并运行对账与端到端测试。
迁移遇到未清理重名会终止,不会自行删除其他环境数据。
恢复须先停止写入、撤销唯一索引,再由备份恢复;不要直接向正常服务恢复重名记录。
### 空间文件工具
-空间文件工具统一由 `core/llm/tools/space_tools.py` 注册,调用名为 `space_list_myspace_files`、`space_stage_myspace_file`、`space_create_folder`、`space_move`、`space_delete`;商业版另提供 `space_list_team_files`。通用 Read/Write/Edit/Glob/Grep 名称保持不变。
+空间文件工具统一由 `core/files/spaces/` 注册,调用名为 `space_list_myspace_files`、`space_stage_myspace_file`、`space_create_folder`、`space_move`、`space_delete`;商业版另提供 `space_list_team_files`。通用 Read/Write/Edit/Glob/Grep 名称保持不变。
收藏会话工具已移除;历史会话仍可通过 `list_related_chats` 和 `read_chat` 访问。
### 挂载目录统一同步
-个人空间监听实现位于 `core/space_sync/`,团队适配位于 `edition_ee/services/space_sync/`。监听回调只入内存队列,后台批量写入挂载目录之外的 SQLite 日志,再执行带权限、版本检查的数据库及对象存储更新。原生移动保留文件和文件夹 ID;空目录、删除、重启恢复及解析缓存失效使用同一同步流程。数据库提交也会刷新已注册的挂载目录,回滚不刷新。冲突保留本地内容并使同步屏障报告失败。网络下载、上传和人工审批时间不属于回调延迟,完整同步延迟须按实际环境测量。
+个人空间监听实现位于 `core/files/spaces/`,团队适配位于 `edition_ee/services/workspace_adapter.py`。监听回调只入内存队列,后台批量写入挂载目录之外的 SQLite 日志,再执行带权限、版本检查的数据库及对象存储更新。原生移动保留文件和文件夹 ID;空目录、删除、重启恢复及解析缓存失效使用同一同步流程。数据库提交也会刷新已注册的挂载目录,回滚不刷新。冲突保留本地内容并使同步屏障报告失败。网络下载、上传和人工审批时间不属于回调延迟,完整同步延迟须按实际环境测量。
### 项目环境
@@ -275,7 +272,7 @@ Config「团队管理」新增成员或调整成员权限时,可选「所有
后端按项目配置限制能力注册、显式调用、历史会话资源恢复、子智能体和插件动态加载。空的自定义环境不回退到全部资源。环境调整从下一轮对话生效;正在执行的响应及其派生作业保留原配置。项目外对话不受影响。基础文件与对话工具仍遵守原有权限规则。
-实现按职责划分:`core/project_environment/` 定义配置校验和纯依赖展开;资源服务读取当前用户获授权的目录;环境服务负责保存和预览;运行策略负责冻结及收窄注册、调用范围。前端编辑器只组织界面,`useProjectEnvironment` 负责资源加载、依赖预览和市场回填,资源选框负责展示显式绑定与推导依赖。
+实现按职责划分:`core/projects/environment/` 定义配置校验和纯依赖展开;资源服务读取当前用户获授权的目录;环境服务负责保存和预览;运行策略负责冻结及收窄注册、调用范围。前端编辑器只组织界面,`useProjectEnvironment` 负责资源加载、依赖预览和市场回填,资源选框负责展示显式绑定与推导依赖。
### 云文档在线编辑
diff --git a/document/zh-CN/modules/prompts.md b/document/zh-CN/modules/prompts.md
index 8c4d77f2..31e4a81c 100644
--- a/document/zh-CN/modules/prompts.md
+++ b/document/zh-CN/modules/prompts.md
@@ -29,7 +29,7 @@ src/backend/prompts/prompt_text/default/system/
└── 40_format.system.md # 输出格式 + [ref:工具名-N] 引用规范
```
-在 base prompt 之后,运行时还会按上下文追加动态段:工具与技能说明(`_TOOLS_AND_SKILLS_NOTICE`)、轻量知识库目录(`prompts/kb_lite_section.py`)、项目模式段(`prompts/project_section.py`,仅项目对话注入)、代码执行段与批量模式提示(由 `factory/build.py` 追加)、子智能体路由表(`core/llm/subagent_tool.py::build_subagent_prompt_section`)。
+在 base prompt 之后,运行时还会按上下文追加动态段:工具与技能说明(`_TOOLS_AND_SKILLS_NOTICE`)、轻量知识库目录(`prompts/kb_lite_section.py`)、项目模式段(`prompts/project_section.py`,仅项目对话注入)、代码执行段与批量模式提示(由 `factory/build.py` 追加)、子智能体路由表(`core/llm/subagents/tool.py::build_subagent_prompt_section`)。
### 缓存设计
@@ -48,7 +48,7 @@ src/backend/prompts/prompt_text/default/system/
### 执行 Manifest 与运行时绑定
-Agent 真正开始执行前,`core/llm/execution_manifest.py::PromptManifestBuilder` 会记录最终请求面。每个有序 PromptSection 都带 `id`、来源、信任级、优先级、缓存类别、预算、token 估算、内容哈希和版本;每个最终 ToolDefinition 同样对名称、完整描述、canonical 参数 schema、权限策略和恢复策略生成稳定哈希。工具定义按稳定 id 排序,提示词分段则保持刻意设计的前缀顺序。
+Agent 真正开始执行前,`core/harness/execution_manifest.py::PromptManifestBuilder` 会记录最终请求面。每个有序 PromptSection 都带 `id`、来源、信任级、优先级、缓存类别、预算、token 估算、内容哈希和版本;每个最终 ToolDefinition 同样对名称、完整描述、canonical 参数 schema、权限策略和恢复策略生成稳定哈希。工具定义按稳定 id 排序,提示词分段则保持刻意设计的前缀顺序。
Builder 最终产出 `prompt_hash`、`prompt_manifest_hash`、`tool_manifest_hash`、`context_hash` 和统一的 `aggregate_hash`。Manifest 构建与 AgentScope 模型请求共用同一份 run-scoped 工具/技能冻结快照,因此“记录的请求面”和“实际发送的请求面”不会错位;渐进式插件激活会让旧快照失效,并在下一次模型调用前发布明确递增的 `surface_generation`。
@@ -58,15 +58,15 @@ Runtime Binder 把这份脱敏 Manifest 挂到本轮递归不可变的 Asset Bun
### 统一 Context IR
-每次模型调用前,`core/llm/context_ir.py` 都会把最终上下文统一转换成不可变的 `ContextItem`。用户输入、助手历史、账户身份、冻结记忆、项目/文件材料、附件、工具调用与结果、压缩摘要、执行中追加指令、Harness 系统提醒,都会携带机器可读的 `kind`、来源、信任级、可见性、优先级、单项预算、截断策略、内容引用/哈希、缓存类别和创建顺序。为兼容模型,记忆和对话中途的提醒仍可渲染成 `user` role,但系统不再靠 XML 标签或 role 猜测它们是谁说的。
+每次模型调用前,`core/llm/context/ir.py` 都会把最终上下文统一转换成不可变的 `ContextItem`。用户输入、助手历史、账户身份、冻结记忆、项目/文件材料、附件、工具调用与结果、压缩摘要、执行中追加指令、Harness 系统提醒,都会携带机器可读的 `kind`、来源、信任级、可见性、优先级、单项预算、截断策略、内容引用/哈希、缓存类别和创建顺序。为兼容模型,记忆和对话中途的提醒仍可渲染成 `user` role,但系统不再靠 XML 标签或 role 猜测它们是谁说的。
`ContextAssembler` 使用确定性顺序,并设置明确的消息预算:模型窗口先预留 15% 给响应,再扣除最终工具 schema 的 canonical token 预算。关键系统规则和当前用户指令不会被低可信内容挤掉;可选内容按优先级选择;文本和工具输出按声明的头尾保留策略截断;工具调用与对应结果则作为一个整体保留、缩短或排除。脱敏 Context Manifest 会逐项说明“纳入、截断、排除”及原因,不保存正文;其哈希会进入本次请求的 `context_hash` 与 `aggregate_hash`,再重新绑定到该 run 的 Asset Bundle。
-`core/llm/context_adapter.py::AgentScopeContextAdapter` 是 Context IR 与 AgentScope 消息之间唯一的转换入口。持久化会话中的来源元数据可在历史回放时保留,最终模型 role 和 SSE 行为继续兼容。`ManifestBoundAgent` 会在 run-scoped 工具/技能请求面冻结后,通过公开的 `Agent.reply` 链路完成最终装配。
+`core/llm/context/adapter.py::AgentScopeContextAdapter` 是 Context IR 与 AgentScope 消息之间唯一的转换入口。持久化会话中的来源元数据可在历史回放时保留,最终模型 role 和 SSE 行为继续兼容。`ManifestBoundAgent` 会在 run-scoped 工具/技能请求面冻结后,通过公开的 `Agent.reply` 链路完成最终装配。
## 提示词版本池(prompt_versions)
-版本池把多套提示词存进单行 `ContentBlock(id="prompt_versions")`,payload 结构 `{active: {kind: version_id}, versions: [...]}`,服务层为 `core/services/prompt_version_service.py`:
+版本池把多套提示词存进单行 `ContentBlock(id="prompt_versions")`,payload 结构 `{active: {kind: version_id}, versions: [...]}`,服务层为 `core/llm/prompt_versions.py`:
- **六类 kind**(`VALID_KINDS`):`system`(主智能体)、`code_exec`(代码执行能力段)、`distillation`(技能蒸馏)、`plan_mode`(计划模式)、`subagents`(平台默认探索员 / 执行员 / 审查员的独立提示词)、`turbo`(极速模式独立提示词)。
- 每个版本含 `(kind, id, name, description, parts[])`,part 即 `{part_id, display_name, content, sort_order, is_enabled}`。
@@ -94,8 +94,8 @@ Runtime Binder 把这份脱敏 Manifest 挂到本轮递归不可变的 Asset Bun
| kind | 运行时消费方 | 解析顺序 |
|---|---|---|
| `code_exec` | `factory/build.py` 在 `CODE_CAPABILITY_ENABLED=true` 时把该段(代码能力提示词)拼到 system prompt 尾部;单一真源为 `prompt_version_service.render_code_capability_segment()`,管理台 preview 同源 | DB 激活版本 → `prompts/prompt_text/code_exec/system/*.system.md` |
-| `distillation` | 技能蒸馏(`core/llm/skill_distiller.py`,把对话轨迹蒸馏为可复用技能) | DB 激活版本 → `prompts/prompt_text/distillation/skill_distiller.system.md` |
-| `plan_mode` | 计划模式生成子智能体(`orchestration/subagents/plan_mode.py::_load_plan_prompt`) | DB `plan_mode` 激活版本 → 旧版 system 版本的 `system/90_plan_mode` 分段 → `prompts/prompt_text/plan_mode/plan_mode.system.md` → 硬编码兜底 |
+| `distillation` | 技能蒸馏(`features/distillation/`,把对话轨迹蒸馏为可复用技能) | DB 激活版本 → `prompts/prompt_text/distillation/skill_distiller.system.md` |
+| `plan_mode` | 计划模式生成子智能体(`orchestration/plans/`) | DB `plan_mode` 激活版本 → 旧版 system 版本的 `system/90_plan_mode` 分段 → `prompts/prompt_text/plan_mode/plan_mode.system.md` → 硬编码兜底 |
| `turbo` | 极速模式(对话模式选「极速」时,`factory/build.py` 以该提示词**整体替换**主系统提示词)。工具集由 Config「系统配置 → 极速模式」动态配置(`turbo.mcp_server_ids`,默认联网搜索/网页抓取/知识库检索三件套,**独立于能力目录的启停**);用户显式呼唤的技能 / @子智能体 / 插件可临时入场(`turbo.manual_invoke_enable`);迭代上限可配(`turbo.max_iters`,默认 4)。单一真源为 `prompt_version_service.render_turbo_system_prompt()` | DB `turbo` 激活版本 → `prompts/prompt_text/turbo/turbo.system.md` → 硬编码兜底 |
子智能体不走版本池整版拼装:`prompt_runtime.py::build_subagent_system_prompt()` 以用户自定义 `system_prompt` 为核心,复用激活版本(或文件)的 `20_tools_policy` / `65_citations` / `60_format` 分段组装,详见 [对话与智能体编排](chat.md)。
@@ -120,18 +120,6 @@ Runtime Binder 把这份脱敏 Manifest 挂到本轮递归不可变的 Asset Bun
两端点接受 `ADMIN_TOKEN` 或 `CONFIG_TOKEN`。快照 schema 经 `PROMPT_BLOCK_MAP` 校验,docs 快照与 prompts 快照不能从错误的端点互导。
-### 脚本(src/backend/scripts/)
-
-```bash
-# 导出(走运行中的后端 API;也支持 --database-url 直连库)
-python scripts/export_content.py --api-url http://localhost:3000/api --only prompts
-# → scripts/exported/prompts_snapshot_.json
-
-# 导入到目标环境(用目标机 .env 的 ADMIN_TOKEN)
-python scripts/import_content.py --api-url http:///api --prompts prompts_snapshot_.json
-# 支持 --no-overwrite / --dry-run
-```
-
离线生产环境(镜像包交付,DB 卷持久化)同样适用:快照文件随镜像包拷入,在 backend 容器内 `curl -X POST .../v1/content/prompts/import` 导入即可,无需重启。换品牌环境迁移时,需人工通读快照逐处改写品牌相关话术后再导入(不要机械查找替换)。
## 相关源码
@@ -141,10 +129,9 @@ python scripts/import_content.py --api-url http:///api --prompts prompts_s
| 运行时装配 + 缓存 | `src/backend/prompts/prompt_runtime.py` |
| Provider(filesystem/inline/minimal) | `src/backend/prompts/provider.py` |
| 配置(provider/parts/PROMPT_DIR) | `src/backend/prompts/prompt_config.py`,`prompts/config/default.json` |
-| 版本池服务 | `src/backend/core/services/prompt_version_service.py` |
+| 版本池服务 | `src/backend/core/llm/prompt_versions.py` |
| 管理台路由 | `src/backend/api/routes/v1/admin_prompts.py` |
-| 迁移接口(export/import) | `src/backend/api/routes/v1/content.py`,`core/content/content_blocks.py` |
-| 迁移脚本 | `src/backend/scripts/export_content.py`,`scripts/import_content.py` |
+| 迁移接口(export/import) | `src/backend/api/routes/v1/content.py`,`core/files/content/content_blocks.py` |
| 系统提示词兜底文件 | `src/backend/prompts/prompt_text/default/system/` |
| 场景提示词兜底 | `src/backend/prompts/prompt_text/{code_exec,distillation,plan_mode,turbo}/` |
| 动态段 | `src/backend/prompts/kb_lite_section.py`,`prompts/project_section.py` |
diff --git a/document/zh-CN/modules/sandbox.md b/document/zh-CN/modules/sandbox.md
index 3421f464..d2db66e6 100644
--- a/document/zh-CN/modules/sandbox.md
+++ b/document/zh-CN/modules/sandbox.md
@@ -63,11 +63,11 @@ Cube 的设计取舍(远端节点版的代价):所有语言统一走"写
框架保留最多 64 个进程记录,仅回收已完成记录;新增输出有界,省略内容通过 `output_omitted_chars` 明示。本地及 script_runner 的完整输出存于当前会话工作区的 `.__process_*/stdout.log` 和 `stderr.log`,通过 `output_files` 返回真实路径;单流超过 64 MiB 时停止命令,已写日志仍可读。OpenSandbox 后台日志合并 stdout/stderr;Cube 总输出超过 64 MiB 时停止。运行期间定期调用沙箱保活:OpenSandbox 可续租,Cube 的兼容服务可能不支持延长服务端 TTL,返回 `lifetime_note` 说明这一边界。普通会话的命令账本保存任务状态与有界输出。OpenSandbox 使用原执行标识和日志游标恢复;不重跑命令。其他 provider 正常退出时停止进程;异常重启后标记 unknown。显式删除会话会停止任务。团队项目在命令终态返回后同步源码;必须等待完成,不能把 running 当成已经保存。
-后台执行由服务端采集状态、保存日志并保持沙箱活动。模型可以执行独立工作;没有其他工作时,模型结束当前运行。完成事件在下一次推理前进入活动会话,或使用现有会话运行入口唤醒空闲模型。write_stdin 用于按需读取新增输出、短时等待和取消,不用于反复轮询。认证日志 API 的读取不消耗模型输出游标。聊天界面不展示独立的后台命令栏。页面预先订阅 /v1/commands/events 长连接,后端在唤醒运行就绪后立即推送运行标识,页面随即接入现有输出流,不等待任务轮询。重连时补发已就绪的运行;已结束的短回复从历史加载。实现边界见 [后台 Bash ADR](../../internal/adr/20261018-background-bash-completion.md)。
+后台执行由服务端采集状态、保存日志并保持沙箱活动。模型可以执行独立工作;没有其他工作时,模型结束当前运行。完成事件在下一次推理前进入活动会话,或使用现有会话运行入口唤醒空闲模型。write_stdin 用于按需读取新增输出、短时等待和取消,不用于反复轮询。认证日志 API 的读取不消耗模型输出游标。聊天界面不展示独立的后台命令栏。页面预先订阅 /v1/commands/events 长连接,后端在唤醒运行就绪后立即推送运行标识,页面随即接入现有输出流,不等待任务轮询。重连时补发已就绪的运行;已结束的短回复从历史加载。
沙箱会话标识由 `resolve_sandbox_session(sandbox_session_id, chat_id)` 解析:非空显式 ID 优先,未传或传空值时统一回落到 `chat_id`。主智能体、计划执行、批量项和所有子智能体因此使用同一个会话沙箱;子智能体结束时不会销毁它。
-**MySpace 实时登记**:写在 `/myspace` 下的文件**就是**用户「我的空间」里的文件,两侧任何时刻看到的都应是同一份状态。登记由**文件系统本身**驱动(`core/myspace/watcher.py`):镜像目录 `myspace_cache/{uid}/` 里发生任何写入或删除,都被登记回 artifact 账本,**不区分是谁写的**。
+**MySpace 实时登记**:写在 `/myspace` 下的文件**就是**用户「我的空间」里的文件,两侧任何时刻看到的都应是同一份状态。登记由**文件系统本身**驱动(`core/files/spaces/`):镜像目录 `myspace_cache/{uid}/` 里发生任何写入或删除,都被登记回 artifact 账本,**不区分是谁写的**。
早先的做法是让会写文件的工具各自登记(write / edit / 文件增删改 / `Bash` 前后拍目录快照)。这套写法的前提是"改动只可能从这几个口子进来",而这个前提不成立:`nohup` 起的后台进程在命令返回之后才写文件,子智能体和批量任务在另一条协程里写,技能里的 CLI 直接写,MCP 服务端也写。每多一个入口就要多补一处登记,漏掉的那一处的表现是——文件躺在用户网盘的磁盘上,界面上看不见也删不掉,而每个新建沙箱都会把这份目录挂进来,成了"上个会话的残留中间文件"。
@@ -84,13 +84,11 @@ Cube 的设计取舍(远端节点版的代价):所有语言统一走"写
其它要点:
-- **登记不挂会话、也不挂产物卡片**:「我的空间」是用户级的一份目录,每个会话的沙箱都挂着它,文件事件里只有路径、没有会话身份——谁写的,这里无从得知。所以登记只记到用户名下(`Artifact.chat_id` 为空),产物区的文件卡片则完全不由登记器挂:卡片只能由**写文件的那一轮自己**挂(模型调 `pin_to_workspace`,或那一轮里的 Write/交付兜底,都走 `core.llm.workspace` 的 ContextVar),那里"哪个会话、哪个文件"都是确定的。曾经拿"这个用户当下随便哪个在跑的会话"顶上,结果是另一个会话写的文件被记到了这个会话名下,还在它的产物区里多出一张卡片。
+- **登记不挂会话、也不挂产物卡片**:「我的空间」是用户级的一份目录,每个会话的沙箱都挂着它,文件事件里只有路径、没有会话身份——谁写的,这里无从得知。所以登记只记到用户名下(`Artifact.chat_id` 为空),产物区的文件卡片则完全不由登记器挂:卡片只能由**写文件的那一轮自己**挂(模型调 `pin_to_workspace`,或那一轮里的 Write/交付兜底,都走 `core.llm.runtime.workspace` 的 ContextVar),那里"哪个会话、哪个文件"都是确定的。曾经拿"这个用户当下随便哪个在跑的会话"顶上,结果是另一个会话写的文件被记到了这个会话名下,还在它的产物区里多出一张卡片。
- **判定基准是 artifact 记录,不是镜像缓存**:开了 bind mount 后沙箱的 `/workspace/myspace/{uid}` 就是后端 `myspace_cache/{uid}` 本身,"沙箱文件与镜像缓存一致"恒真、不能再当作"已登记"。这条拓扑差异曾让回写恒为空转。
- **"磁盘比账本新"是唯一依据**:后端自己往镜像目录写文件(界面上传的物化、否决后的还原)时会把 mtime 对齐到登记时间,于是监听器不会把后端自己的写入当成沙箱的新改动转头再登记一次——不必另外维护一张"这是我自己写的"的表。
- **每个 worker 都监听,认领去重**:确认条活在跑这个会话的那个进程里,所以监听器必须每个 worker 都跑(代价是 inotify watch 按「worker 数 × 目录数」占内核配额 `fs.inotify.max_user_watches`,目录极多的部署要相应调大,超限会**静默丢事件**);同一次改动由 `core.infra.ephemeral` 的认领保证只处理一次,且优先交给托管着该用户会话的进程(它才弹得出确认条)。
-- **沙箱目录不在本机时**(`script_runner` / `cube`:会话工作区在容器内、或整个沙箱在远端)由 `core/myspace/sandbox_sync.py` 在命令结束后把沙箱现状搬进镜像目录,之后的判定与登记完全同路。
-
-历史欠账(本次改造上线前积压的未登记文件 / 该删没删的残留)是**一次性数据迁移**,用 `scripts/reconcile_myspace_mirror.py` 手工跑一次即可(`--dry-run` 先看规模,残留副本的清理加 `--prune-stale`)。它不做成每次启动都执行的扫描:那既是白跑,也会掩盖实时这条路上真正的漏洞。
+- **沙箱目录不在本机时**(`script_runner` / `cube`:会话工作区在容器内、或整个沙箱在远端)由 `core/files/spaces/sandbox.py` 在命令结束后把沙箱现状搬进镜像目录,之后的判定与登记完全同路。
### 大文件产物交付
@@ -175,8 +173,8 @@ artifact 送入沙盒、`Bash` 执行后 `/myspace` 写回同步;backend 与 s
## 管理员沙箱管理
-- **只读监控(安全管理台)**:`api/routes/v1/config_security.py` 暴露 `/v1/config/security/sandbox/*`——总览、实例列表、单实例详情、快照列表、重建历史、生效配置;底层走 provider 的 `admin_*` 接口,按 `admin_capabilities()` 声明裁剪 UI(script_runner 无法枚举实例即置灰对应列)。
-- **依赖重建(商业版 EE)**:`api/routes/v1/admin_sandbox.py`(`/v1/admin/sandbox/*`)聚合所有技能声明的 pip/apt 依赖(`core/services/skill_deps_aggregator.py`),管理员一键触发沙箱镜像重建:`script-runner` / `opensandbox` 目标走本地 `docker compose build`,`cube` 目标走 SSH 到远端节点重建模板并热切(`core/services/sandbox_rebuild_service.py` + `cube_template_builder`),可查看每次 run 的状态与日志。技能新依赖从此"烤入"镜像,无需手工改 Dockerfile。
+- **只读监控(安全管理台)**:`features/security_console/api/config_security.py` 暴露 `/v1/config/security/sandbox/*`——总览、实例列表、单实例详情、快照列表、重建历史、生效配置;底层走 provider 的 `admin_*` 接口,按 `admin_capabilities()` 声明裁剪 UI(script_runner 无法枚举实例即置灰对应列)。
+- **依赖重建(商业版 EE)**:`features/sandbox_ops/api/admin_sandbox.py`(`/v1/admin/sandbox/*`)聚合所有技能声明的 pip/apt 依赖(`core/capabilities/skills/deps/aggregator.py`),管理员一键触发沙箱镜像重建:`script-runner` / `opensandbox` 目标走本地 `docker compose build`,`cube` 目标走 SSH 到远端节点重建模板并热切(`features/sandbox_ops/rebuild.py` + `cube_template_builder`),可查看每次 run 的状态与日志。技能新依赖从此"烤入"镜像,无需手工改 Dockerfile。
## 关键环境变量
@@ -210,17 +208,17 @@ artifact 送入沙盒、`Bash` 执行后 `/myspace` 写回同步;backend 与 s
| `src/backend/core/sandbox/factory.py` | provider 显式选择;初始化错误直接上报 |
| `src/backend/core/sandbox/providers/script_runner/provider.py` | 轻量沙箱(CE) |
| `src/backend/core/sandbox/providers/opensandbox/provider.py` | OpenSandbox 共享 provider 主体 |
-| `src/backend/core/sandbox/providers/opensandbox/session.py` | 会话/快照/park-restore worker(EE) |
+| `src/backend/core/sandbox/providers/opensandbox/` | 会话/快照/park-restore worker(EE) |
| `src/backend/core/sandbox/providers/opensandbox/execution.py` | 执行路径 + idle reaper(EE) |
-| `src/backend/core/sandbox/providers/opensandbox/internals.py` | volume 构造、metadata、user pool(EE) |
+| `src/backend/core/sandbox/providers/opensandbox/` | volume 构造、metadata、user pool(EE) |
| `src/backend/core/sandbox/providers/opensandbox/pool.py` | 双桶预热池 |
| `src/backend/core/sandbox/providers/cube/provider.py` | Cube 远端 MicroVM provider(EE) |
| `src/backend/core/llm/tools/sandbox_tool.py` | Bash / write_stdin / sandbox_put_artifact / sandbox_get_artifact |
-| `src/backend/core/llm/offloader.py` | 超长结果落盘 /workspace/.offload |
-| `src/backend/api/routes/v1/admin_sandbox.py` | 依赖重建管理 API(EE) |
-| `src/backend/api/routes/v1/config_security.py` | 安全管理台沙箱只读视图 |
-| `src/backend/core/services/sandbox_rebuild_service.py` | 镜像/模板重建编排(EE) |
-| `docker/Dockerfile.script-runner` / `docker/Dockerfile.opensandbox` / `docker/Dockerfile.cube-sandbox` | 三种沙箱镜像 |
+| `src/backend/core/llm/context/offloader.py` | 超长结果落盘 /workspace/.offload |
+| `src/backend/features/sandbox_ops/api/admin_sandbox.py` | 依赖重建管理 API(EE) |
+| `src/backend/features/security_console/api/config_security.py` | 安全管理台沙箱只读视图 |
+| `src/backend/features/sandbox_ops/rebuild.py` | 镜像/模板重建编排(EE) |
+| `docker/sandbox/Dockerfile.script-runner` / `docker/sandbox/Dockerfile.opensandbox` / `docker/sandbox/Dockerfile.cube-sandbox` | 三种沙箱镜像 |
相关文档:[技能系统](agent-skills.md) · [MCP 工具系统](mcp-tools.md) · [个人空间](projects-myspace.md) · [版本与许可](../editions/overview.md)
@@ -245,3 +243,24 @@ OpenSandbox 的实例列表与实例计数读取控制器当前运行清单,
控制器查询失败时,计数不可用;不能把查询失败报告成零实例。
连接池诊断保留 scope=worker 与 worker_pid,表示当前后端进程的本地池,
不代表全部进程的用户配额。该修复不把本地池上限改成跨进程全局配额。
+
+## 共享容量与维护所有权
+
+OPENSANDBOX_POOL_MAX_TOTAL 现在由控制器依据 Docker 实例库存统一执行,
+覆盖所有 backend worker、预热实例、会话恢复和计算机实例。各进程池统计仅用于本地诊断。
+控制器在容器创建时持有共享数据卷上的文件锁,不把挂载初始化放入临界区;
+库存达到上限或另一创建正在占用准入锁时返回 429,调用方可稍后重试。
+控制器的 opensandbox_data 数据卷应在同一控制器的进程间共享。
+
+领导进程从共享会话登记发现空闲会话,即使会话由其他 worker 创建或经历过重启,
+也进入快照回收流程。失效登记按身份清理,不阻断其他会话。
+快照任务随领导权停止;GC 只删除本次处理的 snapshot_id 对应记录。
+
+输出读取不再触发挂载准备或对象存储目录写入。目录初始化发生在创建路径,
+执行和文件访问仍核验权限。挂载探测超时或返回不确定错误时保留现有 FUSE helper,
+只有确认缺失或断开时重建。命令观察失败先有限重试,再交给持久恢复;
+没有可恢复引用的进程须确认停止,不能仅因观察失败就丢弃控制句柄。
+
+此机制需要同时更新 backend 和 OpenSandbox 控制器镜像;只更新 backend 无法获得全局容量约束。
+
+技能挂载必须配置 HOST_STORAGE_PATH,指向与后端 STORAGE_PATH 对应的宿主存储目录。缺失配置会明确失败,不再回退到源码目录。重启接管只识别显式归属标签,不根据旧启动命令推断归属。
diff --git a/document/zh-CN/modules/sites.md b/document/zh-CN/modules/sites.md
index 90d351e6..185fb08a 100644
--- a/document/zh-CN/modules/sites.md
+++ b/document/zh-CN/modules/sites.md
@@ -66,13 +66,13 @@
| 环节 | 位置 |
|---|---|
| 发布与查询工具 | `src/backend/extensions/mcp_servers/site_publish_mcp/`(`publish_site` / `list_sites`,由「站点」插件提供) |
-| 内部接口 | `src/backend/api/routes/v1/internal_sites.py`(`/v1/internal/sites/publish`、`/v1/internal/sites/list`) |
-| 站点查询服务 | `src/backend/core/services/site_listing.py`(本机与云端统一记录格式) |
-| 业务服务 | `src/backend/core/services/site_service.py` |
-| 公开托管路由 | `src/backend/api/routes/sites_serve.py`(`GET /site/{slug}/{path}`) |
-| 管理接口 | `src/backend/api/routes/v1/sites.py`(`/v1/sites`) |
-| 访问密码 | `src/backend/core/services/site_password.py`(哈希 + 凭据)、`src/backend/api/routes/site_gate.py`(统一验证页) |
-| 数据表 | `sites`(`core/db/models/site.py`) |
+| 内部接口 | `src/backend/features/sites/api/internal_sites.py`(`/v1/internal/sites/publish`、`/v1/internal/sites/list`) |
+| 站点查询服务 | `src/backend/features/sites/listing.py`(本机与云端统一记录格式) |
+| 业务服务 | `src/backend/features/sites/management.py` |
+| 公开托管路由 | `src/backend/features/sites/api/sites_serve.py`(`GET /site/{slug}/{path}`) |
+| 管理接口 | `src/backend/features/sites/api/sites.py`(`/v1/sites`) |
+| 访问密码 | `src/backend/features/sites/`(哈希 + 凭据)、`src/backend/features/sites/api/site_gate.py`(统一验证页) |
+| 数据表 | `sites`(`features/sites/models.py`) |
| 前端管理面板 | `src/frontend/src/components/sites/SitesPanel.tsx`(实验室 → 站点)、`SitePasswordField.tsx`(密码管理) |
| nginx 转发 | `src/frontend/default.conf.template` 的 `location /site/` |
diff --git a/document/zh-CN/modules/storage.md b/document/zh-CN/modules/storage.md
index d34be521..0adcf949 100644
--- a/document/zh-CN/modules/storage.md
+++ b/document/zh-CN/modules/storage.md
@@ -4,7 +4,7 @@ CE 和 EE 的云端 Docker 部署默认使用 S3 兼容存储。桌面离线本
设置 STORAGE_TYPE=s3,并配置 S3_ENDPOINT、S3_BUCKET、S3_REGION、S3_ACCESS_KEY 和 S3_SECRET_KEY。后端和沙盒控制器均须能够访问该服务地址。
-可选的 docker/compose.object-storage.yml 启动独立 S3 服务。它使用独立数据卷,不发布宿主端口,也不共用 Milvus 存储。
+可选的 docker/object-storage/compose.object-storage.yml 启动独立 S3 服务。它使用独立数据卷,不发布宿主端口,也不共用 Milvus 存储。
在新的 CE checkout 中运行 python3 scripts/configure-ce-docker.py。初始化工具生成独立随机访问密钥和秘密密钥,并拒绝覆盖已有 .env。
@@ -12,7 +12,7 @@ CE 和 EE 的云端 Docker 部署默认使用 S3 兼容存储。桌面离线本
已有部署切换存储前,先备份配置、数据库和文件引用。复制所需对象并逐对象校验 SHA-256。保留源对象。
-存储工厂位于 core/storage/factory.py。S3StorageBackend 使用 boto3 并提供目录操作。LocalStorageBackend 服务桌面本地文件。OSS 和 OBS 适配仍属于 EE。
+存储工厂位于 core/files/storage/factory.py。S3StorageBackend 使用 boto3 并提供目录操作。LocalStorageBackend 服务桌面本地文件。OSS 和 OBS 适配仍属于 EE。
生成物仓库支持 S3、OSS 和 OBS。元数据记录保存在本地,并镜像到对象存储。迁移时保留原存储键,以保持历史下载引用有效。
diff --git a/document/zh-CN/modules/workflow-jobs.md b/document/zh-CN/modules/workflow-jobs.md
index 64a41ab8..53fb7f1b 100644
--- a/document/zh-CN/modules/workflow-jobs.md
+++ b/document/zh-CN/modules/workflow-jobs.md
@@ -23,11 +23,3 @@ run_job 将脚本交给沙箱的受管理进程接口执行。提交时先准备
派生实例采用主智能体的轮数策略,不套用轻量执行器或子智能体的固定轮数上限。完整任务只执行一次;输出格式错误、执行失败或回调结果不确定时,不自动重跑已可能产生副作用的任务。检查已有操作后再显式续跑。
Windows 本机作业使用 `.job/<完整 attempt ID>/`,减少一层目录,避免随包 Python 在深层数据目录导入作业 SDK 时超过传统路径长度限制。每次执行仍保留完整的唯一 attempt ID;续跑和取消使用已保存的实际目录及精确进程引用。
-
-原生桌面端到端验收脚本为 `src/backend/tests/integration/job_agent_local_smoke.py`。使用随包私有 Python 和新建隔离输出目录运行;验收真实 HTTP、作业进程、并发派生主智能体、子智能体、Bash 和文件结果。仅模型决策由本地脚本服务控制,不使用真实账号或数据,不替代安装包和 WebView2 窗口验收。
-
-## 验证
-
-相关单元与回归测试位于 src/backend/tests/orchestration/test_job*.py。显式真实沙箱测试位于 src/backend/tests/integration/job_managed_smoke.py,需指定现有作业作为沙箱上下文。它创建合成作业和临时回调端口,验证并发台账、线程退出、精确取消、跨进程接管、续跑和启动失败,不调用模型,不重新执行原始业务脚本。
-
-部署前须确认容器真正挂载的源码目录。工作区改动与运行容器的源码可能来自不同 checkout;仅重启错误挂载的容器不能使修改生效。
diff --git a/examples/customer-chat/.env.example b/examples/customer-chat/.env.example
deleted file mode 100644
index 0c63ba12..00000000
--- a/examples/customer-chat/.env.example
+++ /dev/null
@@ -1,6 +0,0 @@
-# Optional. Leave empty for offline host preview.
-# Backend must be an origin. Do not append /api.
-HUGAGENT_BACKEND_URL=http://127.0.0.1:3001
-HUGAGENT_DEV_PORT=4310
-# Change only when the backend uses another session cookie name.
-HUGAGENT_SESSION_COOKIE=jx_session
diff --git a/examples/customer-chat/.gitignore b/examples/customer-chat/.gitignore
deleted file mode 100644
index dbec1f16..00000000
--- a/examples/customer-chat/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-node_modules/
-dist/
-standalone-dist/
-*.tgz
-web-dist/
-.env
-.env.local
-vendor/
diff --git a/examples/customer-chat/README.md b/examples/customer-chat/README.md
deleted file mode 100644
index 058c830a..00000000
--- a/examples/customer-chat/README.md
+++ /dev/null
@@ -1,37 +0,0 @@
-# Customer chat example
-
-Two independent consumers are included: a scoped host UI package and a standalone browser application.
-Both import only the published SDK. The current file dependency is a monorepo convenience; replace it with a pinned SDK tarball/version when copying this directory to a customer repository.
-
-```sh
-npm install /path/to/hugagent-chat-sdk-1.0.0.tgz
-npm run build
-npm pack
-```
-
-Install the resulting customer package in src/frontend and build the platform with HUGAGENT_CHAT_CUSTOMIZATION=hugagent-customer-chat-example.
-For this repository's source example:
-
-```sh
-cd src/frontend
-HUGAGENT_CHAT_CUSTOMIZATION=../../examples/customer-chat/src/customization.tsx npm run build
-```
-
-Unset that variable to build the official default interface.
-A customer package must use the same SDK major and React major as the host. The host deduplicates React and the SDK.
-
-The example wraps default welcome/composer/message components. Customer source, CSS and lockfile live independently.
-For a replacement component, use useChatRuntime for draft, messages, attachments and actions.
-Preserve children for rich platform controls that the replacement does not implement.
-
-`npm run standalone` produces standalone-dist, served at /customer/ by the acceptance environment.
-Set VITE_CUSTOMER_API_BASE (default /api) and VITE_CUSTOMER_LOGIN_URL (default /login?redirect=/session.html) when building for your platform.
-The default login link opens the platform's normal login page. It does not depend on mock SSO.
-Use a same-origin API gateway and an authenticated platform session. Do not embed shared API keys in a public build.
-The standalone example renders plain-text replies, paginated history, attachment upload, cancellation, replay, and tool/question gates.
-It intentionally does not implement the entire platform's rich artifact, citation, plan and ontology UI; use default host children for those features.
-
-Customer UI images and bundles must be deployed separately from backend images. Pin both versions and test the desired combination before upgrade.
-The Docker example's customer source is inside the build context; external customer repositories require their own trusted build-stage COPY/install steps.
-
-For the complete no-source developer kit, run node scripts/export-customer-devkit.mjs from the repository root. See document/zh-CN/development/customer-devkit.md. The kit provides npm ci, npm run dev, offline slot preview, backend login, fixed API proxy, API debug and npm run test:package. Component builds bundle imported helpers, CSS and assets.
diff --git a/examples/customer-chat/debug.html b/examples/customer-chat/debug.html
deleted file mode 100644
index 04157231..00000000
--- a/examples/customer-chat/debug.html
+++ /dev/null
@@ -1 +0,0 @@
-API debug Host preview · Live chat · API debug API debug Sign in to backend Sign out Checking session
Requests use the fixed backend configured by your developer. Open browser Network tools to inspect SSE and uploads. No request body or cookies are logged by this kit.
GET endpoint/v1/me /v1/chats?page=1&page_size=10 /ready Send GET
\ No newline at end of file
diff --git a/examples/customer-chat/index.html b/examples/customer-chat/index.html
deleted file mode 100644
index db66376c..00000000
--- a/examples/customer-chat/index.html
+++ /dev/null
@@ -1,9 +0,0 @@
-
-Independent Customer Chat
-Host preview · Live chat · API debug Independent Customer Chat Sign in
-Ready
-
-
-Cancel run Resume
-New chat Attach
-
diff --git a/examples/customer-chat/package-lock.json b/examples/customer-chat/package-lock.json
deleted file mode 100644
index 4735d4f8..00000000
--- a/examples/customer-chat/package-lock.json
+++ /dev/null
@@ -1,1224 +0,0 @@
-{
- "name": "hugagent-customer-chat-example",
- "version": "1.0.0",
- "lockfileVersion": 3,
- "requires": true,
- "packages": {
- "": {
- "name": "hugagent-customer-chat-example",
- "version": "1.0.0",
- "dependencies": {
- "react": "^19.2.0",
- "react-dom": "^19.2.0"
- },
- "devDependencies": {
- "@hugagent/chat-sdk": "file:../../packages/chat-sdk",
- "@types/react": "^19.2.7",
- "@types/react-dom": "^19.2.0",
- "typescript": "~5.9.3",
- "vite": "^7.3.1"
- },
- "peerDependencies": {
- "@hugagent/chat-sdk": "^1.0.0"
- }
- },
- "../../packages/chat-sdk": {
- "name": "@hugagent/chat-sdk",
- "version": "1.0.0",
- "dev": true,
- "license": "UNLICENSED",
- "devDependencies": {
- "@types/react": "^19.2.7",
- "react": "^19.2.0",
- "typescript": "~5.9.3"
- },
- "peerDependencies": {
- "react": ">=19 <20"
- },
- "peerDependenciesMeta": {
- "react": {
- "optional": true
- }
- }
- },
- "node_modules/@esbuild/aix-ppc64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz",
- "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==",
- "cpu": [
- "ppc64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "aix"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/android-arm": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz",
- "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==",
- "cpu": [
- "arm"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "android"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/android-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz",
- "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "android"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/android-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz",
- "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "android"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/darwin-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz",
- "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/darwin-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz",
- "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/freebsd-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz",
- "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "freebsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/freebsd-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz",
- "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "freebsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-arm": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz",
- "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==",
- "cpu": [
- "arm"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz",
- "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-ia32": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz",
- "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==",
- "cpu": [
- "ia32"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-loong64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz",
- "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==",
- "cpu": [
- "loong64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-mips64el": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz",
- "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==",
- "cpu": [
- "mips64el"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-ppc64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz",
- "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==",
- "cpu": [
- "ppc64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-riscv64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz",
- "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==",
- "cpu": [
- "riscv64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-s390x": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz",
- "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==",
- "cpu": [
- "s390x"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/linux-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz",
- "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/netbsd-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz",
- "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "netbsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/netbsd-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz",
- "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "netbsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/openbsd-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz",
- "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "openbsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/openbsd-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz",
- "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "openbsd"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/openharmony-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz",
- "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "openharmony"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/sunos-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz",
- "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "sunos"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/win32-arm64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz",
- "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/win32-ia32": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz",
- "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==",
- "cpu": [
- "ia32"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@esbuild/win32-x64": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz",
- "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@hugagent/chat-sdk": {
- "resolved": "../../packages/chat-sdk",
- "link": true
- },
- "node_modules/@napi-rs/lzma-linux-x64-gnu": {
- "version": "1.5.1",
- "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz",
- "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^22.20 || ^24.12 || >=25"
- }
- },
- "node_modules/@rollup/rollup-android-arm-eabi": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.64.7.tgz",
- "integrity": "sha512-GFF2tHk0NqYVObrK0+/ps/LEXsV/v2q6Lyiy7mZr+Ej24fpQScvqK2ziz9NHeEQYnKWKym38Ml7dZbPtC9tuAw==",
- "cpu": [
- "arm"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "android"
- ]
- },
- "node_modules/@rollup/rollup-android-arm64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.64.7.tgz",
- "integrity": "sha512-BOymcXJFYYXqluDidj6sLelNjXruC3u1BqVxLn2OPCXH9xaPSOTH+MS4uychsGI1NU7tJoiv9KvqijDHFe+hhQ==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "android"
- ]
- },
- "node_modules/@rollup/rollup-darwin-arm64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.64.7.tgz",
- "integrity": "sha512-jgVpGweIbls2H5jfBB70tnX52abMd5pJr3xmtoDILx6JdmGl+Un+tMb9R+MbHPVB46tTBpY8emtn7M71SnQhEQ==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "darwin"
- ]
- },
- "node_modules/@rollup/rollup-darwin-x64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.64.7.tgz",
- "integrity": "sha512-AVMpt94mrXH7WjyWKmd9nT4RC5yM0AN3Lt6A89xVIsuLDi3dDX6dxMytLbW36FlppY9JsoqUoS3LUUpUrYxiMQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "darwin"
- ]
- },
- "node_modules/@rollup/rollup-freebsd-arm64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.64.7.tgz",
- "integrity": "sha512-Nje29c/MyajxRNFGU05oyeTDS6SWdMg/wWMSlZSBTBju5BKz2NVc0kFiFSQGzRELNiB+CBk/ZvAVsWsENgjGag==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "freebsd"
- ]
- },
- "node_modules/@rollup/rollup-freebsd-x64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.64.7.tgz",
- "integrity": "sha512-xxgN5/kMzCNz/NYJ4t49B0/WS6InNlWv6xvgwUQqNnWM9WJMMwMUGWrAxdNEgYWdHL0sMnuh8I87StGrIMTA+w==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "freebsd"
- ]
- },
- "node_modules/@rollup/rollup-linux-arm-gnueabihf": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.64.7.tgz",
- "integrity": "sha512-5U/9gVn/X657I8+NHoHO9Wnfyd0lEXwqbslobRoB9TC8SwYzOitcIr6TLE7531Q/NWs3L4mb+YOGzlf3vHIeYw==",
- "cpu": [
- "arm"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-arm-musleabihf": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.64.7.tgz",
- "integrity": "sha512-hcDS0a/Y8ggM3dcOfzH7UpSaogCViWTtNDSl0EXpWIj8g8/r86ITqptjil2hayTXS0Umzb+Yt3mjgpQVFVf2+A==",
- "cpu": [
- "arm"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-arm64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.64.7.tgz",
- "integrity": "sha512-awbAr4N7Qd/yjZAQBEEPzgcPveYG2Gi5L6gq1/Bp+3SMRjn5F68Wv1zKsntzvvoioBR6yKvhLtWqu11am1TsxA==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-arm64-musl": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.64.7.tgz",
- "integrity": "sha512-Ut7xQJfeLhzu2yN3rHzVvL9ccyPJvN4aCdP1V7hJ5lGxX/eCzgB9SiMk/Md3J+rWWZgvoYsDW4BUFggJr32pBA==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-loong64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.64.7.tgz",
- "integrity": "sha512-trqoni/vHkuM5oo5iQhoKPh6fjjD7E80pMgCXWgGxU8vSRPUlwvG8GeYvEEcB6eJAxNa99WH9iVHPHelgtKiHA==",
- "cpu": [
- "loong64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-loong64-musl": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.64.7.tgz",
- "integrity": "sha512-gvlUy0XFIwtUWiQmhc+4GT2Knm3KosinpUVhr7CtTFA78gFDnE5NWwMPzHrJsYpKkh2LlY7yvBIboN1FAV6Xeg==",
- "cpu": [
- "loong64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-ppc64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.64.7.tgz",
- "integrity": "sha512-8yS/kSXTJ/47Z62JW+C2AJ3vd/8zZw016vFjtNpZwmZlwJumLua4s1i78dkYuSSqQ+0aIMxJcBp/4uIwp2ehjA==",
- "cpu": [
- "ppc64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-ppc64-musl": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.64.7.tgz",
- "integrity": "sha512-mxVSKvvty0hOP1AFo/l4Kpv3i3HK0zx4rYUrirZDToYlWhS3xzmIkq3vUJ7rtrM2cPyXveDyDT6M1FfXZ9hTRA==",
- "cpu": [
- "ppc64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-riscv64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.64.7.tgz",
- "integrity": "sha512-lEse1zra1CagM+uHH2bl28GInDsZZ45NJzHei2OMaXQFzZWKBO1hFo07rS0xB6ayV4WU360UrSmPDSyCMpYsUQ==",
- "cpu": [
- "riscv64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-riscv64-musl": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.64.7.tgz",
- "integrity": "sha512-d/cqCoiG9WMZ5VuJZ5Qdlv9g97QyoGwnmvHccvGhKSacTy7xFrAS+jMcJSBdMNOw1oacSkl6p7K6O/DRYTKL3g==",
- "cpu": [
- "riscv64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-s390x-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.64.7.tgz",
- "integrity": "sha512-kVEA//CeDOZZP1XCH2JBgqXb9+ALlwuA/4civaazMVdgzaSQC9WM38w65zG/LLk0lA7F9Od19CSBZW7J9mCbNg==",
- "cpu": [
- "s390x"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-x64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.64.7.tgz",
- "integrity": "sha512-14oLfDsOSr0TSntT/34jRg/V8B2XW/Jomh9zX9CjIM9gwwNlI1OCymS/GVn0DV32h67eRf7U0GGem0rvfXgNwQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-linux-x64-musl": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.64.7.tgz",
- "integrity": "sha512-dFVtPfjZqAnUPpPY4Bfg+87yo1IBW9HiY8iCFtg49Pn46MfZTebSHZh8PL6SOsgyt9LQP8Nj/sE6jm3aWpySmQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "linux"
- ]
- },
- "node_modules/@rollup/rollup-openbsd-x64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.64.7.tgz",
- "integrity": "sha512-Dm9CZ2SM3u2NgvBj4awW/gGqkOwzH8f6MSYIJE6+bKcKLI+T0+LhpD6vtetTfrIIx+rLXDf5RJqBP6v7V5O+wg==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "openbsd"
- ]
- },
- "node_modules/@rollup/rollup-openharmony-arm64": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.64.7.tgz",
- "integrity": "sha512-oaxsvS2mL6+78741CT2HCORBvy1USg3Wqpt5qvZwy3aMXRMl4zxyDo8maeFRWJkF3dp0Wbi71fKH5Gr9YI3vYA==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "openharmony"
- ]
- },
- "node_modules/@rollup/rollup-win32-arm64-msvc": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.64.7.tgz",
- "integrity": "sha512-UKZz1Y4mxUqkWh60uH7B1aZn6UcAimPk4ipRktpb3wfZq7JrzZYyAXTwiafkb3T+kd5P3Sf+vzcRdsDks4WtjQ==",
- "cpu": [
- "arm64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ]
- },
- "node_modules/@rollup/rollup-win32-ia32-msvc": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.64.7.tgz",
- "integrity": "sha512-WWkSuC1J+mZJnsDh0mRaCAFSWfQdl6y3fDjZeWElI6zHo8CcOSG5lY3yurKJjEq+FrX0WD60sFrVVpbHqzMmrw==",
- "cpu": [
- "ia32"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ]
- },
- "node_modules/@rollup/rollup-win32-x64-gnu": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.64.7.tgz",
- "integrity": "sha512-u9bp2yXCI3/BUjUxY6uWknAY32pfvvDCPysyUTySotx0dj3ydlFHrRQYfn/b/kZgnfjyEwySMtI7Ga054uiysQ==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ]
- },
- "node_modules/@rollup/rollup-win32-x64-msvc": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.64.7.tgz",
- "integrity": "sha512-EhX6N3XPMEnVU5cRfNgzPOKcSucsbu3nNNWuoAaEuKIqmddBKHN5b9d4rPe65T1I+FaOT176JKiZNsUBp3rqbA==",
- "cpu": [
- "x64"
- ],
- "dev": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "win32"
- ]
- },
- "node_modules/@types/estree": {
- "version": "1.0.9",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
- "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/@types/react": {
- "version": "19.3.0",
- "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz",
- "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "csstype": "^3.2.2"
- }
- },
- "node_modules/@types/react-dom": {
- "version": "19.3.0",
- "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz",
- "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==",
- "dev": true,
- "license": "MIT",
- "peerDependencies": {
- "@types/react": "^19.3.0"
- }
- },
- "node_modules/csstype": {
- "version": "3.2.3",
- "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
- "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
- "dev": true,
- "license": "MIT"
- },
- "node_modules/esbuild": {
- "version": "0.28.2",
- "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz",
- "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==",
- "dev": true,
- "hasInstallScript": true,
- "license": "MIT",
- "bin": {
- "esbuild": "bin/esbuild"
- },
- "engines": {
- "node": ">=18"
- },
- "optionalDependencies": {
- "@esbuild/aix-ppc64": "0.28.2",
- "@esbuild/android-arm": "0.28.2",
- "@esbuild/android-arm64": "0.28.2",
- "@esbuild/android-x64": "0.28.2",
- "@esbuild/darwin-arm64": "0.28.2",
- "@esbuild/darwin-x64": "0.28.2",
- "@esbuild/freebsd-arm64": "0.28.2",
- "@esbuild/freebsd-x64": "0.28.2",
- "@esbuild/linux-arm": "0.28.2",
- "@esbuild/linux-arm64": "0.28.2",
- "@esbuild/linux-ia32": "0.28.2",
- "@esbuild/linux-loong64": "0.28.2",
- "@esbuild/linux-mips64el": "0.28.2",
- "@esbuild/linux-ppc64": "0.28.2",
- "@esbuild/linux-riscv64": "0.28.2",
- "@esbuild/linux-s390x": "0.28.2",
- "@esbuild/linux-x64": "0.28.2",
- "@esbuild/netbsd-arm64": "0.28.2",
- "@esbuild/netbsd-x64": "0.28.2",
- "@esbuild/openbsd-arm64": "0.28.2",
- "@esbuild/openbsd-x64": "0.28.2",
- "@esbuild/openharmony-arm64": "0.28.2",
- "@esbuild/sunos-x64": "0.28.2",
- "@esbuild/win32-arm64": "0.28.2",
- "@esbuild/win32-ia32": "0.28.2",
- "@esbuild/win32-x64": "0.28.2"
- }
- },
- "node_modules/fdir": {
- "version": "6.5.0",
- "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
- "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=12.0.0"
- },
- "peerDependencies": {
- "picomatch": "^3 || ^4"
- },
- "peerDependenciesMeta": {
- "picomatch": {
- "optional": true
- }
- }
- },
- "node_modules/fsevents": {
- "version": "2.3.3",
- "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
- "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
- "dev": true,
- "hasInstallScript": true,
- "license": "MIT",
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
- }
- },
- "node_modules/nanoid": {
- "version": "3.3.20",
- "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz",
- "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==",
- "dev": true,
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/ai"
- }
- ],
- "license": "MIT",
- "bin": {
- "nanoid": "bin/nanoid.cjs"
- },
- "engines": {
- "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
- }
- },
- "node_modules/picocolors": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
- "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
- "dev": true,
- "license": "ISC"
- },
- "node_modules/picomatch": {
- "version": "4.0.7",
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz",
- "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">=12"
- },
- "funding": {
- "url": "https://github.com/sponsors/jonschlinkert"
- }
- },
- "node_modules/postcss": {
- "version": "8.5.30",
- "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz",
- "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==",
- "dev": true,
- "funding": [
- {
- "type": "opencollective",
- "url": "https://opencollective.com/postcss/"
- },
- {
- "type": "tidelift",
- "url": "https://tidelift.com/funding/github/npm/postcss"
- },
- {
- "type": "github",
- "url": "https://github.com/sponsors/ai"
- }
- ],
- "license": "MIT",
- "dependencies": {
- "nanoid": "^3.3.20",
- "picocolors": "^1.1.1",
- "source-map-js": "^1.2.2"
- },
- "engines": {
- "node": "^10 || ^12 || >=14"
- }
- },
- "node_modules/react": {
- "version": "19.3.1",
- "resolved": "https://registry.npmjs.org/react/-/react-19.3.1.tgz",
- "integrity": "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw==",
- "license": "MIT",
- "engines": {
- "node": ">=0.10.0"
- }
- },
- "node_modules/react-dom": {
- "version": "19.3.1",
- "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.1.tgz",
- "integrity": "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ==",
- "license": "MIT",
- "dependencies": {
- "scheduler": "^0.28.0"
- },
- "peerDependencies": {
- "react": "^19.3.1"
- }
- },
- "node_modules/rollup": {
- "version": "4.64.7",
- "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.64.7.tgz",
- "integrity": "sha512-aCiJONoZp0lNGJ6sYqspLkh6UXaWEey/7Ko2TZyXQ4kgzAhDUxAgbt/oulXKmD+zFHXw1k+RF6ZZWOrjp7DZnQ==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "@types/estree": "1.0.9"
- },
- "bin": {
- "rollup": "dist/bin/rollup"
- },
- "engines": {
- "node": ">=18.0.0",
- "npm": ">=8.0.0"
- },
- "optionalDependencies": {
- "@napi-rs/lzma-linux-x64-gnu": "1.5.1",
- "@rollup/rollup-android-arm-eabi": "4.64.7",
- "@rollup/rollup-android-arm64": "4.64.7",
- "@rollup/rollup-darwin-arm64": "4.64.7",
- "@rollup/rollup-darwin-x64": "4.64.7",
- "@rollup/rollup-freebsd-arm64": "4.64.7",
- "@rollup/rollup-freebsd-x64": "4.64.7",
- "@rollup/rollup-linux-arm-gnueabihf": "4.64.7",
- "@rollup/rollup-linux-arm-musleabihf": "4.64.7",
- "@rollup/rollup-linux-arm64-gnu": "4.64.7",
- "@rollup/rollup-linux-arm64-musl": "4.64.7",
- "@rollup/rollup-linux-loong64-gnu": "4.64.7",
- "@rollup/rollup-linux-loong64-musl": "4.64.7",
- "@rollup/rollup-linux-ppc64-gnu": "4.64.7",
- "@rollup/rollup-linux-ppc64-musl": "4.64.7",
- "@rollup/rollup-linux-riscv64-gnu": "4.64.7",
- "@rollup/rollup-linux-riscv64-musl": "4.64.7",
- "@rollup/rollup-linux-s390x-gnu": "4.64.7",
- "@rollup/rollup-linux-x64-gnu": "4.64.7",
- "@rollup/rollup-linux-x64-musl": "4.64.7",
- "@rollup/rollup-openbsd-x64": "4.64.7",
- "@rollup/rollup-openharmony-arm64": "4.64.7",
- "@rollup/rollup-win32-arm64-msvc": "4.64.7",
- "@rollup/rollup-win32-ia32-msvc": "4.64.7",
- "@rollup/rollup-win32-x64-gnu": "4.64.7",
- "@rollup/rollup-win32-x64-msvc": "4.64.7",
- "fsevents": "~2.3.2"
- }
- },
- "node_modules/scheduler": {
- "version": "0.28.0",
- "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz",
- "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==",
- "license": "MIT"
- },
- "node_modules/source-map-js": {
- "version": "1.2.2",
- "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz",
- "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==",
- "dev": true,
- "license": "BSD-3-Clause",
- "engines": {
- "node": ">=0.10.0"
- }
- },
- "node_modules/tinyglobby": {
- "version": "0.2.17",
- "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
- "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "fdir": "^6.5.0",
- "picomatch": "^4.0.4"
- },
- "engines": {
- "node": ">=12.0.0"
- },
- "funding": {
- "url": "https://github.com/sponsors/SuperchupuDev"
- }
- },
- "node_modules/typescript": {
- "version": "5.9.3",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
- "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
- "dev": true,
- "license": "Apache-2.0",
- "bin": {
- "tsc": "bin/tsc",
- "tsserver": "bin/tsserver"
- },
- "engines": {
- "node": ">=14.17"
- }
- },
- "node_modules/vite": {
- "version": "7.3.7",
- "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.7.tgz",
- "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==",
- "dev": true,
- "license": "MIT",
- "dependencies": {
- "esbuild": "^0.27.0 || ^0.28.0",
- "fdir": "^6.5.0",
- "picomatch": "^4.0.3",
- "postcss": "^8.5.6",
- "rollup": "^4.43.0",
- "tinyglobby": "^0.2.15"
- },
- "bin": {
- "vite": "bin/vite.js"
- },
- "engines": {
- "node": "^20.19.0 || >=22.12.0"
- },
- "funding": {
- "url": "https://github.com/vitejs/vite?sponsor=1"
- },
- "optionalDependencies": {
- "fsevents": "~2.3.3"
- },
- "peerDependencies": {
- "@types/node": "^20.19.0 || >=22.12.0",
- "jiti": ">=1.21.0",
- "less": "^4.0.0",
- "lightningcss": "^1.21.0",
- "sass": "^1.70.0",
- "sass-embedded": "^1.70.0",
- "stylus": ">=0.54.8",
- "sugarss": "^5.0.0",
- "terser": "^5.16.0",
- "tsx": "^4.8.1",
- "yaml": "^2.4.2"
- },
- "peerDependenciesMeta": {
- "@types/node": {
- "optional": true
- },
- "jiti": {
- "optional": true
- },
- "less": {
- "optional": true
- },
- "lightningcss": {
- "optional": true
- },
- "sass": {
- "optional": true
- },
- "sass-embedded": {
- "optional": true
- },
- "stylus": {
- "optional": true
- },
- "sugarss": {
- "optional": true
- },
- "terser": {
- "optional": true
- },
- "tsx": {
- "optional": true
- },
- "yaml": {
- "optional": true
- }
- }
- }
- }
-}
diff --git a/examples/customer-chat/package.json b/examples/customer-chat/package.json
deleted file mode 100644
index 0838c053..00000000
--- a/examples/customer-chat/package.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "name": "hugagent-customer-chat-example",
- "version": "1.0.0",
- "private": true,
- "type": "module",
- "scripts": {
- "build": "tsc --noEmit && node scripts/build-component.mjs && tsc --emitDeclarationOnly",
- "standalone": "vite build --base /customer/ --outDir standalone-dist",
- "dev": "vite --open /preview.html",
- "dev:headless": "vite",
- "build:web": "vite build",
- "preview:web": "vite preview",
- "prepack": "npm run build",
- "test": "node scripts/test-proxy.mjs",
- "test:package": "node scripts/test-package.mjs"
- },
- "dependencies": {
- "react": "^19.2.0",
- "react-dom": "^19.2.0"
- },
- "devDependencies": {
- "@types/react": "^19.2.7",
- "typescript": "~5.9.3",
- "vite": "^7.3.1",
- "@types/react-dom": "^19.2.0",
- "@hugagent/chat-sdk": "file:../../packages/chat-sdk"
- },
- "exports": {
- ".": {
- "types": "./dist/customization.d.ts",
- "import": "./dist/customization.js"
- }
- },
- "files": [
- "dist",
- "README.md"
- ],
- "peerDependencies": {
- "@hugagent/chat-sdk": "^1.0.0"
- }
-}
diff --git a/examples/customer-chat/preview.html b/examples/customer-chat/preview.html
deleted file mode 100644
index dea4dac0..00000000
--- a/examples/customer-chat/preview.html
+++ /dev/null
@@ -1 +0,0 @@
-Customer host preview Host preview · Live chat · API debug Customer host preview This offline host tests the public slot contract. Use Live chat for backend tests.
\ No newline at end of file
diff --git a/examples/customer-chat/scripts/build-component.mjs b/examples/customer-chat/scripts/build-component.mjs
deleted file mode 100644
index 8e93c500..00000000
--- a/examples/customer-chat/scripts/build-component.mjs
+++ /dev/null
@@ -1,14 +0,0 @@
-import { readFileSync, writeFileSync, existsSync } from 'node:fs';
-import { build } from 'vite';
-await build({
- configFile: false,
- build: {
- outDir: 'dist', emptyOutDir: true, sourcemap: false,
- lib: { entry: 'src/customization.tsx', formats: ['es'], fileName: 'customization', cssFileName: 'theme' },
- rollupOptions: { external: id => id === 'react' || id.startsWith('react/') ||
- id === 'react-dom' || id.startsWith('react-dom/') || id === '@hugagent/chat-sdk' || id.startsWith('@hugagent/chat-sdk/') },
- },
-});
-
-// Vite extracts library CSS. Keep the package entry self-contained for the host.
-if (existsSync('dist/theme.css')) writeFileSync('dist/customization.js', "import './theme.css';\n" + readFileSync('dist/customization.js', 'utf8'));
diff --git a/examples/customer-chat/scripts/developer-proxy.mjs b/examples/customer-chat/scripts/developer-proxy.mjs
deleted file mode 100644
index a4ced863..00000000
--- a/examples/customer-chat/scripts/developer-proxy.mjs
+++ /dev/null
@@ -1,43 +0,0 @@
-import { createHash } from 'node:crypto';
-
-/** A fixed upstream gateway; no client-supplied destination is accepted. */
-export function createDeveloperProxy(env) {
- const raw = env.HUGAGENT_BACKEND_URL;
- if (!raw) return {};
- const target = new URL(raw);
- if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password ||
- target.search || target.hash || target.pathname !== '/') {
- throw new Error('HUGAGENT_BACKEND_URL must be an HTTP(S) origin, without a path, credentials, query or fragment.');
- }
- const upstreamCookie = env.HUGAGENT_SESSION_COOKIE || 'jx_session';
- if (!/^[A-Za-z0-9_-]+$/.test(upstreamCookie)) throw new Error('Invalid session cookie name');
- const localCookie = 'hugagent_dev_' + createHash('sha256').update(target.origin).digest('hex').slice(0, 16);
- const configure = proxy => {
- proxy.on('proxyReq', (outgoing, incoming) => {
- // Cookies are scoped to this upstream; never forward unrelated localhost cookies.
- const session = (incoming.headers.cookie || '').split(';').map(v => v.trim()).find(v => v.startsWith(localCookie + '='));
- outgoing.removeHeader('cookie');
- outgoing.removeHeader('authorization');
- if (session) outgoing.setHeader('cookie', upstreamCookie + '=' + session.slice(localCookie.length + 1));
- outgoing.setHeader('x-forwarded-host', incoming.headers.host || 'localhost');
- outgoing.setHeader('x-forwarded-proto', 'http');
- });
- proxy.on('proxyRes', response => {
- // Keep Secure, HttpOnly, SameSite and expiry. Only rewrite the development cookie name/domain.
- response.headers['set-cookie'] = (response.headers['set-cookie'] || [])
- .filter(cookie => cookie.startsWith(upstreamCookie + '='))
- .map(cookie => {
- const rewritten = cookie.replace(upstreamCookie + '=', localCookie + '=').replace(/;\s*Domain=[^;]*/gi, '');
- return /;\s*HttpOnly(?:;|$)/i.test(rewritten) ? rewritten : rewritten + '; HttpOnly';
- });
- delete response.headers['access-control-allow-origin'];
- delete response.headers['access-control-allow-credentials'];
- });
- };
- const options = { target: target.origin, changeOrigin: true, secure: true, xfwd: false, configure,
- proxyTimeout: 180000, timeout: 180000 };
- return {
- '^/api(?:/|$)': { ...options, rewrite: path => path.replace(/^\/api(?=\/|$)/, '') || '/' },
- '^/login(?:[?]|$)': options,
- };
-}
diff --git a/examples/customer-chat/scripts/test-package.mjs b/examples/customer-chat/scripts/test-package.mjs
deleted file mode 100644
index 1bc62ada..00000000
--- a/examples/customer-chat/scripts/test-package.mjs
+++ /dev/null
@@ -1,30 +0,0 @@
-import { execFileSync } from 'node:child_process';
-import { mkdtempSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join, resolve } from 'node:path';
-import assert from 'node:assert/strict';
-const pkg = JSON.parse(readFileSync('package.json'));
-const sdkFile = pkg.devDependencies['@hugagent/chat-sdk'].replace(/^file:/, '');
-if (!sdkFile.endsWith('.tgz')) throw new Error('Run this check in the exported developer kit');
-const component = resolve(pkg.name.replaceAll('@', '').replaceAll('/', '-') + '-' + pkg.version + '.tgz');
-if (!existsSync(component)) throw new Error('Run npm pack first');
-const host = mkdtempSync(join(tmpdir(), 'hugagent-customer-host-'));
-const react = JSON.parse(readFileSync('node_modules/react/package.json')).version;
-const vite = JSON.parse(readFileSync('node_modules/vite/package.json')).version;
-writeFileSync(join(host, 'package.json'), JSON.stringify({ private: true, type: 'module', dependencies: {
- '@hugagent/chat-sdk': 'file:' + resolve(sdkFile), [pkg.name]: 'file:' + component, react, vite,
-} }, null, 2));
-execFileSync('npm', ['install', '--prefer-offline', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: host, stdio: 'pipe' });
-const installed = JSON.parse(readFileSync(join(host, 'node_modules', pkg.name, 'package.json')));
-assert.equal(installed.dependencies?.['@hugagent/chat-sdk'], undefined);
-assert.ok(installed.peerDependencies['@hugagent/chat-sdk']);
-writeFileSync(join(host, 'entry.js'), "export { default } from " + JSON.stringify(pkg.name) + ";");
-writeFileSync(join(host, 'build.mjs'), "import{build}from'vite';await build({configFile:false,logLevel:'error',build:{lib:{entry:'entry.js',formats:['es'],fileName:'customer',cssFileName:'customer'}}});");
-execFileSync(process.execPath, ['build.mjs'], { cwd: host, stdio: 'pipe' });
-assert.ok(readFileSync(join(host, 'dist/customer.js'), 'utf8').includes('Customer Portal'));
-assert.ok(existsSync(join(host, 'dist/customer.css')));
-if (process.env.HUGAGENT_EXPECT_HELPER) {
- assert.ok(readFileSync(join(host, 'dist/customer.js'), 'utf8').includes('Customer helper included'));
- assert.ok(readFileSync(join(host, 'dist/customer.css'), 'utf8').includes('customer-extra-badge'));
-}
-console.log('PASS customer tarball installs and builds in a fresh host: ' + host);
diff --git a/examples/customer-chat/scripts/test-proxy.mjs b/examples/customer-chat/scripts/test-proxy.mjs
deleted file mode 100644
index 54e2a9f1..00000000
--- a/examples/customer-chat/scripts/test-proxy.mjs
+++ /dev/null
@@ -1,26 +0,0 @@
-import assert from 'node:assert/strict';
-import { EventEmitter } from 'node:events';
-import { createDeveloperProxy } from './developer-proxy.mjs';
-for (const url of ['file:///tmp', 'http://user:password@host', 'https://host/api', 'https://host/?target=other']) {
- assert.throws(() => createDeveloperProxy({ HUGAGENT_BACKEND_URL: url }));
-}
-assert.deepEqual(createDeveloperProxy({}), {});
-const a = createDeveloperProxy({ HUGAGENT_BACKEND_URL: 'https://backend.example' });
-const b = createDeveloperProxy({ HUGAGENT_BACKEND_URL: 'https://other.example' });
-function cookie(proxyConfig) {
- const proxy = new EventEmitter(); proxyConfig['^/api(?:/|$)'].configure(proxy);
- const response = { headers: { 'set-cookie': ['jx_session=secret; Domain=backend.example; Secure; SameSite=Lax', 'unrelated=deny'] } };
- proxy.emit('proxyRes', response);
- const cookies = response.headers['set-cookie'];
- assert.equal(cookies.length, 1); assert.ok(!cookies[0].includes('Domain='));
- assert.ok(cookies[0].includes('; HttpOnly')); assert.ok(cookies[0].includes('Secure; SameSite=Lax'));
- return { proxy, value: cookies[0].split(';')[0] };
-}
-const first = cookie(a), second = cookie(b); assert.notEqual(first.value, second.value);
-const headers = new Map([['authorization', 'secret'], ['cookie', 'unrelated=x']]);
-const request = { removeHeader: name => headers.delete(name), setHeader: (name,value) => headers.set(name,value) };
-first.proxy.emit('proxyReq', request, { headers: { host: 'localhost:4310', cookie: 'unrelated=x; ' + second.value } });
-assert.equal(headers.has('cookie'), false); assert.equal(headers.has('authorization'), false);
-first.proxy.emit('proxyReq', request, { headers: { host: 'localhost:4310', cookie: 'unrelated=x; ' + first.value } });
-assert.equal(headers.get('cookie'), 'jx_session=secret');
-console.log('PASS fixed target validation, Secure/HttpOnly retention, cross-target cookie isolation and authorization filtering');
diff --git a/examples/customer-chat/session.html b/examples/customer-chat/session.html
deleted file mode 100644
index d62e4fb2..00000000
--- a/examples/customer-chat/session.html
+++ /dev/null
@@ -1 +0,0 @@
-Session callback Completing login
Retry login
\ No newline at end of file
diff --git a/examples/customer-chat/src/controls.ts b/examples/customer-chat/src/controls.ts
deleted file mode 100644
index c2ab7600..00000000
--- a/examples/customer-chat/src/controls.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-import { ChatClient, type ChatEvent, type QuestionAnswer } from '@hugagent/chat-sdk';
-
-/** Deliberately uses DOM textContent: server/model text must never become executable HTML. */
-export class Controls {
- private readonly pending = new Map();
- constructor(private readonly client: ChatClient, private readonly root: HTMLElement,
- private readonly chat: () => string, private readonly error: (message: string) => void) {}
- clear() { this.pending.clear(); this.root.replaceChildren(); }
- private remove(id: string) { this.pending.get(id)?.remove(); this.pending.delete(id); }
- private panel(id: string, title: string) {
- if (this.pending.has(id)) return null;
- const panel = document.createElement('section'); panel.dataset.request = id;
- const heading = document.createElement('p'); heading.textContent = title; panel.append(heading);
- this.pending.set(id, panel); this.root.append(panel); return panel;
- }
- private button(panel: HTMLElement, label: string, action: () => Promise) {
- const button = document.createElement('button'); button.type = 'button'; button.textContent = label;
- button.addEventListener('click', () => {
- const buttons = [...panel.querySelectorAll('button')]; buttons.forEach(item => { item.disabled = true; });
- void action().catch(error => {
- this.error(String(error)); buttons.forEach(item => { item.disabled = false; });
- });
- }); panel.append(button);
- }
- consume(event: ChatEvent) {
- if (event.type === 'file_confirm' || event.type === 'design_pick') {
- const id = String(event.confirm_id || ''); if (!id) return;
- if (event.expired || event.resolved) return this.remove(id);
- const panel = this.panel(id, String(event.message || event.question || event.logical_path || event.op || 'Confirm operation'));
- if (!panel) return;
- const decide = async (decision: 'allow' | 'deny' | 'choice' | 'skip', optionId?: string) => {
- const result = await this.client.confirmFile(this.chat(), id, decision, optionId);
- if (!result.ok && !result.stale) throw new Error(result.message || 'Decision was not accepted');
- this.remove(id);
- if (result.stale) this.error(result.message || 'This confirmation has expired');
- };
- if (event.type === 'design_pick') {
- for (const option of Array.isArray(event.options) ? event.options as Record[] : []) {
- this.button(panel, String(option.title || option.id), () => decide('choice', String(option.id)));
- }
- this.button(panel, 'Skip', () => decide('skip'));
- } else {
- this.button(panel, 'Allow', () => decide('allow')); this.button(panel, 'Deny', () => decide('deny'));
- }
- }
- if (event.type === 'user_question_resolved') this.remove(String(event.request_id || ''));
- if (event.type !== 'user_question') return;
- const id = String(event.request_id || ''); if (!id) return;
- const panel = this.panel(id, 'Assistant needs your answer'); if (!panel) return;
- const answers: (() => QuestionAnswer)[] = [];
- for (const question of Array.isArray(event.questions) ? event.questions as Record[] : []) {
- const fieldset = document.createElement('fieldset'), legend = document.createElement('legend');
- legend.textContent = String(question.question || ''); fieldset.append(legend);
- const selections: HTMLInputElement[] = [];
- for (const option of Array.isArray(question.options) ? question.options as Record[] : []) {
- const label = document.createElement('label'), input = document.createElement('input');
- input.type = question.multi_select ? 'checkbox' : 'radio'; input.name = id + ':' + String(question.id);
- input.value = String(option.id); label.append(input, document.createTextNode(String(option.label))); fieldset.append(label); selections.push(input);
- }
- const custom = document.createElement('input'); custom.setAttribute('aria-label', 'Custom answer'); fieldset.append(custom);
- panel.append(fieldset);
- answers.push(() => ({ id: String(question.id), selected: selections.filter(item => item.checked).map(item => item.value), custom: custom.value }));
- }
- this.button(panel, 'Submit answers', async () => {
- const result = await this.client.answerQuestion(this.chat(), id, answers.map(read => read()));
- if (!result.ok && !result.stale) throw new Error(result.message || 'Answer was not accepted');
- this.remove(id); // A successful POST atomically claims and ends the pending request.
- if (result.stale) this.error(result.message || 'This question has expired');
- });
- }
- async recover() {
- const [confirms, questions] = await Promise.all([this.client.pendingConfirms(this.chat()), this.client.pendingQuestions(this.chat())]);
- for (const event of confirms.pendings || []) this.consume({ ...event, type: event.kind === 'design_pick' ? 'design_pick' : 'file_confirm' });
- for (const event of questions.requests || []) this.consume({ ...event, type: 'user_question' });
- }
-}
diff --git a/examples/customer-chat/src/css.d.ts b/examples/customer-chat/src/css.d.ts
deleted file mode 100644
index 35306c6f..00000000
--- a/examples/customer-chat/src/css.d.ts
+++ /dev/null
@@ -1 +0,0 @@
-declare module '*.css';
diff --git a/examples/customer-chat/src/customization.tsx b/examples/customer-chat/src/customization.tsx
deleted file mode 100644
index 3be05a17..00000000
--- a/examples/customer-chat/src/customization.tsx
+++ /dev/null
@@ -1,29 +0,0 @@
-import { defineChatCustomization, useChatRuntime, type ChatSlotProps, type MessageSlotProps } from '@hugagent/chat-sdk/react';
-import './theme.css';
-
-function Welcome({ children }: ChatSlotProps) {
- const { view, actions } = useChatRuntime();
- return
- Customer Portal {view.productName}
- actions.setDraft('Please summarize this document.')}>Use a template
-
- {children}
- ;
-}
-function Message({ message, children }: MessageSlotProps) {
- return
- {message.role === 'assistant' ? 'Portal Assistant' : 'You'}
- {children}
-
;
-}
-function Composer({ children }: ChatSlotProps) {
- const { view } = useChatRuntime();
- return
- {view.conversation.sending ? 'Working' : 'Ready'}
- {children}
- ;
-}
-export default defineChatCustomization({
- apiVersion: 1, id: 'customer-portal',
- slots: { welcome: Welcome, message: Message, composer: Composer },
-});
diff --git a/examples/customer-chat/src/debug.ts b/examples/customer-chat/src/debug.ts
deleted file mode 100644
index a1cb48b2..00000000
--- a/examples/customer-chat/src/debug.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import './login.js';
-import './developer.css';
-const status = document.getElementById('session')!;
-const result = document.getElementById('result')!;
-async function session() {
- try {
- const response = await fetch('/api/v1/me', { credentials: 'same-origin' });
- if (response.status === 401) { status.textContent = 'Signed out'; return; }
- const body = await response.json();
- status.textContent = response.ok && body.code >= 10000 && body.code < 20000 && body.data?.user_id
- ? 'Signed in' : 'Backend error: ' + response.status;
- } catch { status.textContent = 'Backend unavailable. Configure HUGAGENT_BACKEND_URL and restart npm run dev.'; }
-}
-document.getElementById('request')!.addEventListener('click', () => {
- void (async () => {
- try {
- const path = (document.getElementById('endpoint') as HTMLSelectElement).value;
- const response = await fetch('/api' + path, { credentials: 'same-origin' });
- result.textContent = 'HTTP ' + response.status + '\n' + await response.text();
- } catch (error) { result.textContent = String(error); }
- })();
-});
-document.getElementById('logout')!.addEventListener('click', () => {
- void (async () => {
- try {
- const response = await fetch('/api/v1/auth/logout', { method: 'POST', credentials: 'same-origin' });
- if (!response.ok) throw new Error('Logout failed: ' + response.status);
- await session();
- } catch (error) { status.textContent = String(error); }
- })();
-});
-void session();
diff --git a/examples/customer-chat/src/developer.css b/examples/customer-chat/src/developer.css
deleted file mode 100644
index afec40f6..00000000
--- a/examples/customer-chat/src/developer.css
+++ /dev/null
@@ -1,3 +0,0 @@
-body{font-family:system-ui,sans-serif;max-width:1100px;margin:24px auto;padding:16px;color:#172b4d;background:#f5f8fc}nav{display:flex;gap:12px;flex-wrap:wrap}button,input,textarea,select{font:inherit;margin:6px;padding:8px}textarea{display:block;width:90%;min-height:90px}button{cursor:pointer}button:disabled{cursor:default}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:white;padding:16px}.preview-layout{display:grid;grid-template-columns:220px minmax(0,1fr);gap:24px}aside button{display:block}article,section.customer-composer{padding:12px;background:white;border-radius:10px;margin:10px 0}@media(max-width:650px){.preview-layout{grid-template-columns:1fr}body{margin:0}}
-
-*{box-sizing:border-box}[data-chat-customization] .preview-layout .customer-welcome{height:auto}[data-chat-customization]{--color-primary:#1554ad;--color-text:#172b4d;--color-text-secondary:#526581;--color-bg-container:white;--color-border:#c9d5e6;--radius-sm:6px}.preview-layout>section{min-width:0}
diff --git a/examples/customer-chat/src/login.ts b/examples/customer-chat/src/login.ts
deleted file mode 100644
index 5f03566d..00000000
--- a/examples/customer-chat/src/login.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-// Bind a login callback to the tab that started it; this value is not a session credential.
-for (const link of document.querySelectorAll('a[data-login], a#login')) {
- link.addEventListener('click', event => {
- event.preventDefault();
- const state = crypto.randomUUID();
- sessionStorage.setItem('hugagent-login', JSON.stringify({ state, started: Date.now() }));
- const redirect = '/session.html?state=' + encodeURIComponent(state);
- location.assign('/login?redirect=' + encodeURIComponent(redirect));
- });
-}
-export {};
diff --git a/examples/customer-chat/src/preview.tsx b/examples/customer-chat/src/preview.tsx
deleted file mode 100644
index 5f946843..00000000
--- a/examples/customer-chat/src/preview.tsx
+++ /dev/null
@@ -1,5 +0,0 @@
-import { createRoot } from 'react-dom/client';
-import { PreviewHost } from '@hugagent/chat-sdk/preview';
-import customization from './customization.js';
-import './developer.css';
-createRoot(document.getElementById('root')!).render( );
diff --git a/examples/customer-chat/src/session.ts b/examples/customer-chat/src/session.ts
deleted file mode 100644
index 0df0bc11..00000000
--- a/examples/customer-chat/src/session.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-// Remove one-time credentials before any API call or further navigation.
-const url = new URL(location.href);
-const state = url.searchParams.get('state');
-const started = sessionStorage.getItem('hugagent-login');
-sessionStorage.removeItem('hugagent-login');
-const code = url.searchParams.get('code') || url.searchParams.get('ticket');
-history.replaceState(null, '', '/session.html');
-void (async () => {
- try {
- const expected = started ? JSON.parse(started) : null;
- if (!state || expected?.state !== state || Date.now() - expected.started > 300000) throw new Error('Login state missing or expired. Start login from API debug.');
- if (!code) throw new Error('No login credential. Use the login link.');
- const response = await fetch('/api/v1/auth/ticket/exchange', { method: 'POST', credentials: 'same-origin',
- headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ code }) });
- const body = await response.json();
- if (!response.ok || body.code < 10000 || body.code >= 20000) throw new Error(body.message || 'Login exchange failed');
- location.replace('/debug.html');
- } catch (error) { document.getElementById('session')!.textContent = String(error); }
-})();
-export {};
diff --git a/examples/customer-chat/src/standalone.css b/examples/customer-chat/src/standalone.css
deleted file mode 100644
index f61c6e4d..00000000
--- a/examples/customer-chat/src/standalone.css
+++ /dev/null
@@ -1,6 +0,0 @@
-/* Independent consumer styles; no host selectors or assets. */
-:root { color-scheme: light dark; font-family: system-ui, sans-serif; }
-main { max-width: 840px; margin: 32px auto; padding: 16px; }
-textarea { display: block; width: 100%; min-height: 100px; box-sizing: border-box; }
-button { padding: 8px 16px; margin: 8px 8px 8px 0; }
-[data-role="assistant"] { border-inline-start: 3px solid currentColor; padding-inline-start: 16px; white-space: pre-wrap; }
diff --git a/examples/customer-chat/src/standalone.ts b/examples/customer-chat/src/standalone.ts
deleted file mode 100644
index 346e522c..00000000
--- a/examples/customer-chat/src/standalone.ts
+++ /dev/null
@@ -1,142 +0,0 @@
-import './login.js';
-import { Controls } from './controls.js';
-import { ChatClient, type ChatEvent, type ChatHistoryMessage } from '@hugagent/chat-sdk';
-import './standalone.css';
-
-// Serve this independent application behind the platform's same-origin API gateway.
-const client = new ChatClient({ baseUrl: import.meta.env.VITE_CUSTOMER_API_BASE || '/api' });
-const element = (id: string) => document.getElementById(id) as T;
-const status = element('status');
-element('login').href = import.meta.env.VITE_CUSTOMER_LOGIN_URL || '/login?redirect=/session.html';
-const input = element('message');
-const send = element('send');
-const cancel = element('cancel');
-const resume = element('resume');
-const messages = element('messages');
-const controls = new Controls(client, element('controls'), () => chatId, text => { status.textContent = text; });
-let chatId = new URL(location.href).searchParams.get('chat') || '';
-let runId = '', offset = 0, assistant: HTMLElement | null = null;
-let running = false;
-let attachment: { file_id: string; name: string; mime_type: string; download_url: string } | null = null;
-let controller: AbortController | undefined;
-function row(role: string, content: string) {
- const item = document.createElement('p'); item.dataset.role = role; item.textContent = content;
- messages.append(item); return item;
-}
-function busy(value: boolean) {
- running = value; send.disabled = value; cancel.disabled = !value || !runId;
- element('new').disabled = value;
- element('file').disabled = value;
- resume.disabled = value || !runId;
-}
-function rememberChat(id: string) {
- chatId = id; const url = new URL(location.href); url.searchParams.set('chat', id); history.replaceState(null, '', url);
-}
-async function historyMessages() {
- if (!chatId) return;
- const historyChatId = chatId;
- const history = { items: [] as ChatHistoryMessage[] };
- for (let page = 1; page <= 10000; page++) {
- const batch = await client.messages(historyChatId, page, 100);
- history.items.push(...batch.items);
- if (batch.items.length < 100) break;
- if (page === 10000) throw new Error('History limit reached');
- }
- if (chatId !== historyChatId) return;
- messages.replaceChildren();
- [...history.items].sort((a: ChatHistoryMessage, b: ChatHistoryMessage) => (a.chat_seq || 0) - (b.chat_seq || 0))
- .forEach(item => row(item.role, item.content));
-}
-function consume(event: ChatEvent) {
- controls.consume(event);
- if (event.type === 'run_snapshot' && event.state && typeof event.state === 'object') {
- const snapshot = event.state as Record;
- const blocks = Array.isArray(snapshot.blocks) ? snapshot.blocks as Record[] : [];
- let content = '';
- for (const block of blocks) {
- if (block.kind === 'answer') content = String(block.content || '');
- else if (block.kind === 'text') content += String(block.content || '');
- }
- assistant ??= messages.lastElementChild?.getAttribute('data-role') === 'assistant'
- ? messages.lastElementChild as HTMLElement : row('assistant', '');
- assistant.textContent = content;
- const signals = snapshot.signals && typeof snapshot.signals === 'object' ? Object.values(snapshot.signals) : [];
- for (const signal of signals) controls.consume(signal as ChatEvent);
- }
- if (event.type === 'run_started') { runId = String(event.run_id); cancel.disabled = false; }
- if (typeof event.event_offset === 'number') {
- if (event.event_offset <= offset) return;
- offset = event.event_offset;
- }
- if (event.type === 'content' || event.type === 'ai_message') {
- assistant ??= row('assistant', '');
- assistant.textContent += String(event.delta || event.content || '');
- }
- if (event.type === 'content_replace') {
- assistant ??= row('assistant', ''); assistant.textContent = String(event.content || event.text || '');
- }
- if (event.type === 'error') throw new Error(String(event.message || event.error || 'Run failed'));
-}
-async function follow(events: AsyncIterable) {
- busy(true); status.textContent = 'Working';
- try {
- for await (const event of events) consume(event);
- status.textContent = 'Complete'; runId = ''; await historyMessages();
- } catch (error) {
- if (error instanceof DOMException && error.name === 'AbortError') { status.textContent = 'Cancelled'; runId = ''; }
- else status.textContent = String(error);
- }
- finally { busy(false); }
-}
-element('chat-form').addEventListener('submit', event => {
- event.preventDefault(); if (running) return;
- const text = input.value.trim(); if (!text) return;
- void (async () => {
- busy(true);
- try {
- if (!chatId) rememberChat((await client.createChat()).chat_id);
- input.value = ''; row('user', text); assistant = null; offset = 0; controller = new AbortController();
- const files = attachment ? [attachment] : [];
- attachment = null; element('attachment').textContent = '';
- await follow(client.send({ chat_id: chatId, message: text, ...(files.length ? { attachments: files } : {}) }, controller.signal));
- } catch (error) { status.textContent = String(error); busy(false); }
- })();
-});
-cancel.addEventListener('click', () => {
- void (async () => {
- try { await client.cancel(runId); controller?.abort(); status.textContent = 'Cancelled'; }
- catch (error) { status.textContent = String(error); }
- })();
-});
-resume.addEventListener('click', () => { controller = new AbortController(); void follow(client.resume(runId, offset, controller.signal)); });
-element('new').addEventListener('click', () => {
- if (running) return; rememberChat(''); runId = ''; offset = 0; attachment = null;
- element('attachment').textContent = ''; input.value = '';
- (element('file')).value = '';
- messages.replaceChildren(); controls.clear(); assistant = null; status.textContent = 'Ready'; busy(false);
-});
-element('file').addEventListener('change', event => {
- const file = (event.target as HTMLInputElement).files?.[0]; if (!file || running) return;
- busy(true);
- void (async () => {
- try {
- if (!chatId) rememberChat((await client.createChat()).chat_id);
- attachment = await client.upload(file, chatId); element('attachment').textContent = attachment.name;
- } catch (error) { status.textContent = String(error); }
- finally { busy(false); }
- })();
-});
-void (async () => {
- busy(true);
- try {
- await historyMessages();
- if (!chatId) return;
- await controls.recover();
- const active = await client.activeRun(chatId);
- if (active?.run_id && !['completed', 'failed', 'cancelled'].includes(String(active.status))) {
- runId = String(active.run_id); controller = new AbortController();
- await follow(client.subscribe(runId, controller.signal));
- }
- } catch (error) { status.textContent = String(error); }
- finally { busy(false); }
-})();
diff --git a/examples/customer-chat/src/theme.css b/examples/customer-chat/src/theme.css
deleted file mode 100644
index f708d6b3..00000000
--- a/examples/customer-chat/src/theme.css
+++ /dev/null
@@ -1,19 +0,0 @@
-[data-chat-customization="customer-portal"] .customer-welcome {
- display: flex; flex-direction: column; height: 100%; min-height: 0;
-}
-[data-chat-customization="customer-portal"] .customer-welcome > header {
- display: flex; flex-wrap: wrap; align-items: center; gap: 16px; padding: 16px;
- color: var(--color-text); background: var(--color-bg-container); border-bottom: 1px solid var(--color-border);
-}
-[data-chat-customization="customer-portal"] button {
- color: var(--color-primary); background: var(--color-bg-container); border: 1px solid var(--color-border);
- border-radius: var(--radius-sm); padding: 8px 16px; cursor: pointer;
-}
-[data-chat-customization="customer-portal"] .customer-message { min-width: 0; }
-[data-chat-customization="customer-portal"] .customer-message-label {
- display: block; margin: 8px 20px 0; color: var(--color-text-secondary); font-size: 12px;
-}
-[data-chat-customization="customer-portal"] .customer-composer { width: 100%; min-width: 0; }
-[data-chat-customization="customer-portal"] .customer-composer-status {
- display: block; text-align: right; color: var(--color-text-secondary); font-size: 12px; padding: 0 16px;
-}
diff --git a/examples/customer-chat/tsconfig.json b/examples/customer-chat/tsconfig.json
deleted file mode 100644
index 19a17e5e..00000000
--- a/examples/customer-chat/tsconfig.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "compilerOptions": {
- "target": "ES2022",
- "module": "NodeNext",
- "moduleResolution": "NodeNext",
- "lib": [
- "ES2022",
- "DOM",
- "DOM.Iterable"
- ],
- "strict": true,
- "declaration": true,
- "outDir": "dist",
- "rootDir": "src",
- "jsx": "react-jsx",
- "verbatimModuleSyntax": true,
- "skipLibCheck": true,
- "types": [
- "vite/client"
- ]
- },
- "include": [
- "src"
- ]
-}
diff --git a/examples/customer-chat/vite.config.mjs b/examples/customer-chat/vite.config.mjs
deleted file mode 100644
index 9fbfbe95..00000000
--- a/examples/customer-chat/vite.config.mjs
+++ /dev/null
@@ -1,30 +0,0 @@
-import { defineConfig, loadEnv } from 'vite';
-import { resolve } from 'node:path';
-import { createDeveloperProxy } from './scripts/developer-proxy.mjs';
-
-export default defineConfig(({ mode }) => {
- const env = { ...loadEnv(mode, process.cwd(), 'HUGAGENT_'), ...process.env };
- const host = env.HUGAGENT_DEV_HOST || '127.0.0.1';
- const settings = { host, port: Number(env.HUGAGENT_DEV_PORT || 4310), strictPort: true,
- allowedHosts: ['localhost'], cors: false, fs: { strict: true, allow: [process.cwd()] } };
- const originGuard = { name: 'developer-origin-guard', configureServer: guard, configurePreviewServer: guard };
- function guard(server) {
- server.middlewares.use((req, res, next) => {
- if (!/^\/(api(?:\/|$)|login(?:[?]|$))/.test(req.url || '')) return next();
- const origin = req.headers.origin;
- if (req.headers['sec-fetch-site'] === 'cross-site' || (origin && origin !== 'http://' + req.headers.host)) {
- res.statusCode = 403; res.end('Cross-origin developer API request refused'); return;
- }
- next();
- });
- }
- return {
- plugins: [originGuard],
- server: { ...settings, proxy: createDeveloperProxy(env) },
- preview: { host, port: Number(env.HUGAGENT_DEV_PORT || 4310), strictPort: true, proxy: createDeveloperProxy(env) },
- resolve: { dedupe: ['react', 'react-dom', '@hugagent/chat-sdk'] },
- build: { outDir: 'web-dist', sourcemap: false, rollupOptions: { input: {
- chat: resolve('index.html'), preview: resolve('preview.html'), debug: resolve('debug.html'), session: resolve('session.html'),
- } } },
- };
-});
diff --git a/install.sh b/install.sh
index b1333327..94ebc093 100755
--- a/install.sh
+++ b/install.sh
@@ -339,7 +339,7 @@ pip_install --upgrade \
"pymilvus==2.5.18" \
"milvus-lite==3.1.0"
info "Installing Agent Skills Python dependencies"
-pip_install -r docker/requirements-script-runner.txt
+pip_install -r docker/sandbox/requirements-script-runner.txt
pip_install --no-deps -e .
SKILL_NODE_DIR="${HUGAGENT_DATA_DIR}/node"
diff --git a/packages/cli/DESIGN.md b/packages/cli/DESIGN.md
new file mode 100644
index 00000000..2abda9cf
--- /dev/null
+++ b/packages/cli/DESIGN.md
@@ -0,0 +1,164 @@
+# hugagent 终端客户端设计(对标 Codex CLI / pi)
+
+状态:草案,未实现。已定取舍:TUI 用 TypeScript + `@mariozechner/pi-tui`;local 模式复用现有后端运行时。
+
+## 1. 目标
+
+- 交互体验对标 pi:内联差分渲染(保留终端原生回滚,不占全屏),多行编辑器,流式 Markdown,工具调用一行一条,运行中可插话。
+- 能力对标 Codex CLI:审批与沙箱档位、会话恢复、非交互 `exec`、JSONL 事件输出、MCP、`/review` `/diff`。
+- `/mode` 切换执行位置:
+ - `local`:全部是本机进程,agent 直接操作当前目录。
+ - `cloud`:连接远端 HugAgentOS 服务。
+ - `device:`:远程运维另一台已配对的电脑。
+- 两种模式用同一套界面、同一套事件协议,切换后体验一致。
+
+不做:在 TypeScript 里重写 agent 循环。agent、工具、沙箱、压缩、技能、记忆全部由后端负责。
+
+## 2. 架构
+
+```
+┌──────────────── hugagent(Node,packages/cli)───────────────┐
+│ 入口:interactive │ -p/exec │ --output json │ --output rpc │
+│ TUI:pi-tui(Editor / Markdown / SelectList / Loader) │
+│ Engine 接口 ── send / subscribe / steer / cancel / 审批 / 会话 │
+│ ├─ LocalEngine → http://127.0.0.1:<随机端口>(本机子进程) │
+│ └─ RemoteEngine → https://(cloud / device:) │
+└──────────────────────────────┬───────────────────────────────┘
+ │ 同一套 HTTP + SSE(@hugagent/chat-sdk)
+┌──────────────────────────────┴───────────────────────────────┐
+│ Python 后端(现有):agent 运行时 · 工具 · OS 沙箱 · 压缩 · 技能 · MCP │
+│ local:hugagent-server engine(SQLite,~/.hugagent,绑定 loopback)│
+└──────────────────────────────────────────────────────────────┘
+```
+
+### 为什么本机引擎走 loopback HTTP,而不是 stdio RPC
+
+调整说明:讨论取舍时写的是 stdio JSON-RPC,这里改成 loopback HTTP。理由如下:
+
+- 后端已有完整的 SSE 事件协议,包括 `run_snapshot` 断线续播、`run.steer` 插话,以及 `tool-approval`、`file-confirm`、`user-questions` 带外审批。`@hugagent/chat-sdk` 已经封装了 `send / resume / subscribe`。走 HTTP,local 和 cloud 之间只差 base URL 和凭据。
+- 如果改成 stdio RPC,就要为同一套能力再定义第二种协议,两边会慢慢分叉。
+- 桌面端已经在 32101 端口跑着本机后端。CLI 检测到它(并且数据目录相同)时可以直接接入,和桌面端共享会话、不再重复启动。
+- pi 风格的 `--output rpc` 仍然保留,但它是 CLI **对外**提供的嵌入协议,不是 CLI 和后端之间的协议。
+
+### LocalEngine 生命周期
+
+1. 先找可复用的后端:读 `~/.hugagent/engine.json`(端口、令牌、PID),健康检查通过就直接接入。
+2. 找不到就启动 `hugagent-server engine --port 0 --token-file …`:
+ - 无头运行,不开浏览器、不托管前端静态文件;
+ - 只绑定 127.0.0.1;
+ - 就绪后把端口写进就绪文件。
+3. 鉴权:一次性随机令牌,复用桌面桥的机制(`HUGAGENT_DESKTOP_BRIDGE_SECRET` 这一类);不需要本机账号密码。
+4. 引擎空闲 N 分钟后自动退出;CLI 退出不强制关闭它,下次启动就是热启动。
+5. 当前目录映射:按 realpath 查找或创建 `kind=local` 的项目,挂载到 `/workspace/local/`(`core/sandbox/workspace/mounts.py`)。
+ - 新会话都建在这个项目里,根目录的 `AGENTS.md` 由现有逻辑读取(`core/projects/instructions.py`)。
+ - 命令执行用现有 OS 沙箱:Seatbelt / bwrap / Windows 低权限令牌。
+6. 首次使用时没有模型配置:TUI 内引导完成(复用 `cli.py` 的 `configure_model` 逻辑),不跳到网页。
+
+### RemoteEngine
+
+- `cloud`:用 `/login` 获取个人 API Key 并存进系统钥匙串,回退到 `~/.hugagent/cli.json`(权限 0600)。默认要求 HTTPS,只有 loopback 和显式允许的内网地址例外。
+- 云端 agent **碰不到本机当前目录**。状态栏必须常驻显示 `cloud · 无本地文件访问`,`@文件` 只能以附件上传的方式提供。
+- `device:`:沿用现有 session-manager 操作队列(`hugagent_cli/contract.py`)。这台设备上的对话由那台机器执行,TUI 只能轮询结果,不能实时流式显示(设备链路目前没有 SSE)。
+
+## 3. `/mode` 语义
+
+| 操作 | 行为 |
+|---|---|
+| `/mode` | 弹出选择列表:local / cloud / 在线设备,并标出当前项 |
+| `/mode local\|cloud\|device:` | 直接切换。当前会话留在原来那一侧,目标侧开新会话;`/resume` 只列目标侧、当前项目的会话 |
+| `--mode` 启动参数 / 配置文件 `defaultMode` | 启动时的默认值;不指定时,如果当前目录是 git 仓库就用 local,否则用上次的选择 |
+| 状态栏 | `local · ~/repo · deepseek-chat · 审批:按需 · ctx 42%` |
+
+- 第一版不支持把会话从一侧搬到另一侧。以后可以加 `/handoff`:把摘要和关键文件带到另一侧,开一个新会话。
+- 命名提醒:pi 的 `--mode` 表示输出格式。我们的 `--mode` 表示执行位置,输出格式统一改用 `--output text|json|rpc`。
+
+## 4. 功能对标矩阵
+
+Codex / pi 两列按公开版本归纳,细节以官方文档为准。"后端"列表示后端现在是否已经具备这项能力。
+
+| 能力 | Codex | pi | 我们的做法 | 后端 | 阶段 |
+|---|---|---|---|---|---|
+| 内联差分渲染、原生回滚 | 全屏 TUI | ✓ | pi-tui | — | P1 |
+| 多行编辑器、历史、快捷键 | ✓ | ✓ | pi-tui Editor | — | P1 |
+| 流式 Markdown、思考折叠 | ✓ | ✓ | 渲染 `content` / `thinking` 事件 | ✓ | P1 |
+| 工具调用一行显示、可展开 | ✓ | ✓ | `tool_call_*` / `tool_result` / `subagent_event` | ✓ | P1 |
+| Esc 中断 | ✓ | ✓ | `run.cancel` | ✓ | P1 |
+| 运行中插话 / 排到下一条 | 排队 | Enter 插话 / Alt+Enter 排队 | `run.steer`,`delivery_mode=steer\|followUp\|nextRun` | ✓ | P1 |
+| 工具与命令审批 | approval policy | 默认不拦 | 终端内回应 `tool-approval` / `file-confirm` | ✓ | P1 |
+| 向用户提问 | — | — | `user_question` 内联选择 | ✓ | P1 |
+| 沙箱档位 `/permissions` | read-only / workspace-write / full | — | 映射现有权限预设 + OS 沙箱 | ✓ | P2 |
+| `/model`、思考档位 | ✓ | ✓(Ctrl+P / Shift+Tab) | `ChatRequest.model_name / model_provider_id`,思考档位走偏好设置 | ✓ | P1 |
+| 会话恢复 `-c` / `-r` | `resume` | ✓ | 服务端对话按项目(当前目录)过滤 | ✓ | P1 |
+| 分支 `/fork`、`/tree`、回到旧消息重写 | Esc Esc 回退 | ✓ | `POST /chats/{id}/fork` | ✓ | P2 |
+| 手动 `/compact` | ✓ | ✓ | 后端只有自动压缩,**需新增接口** | 部分 | P2 |
+| 上下文用量显示 | ✓ | ✓ | `context_usage` 事件 | ✓ | P1 |
+| AGENTS.md | 逐级向上 + 全局 | 逐级向上 + 全局 | 只读项目根目录,**需补逐级合并和全局文件** | 部分 | P2 |
+| `@文件` 模糊引用 | ✓ | ✓ | local:路径交给 agent;cloud:作为附件上传 | 部分 | P2 |
+| `!命令` 直接执行 | ✓ | ✓ | local 在本机沙箱执行,把输出注入上下文 | — | P2 |
+| 粘贴图片 | ✓ | ✓ | 附件 + 视觉桥 | ✓ | P2 |
+| `/diff`、`/review` | ✓ | 扩展实现 | local 由客户端读 git;review 作为一种对话模式 | — | P2 |
+| 非交互 `exec` / `-p` | ✓ | ✓ | `hugagent -p "…"`,退出码反映运行结果 | ✓ | P0 |
+| JSONL 事件流 | `--json` | `--mode json` | `--output json`,原样输出 SSE 事件 | ✓ | P0 |
+| 嵌入协议 | app-server | `--mode rpc` | `--output rpc`(stdin/stdout JSON) | — | P2 |
+| MCP | ✓ | 不内置 | 后端插件市场;提供 `hugagent mcp list/add` | ✓ | P3 |
+| 技能 / 提示模板 / 扩展 | ✓ | ✓ | 后端技能 + 插件;TUI 侧扩展暂不做 | ✓ | P3 |
+| 登录 | ChatGPT / API Key | 订阅 / API Key | `/login`(cloud);local 的模型凭据存在本机库 | 部分 | P1 |
+| 主题、按键绑定 | 部分 | ✓ | `~/.hugagent/cli.json` | — | P3 |
+| 导出 / 分享 | — | HTML / gist | `/export` Markdown 或 HTML | — | P3 |
+| 远程运维 | — | — | `/mode device:`、`/sessions`、`/runs`、`/devices` | ✓ | P3 |
+| 子 agent、计划、自主循环 | 部分 | 扩展 | 渲染 `subagent_event` / `plan_update`(我们的优势) | ✓ | P2 |
+| 分发、升级 | npm / brew | npm | npm 全局安装;local 引擎复用桌面端自带的 Python 运行时,或 `pip install hugagent` | — | P3 |
+
+## 5. 后端需要补的东西
+
+1. `hugagent-server engine` 子命令:无头运行、`--port 0`、就绪文件、loopback 令牌鉴权、空闲退出、父进程监护。
+2. 按路径查找或创建本地项目的接口(目前只有桌面端授权文件夹的流程)。
+3. 手动压缩接口,对应 `/compact`。
+4. `ChatRequest.message` 的长度上限 10000 字符,对粘贴代码太小。附件可以绕开,但 `-p` 和管道输入需要更大的上限,或者自动转成附件。
+5. AGENTS.md 逐级向上合并 + 全局文件 `~/.hugagent/AGENTS.md`。
+6. 统一命名:
+ - Python 的 console script 改名 `hugagent-server`(负责 onboard / serve / doctor / engine);
+ - npm 包占用 `hugagent`;`hugagent serve` 等命令转发给 `hugagent-server`;
+ - 云电脑镜像里的零依赖会话客户端 `hugagent_cli` 保持不变,因为 Personal Agent 依赖它。
+ - 上次审查发现的入口路由、错误详情丢失、空响应、操作表清理等问题,在这一步一起修。
+
+## 6. 包结构
+
+```
+packages/cli/
+ src/
+ main.ts 参数解析,分发到各模式
+ config.ts ~/.hugagent/cli.json、钥匙串
+ engine/
+ engine.ts Engine 接口 + 统一事件类型(复用 chat-sdk 类型)
+ local.ts 发现 / 启动 / 接入本机引擎,当前目录 → 本地项目
+ remote.ts cloud 与 device:
+ modes/
+ interactive.ts TUI 主循环
+ print.ts -p / exec
+ json.ts --output json
+ rpc.ts --output rpc
+ tui/
+ transcript.ts 事件 → 组件(正文 / 思考 / 工具行 / 计划 / 子 agent)
+ footer.ts 状态栏
+ prompts/ 审批、提问、选择列表
+ commands/ /mode /model /resume /new /compact /permissions /fork /diff …
+ test/
+```
+
+## 7. 分期与验收
+
+- **P0 骨架**:包结构、两种 Engine、`-p` 和 `--output json`。
+ - 验收:在任意仓库目录执行 `hugagent -p "列出这个目录的文件"`,local 模式由本机引擎在当前目录完成;`--mode cloud` 走远端服务;两边的 JSONL 事件一致。
+- **P1 交互 TUI**:编辑器、流式渲染、工具行、状态栏、Esc 中断、插话 / 排队、审批与提问内联处理、`/mode` `/model` `/new` `/resume` `/login` `/quit`。
+ - 验收:一次完整的"改代码 → 审批 → 跑测试"闭环不离开终端。
+- **P2 对标补齐**:`@文件`、`!命令`、图片、`/compact`、`/permissions`、`/fork` `/tree`、`/diff` `/review`、AGENTS.md 层级、`--output rpc`。
+- **P3 生态与运维**:`device:`、`/sessions` `/runs` `/devices`、MCP / 技能命令、主题按键、导出、npm 发布与升级。
+
+## 8. 风险
+
+- **Python 引擎冷启动慢**:完整 lifespan 要数秒。先实测;用"接入已运行的后端 + 空闲保活"缓解;engine 子命令只启动 agent 运行需要的步骤。
+- **两套运行时**:只用 cloud 的用户只需要 Node;local 需要 Python 运行时,优先复用桌面端已安装的那份。
+- **pi-tui 还是 0.x**:锁定精确版本,用自己的组件层隔离它的 API。
+- **终端兼容**:中文输入法和宽字符(pi-tui 依赖 get-east-asian-width),Windows Terminal,SSH 下的剪贴板。
+- **cloud 不能访问本机目录**:必须在界面上一直说清楚,否则用户会以为云端改了本地代码。
diff --git a/pyproject.toml b/pyproject.toml
index c774c665..fad4f86f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ dev = [
"black>=23.0.0",
"isort>=5.12.0",
"mypy>=1.0.0",
+ "import-linter>=2.15,<3",
]
# Console entry for the no-Docker local profile: `hugagent` (onboard / serve /
@@ -47,7 +48,7 @@ dependencies = {file = ["requirements.txt"]}
where = ["src/backend"]
include = [
"api*", "core*", "orchestration*", "prompts*", "extensions*",
- "hugagent_cli*", "edition_ee*", "personal_agent*",
+ "hugagent_cli*", "edition_ee*", "personal_agent*", "features*", "integrations*",
]
# alembic migrations / tests / uploads / scripts are not import-time runtime deps.
exclude = ["tests*", "alembic*", "uploads*", "scripts*"]
@@ -55,7 +56,7 @@ exclude = ["tests*", "alembic*", "uploads*", "scripts*"]
# Bundle manifests, prompts, scripts and binary assets are runtime resources.
[tool.setuptools.package-data]
extensions = ["README.md", "skills/**/*", "agents/**/*", "plugins/**/*", "mcp_servers/**/*.md"]
-"core.applications" = ["schemas/*.json"]
+"features.workbench" = ["schemas/*.json"]
"core.config" = ["*.json"]
prompts = ["config/*.json", "prompt_text/**/*.md"]
"personal_agent.workspace" = ["prompts/*.md"]
@@ -111,3 +112,12 @@ markers = [
# GCE 门禁类测试:跑得起但不属于快速内环,CI 全量跑。
"slow: gate / benchmark tests excluded from the fast inner loop",
]
+
+# Core boundary checks use the migration manifest to classify legacy locations.
+[tool.importlinter]
+root_packages = ["core", "api", "orchestration", "personal_agent", "edition_ee", "features", "integrations"]
+contract_types = ["core-boundary: scripts.refactor.import_contract.CoreBoundaryContract"]
+
+[[tool.importlinter.contracts]]
+name = "Core boundaries and layer ratchet"
+type = "core-boundary"
diff --git a/requirements.txt b/requirements.txt
index f38daea0..0eaea501 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
# Core Web Framework
fastapi==0.115.12
uvicorn>=0.27.0
-pydantic>=2.6.0
+pydantic>=2.12.0
# Async & Utilities
-anyio>=4.2.0
+anyio>=4.9.0
python-dotenv>=1.0.0
requests>=2.31.0
httpx>=0.27.0
@@ -13,11 +13,11 @@ httpx>=0.27.0
jsonschema>=4.20.0
# AgentScope (replaces LangChain ecosystem)
-# 已迁移到 2.0:Agent/Toolkit/Middleware/MCP 全面重写,固定 ==2.0.0(GitHub main 已与 2.0.0 分叉,不兼容)
-agentscope==2.0.0
+# Pin the reviewed MCP 2.x implementation for reproducible builds.
+agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz
# 多厂商模型渠道所需 SDK(均为惰性 import,未装只影响对应渠道,不影响其余功能):
-# - anthropic / dashscope / openai 已随 agentscope==2.0.0 内置,无需在此重复声明
+# - anthropic / dashscope / openai 已由 AgentScope 声明依赖,无需在此重复声明
# - 以下为 agentscope 未捆绑、需显式安装的渠道 SDK:
# litellm —— 非 OpenAI 协议且 AgentScope 无原生类的厂商(如 AWS Bedrock)
litellm>=1.50.0
@@ -27,7 +27,9 @@ google-genai>=0.3.0
ollama>=0.3.0
# MCP Integration
-mcp>=1.0.0,<2.0.0
+mcp==2.3.0
+# Keep the SSE dependency within FastAPI 0.115 Starlette constraints.
+sse-starlette==3.0.3
# Additional Tools
tavily-python>=0.5.0
@@ -67,9 +69,8 @@ botocore>=1.34.0
werkzeug>=3.0.0
python-multipart>=0.0.9
-# Logging & Rate Limiting
+# Logging
structlog>=23.2.0
-slowapi>=0.1.9
# Security
safety>=3.0.0
diff --git a/scripts/performance/benchmark_preview.py b/scripts/performance/benchmark_preview.py
index f319cc0d..80e51c3a 100644
--- a/scripts/performance/benchmark_preview.py
+++ b/scripts/performance/benchmark_preview.py
@@ -8,7 +8,7 @@
import time
from docx import Document
from api.routes.files import _convert_office_to_pdf
-from core.content.office_preview_cache import cached_office_pdf
+from core.files.content.office_preview_cache import cached_office_pdf
with tempfile.TemporaryDirectory(prefix="perf172-office-") as directory:
root = Path(directory)
diff --git a/scripts/performance/serve_fixture.py b/scripts/performance/serve_fixture.py
index a6c20c0c..5a017e70 100644
--- a/scripts/performance/serve_fixture.py
+++ b/scripts/performance/serve_fixture.py
@@ -11,7 +11,7 @@
import time
from types import SimpleNamespace
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
import uvicorn
from api.routes.v1 import chats
from core.auth.backend import get_current_user
@@ -171,7 +171,7 @@ async def chunks():
# Isolate the slow database boundary for an HTTP concurrency experiment. The
# real get_pending_user_questions route/worker scheduling remains unchanged.
-OriginalService = chat_services.ChatService
+OriginalService = chat_services_ChatService_module.ChatService
class MeasuredService(OriginalService):
@@ -182,7 +182,7 @@ def get_session(self, chat_id, user_id):
return super().get_session(chat_id, user_id)
-chat_services.ChatService = MeasuredService
+chat_services_ChatService_module.ChatService = MeasuredService
async def no_questions(chat_id):
diff --git a/scripts/refactor/__init__.py b/scripts/refactor/__init__.py
new file mode 100644
index 00000000..c782d1ba
--- /dev/null
+++ b/scripts/refactor/__init__.py
@@ -0,0 +1 @@
+"""Core restructuring verification tools."""
diff --git a/scripts/refactor/check_ce_paths.py b/scripts/refactor/check_ce_paths.py
new file mode 100644
index 00000000..4a414b0d
--- /dev/null
+++ b/scripts/refactor/check_ce_paths.py
@@ -0,0 +1,79 @@
+"""Validate source paths in the CE derivation manifest."""
+from __future__ import annotations
+
+import ast
+import fnmatch
+import subprocess
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def path_errors(root: Path, manifest: dict, tracked: list[str]) -> list[str]:
+ errors = []
+ optional = set(manifest.get("optional_safety_excludes", []))
+ excludes = manifest.get("exclude", [])
+ for pattern in optional - set(excludes):
+ errors.append(f"Safety exception has no exclusion: {pattern}")
+ for pattern in excludes:
+ if pattern not in optional and not any(fnmatch.fnmatch(p, pattern) and (root / p).is_file() for p in tracked):
+ errors.append(f"exclude: no source file matches {pattern}")
+ overlay = root / manifest["overlay"]
+ for path in manifest.get("split", []):
+ if not (overlay / path).is_file():
+ errors.append(f"split: missing overlay {path}")
+ # Prunes run before overlay. Check only declared input paths, not output
+ # names, replacement text, dictionary values, or forbidden module names.
+ for name, rule in manifest.get("prunes", {}).items():
+ inputs = []
+ if "path" in rule:
+ inputs.append(rule["path"])
+ inputs.extend(rule.get("paths", []))
+ inputs.extend(rule.get("dockerfiles", []))
+ inputs.extend(rule.get("text_paths", []))
+ inputs.extend(item["path"] for item in rule.get("files", []))
+ for path in inputs:
+ if not (root / path).exists():
+ errors.append(f"prunes.{name}: missing source {path}")
+ generated = {
+ rule["optional_out"] for rule in manifest.get("prunes", {}).values()
+ if "optional_out" in rule
+ }
+ generated.update(pair[1] for pair in manifest.get("renames", []))
+ for path in manifest.get("required_files", []):
+ if path not in generated and not (root / path).is_file() and not (overlay / path).is_file():
+ errors.append(f"required_files: no source or overlay {path}")
+ for source, _target in manifest.get("renames", []):
+ if not (root / source).exists():
+ errors.append(f"renames: missing source {source}")
+ # CE overlays must not restore old aggregate exports after ownership moves.
+ for relative in ("src/backend/core/db/models/__init__.py", "src/backend/core/db/repository/__init__.py"):
+ path = overlay / relative
+ if not path.is_file():
+ continue
+ for node in ast.walk(ast.parse(path.read_text())):
+ modules = ([node.module or ""] if isinstance(node, ast.ImportFrom)
+ else [alias.name for alias in node.names] if isinstance(node, ast.Import) else [])
+ for module in modules:
+ if module.split(".")[0] in {"features", "integrations", "personal_agent"}:
+ errors.append(f"overlay aggregate forwards moved ownership: {relative} -> {module}")
+ return sorted(errors)
+
+
+def main():
+ manifest = yaml.safe_load((ROOT / "ce/manifest.yaml").read_text())
+ tracked = subprocess.check_output(
+ ["git", "ls-files", "-z"], cwd=ROOT, text=True
+ ).split("\0")
+ errors = path_errors(ROOT, manifest, tracked)
+ for error in errors:
+ print(error)
+ if not errors:
+ print("CE manifest paths: passed")
+ return bool(errors)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/refactor/check_line_budget.py b/scripts/refactor/check_line_budget.py
new file mode 100644
index 00000000..2b78b2c0
--- /dev/null
+++ b/scripts/refactor/check_line_budget.py
@@ -0,0 +1,37 @@
+"""Reject new oversized Python modules and growth of existing exceptions."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+BASELINE = ROOT / "internal design docs"
+
+
+def violations(root: Path, baseline: dict[str, int]) -> list[str]:
+ errors = []
+ for directory in ("src/backend", "ce/overlay/src/backend", "scripts/refactor"):
+ for path in sorted((root / directory).rglob("*.py")):
+ relative = path.relative_to(root).as_posix()
+ count = len(path.read_text(encoding="utf-8-sig").splitlines())
+ limit = max(500, baseline.get(relative, 500))
+ if count > limit:
+ errors.append(f"{relative}: {count} lines exceeds {limit}")
+ return errors
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--baseline", type=Path, default=BASELINE)
+ args = parser.parse_args()
+ errors = violations(ROOT, json.loads(args.baseline.read_text()))
+ for error in errors:
+ print(error)
+ if not errors:
+ print("Line budget: passed (existing exceptions may only shrink)")
+ return bool(errors)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/refactor/check_test_baseline.py b/scripts/refactor/check_test_baseline.py
new file mode 100644
index 00000000..84c01661
--- /dev/null
+++ b/scripts/refactor/check_test_baseline.py
@@ -0,0 +1,75 @@
+"""Compare a completed pytest run against the original failure set."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import re
+
+
+def result(log: str) -> dict:
+ summaries = re.findall(r"^=+ (.*\bin [0-9.]+s(?: \([^\n]*\))?) =+$", log, re.MULTILINE)
+ if not summaries or not re.search(r"\d+ (?:passed|failed|error|skipped)", summaries[-1]):
+ raise ValueError("pytest has not produced a complete final summary")
+ if "Interrupted:" in log or "INTERNALERROR>" in log:
+ raise ValueError("interrupted or internally failed pytest run is not a baseline")
+ failures = []
+ # Application logs can also start with ERROR; only parse pytest's final list.
+ failure_section = log.rsplit("short test summary info", 1)[-1]
+ for line in failure_section.splitlines():
+ if line.startswith(("FAILED ", "ERROR ")):
+ failures.append(line.split(" ", 1)[1].split(" - ", 1)[0])
+ reported = re.search(r"(\d+) failed", summaries[-1])
+ errors = re.search(r"(\d+) errors?", summaries[-1])
+ expected = sum(int(match[1]) for match in (reported, errors) if match)
+ if len(failures) != expected:
+ raise ValueError("failure list is incomplete; keep the full pytest report")
+ return {"summary": summaries[-1], "failures": sorted(set(failures))}
+
+
+def canonical_failures(failures: list[str], moves: dict[str, str]) -> set[str]:
+ """Compare original node IDs after explicit test-file moves, keeping parameters intact."""
+ destinations = set()
+ for old, new in moves.items():
+ for path in (old, new):
+ if not isinstance(path, str) or not path.startswith("src/backend/tests/") or not path.endswith(".py") or ".." in Path(path).parts or "\\" in path:
+ raise ValueError(f"Invalid test migration path: {path!r}")
+ if old == new or new in destinations or new in moves:
+ raise ValueError(f"Ambiguous test migration: {old} -> {new}")
+ destinations.add(new)
+ reverse = {new: old for old, new in moves.items()}
+ normalized = set()
+ for failure in failures:
+ path, separator, test = failure.partition("::")
+ normalized.add(reverse.get(path, path) + separator + test)
+ return normalized
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("log", type=Path)
+ mode = parser.add_mutually_exclusive_group(required=True)
+ mode.add_argument("--save", type=Path)
+ mode.add_argument("--baseline", type=Path)
+ parser.add_argument("--test-moves", type=Path, help="Reviewed original-to-current test file mapping")
+ args = parser.parse_args()
+ if args.save and args.test_moves:
+ parser.error("save the original baseline without migration normalization")
+ actual = result(args.log.read_text())
+ if args.save:
+ if args.save.exists():
+ parser.error("baseline already exists; preserve the original evidence")
+ args.save.write_text(json.dumps(actual, ensure_ascii=False, indent=2) + "\n")
+ else:
+ expected = json.loads(args.baseline.read_text())
+ moves = json.loads(args.test_moves.read_text()) if args.test_moves else {}
+ introduced = sorted(canonical_failures(actual["failures"], moves) - set(expected["failures"]))
+ if introduced:
+ print("New test failures:\n" + "\n".join(introduced))
+ return 1
+ print(actual["summary"])
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/refactor/import_contract.py b/scripts/refactor/import_contract.py
new file mode 100644
index 00000000..abf1c1ca
--- /dev/null
+++ b/scripts/refactor/import_contract.py
@@ -0,0 +1,209 @@
+"""Import-linter contract for the core migration's dependency ratchet."""
+from __future__ import annotations
+
+import ast
+import json
+from pathlib import Path
+
+from importlinter.application import output
+from importlinter.domain.contract import Contract, ContractCheck
+
+ROOT = Path(__file__).resolve().parents[2]
+DATA = ROOT / "internal design docs"
+LAYERS = (
+ ("core.agent_runtime",),
+ ("core.runs", "core.jobs", "core.chat", "core.sandbox.commands"),
+ ("core.llm",),
+ ("core.evolution", "core.ontology"),
+ ("core.memory", "core.sandbox", "core.capabilities", "core.files",
+ "core.projects", "core.auth"),
+ ("core.models",),
+ ("core.db",),
+ ("core.config", "core.infra"),
+)
+FEATURE_EDGES = {
+ ("desktop", "marketplace"), ("desktop", "knowledge"),
+ ("sites", "desktop"), ("workbench", "desktop"), ("hosted_apps", "desktop"),
+ ("workbench", "sites"), ("hosted_apps", "sites"),
+ ("automation", "desktop"), # Authorized local task dispatch; see module README.
+ ("distillation", "marketplace"), # Explicit draft publication; see module README.
+ ("security_console", "sandbox_ops"), # Read-only rebuild status; see module README.
+}
+OUTER = ("features", "integrations", "personal_agent", "orchestration", "api")
+
+
+def module_name(path):
+ name = path.removesuffix(".py").replace("/", ".")
+ return name.removesuffix(".__init__")
+
+
+def belongs(name, prefix):
+ return name == prefix or name.startswith(prefix + ".")
+
+
+def canonical_names():
+ # Apply the accepted manifest virtually before moves. Thus the baseline
+ # already exposes dependency debt concealed by the old services directory.
+ result = {}
+ rows = json.loads((DATA / "manifest.json").read_text())
+ supplemental = DATA / "supplemental_moves.json"
+ if supplemental.exists():
+ rows.extend(json.loads(supplemental.read_text()))
+ for row in rows:
+ if row["action"] == "move":
+ result[module_name(row["src"])] = module_name(row["dst"])
+ routes = DATA / "routes.json"
+ if routes.exists():
+ for row in json.loads(routes.read_text())["moves"]:
+ result[module_name(row["src"])] = module_name(row["dst"])
+ # A completed move may reuse an earlier source name (D11: capabilities).
+ # Live destination modules represent their new owner, never the historical
+ # alias. Preserve virtual classification only for paths not yet migrated.
+ destinations = {module_name(row["dst"]) for row in rows if row["action"] == "move"}
+ for name in destinations:
+ module_path = ROOT / "src/backend" / Path(*name.split("."))
+ if module_path.with_suffix(".py").is_file() or (module_path / "__init__.py").is_file():
+ result.pop(name, None)
+ test_moves = DATA / "test_moves.json"
+ if test_moves.exists():
+ from scripts.refactor.check_test_baseline import canonical_failures
+ moves = json.loads(test_moves.read_text())
+ canonical_failures([], moves) # Validate one-to-one test-file migrations.
+ for old, new in moves.items():
+ result[module_name(new.removeprefix("src/backend/"))] = module_name(old.removeprefix("src/backend/"))
+ return result
+
+
+def layer(name):
+ candidates = [
+ (len(prefix), index)
+ for index, prefixes in enumerate(LAYERS)
+ for prefix in prefixes if belongs(name, prefix)
+ ]
+ return max(candidates)[1] if candidates else None
+
+
+def edge_rule(source, target):
+ if belongs(source, "core") and any(belongs(target, p) for p in OUTER):
+ return "core-outward"
+ if source.startswith("features.") and target.startswith("features."):
+ a, b = source.split(".")[1], target.split(".")[1]
+ if a != b and (a, b) not in FEATURE_EDGES:
+ return "feature-direction"
+ a, b = layer(source), layer(target)
+ if a is not None and b is not None and a > b:
+ return "core-layers"
+ return None
+
+
+def canonical(name, aliases):
+ candidates = [key for key in aliases if belongs(name, key)]
+ if not candidates:
+ return name
+ prefix = max(candidates, key=len)
+ # A destination nested under its former module is already canonical.
+ if any(belongs(name, target) and len(target) >= len(prefix) for target in aliases.values()):
+ return name
+ return aliases[prefix] + name[len(prefix):]
+
+
+def private_imports_in_module(relative, text, aliases):
+ physical_source = module_name(relative)
+ source = canonical(physical_source, aliases)
+ if belongs(source, "core"):
+ return set()
+ package = physical_source if relative.endswith("/__init__.py") else physical_source.rpartition(".")[0]
+ result, bindings = set(), {}
+ tree = ast.parse(text)
+
+ def bind(name, target):
+ # Keep every candidate across scopes: a later local alias must never
+ # erase an outer import and hide a boundary violation.
+ bindings.setdefault(name, set()).add(target)
+
+ def record(target, symbol):
+ target = canonical(target, aliases)
+ if belongs(target, "core") and symbol.startswith("_"):
+ result.add(f"private-symbol: {source} -> {target}.{symbol}")
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ bind(alias.asname or alias.name.split(".")[0], alias.name if alias.asname else alias.name.split(".")[0])
+ if not isinstance(node, ast.ImportFrom):
+ continue
+ target = node.module or ""
+ if node.level:
+ parts = package.split(".")
+ target = ".".join(parts[:len(parts) - node.level + 1] + ([target] if target else []))
+ for alias in node.names:
+ record(target, alias.name)
+ bind(alias.asname or alias.name, target + "." + alias.name)
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Attribute) or not node.attr.startswith("_"):
+ continue
+ parts, value = [], node.value
+ while isinstance(value, ast.Attribute):
+ parts.insert(0, value.attr)
+ value = value.value
+ if isinstance(value, ast.Name) and value.id in bindings:
+ for target in bindings[value.id]:
+ record(".".join([target, *parts]), node.attr)
+ return result
+
+
+def private_imports():
+ result = set()
+ backend = ROOT / "src/backend"
+ aliases = canonical_names()
+ for path in backend.rglob("*.py"):
+ relative = path.relative_to(backend).as_posix()
+ result.update(private_imports_in_module(relative, path.read_text(encoding="utf-8-sig"), aliases))
+ return result
+
+
+def package_marker_errors(backend):
+ """Keep namespace directories from silently disappearing from grimp's graph."""
+ missing = set()
+ for root_name in ("core", "features", "integrations"):
+ root = backend / root_name
+ for path in root.rglob("*.py"):
+ if "__pycache__" in path.parts:
+ continue
+ parent = path.parent
+ while parent != backend:
+ if not (parent / "__init__.py").is_file():
+ missing.add("package-marker: " + parent.relative_to(backend).as_posix())
+ parent = parent.parent
+ return missing
+
+
+def violations(graph):
+ aliases = canonical_names()
+ result = private_imports() | package_marker_errors(ROOT / "src/backend")
+ for source in sorted(graph.modules):
+ for target in graph.find_modules_directly_imported_by(source):
+ rule = edge_rule(aliases.get(source, source), aliases.get(target, target))
+ if rule:
+ result.add(f"{rule}: {aliases.get(source, source)} -> {aliases.get(target, target)}")
+ return result
+
+
+class CoreBoundaryContract(Contract):
+ """Require existing architecture violations to decrease, never increase."""
+
+ def check(self, graph, verbose):
+ current = violations(graph)
+ baseline = set(json.loads((DATA / "import-baseline.json").read_text()))
+ introduced = sorted(current - baseline)
+ removed = sorted(baseline - current)
+ return ContractCheck(
+ kept=not introduced,
+ metadata={"introduced": introduced},
+ ignored_import_count=len(current & baseline),
+ warnings=[f"Remove {len(removed)} resolved baseline entries"] if removed else [],
+ )
+
+ def render_broken_contract(self, check):
+ for violation in check.metadata["introduced"]:
+ output.print_error(violation)
diff --git a/scripts/refactor/rewrite_imports.py b/scripts/refactor/rewrite_imports.py
new file mode 100644
index 00000000..59ffead9
--- /dev/null
+++ b/scripts/refactor/rewrite_imports.py
@@ -0,0 +1,357 @@
+"""AST-based import rewrite planner. Preview by default; --apply writes changes."""
+from __future__ import annotations
+
+import argparse
+import ast
+import importlib.util
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+DATA = ROOT / "internal design docs"
+
+
+def module_name(path: str) -> str:
+ return path.removesuffix(".py").replace("/", ".").removesuffix(".__init__")
+
+
+def mapped(value: str, moves: dict[str, str]) -> str:
+ for old in sorted(moves, key=len, reverse=True):
+ if value == old or value.startswith(old + "."):
+ return moves[old] + value[len(old):]
+ return value
+
+
+def dotted(node):
+ if isinstance(node, ast.Name):
+ return node.id
+ if isinstance(node, ast.Attribute):
+ prefix = dotted(node.value)
+ if prefix:
+ return prefix + "." + node.attr
+ return None
+
+
+def rewrite(source, old_module, moves, symbols=None, *, package=False):
+ """Return source and unresolved string references; do not execute imports."""
+ symbols = symbols or {}
+ tree = ast.parse(source)
+ # AST offsets are UTF-8 bytes, not character positions.
+ lines = source.encode("utf-8").splitlines(keepends=True)
+ offsets = [0]
+ for line in lines:
+ offsets.append(offsets[-1] + len(line))
+ patches = []
+ unresolved = []
+ parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
+
+ imported_names = {}
+ shadowed_names = set()
+ wildcard_import = False
+
+ def record_import(binding, target):
+ previous = imported_names.get(binding)
+ if previous is not None and previous != target:
+ shadowed_names.add(binding)
+ imported_names[binding] = target
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ record_import(
+ alias.asname or alias.name.split(".")[0],
+ alias.name if alias.asname else alias.name.split(".")[0],
+ )
+ elif isinstance(node, ast.ImportFrom):
+ for alias in node.names:
+ if alias.name == "*":
+ wildcard_import = True
+ record_import(alias.asname or alias.name, "." * node.level + (node.module or "") + "." + alias.name)
+ elif isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
+ shadowed_names.add(node.id)
+ elif isinstance(node, ast.arg):
+ shadowed_names.add(node.arg)
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ shadowed_names.add(node.name)
+ elif isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)):
+ shadowed_names.add(node.name)
+ elif isinstance(node, ast.MatchMapping):
+ shadowed_names.add(node.rest)
+
+ def patch_target(node):
+ parent = parents.get(node)
+ call = parents.get(parent) if isinstance(parent, ast.keyword) else parent
+ if not isinstance(call, ast.Call):
+ return False
+ first_argument = bool(call.args) and call.args[0] is node
+ target_keyword = isinstance(parent, ast.keyword) and parent.arg == "target"
+ if not (first_argument or target_keyword):
+ return False
+ name = dotted(call.func) or ""
+ head, _, suffix = name.partition(".")
+ resolved = imported_names.get(head, head) + ("." + suffix if suffix else "")
+ if not wildcard_import and head not in shadowed_names and resolved in {
+ "unittest.mock.patch", "unittest.mock.patch.dict", "unittest.mock.patch.multiple"
+ }:
+ return True
+ if name != "monkeypatch.setattr":
+ return False
+ ancestor = call
+ while ancestor in parents:
+ ancestor = parents[ancestor]
+ if isinstance(ancestor, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ args = ancestor.args.posonlyargs + ancestor.args.args + ancestor.args.kwonlyargs
+ if not any(arg.arg == "monkeypatch" for arg in args):
+ continue
+ rebound = any(
+ (isinstance(child, ast.Name) and child.id == "monkeypatch"
+ and isinstance(child.ctx, (ast.Store, ast.Del)))
+ or (isinstance(child, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar))
+ and child.name == "monkeypatch")
+ or (isinstance(child, ast.MatchMapping) and child.rest == "monkeypatch")
+ or (isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+ and child.name == "monkeypatch")
+ or (isinstance(child, ast.alias)
+ and (child.asname or child.name.split(".")[0]) in {"monkeypatch", "*"})
+ for child in ast.walk(ancestor)
+ )
+ return not rebound
+ return False
+
+ def replace(node, text):
+ start = offsets[node.lineno - 1] + node.col_offset
+ end = offsets[node.end_lineno - 1] + node.end_col_offset
+ patches.append((start, end, text.encode("utf-8")))
+
+ def import_lines(node, statements):
+ prefix = lines[node.lineno - 1][:node.col_offset].decode("utf-8")
+ separator = "; " if prefix.strip() else "\n" + prefix
+ replacement = separator.join(statements)
+ original = ast.get_source_segment(source, node)
+ if replacement != original:
+ replace(node, replacement)
+
+ unaliased = {}
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ statements = []
+ for alias in node.names:
+ target = mapped(alias.name, moves)
+ statements.append("import " + target + (" as " + alias.asname if alias.asname else ""))
+ if target != alias.name and not alias.asname:
+ unaliased[alias.name] = target
+ # Only rewrite changed imports, preserving comments/format otherwise.
+ if any(mapped(a.name, moves) != a.name for a in node.names):
+ import_lines(node, statements)
+ elif isinstance(node, ast.ImportFrom):
+ original_module = node.module or ""
+ if node.level:
+ containing = old_module if package else old_module.rpartition(".")[0]
+ original_module = importlib.util.resolve_name(
+ "." * node.level + original_module, containing
+ )
+ groups = {}
+ for alias in node.names:
+ submodule = original_module + "." + alias.name
+ if submodule in moves:
+ dest = moves[submodule]
+ parent, _, name = dest.rpartition(".")
+ renamed = ast.alias(name=name, asname=alias.asname or (
+ alias.name if name != alias.name else None
+ ))
+ groups.setdefault(parent, []).append(renamed)
+ else:
+ target = mapped(original_module, moves)
+ if original_module == "core.db.models" or original_module.startswith("core.db.models."):
+ target = symbols.get(alias.name, target)
+ groups.setdefault(target, []).append(alias)
+ moved_self = old_module in moves
+ changed = any(target != original_module for target in groups)
+ changed |= any(original_module + "." + a.name in moves for a in node.names)
+ if changed or (node.level and moved_self):
+ statements = [
+ "from " + target + " import " + ", ".join(
+ a.name + (" as " + a.asname if a.asname else "") for a in aliases
+ )
+ for target, aliases in groups.items()
+ ]
+ import_lines(node, statements)
+
+ protected_roots = {
+ name.split(".")[0] for pair in unaliased.items() for name in pair
+ }
+ for node in ast.walk(tree):
+ binding = None
+ if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
+ binding = node.id
+ elif isinstance(node, ast.arg):
+ binding = node.arg
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ binding = node.name
+ elif isinstance(node, ast.alias):
+ parent = parents.get(node)
+ if node.asname:
+ binding = node.asname
+ elif isinstance(parent, ast.ImportFrom):
+ binding = node.name
+ elif isinstance(node, ast.ExceptHandler):
+ binding = node.name
+ elif isinstance(node, (ast.MatchAs, ast.MatchStar)):
+ binding = node.name
+ elif isinstance(node, ast.MatchMapping):
+ binding = node.rest
+ if binding in protected_roots:
+ raise ValueError(f"ambiguous binding {binding!r} at line {node.lineno}; manual rewrite required")
+
+ changed_roots = {
+ old.split(".")[0] for old, new in unaliased.items()
+ if old.split(".")[0] != new.split(".")[0]
+ }
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Name) or not isinstance(node.ctx, ast.Load):
+ continue
+ if node.id not in changed_roots:
+ continue
+ usage = node
+ while isinstance(parents.get(usage), ast.Attribute):
+ usage = parents[usage]
+ name = dotted(usage)
+ if mapped(name, unaliased) == name:
+ raise ValueError(f"unresolved root use {name!r} at line {node.lineno}; manual rewrite required")
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Attribute) and not isinstance(parents.get(node), ast.Attribute):
+ name = dotted(node)
+ if name and mapped(name, unaliased) != name:
+ replace(node, mapped(name, unaliased))
+ elif isinstance(node, ast.Constant) and isinstance(node.value, str):
+ target = node.value
+ for old in sorted(moves, key=len, reverse=True):
+ target = re.sub(
+ r"(? end_of_previous:
+ raise ValueError("Overlapping AST edits; manual review required")
+ result = result[:start] + replacement + result[end:]
+ end_of_previous = start
+ result = result.decode("utf-8")
+ ast.parse(result)
+ return result, unresolved
+
+
+def load_moves(batch):
+ rows = json.loads((DATA / "manifest.json").read_text())
+ supplemental = DATA / "supplemental_moves.json"
+ if supplemental.exists():
+ for extra in json.loads(supplemental.read_text()):
+ if extra.get("override") is not True:
+ rows.append(extra)
+ continue
+ matches = [i for i, row in enumerate(rows) if row["src"] == extra["src"]]
+ if (len(matches) != 1 or not extra.get("note", "").strip()
+ or rows[matches[0]]["batch"] != extra["batch"]
+ or rows[matches[0]]["action"] != extra["action"]):
+ raise ValueError("Invalid reviewed migration override")
+ rows[matches[0]] = extra
+ routes = json.loads((DATA / "routes.json").read_text())["moves"]
+ sources, destinations, destination_modules = set(), set(), set()
+ for row in rows + routes:
+ if row.get("action", "move") != "move":
+ continue
+ if row["src"] in sources or row["dst"] in destinations or module_name(row["dst"]) in destination_modules:
+ raise ValueError(f"Conflicting migration path: {row['src']} -> {row['dst']}")
+ sources.add(row["src"])
+ destinations.add(row["dst"])
+ destination_modules.add(module_name(row["dst"]))
+ return {
+ module_name(row["src"]): module_name(row["dst"])
+ for row in rows + routes
+ if row["batch"] == batch and row.get("action", "move") == "move"
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--batch", required=True)
+ parser.add_argument("--apply", action="store_true")
+ parser.add_argument("--symbols", type=Path, help="Reviewed symbol subset for this batch")
+ parser.add_argument("--report", type=Path, required=True)
+ args = parser.parse_args()
+ moves = load_moves(args.batch)
+ if not moves:
+ parser.error("Batch has no pure file moves; provide a supported migration batch")
+ if args.symbols:
+ symbols = json.loads(args.symbols.read_text())
+ else:
+ owner = args.batch.split("-", 1)[-1]
+ symbols = {
+ name: target
+ for name, target in json.loads((DATA / "symbol_moves.json").read_text()).items()
+ if target in {f"features.{owner}.models", f"integrations.{owner}.models"}
+ }
+ reverse = {target: old for old, target in moves.items()}
+ changes = []
+ report = {
+ "batch": args.batch, "changed_files": [], "unresolved_strings": [],
+ "core_string_inventory": [], "non_runtime_resources": [],
+ }
+ for base in ("src/backend", "ce/overlay/src/backend", "evaluation"):
+ directory = ROOT / base
+ for path in sorted(directory.rglob("*.py")):
+ relative = path.relative_to(directory).as_posix()
+ module = module_name(relative)
+ if base == "evaluation":
+ module = "evaluation" if relative == "__init__.py" else "evaluation." + module
+ old = reverse.get(module, module)
+ source = path.read_text(encoding="utf-8")
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ if "core." in source or "from core" in source or old in moves:
+ raise ValueError(f"Cannot parse migration source: {path}") from None
+ report["non_runtime_resources"].append(path.relative_to(ROOT).as_posix())
+ continue
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Constant) and isinstance(node.value, str) and "core." in node.value:
+ report["core_string_inventory"].append({
+ "file": path.relative_to(ROOT).as_posix(),
+ "line": node.lineno, "value": node.value,
+ })
+ try:
+ result, unresolved = rewrite(
+ source, old, moves, symbols, package=path.name == "__init__.py"
+ )
+ except ValueError as error:
+ raise ValueError(f"{path.relative_to(ROOT)}: {error}") from error
+ if result != source:
+ changes.append((path, result))
+ report["changed_files"].append(path.relative_to(ROOT).as_posix())
+ for item in unresolved:
+ report["unresolved_strings"].append({
+ "file": path.relative_to(ROOT).as_posix(), **item
+ })
+ args.report.parent.mkdir(parents=True, exist_ok=True)
+ args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
+ if args.apply and report["unresolved_strings"]:
+ print("No files written: review unresolved module strings in", args.report)
+ return 2
+ if args.apply:
+ for path, result in changes:
+ path.write_text(result)
+ print(f"{len(changes)} files planned; {len(report['unresolved_strings'])} strings need review")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/refactor/snapshot_routes.py b/scripts/refactor/snapshot_routes.py
new file mode 100644
index 00000000..cc03fe92
--- /dev/null
+++ b/scripts/refactor/snapshot_routes.py
@@ -0,0 +1,35 @@
+"""Print a deterministic route contract without starting application workers."""
+from __future__ import annotations
+
+import contextlib
+import json
+import sys
+
+
+def snapshot(app):
+ """Retain route order and duplicate routes, including non-OpenAPI endpoints."""
+ return {
+ "openapi_paths": sorted(app.openapi()["paths"]),
+ "routes": [
+ {
+ "order": index,
+ "path": getattr(route, "path", None),
+ "methods": sorted(getattr(route, "methods", None) or []),
+ "kind": type(route).__name__,
+ }
+ for index, route in enumerate(app.routes)
+ ],
+ }
+
+
+def main():
+ # Third-party import diagnostics must not contaminate the JSON stream.
+ with contextlib.redirect_stdout(sys.stderr):
+ from api.app import app
+
+ result = snapshot(app)
+ print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/refactor/snapshot_schema.py b/scripts/refactor/snapshot_schema.py
new file mode 100644
index 00000000..319e6eab
--- /dev/null
+++ b/scripts/refactor/snapshot_schema.py
@@ -0,0 +1,81 @@
+"""Snapshot registered ORM structure without connecting to a database."""
+from __future__ import annotations
+
+import contextlib
+import json
+import sys
+
+
+def default_value(default):
+ if default is None:
+ return None
+ if hasattr(default, "sqltext"):
+ return {"computed": str(default.sqltext), "persisted": default.persisted}
+ value = getattr(default, "arg", default)
+ if callable(value):
+ return {"callable": value.__qualname__}
+ return str(value)
+
+
+def constraint_snapshot(constraint):
+ value = {
+ "type": type(constraint).__name__,
+ "name": constraint.name,
+ "columns": [column.name for column in constraint.columns],
+ "deferrable": constraint.deferrable,
+ "initially": constraint.initially,
+ }
+ if hasattr(constraint, "sqltext"):
+ value["sqltext"] = str(constraint.sqltext)
+ if hasattr(constraint, "elements"):
+ value["references"] = [
+ {"target": item.target_fullname, "ondelete": item.ondelete,
+ "onupdate": item.onupdate, "match": item.match}
+ for item in constraint.elements
+ ]
+ return value
+
+
+def snapshot(metadata):
+ return {
+ table.fullname: {
+ "columns": [
+ {
+ "name": column.name,
+ "type": str(column.type),
+ "nullable": column.nullable,
+ "primary_key": column.primary_key,
+ "default": default_value(column.default),
+ "server_default": default_value(column.server_default),
+ "onupdate": default_value(column.onupdate),
+ "server_onupdate": default_value(column.server_onupdate),
+ "foreign_keys": sorted(
+ (fk.target_fullname, fk.ondelete, fk.onupdate)
+ for fk in column.foreign_keys
+ ),
+ }
+ for column in table.columns
+ ],
+ "constraints": sorted(
+ (constraint_snapshot(item) for item in table.constraints),
+ key=lambda item: json.dumps(item, sort_keys=True),
+ ),
+ "indexes": sorted(
+ (index.name, bool(index.unique), tuple(str(e) for e in index.expressions))
+ for index in table.indexes
+ ),
+ }
+ for table in sorted(metadata.tables.values(), key=lambda item: item.fullname)
+ }
+
+
+def main():
+ with contextlib.redirect_stdout(sys.stderr):
+ # The app is the composition root for installed feature schemas.
+ from api.app import app # noqa: F401
+ from core.db.engine import Base
+ print(json.dumps(snapshot(Base.metadata), ensure_ascii=False, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/refactor/tests/test_edition_seams.py b/scripts/refactor/tests/test_edition_seams.py
new file mode 100644
index 00000000..e8fac12b
--- /dev/null
+++ b/scripts/refactor/tests/test_edition_seams.py
@@ -0,0 +1,33 @@
+"""Enforce the completed P7 boundary instead of relying on reviewer memory."""
+import ast
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def test_core_and_feature_overlays_select_only_edition_policy():
+ overlay = ROOT / "ce/overlay/src/backend"
+ violations = []
+ for domain in ("core", "features"):
+ for path in (overlay / domain).rglob("*.py"):
+ if path == overlay / "features/installed.py":
+ continue # D10's installed-module selection is explicitly overlaid.
+ if path.name != "edition.py" or len(path.read_text().splitlines()) > 150:
+ violations.append(str(path.relative_to(ROOT)))
+ assert violations == []
+
+
+def test_enterprise_imports_are_confined_to_edition_entry_points():
+ violations = []
+ backend = ROOT / "src/backend"
+ for domain in ("core", "features", "integrations", "api", "orchestration", "personal_agent"):
+ for path in (backend / domain).rglob("*.py"):
+ if "edition" in path.stem:
+ continue
+ for node in ast.walk(ast.parse(path.read_text())):
+ modules = ([node.module or ""] if isinstance(node, ast.ImportFrom)
+ else [alias.name for alias in node.names] if isinstance(node, ast.Import)
+ else [])
+ if any(name == "edition_ee" or name.startswith("edition_ee.") for name in modules):
+ violations.append((str(path.relative_to(ROOT)), node.lineno))
+ assert violations == []
diff --git a/scripts/refactor/tests/test_guards.py b/scripts/refactor/tests/test_guards.py
new file mode 100644
index 00000000..ffdff686
--- /dev/null
+++ b/scripts/refactor/tests/test_guards.py
@@ -0,0 +1,127 @@
+"""Behavioral checks for the refactoring gates."""
+from pathlib import Path
+from types import SimpleNamespace
+
+from scripts.refactor.check_ce_paths import path_errors
+from scripts.refactor.check_line_budget import violations
+from scripts.refactor.import_contract import edge_rule, layer
+from scripts.refactor.snapshot_routes import snapshot
+
+
+def test_routes_keep_duplicates_methods_and_registration_order():
+ def route(path, methods):
+ return SimpleNamespace(path=path, methods=set(methods))
+ app = SimpleNamespace(
+ routes=[route("/a", ["POST", "GET"]), route("/a", ["GET"]), route("/b", [])],
+ openapi=lambda: {"paths": {"/b": {}, "/a": {}}},
+ )
+ result = snapshot(app)
+ assert result["openapi_paths"] == ["/a", "/b"]
+ assert [r["path"] for r in result["routes"]] == ["/a", "/a", "/b"]
+ assert [r["order"] for r in result["routes"]] == [0, 1, 2]
+ assert result["routes"][0]["methods"] == ["GET", "POST"]
+
+
+def test_line_budget_blocks_growth_but_allows_reduction(tmp_path):
+ path = tmp_path / "src/backend/core/example.py"
+ path.parent.mkdir(parents=True)
+ path.write_text("# line\n" * 600)
+ key = "src/backend/core/example.py"
+ assert violations(tmp_path, {key: 600}) == []
+ assert violations(tmp_path, {key: 601}) == []
+ assert len(violations(tmp_path, {key: 599})) == 1
+ assert len(violations(tmp_path, {})) == 1
+
+
+def test_new_file_at_limit_passes(tmp_path):
+ path = tmp_path / "src/backend/new.py"
+ path.parent.mkdir(parents=True)
+ path.write_text("# line\n" * 500)
+ assert violations(tmp_path, {}) == []
+
+
+def test_ce_missing_business_path_is_not_silently_ignored(tmp_path):
+ manifest = {"overlay": "overlay", "exclude": ["src/missing.py"]}
+ assert path_errors(tmp_path, manifest, []) == [
+ "exclude: no source file matches src/missing.py"
+ ]
+
+
+def test_ce_safety_exception_retains_exclusion(tmp_path):
+ manifest = {
+ "overlay": "overlay", "exclude": ["private.key"],
+ "optional_safety_excludes": ["private.key"],
+ }
+ assert path_errors(tmp_path, manifest, []) == []
+ manifest["exclude"] = []
+ assert path_errors(tmp_path, manifest, []) == [
+ "Safety exception has no exclusion: private.key"
+ ]
+
+
+def test_ce_required_overlay_and_prune_input_are_different_phases(tmp_path):
+ overlay = tmp_path / "overlay/src"
+ overlay.mkdir(parents=True)
+ (overlay / "only.py").write_text("")
+ manifest = {
+ "overlay": "overlay", "required_files": ["src/only.py"],
+ "split": ["src/only.py"], "prunes": {"example": {"path": "src/only.py"}},
+ }
+ assert path_errors(tmp_path, manifest, []) == [
+ "prunes.example: missing source src/only.py"
+ ]
+
+
+def test_dependency_directions_cover_feature_and_core_boundaries():
+ assert edge_rule("core.llm.runner", "features.desktop.service") == "core-outward"
+ assert edge_rule("features.desktop.service", "features.sites.service") == "feature-direction"
+ assert edge_rule("features.sites.service", "features.desktop.service") is None
+ assert edge_rule("core.db.engine", "core.models.client") == "core-layers"
+ assert edge_rule("core.models.client", "core.db.engine") is None
+
+
+def test_nested_command_layer_takes_priority_over_sandbox():
+ assert layer("core.sandbox.commands.runtime") == 1
+ assert layer("core.sandbox.providers.local") == 4
+ assert edge_rule("core.llm.tools", "core.sandbox.commands.runtime") == "core-layers"
+
+
+def test_import_contract_rejects_new_edge_even_when_existing_debt_is_allowed(monkeypatch, tmp_path):
+ import json
+ import scripts.refactor.import_contract as contract
+
+ (tmp_path / "import-baseline.json").write_text(json.dumps(["existing"]))
+ monkeypatch.setattr(contract, "DATA", tmp_path)
+ monkeypatch.setattr(contract, "violations", lambda graph: {"existing", "new-edge"})
+ gate = contract.CoreBoundaryContract("test", {}, {})
+ check = gate.check(None, False)
+ assert not check.kept
+ assert check.metadata["introduced"] == ["new-edge"]
+ assert check.ignored_import_count == 1
+
+
+def test_deleted_tracked_file_cannot_satisfy_ce_path_gate(tmp_path):
+ manifest = {"overlay": "overlay", "exclude": ["src/business.py"]}
+ assert path_errors(tmp_path, manifest, ["src/business.py"]) == [
+ "exclude: no source file matches src/business.py"
+ ]
+
+
+def test_ce_overlay_cannot_restore_moved_repository_exports(tmp_path):
+ from scripts.refactor.check_ce_paths import path_errors
+ path = tmp_path / "overlay/src/backend/core/db/repository/__init__.py"
+ path.parent.mkdir(parents=True)
+ path.write_text("from integrations.email.repository import EmailConnectionRepository\n")
+ errors = path_errors(tmp_path, {"overlay": "overlay"}, [])
+ assert len(errors) == 1 and "forwards moved ownership" in errors[0]
+ path.write_text("from core.db.repository.user import UserRepository\n")
+ assert path_errors(tmp_path, {"overlay": "overlay"}, []) == []
+
+
+def test_canonical_does_not_repeat_nested_destination():
+ from scripts.refactor.import_contract import canonical
+ moves = {"core.evolution.settlement": "core.evolution.settlement.settlement",
+ "core.evolution.settlement_runner": "core.evolution.settlement.settlement_runner"}
+ assert canonical("core.evolution.settlement", moves) == "core.evolution.settlement.settlement"
+ assert canonical("core.evolution.settlement.settlement", moves) == "core.evolution.settlement.settlement"
+ assert canonical("core.evolution.settlement.settlement_runner", moves) == "core.evolution.settlement.settlement_runner"
diff --git a/scripts/refactor/tests/test_package_markers.py b/scripts/refactor/tests/test_package_markers.py
new file mode 100644
index 00000000..c393c077
--- /dev/null
+++ b/scripts/refactor/tests/test_package_markers.py
@@ -0,0 +1,20 @@
+from scripts.refactor.import_contract import package_marker_errors
+
+
+def test_missing_domain_and_nested_package_cannot_escape_scan(tmp_path):
+ nested = tmp_path / "features/marketplace/versions"
+ nested.mkdir(parents=True)
+ (nested / "selection.py").write_text("value = 1")
+ assert package_marker_errors(tmp_path) == {
+ "package-marker: features", "package-marker: features/marketplace",
+ "package-marker: features/marketplace/versions",
+ }
+ for path in (tmp_path / "features", nested.parent, nested):
+ (path / "__init__.py").write_text("")
+ assert package_marker_errors(tmp_path) == set()
+
+
+def test_non_domain_scripts_do_not_need_package_markers(tmp_path):
+ (tmp_path / "scripts").mkdir()
+ (tmp_path / "scripts/job.py").write_text("pass")
+ assert package_marker_errors(tmp_path) == set()
diff --git a/scripts/refactor/tests/test_private_imports.py b/scripts/refactor/tests/test_private_imports.py
new file mode 100644
index 00000000..22f774c5
--- /dev/null
+++ b/scripts/refactor/tests/test_private_imports.py
@@ -0,0 +1,41 @@
+import importlib.util
+from pathlib import Path
+
+from scripts.refactor import import_contract as contract
+ALIASES = {"core.services.market": "features.marketplace.source", "core.plugins.archive": "core.capabilities.plugins.archive", "core.services.other": "features.marketplace.other"}
+
+
+def test_future_external_direct_private_import():
+ assert contract.private_imports_in_module("core/services/market.py", "from core.capabilities.plugins.archive import _extract", ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"}
+
+
+def test_module_alias_cannot_hide_private_access():
+ for source in ("import core.capabilities.plugins.archive as archive; archive._extract()", "from core.capabilities.plugins import archive as archive; archive._extract()", "import core.capabilities.plugins.archive; core.capabilities.plugins.archive._extract()"):
+ assert contract.private_imports_in_module("core/services/market.py", source, ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"}
+
+
+def test_relative_target_resolves_before_ownership():
+ assert contract.private_imports_in_module("core/services/market.py", "from ..plugins.archive import _extract", ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"}
+
+
+def test_future_feature_target_is_not_a_core_private_import():
+ assert not contract.private_imports_in_module("api/host.py", "from core.services.other import _helper", ALIASES)
+
+
+def test_internal_core_and_public_interfaces_are_allowed():
+ assert not contract.private_imports_in_module("core/internal.py", "from core.capabilities.plugins.archive import _extract", ALIASES)
+ assert not contract.private_imports_in_module("core/services/market.py", "from core.capabilities.plugins.archive import extract", ALIASES)
+
+
+def test_unrelated_scope_alias_cannot_hide_outer_import():
+ source = "import core.capabilities.plugins.archive as archive\narchive._extract()\ndef unrelated():\n import pathlib as archive\n"
+ assert contract.private_imports_in_module("core/services/market.py", source, ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"}
+
+
+def test_dependency_identity_survives_a_physical_move(monkeypatch):
+ from types import SimpleNamespace
+ monkeypatch.setattr(contract, "canonical_names", lambda: {"core.old": "core.models.client"})
+ monkeypatch.setattr(contract, "private_imports", lambda: set())
+ def graph(source):
+ return SimpleNamespace(modules=[source], find_modules_directly_imported_by=lambda _: ["core.llm.stub"])
+ assert contract.violations(graph("core.old")) == contract.violations(graph("core.models.client")) == {"core-layers: core.models.client -> core.llm.stub"}
diff --git a/scripts/refactor/tests/test_rewrite_imports.py b/scripts/refactor/tests/test_rewrite_imports.py
new file mode 100644
index 00000000..dcd2db9c
--- /dev/null
+++ b/scripts/refactor/tests/test_rewrite_imports.py
@@ -0,0 +1,142 @@
+"""The rewriter changes ownership without executing or changing module logic."""
+from scripts.refactor.rewrite_imports import rewrite
+
+MOVES = {"core.services.old_service": "features.example.service"}
+
+
+def apply(source, old="caller", moves=None, **kwargs):
+ return rewrite(source, old, moves or MOVES, **kwargs)
+
+
+def test_function_local_import_keeps_alias():
+ result, review = apply("def run():\n from core.services.old_service import get as read\n return read()\n")
+ assert " from features.example.service import get as read" in result
+ assert not review
+
+
+def test_importing_submodule_from_package_preserves_local_binding():
+ result, _ = apply("from core.services import old_service, other\n")
+ assert result == "from features.example import service as old_service\nfrom core.services import other\n"
+
+
+def test_relative_import_in_moved_module_resolves_against_old_package():
+ result, _ = apply("from .helpers import helper\n", old="core.services.old_service")
+ assert result == "from core.services.helpers import helper\n"
+
+
+def test_relative_import_with_parent_keeps_original_dependency():
+ result, _ = apply("from ..infra import logging\n", old="core.services.old_service")
+ assert result == "from core.infra import logging\n"
+
+
+def test_patch_and_monkeypatch_targets_are_updated():
+ source = "from unittest import mock\ndef test_x(monkeypatch):\n with mock.patch('core.services.old_service.get'):\n monkeypatch.setattr('core.services.old_service.value', 1)\n"
+ result, review = apply(source)
+ assert "'features.example.service.get'" in result
+ assert "'features.example.service.value'" in result
+ assert not review
+
+
+def test_unrecognized_string_is_not_changed_and_requires_review():
+ source = "target = 'core.services.old_service.get'\n"
+ result, review = apply(source)
+ assert result == source
+ assert review == [{"line": 1, "old": "core.services.old_service.get", "new": "features.example.service.get"}]
+
+
+def test_unicode_and_multiple_imports_keep_valid_offsets():
+ source = "标签 = '中文'; import core.services.old_service as svc\n"
+ result, _ = apply(source)
+ assert result == "标签 = '中文'; import features.example.service as svc\n"
+
+
+def test_unaliased_dotted_import_updates_use_sites():
+ result, _ = apply("import core.services.old_service\nx = core.services.old_service.get()\n")
+ assert result == "import features.example.service\nx = features.example.service.get()\n"
+
+
+def test_orm_symbol_moves_do_not_move_unrelated_models():
+ result, _ = apply(
+ "from core.db.models import Site, UserShadow\n",
+ symbols={"Site": "features.sites.models"},
+ )
+ assert result == "from features.sites.models import Site\nfrom core.db.models import UserShadow\n"
+
+
+def test_unrelated_import_preserves_multiline_comments():
+ source = "from core.db.models import (\n UserShadow, # identity remains in core\n)\n"
+ result, _ = apply(source)
+ assert result == source
+
+
+def test_embedded_module_reference_requires_review():
+ source = "message = 'Cannot import core.services.old_service.get'\n"
+ result, review = apply(source)
+ assert result == source
+ assert review[0]["new"] == "Cannot import features.example.service.get"
+
+
+def test_split_import_after_semicolon_is_valid():
+ result, _ = apply("x = 1; from core.services import old_service, other\n")
+ assert result == "x = 1; from features.example import service as old_service; from core.services import other\n"
+
+
+def test_parameter_shadowing_rejects_unaliased_rewrite():
+ import pytest
+ with pytest.raises(ValueError, match="ambiguous binding"):
+ apply("import core.services.old_service\ndef read(core):\n return core.services.old_service.get()\n")
+
+
+def test_destination_root_shadowing_is_also_rejected():
+ import pytest
+ with pytest.raises(ValueError, match="ambiguous binding"):
+ apply("import core.services.old_service\ndef read(features):\n return core.services.old_service.get()\n")
+
+
+def test_business_patch_function_is_not_a_mock():
+ source = "def patch(value):\n return value\nx = patch('core.services.old_service')\n"
+ result, review = apply(source)
+ assert result == source
+ assert len(review) == 1
+
+
+def test_imported_mock_alias_and_keyword_target_are_recognized():
+ source = "from unittest.mock import patch as replace\nwith replace(target='core.services.old_service.get'):\n pass\n"
+ result, review = apply(source)
+ assert "target='features.example.service.get'" in result
+ assert not review
+
+
+def test_nested_import_cannot_reclassify_outer_business_patch():
+ source = "from application import patch\ndef test_real():\n from unittest.mock import patch\n return patch('core.services.old_service')\ndef unrelated():\n return patch('core.services.old_service')\n"
+ result, review = apply(source)
+ assert result == source
+ assert len(review) == 2
+
+
+def test_wildcard_import_makes_mock_binding_ambiguous():
+ source = "from unittest.mock import patch\nfrom business import *\npatch('core.services.old_service')\n"
+ result, review = apply(source)
+ assert result == source
+ assert len(review) == 1
+
+
+def test_relative_import_cannot_reclassify_business_patch():
+ source = "from unittest.mock import patch\ndef f():\n from .application import patch\n return patch('core.services.old_service')\n"
+ result, review = apply(source, old="core.example.caller")
+ assert result == source
+ assert len(review) == 1
+
+
+def test_rebound_monkeypatch_fixture_requires_manual_review():
+ for binding in ("from application import monkeypatch", "def monkeypatch(): pass", "class monkeypatch: pass"):
+ source = "def test_x(monkeypatch):\n " + binding + "\n monkeypatch.setattr('core.services.old_service', 1)\n"
+ result, review = apply(source)
+ assert result == source
+ assert len(review) == 1
+
+
+def test_root_escape_after_namespace_move_requires_manual_review():
+ import pytest
+ with pytest.raises(ValueError, match="unresolved root use"):
+ apply("import core.services.old_service\nmodule = core\n")
diff --git a/scripts/refactor/tests/test_snapshot_schema.py b/scripts/refactor/tests/test_snapshot_schema.py
new file mode 100644
index 00000000..f9635dad
--- /dev/null
+++ b/scripts/refactor/tests/test_snapshot_schema.py
@@ -0,0 +1,33 @@
+from sqlalchemy import Column, Integer, MetaData, Table, UniqueConstraint, CheckConstraint, ForeignKeyConstraint
+from scripts.refactor.snapshot_schema import snapshot
+
+
+def test_snapshot_detects_removed_composite_unique_constraint():
+ metadata = MetaData()
+ table = Table("example", metadata, Column("owner", Integer), Column("name", Integer), UniqueConstraint("owner", "name"))
+ before = snapshot(metadata)
+ table.constraints.remove(next(c for c in table.constraints if isinstance(c, UniqueConstraint)))
+ assert snapshot(metadata) != before
+
+
+def test_snapshot_detects_check_foreign_key_and_default_changes():
+ metadata = MetaData()
+ Table("parent", metadata, Column("a", Integer, primary_key=True), Column("b", Integer, primary_key=True))
+ table = Table("child", metadata, Column("a", Integer, server_default="1"), Column("b", Integer), CheckConstraint("a > 0"), ForeignKeyConstraint(["a", "b"], ["parent.a", "parent.b"], ondelete="CASCADE"))
+ before = snapshot(metadata)
+ table.c.a.server_default = None
+ assert snapshot(metadata) != before
+ foreign = next(c for c in before["child"]["constraints"] if c["type"] == "ForeignKeyConstraint")
+ assert foreign["columns"] == ["a", "b"]
+ assert [item["target"] for item in foreign["references"]] == ["parent.a", "parent.b"]
+ assert all(item["ondelete"] == "CASCADE" for item in foreign["references"])
+ assert any(c.get("sqltext") == "a > 0" for c in before["child"]["constraints"])
+
+
+def test_computed_defaults_are_stable_across_metadata_instances():
+ from sqlalchemy import Computed
+ def schema():
+ metadata = MetaData()
+ Table("example", metadata, Column("a", Integer), Column("b", Integer, Computed("a + 1", persisted=True)))
+ return snapshot(metadata)
+ assert schema() == schema()
diff --git a/scripts/refactor/tests/test_supplemental_moves.py b/scripts/refactor/tests/test_supplemental_moves.py
new file mode 100644
index 00000000..85832268
--- /dev/null
+++ b/scripts/refactor/tests/test_supplemental_moves.py
@@ -0,0 +1,47 @@
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.refactor import rewrite_imports as rewrite
+
+
+def manifests(tmp_path, monkeypatch, supplemental):
+ (tmp_path / "manifest.json").write_text(json.dumps([{"src": "core/old.py", "dst": "features/one/old.py", "batch": "P2-one", "action": "move"}]))
+ (tmp_path / "routes.json").write_text(json.dumps({"moves": []}))
+ (tmp_path / "supplemental_moves.json").write_text(json.dumps(supplemental))
+ monkeypatch.setattr(rewrite, "DATA", tmp_path)
+
+
+def test_extracted_module_is_included_in_its_own_batch(tmp_path, monkeypatch):
+ manifests(tmp_path, monkeypatch, [{"src": "core/extracted.py", "dst": "features/two/extracted.py", "batch": "P2-two", "action": "move"}])
+ assert rewrite.load_moves("P2-two") == {"core.extracted": "features.two.extracted"}
+ assert rewrite.load_moves("P2-one") == {"core.old": "features.one.old"}
+
+
+@pytest.mark.parametrize("source,target", [("core/old.py", "features/two/new.py"), ("core/new.py", "features/one/old.py")])
+def test_ambiguous_source_or_destination_is_rejected(tmp_path, monkeypatch, source, target):
+ manifests(tmp_path, monkeypatch, [{"src": source, "dst": target, "batch": "P2-two", "action": "move"}])
+ with pytest.raises(ValueError, match="Conflicting migration path"):
+ rewrite.load_moves("P2-two")
+
+
+def test_reviewed_override_preserves_source_and_batch(tmp_path, monkeypatch):
+ manifests(tmp_path, monkeypatch, [{"src": "core/old.py", "dst": "features/one/service.py", "batch": "P2-one", "action": "move", "override": True, "note": "Resolve a module/package collision"}])
+ assert rewrite.load_moves("P2-one") == {"core.old": "features.one.service"}
+
+
+@pytest.mark.parametrize("change", [{"src": "core/missing.py"}, {"batch": "P2-other"}, {"action": "split"}, {"note": ""}])
+def test_override_requires_existing_source_same_batch_and_reason(tmp_path, monkeypatch, change):
+ entry = {"src": "core/old.py", "dst": "features/one/service.py", "batch": "P2-one", "action": "move", "override": True, "note": "Resolve a module/package collision"}
+ entry.update(change)
+ manifests(tmp_path, monkeypatch, [entry])
+ with pytest.raises(ValueError, match="Invalid reviewed migration override"):
+ rewrite.load_moves("P2-one")
+
+
+def test_module_and_package_destination_collision_is_rejected(tmp_path, monkeypatch):
+ manifests(tmp_path, monkeypatch, [{"src": "core/new.py", "dst": "features/one/old/__init__.py", "batch": "P2-two", "action": "move"}])
+ with pytest.raises(ValueError, match="Conflicting migration path"):
+ rewrite.load_moves("P2-two")
diff --git a/scripts/refactor/tests/test_test_baseline.py b/scripts/refactor/tests/test_test_baseline.py
new file mode 100644
index 00000000..9bbea723
--- /dev/null
+++ b/scripts/refactor/tests/test_test_baseline.py
@@ -0,0 +1,22 @@
+import pytest
+from scripts.refactor.check_test_baseline import result
+
+
+def test_completed_failure_summary_keeps_parameter_spaces():
+ parsed = result("FAILED tests/test_x.py::test_case[some value] - AssertionError\n==== 1 failed, 3 passed in 2.02s ====\n")
+ assert parsed["failures"] == ["tests/test_x.py::test_case[some value]"]
+
+
+def test_incomplete_or_truncated_run_is_rejected():
+ for log in ("tests/test_x.py F", "==== 1 failed in 2.02s ====", "INTERNALERROR> failure\n==== 1 passed in 2.02s ===="):
+ with pytest.raises(ValueError):
+ result(log)
+
+
+def test_passed_run_has_empty_failure_set():
+ assert result("==== 3 passed, 1 skipped in 2.02s ====\n")["failures"] == []
+
+
+def test_application_error_logs_are_not_test_outcomes():
+ log = "ERROR unrelated application log\n==== short test summary info ====\nFAILED tests/test_x.py::test_case\n==== 1 failed in 2.02s ====\n"
+ assert result(log)["failures"] == ["tests/test_x.py::test_case"]
diff --git a/scripts/refactor/tests/test_test_moves.py b/scripts/refactor/tests/test_test_moves.py
new file mode 100644
index 00000000..55ff44b7
--- /dev/null
+++ b/scripts/refactor/tests/test_test_moves.py
@@ -0,0 +1,45 @@
+"""Test moves preserve exact failure identities without masking regressions."""
+import pytest
+from scripts.refactor.check_test_baseline import canonical_failures
+
+OLD = "src/backend/tests/services/test_source.py"
+NEW = "src/backend/tests/features/datasources/test_source.py"
+
+
+def test_only_file_prefix_is_normalized():
+ node = NEW + "::test_query[" + NEW + "]"
+ assert canonical_failures([node], {OLD: NEW}) == {OLD + "::test_query[" + NEW + "]"}
+
+
+def test_new_tests_remain_new_failures():
+ assert canonical_failures([NEW + "::test_new"], {OLD: NEW}) == {OLD + "::test_new"}
+
+
+@pytest.mark.parametrize("moves", [
+ {OLD: OLD},
+ {OLD: NEW, "src/backend/tests/other.py": NEW},
+ {OLD: NEW, NEW: "src/backend/tests/final.py"},
+ {OLD: "src/backend/tests/../app.py"},
+ {OLD: "src/backend/api/app.py"},
+])
+def test_ambiguous_or_non_test_mappings_are_rejected(moves):
+ with pytest.raises(ValueError):
+ canonical_failures([], moves)
+
+
+def test_dependency_baseline_keeps_original_test_identity(tmp_path, monkeypatch):
+ import json
+ from scripts.refactor import import_contract
+ (tmp_path / "manifest.json").write_text("[]")
+ (tmp_path / "test_moves.json").write_text(json.dumps({OLD: NEW}))
+ monkeypatch.setattr(import_contract, "DATA", tmp_path)
+ assert import_contract.canonical_names()["tests.features.datasources.test_source"] == "tests.services.test_source"
+
+
+def test_route_moves_are_in_dependency_baseline(tmp_path, monkeypatch):
+ import json
+ from scripts.refactor import import_contract
+ (tmp_path / "manifest.json").write_text("[]")
+ (tmp_path / "routes.json").write_text(json.dumps({"moves": [{"src": "api/routes/v1/sites.py", "dst": "features/sites/api/sites.py"}]}))
+ monkeypatch.setattr(import_contract, "DATA", tmp_path)
+ assert import_contract.canonical_names()["api.routes.v1.sites"] == "features.sites.api.sites"
diff --git a/scripts/repair_artifact_storage_keys.py b/scripts/repair_artifact_storage_keys.py
index dc557b8f..2e087b6b 100644
--- a/scripts/repair_artifact_storage_keys.py
+++ b/scripts/repair_artifact_storage_keys.py
@@ -4,10 +4,10 @@
import argparse
import json
-from core.content.artifact_key_repair import repair_legacy_key, rollback_legacy_key, verified_legacy_key
+from core.files.artifacts.key_repair import repair_legacy_key, rollback_legacy_key, verified_legacy_key
from core.db.engine import SessionLocal
from core.db.models import Artifact
-from core.storage import get_storage
+from core.files.storage import get_storage
def main():
diff --git a/scripts/sync-workbench-schema.mjs b/scripts/sync-workbench-schema.mjs
index 80e398b1..f4a546d5 100644
--- a/scripts/sync-workbench-schema.mjs
+++ b/scripts/sync-workbench-schema.mjs
@@ -3,11 +3,11 @@ import {fileURLToPath} from 'node:url';
const root = fileURLToPath(new URL('../', import.meta.url));
for (const name of ['application.schema.json', 'module.schema.json', 'host.json']) {
const source = await readFile(root + 'packages/workbench-protocol/schemas/' + name, 'utf8');
- const destination = root + 'src/backend/core/applications/schemas/' + name;
+ const destination = root + 'src/backend/features/workbench/schemas/' + name;
if (process.argv.includes('--check')) {
if (await readFile(destination, 'utf8') !== source) throw new Error('Workbench Schema artifact drift: ' + name);
} else {
- await mkdir(root + 'src/backend/core/applications/schemas', {recursive: true});
+ await mkdir(root + 'src/backend/features/workbench/schemas', {recursive: true});
await writeFile(destination, source);
}
}
diff --git a/scripts/test-workbench-build.mjs b/scripts/test-workbench-build.mjs
index 5eb6bb86..cb00ac43 100644
--- a/scripts/test-workbench-build.mjs
+++ b/scripts/test-workbench-build.mjs
@@ -17,7 +17,7 @@ function fixture(name) {
cpSync(join(repo, 'packages'), join(root, 'packages'), {recursive: true,
filter: path => !['node_modules', 'dist'].includes(path.split('/').at(-1))});
for (const path of ['src/frontend/package.json', 'src/frontend/package-lock.json',
- 'src/backend/core/applications/schemas']) {
+ 'src/backend/features/workbench/schemas']) {
mkdirSync(dirname(join(root, path)), {recursive: true});
cpSync(join(repo, path), join(root, path), {recursive: true});
}
diff --git a/src/backend/alembic/env.py b/src/backend/alembic/env.py
index bc4b71fb..a75537aa 100644
--- a/src/backend/alembic/env.py
+++ b/src/backend/alembic/env.py
@@ -26,6 +26,10 @@
from core.db.connection import utc_connect_args
from core.db.utc_datetime import render_utc_type
from core.db import models as db_models # noqa: F401
+from core.harness.registry import load_models
+from features.installed import INSTALLED_FEATURES
+
+load_models(INSTALLED_FEATURES)
config.set_main_option("sqlalchemy.url", settings.db.url)
diff --git a/src/backend/alembic/versions/ce_0001_initial.py b/src/backend/alembic/versions/ce_0001_initial.py
index 47ec1f45..6313bacc 100644
--- a/src/backend/alembic/versions/ce_0001_initial.py
+++ b/src/backend/alembic/versions/ce_0001_initial.py
@@ -24,7 +24,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_create_all
+ from core.db.edition import ce_create_all
ce_create_all(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0002_reconcile_schema.py b/src/backend/alembic/versions/ce_0002_reconcile_schema.py
index b586d38b..8ff9b8cf 100644
--- a/src/backend/alembic/versions/ce_0002_reconcile_schema.py
+++ b/src/backend/alembic/versions/ce_0002_reconcile_schema.py
@@ -14,7 +14,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0003_mcp_marketplace.py b/src/backend/alembic/versions/ce_0003_mcp_marketplace.py
index 2082dc09..7a39d7e0 100644
--- a/src/backend/alembic/versions/ce_0003_mcp_marketplace.py
+++ b/src/backend/alembic/versions/ce_0003_mcp_marketplace.py
@@ -17,7 +17,7 @@ def upgrade() -> None:
# CE intentionally owns an independent migration chain. Reusing the CE
# metadata reconciler creates the new core marketplace tables/columns while
# continuing to omit every EE-only table and foreign-key dependency.
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0009_desktop_delivery.py b/src/backend/alembic/versions/ce_0009_desktop_delivery.py
index 21567665..95db8694 100644
--- a/src/backend/alembic/versions/ce_0009_desktop_delivery.py
+++ b/src/backend/alembic/versions/ce_0009_desktop_delivery.py
@@ -8,7 +8,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py b/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py
index a8f644d2..349e3cb9 100644
--- a/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py
+++ b/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py
@@ -8,7 +8,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0012_model_steps.py b/src/backend/alembic/versions/ce_0012_model_steps.py
index fce26851..b00ca229 100644
--- a/src/backend/alembic/versions/ce_0012_model_steps.py
+++ b/src/backend/alembic/versions/ce_0012_model_steps.py
@@ -8,11 +8,10 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
def downgrade() -> None:
raise NotImplementedError("Model-step reconciliation preserves conversation history")
-
diff --git a/src/backend/alembic/versions/ce_0013_tool_media.py b/src/backend/alembic/versions/ce_0013_tool_media.py
index 1fd478ab..6d9d8009 100644
--- a/src/backend/alembic/versions/ce_0013_tool_media.py
+++ b/src/backend/alembic/versions/ce_0013_tool_media.py
@@ -8,7 +8,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0014_site_access_password.py b/src/backend/alembic/versions/ce_0014_site_access_password.py
index 69c10300..e707944d 100644
--- a/src/backend/alembic/versions/ce_0014_site_access_password.py
+++ b/src/backend/alembic/versions/ce_0014_site_access_password.py
@@ -8,7 +8,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0019_history_display.py b/src/backend/alembic/versions/ce_0019_history_display.py
index 4beba214..ef9bb570 100644
--- a/src/backend/alembic/versions/ce_0019_history_display.py
+++ b/src/backend/alembic/versions/ce_0019_history_display.py
@@ -8,7 +8,7 @@
def upgrade() -> None:
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0020_plugin_resources.py b/src/backend/alembic/versions/ce_0020_plugin_resources.py
index b7e8e9a4..8aba8436 100644
--- a/src/backend/alembic/versions/ce_0020_plugin_resources.py
+++ b/src/backend/alembic/versions/ce_0020_plugin_resources.py
@@ -8,7 +8,7 @@
def upgrade():
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
ce_reconcile_schema(op.get_bind())
diff --git a/src/backend/alembic/versions/ce_0021_retire_site_storage.py b/src/backend/alembic/versions/ce_0021_retire_site_storage.py
index eb623ef2..d502271e 100644
--- a/src/backend/alembic/versions/ce_0021_retire_site_storage.py
+++ b/src/backend/alembic/versions/ce_0021_retire_site_storage.py
@@ -36,7 +36,7 @@ def downgrade():
raise RuntimeError("Set SITE_STORAGE_RESTORE_FILE to the verified retirement archive")
manifest = json.loads(Path(filename).read_text(encoding="utf-8"))
payload = manifest["tables"]
- from core.db.site_storage_retirement import _digest
+ from core.db.site_storage_retirement import digest as _digest
if manifest.get("revision") != "site05retire" or manifest.get("sha256") != _digest(payload):
raise RuntimeError("Retired storage archive is invalid")
diff --git a/src/backend/alembic/versions/ce_0028_personal_agent.py b/src/backend/alembic/versions/ce_0028_personal_agent.py
index 63804b17..25eaaac4 100644
--- a/src/backend/alembic/versions/ce_0028_personal_agent.py
+++ b/src/backend/alembic/versions/ce_0028_personal_agent.py
@@ -16,8 +16,8 @@
def upgrade():
# CE bootstrap also creates current metadata; checkfirst handles fresh installs.
- from core.db.edition_tables import _ce_metadata
- metadata = _ce_metadata()
+ from core.db.personal_schema import ce_metadata
+ metadata = ce_metadata()
metadata.create_all(op.get_bind(), tables=[metadata.tables[name] for name in TABLES], checkfirst=True)
diff --git a/src/backend/alembic/versions/ce_0031_automation_lease_token.py b/src/backend/alembic/versions/ce_0031_automation_lease_token.py
new file mode 100644
index 00000000..3a30147d
--- /dev/null
+++ b/src/backend/alembic/versions/ce_0031_automation_lease_token.py
@@ -0,0 +1,19 @@
+"""Fence scheduled run completion with the current execution token."""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "ce_0031"
+down_revision = "ce_0030"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ columns = {c["name"] for c in sa.inspect(op.get_bind()).get_columns("scheduled_task_runs")}
+ if "lease_token" not in columns:
+ op.add_column("scheduled_task_runs", sa.Column("lease_token", sa.String(64), nullable=True))
+
+
+def downgrade():
+ op.drop_column("scheduled_task_runs", "lease_token")
diff --git a/src/backend/alembic/versions/ce_0032_automation_queue.py b/src/backend/alembic/versions/ce_0032_automation_queue.py
new file mode 100644
index 00000000..80d7ef2c
--- /dev/null
+++ b/src/backend/alembic/versions/ce_0032_automation_queue.py
@@ -0,0 +1,25 @@
+"""Durable queued automation invocations."""
+from alembic import op
+
+revision = "ce_0032"
+down_revision = "ce_0031"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ from features.automation.schema import upgrade_queue
+ upgrade_queue(op.get_bind())
+
+
+def downgrade():
+ # Refuse to erase queued work while downgrading the status contract.
+ from sqlalchemy import text
+ if op.get_bind().execute(text(
+ "SELECT 1 FROM scheduled_task_runs WHERE status = 'pending' LIMIT 1"
+ )).first():
+ raise RuntimeError("Drain pending automation runs before downgrade")
+ with op.batch_alter_table("scheduled_task_runs") as batch:
+ batch.drop_constraint("scheduled_task_runs_status_check", type_="check")
+ batch.create_check_constraint("scheduled_task_runs_status_check",
+ "status IN ('running', 'success', 'failed')")
diff --git a/src/backend/alembic/versions/ce_0033_automation_targets.py b/src/backend/alembic/versions/ce_0033_automation_targets.py
new file mode 100644
index 00000000..e4b38e6d
--- /dev/null
+++ b/src/backend/alembic/versions/ce_0033_automation_targets.py
@@ -0,0 +1,17 @@
+"""Normalize stored automation delivery destinations."""
+from alembic import op
+
+revision = "ce_0033"
+down_revision = "ce_0032"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ from features.automation.schema import upgrade_delivery_targets
+ upgrade_delivery_targets(op.get_bind())
+
+
+def downgrade():
+ # The earlier schema already reads delivery_targets; no lossy reverse conversion.
+ pass
diff --git a/src/backend/api/__init__.py b/src/backend/api/__init__.py
index aa95844f..ea4958f5 100644
--- a/src/backend/api/__init__.py
+++ b/src/backend/api/__init__.py
@@ -1,5 +1,20 @@
-"""API layer for HugAgentOS."""
+"""API layer for HugAgentOS.
-from .app import app
+``app`` is resolved on first access: importing a submodule such as
+``api.deps`` from a route module must not compose the whole application,
+which would import that same, still half-initialised route module.
+"""
__all__ = ["app"]
+
+
+def __getattr__(name):
+ if name == "app":
+ import importlib
+
+ # Importing the submodule binds ``api.app`` to the module; rebind it to
+ # the FastAPI instance, as the former eager ``from .app import app`` did.
+ instance = importlib.import_module("api.app").app
+ globals()["app"] = instance
+ return instance
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/src/backend/api/app.py b/src/backend/api/app.py
index 8b1944ea..77e61be8 100644
--- a/src/backend/api/app.py
+++ b/src/backend/api/app.py
@@ -13,6 +13,17 @@
import os
import sys
+from orchestration.configuration import register_configuration
+register_configuration()
+
+from features import register_run_capabilities
+
+register_run_capabilities()
+from features import register_model_credentials
+register_model_credentials()
+from integrations import register_credentials
+register_credentials()
+
from api.lifecycle import lifespan
from api.openapi import _build_custom_openapi, _docs_title
from api.health import router as health_router
@@ -20,7 +31,8 @@
from api.middleware.edition import edition_router_dependencies, setup_edition_middleware
from api.middleware.error_handler import setup_error_handlers
from api.middleware.logging import setup_logging_middleware
-from api.routes import files_router, sites_serve_router
+from api.routes import files_router
+from features.sites.api.sites_serve import router as sites_serve_router
from api.routes.v1 import (
CE_ROUTERS,
EE_ROUTERS,
@@ -29,9 +41,10 @@
mock_sso_router,
)
from core.config.settings import settings
-from core.extensions import load_extensions
-from core.extensions import routers as extension_routers
+from core.harness.registry import load_extensions
+from core.harness.registry import routers as extension_routers
from core.infra.logging import get_logger
+from features.installed import INSTALLED_FEATURES
from fastapi import FastAPI
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
@@ -184,7 +197,14 @@ async def root():
dependencies=edition_router_dependencies(_feature),
)
# Optional modules mount their routers only when switched on.
-load_extensions()
+from features import register_product_tools, register_package_sources, register_resource_owners
+
+register_product_tools()
+from features import register_catalog_options
+register_catalog_options()
+register_package_sources()
+register_resource_owners()
+load_extensions(INSTALLED_FEATURES)
for _router, _feature in extension_routers():
app.include_router(_router, dependencies=edition_router_dependencies(_feature))
diff --git a/src/backend/api/deps.py b/src/backend/api/deps.py
index 68c8509b..237b2882 100644
--- a/src/backend/api/deps.py
+++ b/src/backend/api/deps.py
@@ -5,9 +5,9 @@
from core.auth.backend import UserContext, require_auth
from core.config.settings import settings
-from core.db.engine import get_db
from core.db.models import UserShadow
from core.db.repository import AuditLogRepository
+from core.db.threads import run_in_session
from fastapi import Depends, Header, HTTPException, Request
from sqlalchemy.orm import Session
@@ -30,14 +30,15 @@ def _require_token_or_flag(
``action_base`` (e.g. ``"admin"`` / ``"config"``) derives the paired audit
actions ``{base}.access_granted`` / ``{base}.access_denied``, so the two
action names are not string-derived from each other (fragile coupling).
+
+ Like every dependency here, it resolves the session (Redis) on the event
+ loop and does all of its database work in one ``run_in_session`` call.
"""
granted_action = f"{action_base}.access_granted"
denied_action = f"{action_base}.access_denied"
- async def dependency(
- request: Request,
- authorization: Optional[str] = Header(None),
- db: Session = Depends(get_db),
+ def authorize(
+ db: Session, request: Request, authorization: Optional[str], user_id: Optional[str]
) -> None:
token = os.getenv(env_var) or getattr(settings.auth, settings_attr, "")
# ① Static token match
@@ -45,19 +46,19 @@ async def dependency(
_audit_grant(db, request, granted_action, user_id=None, via="token")
return
# ② Session user carries the corresponding capability flag (or super_admin)
- user_id = await _resolve_session_user_id(request)
if user_id and _user_has_meta_flag(db, user_id, meta_flag):
_audit_grant(db, request, granted_action, user_id=user_id, via="session")
return
- # ②b 桌面桥接(混合架构):桌面壳孵化的本机后端上,壳注入的云端身份携带
- # 对应管理能力标志时放行——config 控制台在桌面双模式下可读本机数据。
- # 云端部署不设桥接 env,此分支恒短路(零行为变化)。
- from core.auth.desktop_bridge import resolve_bridge_user
-
- bridge_user = resolve_bridge_user(request, db)
- if bridge_user is not None and _user_has_meta_flag(db, bridge_user.user_id, meta_flag):
+ # ②b Host identity (desktop bridge): the shell-injected cloud identity is
+ # accepted when it carries the same flag. Deployments without a host
+ # resolver, or requests it does not claim, skip this without any I/O.
+ from core.harness.registry.authentication import host_user_resolver
+
+ resolver = host_user_resolver(request)
+ host_user = resolver(request, db) if resolver is not None else None
+ if host_user is not None and _user_has_meta_flag(db, host_user.user_id, meta_flag):
_audit_grant(
- db, request, granted_action, user_id=bridge_user.user_id, via="desktop_bridge"
+ db, request, granted_action, user_id=host_user.user_id, via="desktop_bridge"
)
return
# ③ Token not configured at all and no session grant → report explicitly that it is not configured
@@ -79,6 +80,16 @@ async def dependency(
)
raise HTTPException(status_code=401, detail="Unauthorized")
+ async def dependency(
+ request: Request,
+ authorization: Optional[str] = Header(None),
+ ) -> None:
+ token = os.getenv(env_var) or getattr(settings.auth, settings_attr, "")
+ user_id = None
+ if not (token and authorization == f"Bearer {token}"):
+ user_id = await _resolve_session_user_id(request)
+ await run_in_session(authorize, request, authorization, user_id)
+
return dependency
@@ -131,7 +142,6 @@ def _audit_grant(
async def require_admin_or_config(
request: Request,
authorization: Optional[str] = Header(None),
- db: Session = Depends(get_db),
) -> None:
"""Accept either ADMIN_TOKEN or CONFIG_TOKEN.
@@ -151,14 +161,16 @@ async def require_admin_or_config(
return
if config_token and authorization == f"Bearer {config_token}":
return
- AuditLogRepository(db).log_denial(
- user_id=None,
- action="prompt_snapshot.access_denied",
- reason="invalid_token",
- required="admin_or_config_token",
- actual="mismatch",
- resource_type="token",
- request=request,
+ await run_in_session(
+ lambda db: AuditLogRepository(db).log_denial(
+ user_id=None,
+ action="prompt_snapshot.access_denied",
+ reason="invalid_token",
+ required="admin_or_config_token",
+ actual="mismatch",
+ resource_type="token",
+ request=request,
+ )
)
raise HTTPException(status_code=401, detail="Unauthorized")
@@ -194,6 +206,15 @@ async def _resolve_session_user_id(request: Request) -> Optional[str]:
return None
+async def console_actor(request: Request) -> str:
+ """Who is acting on a console endpoint: the session user, else the static token.
+
+ A dependency, so the console routes that record it can stay plain ``def``
+ routes and do their database work in FastAPI's worker threads.
+ """
+ return await _resolve_session_user_id(request) or "config_token"
+
+
def _is_super_admin(db: Session, user_id: str) -> bool:
shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first()
return bool(shadow and (shadow.extra_data or {}).get("role") == "super_admin")
@@ -248,97 +269,98 @@ def user_can_manage_ontology_governance(db: Session, user_id: Optional[str]) ->
) or user_can_manage_system_settings(db, user_id)
-async def require_system_settings(
+def _authorize_delegated(
+ db: Session,
request: Request,
- db: Session = Depends(get_db),
- user: Optional[UserContext] = Depends(require_auth(False)),
+ user: Optional[UserContext],
+ *,
+ token: str,
+ token_grant: str,
+ allowed,
+ action: str,
+ required: str,
+ forbidden: str,
) -> str:
- """System-settings seam (the CE delegated version of ``require_config``): any of three paths grants access.
+ """Shared body of the delegated console gates (runs in a worker thread).
- 1. Static CONFIG_TOKEN (script / ops fallback, equivalent to ``require_config``);
- 2. Session/mock user with the ``can_system_config`` capability flag (implied by super_admin);
- 3. CE + mock single trust domain (see ``user_can_manage_system_settings``).
-
- Key difference from ``require_config``: the user is resolved via the full
- ``get_current_user`` chain — the CE derived tree has no
- ``core/auth/session.py``, so ``require_config``'s cookie-validation path
- always fails on CE; this gate can resolve a user in mock / local-session /
- API-Key modes alike.
+ A matching static token wins; otherwise the resolved user must pass
+ ``allowed(db, user_id)``. Grants are audited for writes, denials whenever a
+ credential or user was presented.
"""
- token = os.getenv("CONFIG_TOKEN") or getattr(settings.auth, "config_token", "")
authorization = request.headers.get("authorization")
if token and authorization == f"Bearer {token}":
- _audit_grant(db, request, "system_settings.access_granted", user_id=None, via="token")
- return "config_token"
- if user is not None and user_can_manage_system_settings(db, user.user_id):
- _audit_grant(
- db, request, "system_settings.access_granted", user_id=user.user_id, via="session"
- )
+ _audit_grant(db, request, f"{action}.access_granted", user_id=None, via="token")
+ return token_grant
+ if user is not None and allowed(db, user.user_id):
+ _audit_grant(db, request, f"{action}.access_granted", user_id=user.user_id, via="session")
return user.user_id
if authorization or user is not None:
AuditLogRepository(db).log_denial(
user_id=user.user_id if user else None,
- action="system_settings.access_denied",
+ action=f"{action}.access_denied",
reason="insufficient_permission",
- required="can_system_config",
+ required=required,
actual="regular_user" if user else "anonymous",
resource_type="backend_console",
request=request,
)
if user is None:
raise HTTPException(status_code=401, detail="未登录")
- raise HTTPException(status_code=403, detail="需要系统配置权限")
+ raise HTTPException(status_code=403, detail=forbidden)
-async def require_ontology_governance(
+async def require_system_settings(
request: Request,
- db: Session = Depends(get_db),
user: Optional[UserContext] = Depends(require_auth(False)),
) -> str:
- """Authorize the CE settings-page ontology governance surface.
+ """System-settings seam (the CE delegated version of ``require_config``): any of three paths grants access.
- ``ADMIN_TOKEN`` remains an operational fallback. Browser access is limited
- to users accepted by :func:`user_can_manage_ontology_governance`.
+ 1. Static CONFIG_TOKEN (script / ops fallback, equivalent to ``require_config``);
+ 2. Session/mock user with the ``can_system_config`` capability flag (implied by super_admin);
+ 3. CE + mock single trust domain (see ``user_can_manage_system_settings``).
+
+ Key difference from ``require_config``: the user is resolved via the full
+ ``get_current_user`` chain — the CE derived tree has no
+ ``core/auth/session.py``, so ``require_config``'s cookie-validation path
+ always fails on CE; this gate can resolve a user in mock / local-session /
+ API-Key modes alike.
"""
- token = os.getenv("ADMIN_TOKEN") or getattr(settings.auth, "admin_token", "")
- authorization = request.headers.get("authorization")
- if token and authorization == f"Bearer {token}":
- _audit_grant(db, request, "ontology_governance.access_granted", user_id=None, via="token")
- return "admin_token"
- if user is not None and user_can_manage_ontology_governance(db, user.user_id):
- _audit_grant(
- db,
- request,
- "ontology_governance.access_granted",
- user_id=user.user_id,
- via="session",
- )
- return user.user_id
- if authorization or user is not None:
- AuditLogRepository(db).log_denial(
- user_id=user.user_id if user else None,
- action="ontology_governance.access_denied",
- reason="insufficient_permission",
- required="ce_ontology_governance",
- actual="regular_user" if user else "anonymous",
- resource_type="backend_console",
- request=request,
- )
- if user is None:
- raise HTTPException(status_code=401, detail="未登录")
- raise HTTPException(status_code=403, detail="需要 CE 本体治理权限")
+ return await run_in_session(
+ _authorize_delegated,
+ request,
+ user,
+ token=os.getenv("CONFIG_TOKEN") or getattr(settings.auth, "config_token", ""),
+ token_grant="config_token",
+ allowed=user_can_manage_system_settings,
+ action="system_settings",
+ required="can_system_config",
+ forbidden="需要系统配置权限",
+ )
-async def require_super_admin(
+async def require_ontology_governance(
request: Request,
- db: Session = Depends(get_db),
+ user: Optional[UserContext] = Depends(require_auth(False)),
) -> str:
- """Require the current session user to be super_admin; a valid ADMIN_TOKEN serves as fallback."""
- admin_token = os.getenv("ADMIN_TOKEN") or settings.auth.admin_token
- authorization = request.headers.get("authorization")
- if admin_token and authorization == f"Bearer {admin_token}":
- return "admin_token"
+ """Authorize the CE settings-page ontology governance surface.
+ ``ADMIN_TOKEN`` remains an operational fallback. Browser access is limited
+ to users accepted by :func:`user_can_manage_ontology_governance`.
+ """
+ return await run_in_session(
+ _authorize_delegated,
+ request,
+ user,
+ token=os.getenv("ADMIN_TOKEN") or getattr(settings.auth, "admin_token", ""),
+ token_grant="admin_token",
+ allowed=user_can_manage_ontology_governance,
+ action="ontology_governance",
+ required="ce_ontology_governance",
+ forbidden="需要 CE 本体治理权限",
+ )
+
+
+def _authorize_super_admin(db: Session, request: Request) -> str:
user_id = _resolve_current_user_id(request)
if not user_id:
AuditLogRepository(db).log_denial(
@@ -361,3 +383,12 @@ async def require_super_admin(
)
raise HTTPException(status_code=403, detail="需要 super_admin 权限")
return user_id
+
+
+async def require_super_admin(request: Request) -> str:
+ """Require the current session user to be super_admin; a valid ADMIN_TOKEN serves as fallback."""
+ admin_token = os.getenv("ADMIN_TOKEN") or settings.auth.admin_token
+ authorization = request.headers.get("authorization")
+ if admin_token and authorization == f"Bearer {admin_token}":
+ return "admin_token"
+ return await run_in_session(_authorize_super_admin, request)
diff --git a/src/backend/api/edition.py b/src/backend/api/edition.py
new file mode 100644
index 00000000..3acc72a7
--- /dev/null
+++ b/src/backend/api/edition.py
@@ -0,0 +1,13 @@
+"""Community API providers."""
+EE_ROUTERS = ()
+
+
+def __getattr__(name):
+ if name in ("router", "_require_can_add_agent"):
+ import api.personal_agents as agents
+ return getattr(agents, name)
+ raise AttributeError(name)
+
+
+def gateway_service(db):
+ return None
diff --git a/src/backend/api/health.py b/src/backend/api/health.py
index 5f4ecea2..ebcefcb0 100644
--- a/src/backend/api/health.py
+++ b/src/backend/api/health.py
@@ -3,16 +3,15 @@
from datetime import datetime
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter
from fastapi.responses import JSONResponse
-from sqlalchemy.orm import Session
from sqlalchemy import text
from core.chat.schemas import HealthResponse
from core.config.settings import settings
-from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.logging import get_logger
-from core.storage import get_storage
+from core.files.storage import get_storage
logger = get_logger(__name__)
@@ -35,7 +34,7 @@ async def health_check():
@router.get("/ready", summary="就绪探针")
-async def readiness_check(db: Session = Depends(get_db)):
+async def readiness_check():
"""
就绪检查(包含依赖检查)
@@ -46,7 +45,7 @@ async def readiness_check(db: Session = Depends(get_db)):
# Check database
try:
- db.execute(text("SELECT 1"))
+ await run_in_session(lambda db: db.execute(text("SELECT 1")))
checks["database"] = True
except Exception as e:
logger.error("database_health_check_failed", error=str(e))
diff --git a/src/backend/api/lifecycle.py b/src/backend/api/lifecycle.py
index e16173da..b6c37b3f 100644
--- a/src/backend/api/lifecycle.py
+++ b/src/backend/api/lifecycle.py
@@ -9,7 +9,6 @@
from api.startup.workers import (
_env_flag,
_startup_preload_enabled,
- _mcp_warmup_enabled,
_shutdown_automation_scheduler,
_shutdown_idle_session_reaper,
_startup_idle_session_reaper,
@@ -32,6 +31,7 @@
_startup_seed_roles,
)
from api.startup.runtime import (
+ _startup_thread_budget,
_startup_watch_capability_changes,
_startup_recover_chat_runs,
_startup_resume_loops,
@@ -42,6 +42,8 @@
_shutdown_orphan_job_reaper,
_shutdown_channel_manager,
_startup_warm_sandbox_pool,
+ _shutdown_sandbox_maintenance,
+ _shutdown_sandbox_provider,
_startup_local_sidecars,
_shutdown_local_sidecars,
_startup_device_link,
@@ -83,7 +85,7 @@
def _runtime_role() -> str:
- from core.auth.desktop_bridge import bridge_enabled
+ from features.desktop.auth.bridge import bridge_enabled
return EXECUTION_PLANE if bridge_enabled() else SERVICE
@@ -121,6 +123,7 @@ def _startup_steps():
"""
return (
# (step, stop, gate, roles, scope)
+ (_startup_thread_budget, None, True, _ALL_ROLES, _PER_WORKER),
(_startup_ensure_tables, None, True, _ALL_ROLES, _SINGLETON),
(_startup_watch_capability_changes, None, True, _ALL_ROLES, _PER_WORKER),
(_startup_desktop_observability, None, True, _ALL_ROLES, _PER_WORKER),
@@ -148,7 +151,7 @@ def _startup_steps():
(_startup_recover_jobs, None, False, _ALL_ROLES, _SINGLETON),
(_startup_orphan_job_reaper, _shutdown_orphan_job_reaper, False, _ALL_ROLES, _SINGLETON),
(_startup_stale_run_reaper, _shutdown_stale_run_reaper, False, _ALL_ROLES, _SINGLETON),
- (_startup_warm_sandbox_pool, None, False, _ALL_ROLES, _SINGLETON),
+ (_startup_warm_sandbox_pool, _shutdown_sandbox_maintenance, False, _ALL_ROLES, _SINGLETON),
(
_startup_idle_session_reaper,
_shutdown_idle_session_reaper,
@@ -214,13 +217,17 @@ async def _stop_singleton_work() -> None:
the role. The handlers come from the registry, so this cannot stop a
different set than the one the leader started.
"""
- for _step, stop, _gate, _roles, scope in _startup_steps():
+ failures = []
+ for _step, stop, _gate, _roles, scope in reversed(_startup_steps()):
if scope is not _SINGLETON or stop is None:
continue
try:
await stop()
except Exception as exc: # noqa: BLE001 - one failure must not skip the rest
logger.warning("leader_stop_failed", step=stop.__name__, error=str(exc))
+ failures.append(exc)
+ if failures:
+ raise ExceptionGroup("Singleton work did not stop", failures)
async def _run_startup_steps(steps) -> None:
@@ -297,7 +304,7 @@ async def lifespan(app: FastAPI):
await _shutdown_stale_run_reaper()
await _shutdown_memory_outbox_worker()
await _shutdown_desktop_observability()
- from core.services.desktop_gateway_observability import drain
+ from features.desktop.gateway.observability import drain
await drain()
await _shutdown_orphan_job_reaper()
@@ -308,4 +315,5 @@ async def lifespan(app: FastAPI):
await _shutdown_datasource_sidecar_recovery()
await _shutdown_mcp_market_monitor()
await _shutdown_local_sidecars()
+ await _shutdown_sandbox_provider()
await _shutdown_pools()
diff --git a/src/backend/api/middleware/agent_api_scope.py b/src/backend/api/middleware/agent_api_scope.py
index d7a16926..5c895dcf 100644
--- a/src/backend/api/middleware/agent_api_scope.py
+++ b/src/backend/api/middleware/agent_api_scope.py
@@ -16,13 +16,8 @@
from core.auth.backend import UserContext
from core.db.engine import SessionLocal
from core.infra.responses import error_response
-from core.services.api_key_service import api_key_token_from_header, resolve_api_key_identity
-from core.services.agent_api_service import (
- begin_agent_api_call,
- call_log_tracking,
- fail_agent_api_call,
- make_agent_api_scope,
-)
+from core.auth.api_keys import api_key_token_from_header, resolve_api_key_identity
+from core.auth.agent_api import begin_agent_api_call, call_log_tracking, fail_agent_api_call, make_agent_api_scope
class AgentApiScopeMiddleware:
@@ -83,7 +78,7 @@ def authorize():
status_code=503,
)
return await response(scope, receive, send)
- from core.services.api_key_runs import request_api_key
+ from core.auth.api_key_runs import request_api_key
credential_token = request_api_key.set((context.user_id, context.api_key_id))
try:
diff --git a/src/backend/api/middleware/logging.py b/src/backend/api/middleware/logging.py
index 72ffe64a..71c09f70 100644
--- a/src/backend/api/middleware/logging.py
+++ b/src/backend/api/middleware/logging.py
@@ -80,7 +80,7 @@ async def send_wrapper(message: dict) -> None:
headers.append((b"x-trace-id", trace_id.encode()))
# 能力变更号:桌面本机端据此发现云端能力改动并同步一次,
# 免掉定时轮询(core/capabilities/change_signal.py)。
- from core.capabilities.change_signal import current
+ from features.desktop.capabilities.change_signal import current
headers.append((b"x-hugagent-capability-epoch", current().encode()))
message = {**message, "headers": headers}
diff --git a/src/backend/api/openapi.py b/src/backend/api/openapi.py
index f592eef5..90cce631 100644
--- a/src/backend/api/openapi.py
+++ b/src/backend/api/openapi.py
@@ -151,7 +151,7 @@ def _docs_title() -> str:
removing the hard-coded brand. Falls back to app.title when the DB read
fails (already the env brand, no literal "HugAgentOS")."""
try:
- from core.content.content_blocks import get_admin_platform_info
+ from core.files.content.branding import get_admin_platform_info
info = get_admin_platform_info()
name = (info.get("product_name") or "").strip()
diff --git a/src/backend/api/routes/v1/agents.py b/src/backend/api/personal_agents.py
similarity index 94%
rename from src/backend/api/routes/v1/agents.py
rename to src/backend/api/personal_agents.py
index 56786c70..ebc1e61e 100644
--- a/src/backend/api/routes/v1/agents.py
+++ b/src/backend/api/personal_agents.py
@@ -11,8 +11,8 @@
from core.db.engine import get_db
from core.infra.exceptions import AccessDeniedError, BadRequestError
from core.infra.responses import error_response, success_response
-from core.services.agent_markdown import agent_filename, agent_to_markdown, parse_agents_upload
-from core.services.user_agent_service import UserAgentService
+from core.capabilities.agents.markdown import agent_filename, agent_to_markdown, parse_agents_upload
+from core.capabilities.agents.edition import UserAgentService
from fastapi import APIRouter, Depends, File, Response, UploadFile
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -71,12 +71,12 @@ class AgentUpdateRequest(BaseModel):
@router.get("", summary="列出当前用户可见的所有子智能体")
-async def list_agents(
+def list_agents(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
# 本机后端:云端账号的智能体登记在本机登记表里,业务库没有对应行。
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
agents = device_catalog.merge_items(
UserAgentService(db).list_for_user(user.user_id),
@@ -87,7 +87,7 @@ async def list_agents(
@router.get("/available-resources", summary="可绑定到子智能体的资源列表")
-async def available_resources(
+def available_resources(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
@@ -117,7 +117,7 @@ async def available_resources(
@router.post("/import", summary="导入子智能体(markdown / zip)")
-async def import_agents(
+def import_agents(
file: UploadFile = File(...),
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -128,7 +128,7 @@ async def import_agents(
JSON 数组文件。需 ``can_add_agent`` 权限。
"""
_require_can_add_agent(str(user.user_id), db)
- raw = await file.read()
+ raw = file.file.read()
try:
items = parse_agents_upload(file.filename or "", raw)
except ValueError as exc:
@@ -154,7 +154,7 @@ async def import_agents(
@router.get("/{agent_id}/export", summary="导出子智能体为 markdown")
-async def export_agent(
+def export_agent(
agent_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -162,7 +162,7 @@ async def export_agent(
"""把单个子智能体导出为 frontmatter markdown 文件(正文即 system prompt),
对齐 Claude Code / pi agent 一类 harness 的 subagent 文件格式,可直接重新导入。
"""
- from core.llm.builtin_subagents import list_builtin_subagents
+ from core.capabilities.agents.builtin import list_builtin_subagents
agent = next(
(
@@ -193,7 +193,7 @@ async def export_agent(
@router.get("/{agent_id}", summary="子智能体详情")
-async def get_agent(
+def get_agent(
agent_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -208,7 +208,7 @@ async def get_agent(
@router.post("", summary="创建个人子智能体")
-async def create_agent(
+def create_agent(
body: AgentCreateRequest,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -229,7 +229,7 @@ async def create_agent(
@router.put("/{agent_id}", summary="更新个人子智能体")
-async def update_agent(
+def update_agent(
agent_id: str,
body: AgentUpdateRequest,
user: UserContext = Depends(get_current_user),
@@ -238,7 +238,7 @@ async def update_agent(
data = body.model_dump(exclude_none=True)
# 云端同步来的智能体登记在本机登记表里,本机业务库没有对应行——不先拦下来,
# 卡片上的开关会打成 404。本机只允许改启停,定义仍以云端那份为准。
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
projected = device_catalog.toggle_projected_agent(agent_id, data)
if projected is not None:
@@ -260,7 +260,7 @@ async def update_agent(
@router.delete("/{agent_id}", summary="删除个人子智能体")
-async def delete_agent(
+def delete_agent(
agent_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
diff --git a/src/backend/api/routes/__init__.py b/src/backend/api/routes/__init__.py
index ddad8534..01e58b53 100644
--- a/src/backend/api/routes/__init__.py
+++ b/src/backend/api/routes/__init__.py
@@ -1,10 +1,17 @@
"""API routes.
v1 routes are registered via the edition registry in ``api.routes.v1`` (seam C1);
-this package only keeps the re-export of non-v1 file download routes.
+this package only keeps the re-export of non-v1 file download routes, resolved
+lazily so that importing a sibling module (``api.routes.file_location``) does not
+load ``files`` and the feature routers it mounts.
"""
-from .files import router as files_router
-from .sites_serve import router as sites_serve_router
+__all__ = ["files_router"]
-__all__ = ["files_router", "sites_serve_router"]
+
+def __getattr__(name):
+ if name == "files_router":
+ from api.routes.files import router
+
+ return router
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/src/backend/api/routes/file_archive.py b/src/backend/api/routes/file_archive.py
index dac000d5..aeab0bda 100644
--- a/src/backend/api/routes/file_archive.py
+++ b/src/backend/api/routes/file_archive.py
@@ -6,28 +6,24 @@
import tempfile
from urllib.parse import quote
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request
from fastapi.responses import Response
from sqlalchemy.orm import Session
from core.auth.backend import UserContext, get_current_user
-from core.content.archive_browser import (
- ArchiveError,
- MAX_ARCHIVE_BYTES,
- inspect_archive,
- read_archive_member,
-)
+from core.files.content.archive_browser import ArchiveError, MAX_ARCHIVE_BYTES, inspect_archive, read_archive_member
from core.db.engine import get_db
+from core.infra.byte_range import byte_range_response, is_continuation_range
from core.infra.exceptions import StorageError
from core.infra.responses import success_response
-from core.storage import get_storage
-from core.storage.local import LocalStorageBackend
+from core.files.storage import get_storage
+from core.files.storage.local import LocalStorageBackend
router = APIRouter()
@contextmanager
-def _source(file_id, user, db):
+def _source(file_id, user, db, *, audit: bool = True):
from api.routes.files import _authorize_access, _load_artifact_item, _record_audit
item = _load_artifact_item(file_id, db)
@@ -59,6 +55,8 @@ def _source(file_id, user, db):
raise HTTPException(exc.status, detail=exc.code) from exc
except (OSError, KeyError, StorageError) as exc:
raise HTTPException(503, detail="archive_read_failed") from exc
+ if not audit:
+ return
_record_audit(
user=user,
db=db,
@@ -83,13 +81,16 @@ def list_archive(
@router.get("/{file_id}/archive/member", summary="读取压缩包内文件")
def archive_member(
+ request: Request,
file_id: str,
path: str = Query(min_length=1, max_length=1024),
inline: bool = False,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- with _source(file_id, user, db) as (source, name, _):
+ range_header = request.headers.get("range")
+ # A player's mid-file seeks re-read the member; audit only the first read.
+ with _source(file_id, user, db, audit=not is_continuation_range(range_header)) as (source, name, _):
data = read_archive_member(source, name, path)
filename = PurePosixPath(path).name
mime = mimetypes.guess_type(filename)[0] or "application/octet-stream"
@@ -104,8 +105,9 @@ def archive_member(
if mime in ("text/html", "image/svg+xml"):
mime = "text/plain"
safe_inline = inline
- return Response(
+ return byte_range_response(
data,
+ range_header,
media_type=mime,
headers={
"Cache-Control": "no-store",
diff --git a/src/backend/api/routes/file_location.py b/src/backend/api/routes/file_location.py
index 06a1911f..531aa629 100644
--- a/src/backend/api/routes/file_location.py
+++ b/src/backend/api/routes/file_location.py
@@ -5,7 +5,7 @@
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.db.models import Artifact
-from core.services.artifact_location import artifact_location
+from core.files.artifacts.location import artifact_location
router = APIRouter()
diff --git a/src/backend/api/routes/files.py b/src/backend/api/routes/files.py
index 447e28cc..d348ae0e 100644
--- a/src/backend/api/routes/files.py
+++ b/src/backend/api/routes/files.py
@@ -8,33 +8,41 @@
from pathlib import Path
from typing import Any, Optional
-from core.artifacts.store import get_artifact
+from core.files.artifacts.store import get_artifact
from core.auth.backend import UserContext, require_auth
-from core.content.office import find_libreoffice_binary
-from core.content.office_preview_cache import cached_office_pdf
+from core.files.content.office import find_libreoffice_binary
+from core.files.content.office_preview_cache import cached_office_pdf
from core.config.local_mode import local_mode_enabled
-from core.storage.local import LocalStorageBackend
+from core.files.storage.local import LocalStorageBackend
from core.db.engine import get_db
from core.db.repository import AuditLogRepository
from core.infra.exceptions import StorageError
from core.infra.proc import no_window_kwargs
-from core.services.artifact_edition import artifact_access_metadata, can_access_artifact_metadata
-from core.storage import get_storage
+from core.files.edition import artifact_access_metadata, can_access_artifact_metadata
+from core.files.storage import get_storage
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
from fastapi.responses import FileResponse, JSONResponse
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
from api.routes.file_archive import router as archive_router
-from api.routes.local_file_actions import router as local_actions_router
+from features.desktop.api.local_file_actions import router as local_actions_router
router = APIRouter(prefix="/files", tags=["files"])
router.include_router(archive_router)
router.include_router(local_actions_router)
-POWERPOINT_EXTENSIONS = {".ppt", ".pptx"}
-POWERPOINT_MIME_MARKERS = ("presentationml", "powerpoint")
-WORD_EXTENSIONS = {".doc", ".docx"}
-WORD_MIME_MARKERS = ("wordprocessingml", "msword")
+# Documents rendered to PDF by LibreOffice for browser preview: legacy binary
+# Word/PowerPoint, OOXML, OpenDocument text/presentation and RTF.
+OFFICE_PREVIEW_EXTENSIONS = {".doc", ".docx", ".rtf", ".odt", ".ppt", ".pptx", ".odp"}
+OFFICE_PREVIEW_MIME_MARKERS = (
+ "wordprocessingml",
+ "msword",
+ "presentationml",
+ "powerpoint",
+ "opendocument.text",
+ "opendocument.presentation",
+ "application/rtf",
+)
def _load_artifact_item(file_id: str, db: Session) -> dict[str, Any]:
@@ -208,28 +216,16 @@ def _prepare_local_file(
return str(temp_path)
-def _is_powerpoint_file(item: dict[str, Any]) -> bool:
- name = str(item.get("name", ""))
+def _is_office_previewable(item: dict[str, Any]) -> bool:
+ ext = Path(str(item.get("name", ""))).suffix.lower()
mime_type = str(item.get("mime_type", "")).lower()
- ext = Path(name).suffix.lower()
- return ext in POWERPOINT_EXTENSIONS or any(
- marker in mime_type for marker in POWERPOINT_MIME_MARKERS
+ return ext in OFFICE_PREVIEW_EXTENSIONS or any(
+ marker in mime_type for marker in OFFICE_PREVIEW_MIME_MARKERS
)
-def _is_word_file(item: dict[str, Any]) -> bool:
- name = str(item.get("name", ""))
- mime_type = str(item.get("mime_type", "")).lower()
- ext = Path(name).suffix.lower()
- return ext in WORD_EXTENSIONS or any(marker in mime_type for marker in WORD_MIME_MARKERS)
-
-
-def _is_office_previewable(item: dict[str, Any]) -> bool:
- return _is_powerpoint_file(item) or _is_word_file(item)
-
-
def _convert_office_to_pdf(source_path: str, file_id: str) -> tuple[str, str]:
- """Render an Office document (PPT/PPTX/DOC/DOCX) to PDF via headless LibreOffice."""
+ """Render an Office document (DOC/DOCX/RTF/ODT/PPT/PPTX/ODP) to PDF via headless LibreOffice."""
libreoffice = find_libreoffice_binary()
if not libreoffice:
raise RuntimeError(
@@ -421,7 +417,7 @@ def preview_file(
user: Optional[UserContext] = Depends(require_auth(required=False)),
db: Session = Depends(get_db),
):
- """将 Office 文件(PPT/PPTX/DOC/DOCX)经 LibreOffice 转为可在浏览器内联预览的 PDF。
+ """将 Office 文件(DOC/DOCX/RTF/ODT/PPT/PPTX/ODP)经 LibreOffice 转为可在浏览器内联预览的 PDF。
需通过文件访问鉴权;目前 format 仅支持 pdf。
"""
@@ -439,7 +435,7 @@ def preview_file(
if not _is_office_previewable(item):
raise HTTPException(
status_code=400,
- detail="Only Word (DOC/DOCX) and PowerPoint (PPT/PPTX) files support preview",
+ detail="Only Word (DOC/DOCX/RTF/ODT) and PowerPoint (PPT/PPTX/ODP) files support preview",
)
source_path = _prepare_local_file(
diff --git a/src/backend/api/routes/v1/__init__.py b/src/backend/api/routes/v1/__init__.py
index b941b623..6bffc440 100644
--- a/src/backend/api/routes/v1/__init__.py
+++ b/src/backend/api/routes/v1/__init__.py
@@ -12,7 +12,7 @@
("users", "router"),
("user_preferences", "router"),
("catalog", "router"),
- ("kb", "router"),
+ ("features.knowledge.api.kb", "router"),
("summary", "router"),
("classify", "router"),
("config", "router"),
@@ -28,13 +28,13 @@
("agent_responses", "router"),
("agent_api_keys", "router"),
("project_api_keys", "router"),
- ("agents", "router"),
+ ("api.edition", "router"),
("artifacts", "router"),
("space_files", "router"),
("space_previews", "router"),
("plans", "router"),
("loops", "router"),
- ("automations", "router"),
+ ("features.automation.api.automations", "router"),
("chat_runs", "router"),
("me_system", "router"),
("me_logs", "router"),
@@ -45,42 +45,42 @@
("internal_jobs", "router"),
("jobs", "router"),
("commands", "router"),
- ("internal_sites", "router"),
- ("applications", "router"),
- ("workbench", "router"),
- ("application_transports", "internal_router"),
- ("application_transports", "public_router"),
- ("application_transports", "mcp_router"),
+ ("features.sites.api.internal_sites", "router"),
+ ("features.hosted_apps.api.applications", "router"),
+ ("features.workbench.api.workbench", "router"),
+ ("features.hosted_apps.api.application_transports", "internal_router"),
+ ("features.hosted_apps.api.application_transports", "public_router"),
+ ("features.hosted_apps.api.application_transports", "mcp_router"),
("project_environments", "router"),
("projects", "router"),
("api_keys", "router"),
("me_capabilities", "router"),
- ("marketplace", "router"),
- ("mcp_marketplace", "router"),
- ("agent_marketplace", "router"),
- ("plugin_ui", "router"),
+ ("features.marketplace.api.marketplace", "router"),
+ ("features.marketplace.api.mcp_marketplace", "router"),
+ ("features.marketplace.api.agent_marketplace", "router"),
+ ("features.workbench.api.plugin_ui", "router"),
("plugin_resources", "router"),
("plugins", "router"),
- ("integrations", "router"),
- ("channels", "router"),
- ("channel_desktop", "router"),
+ ("integrations.api.integrations", "router"),
+ ("integrations.channels.api.channels", "router"),
+ ("integrations.channels.api.channel_desktop", "router"),
("meta", "router"),
# 个人进化的证据面与偏好开关(结算查询 / prefs / 个人候选审批)。CE 完整
# 可用;控制面(管理员审批/发布)在 EE。此前 CE 漏挂本路由,首启向导的
# PATCH /v1/evolution/prefs 落到桌面本机 server 的 GET catch-all 上,
# 「启动进化」直接 405。
("evolution", "router"),
- ("sites", "router"),
- ("desktop", "router"),
- ("desktop_login", "router"),
+ ("features.sites.api.sites", "router"),
+ ("features.desktop.api.desktop", "router"),
+ ("features.desktop.api.desktop_login", "router"),
# The desktop shell bridge and device capability management are CE runtime APIs.
- ("desktop_capability", "router"),
- ("desktop_observability", "router"),
- ("desktop_capabilities", "router"),
+ ("features.desktop.api.desktop_capability", "router"),
+ ("features.desktop.api.desktop_observability", "router"),
+ ("features.desktop.api.desktop_capabilities", "router"),
# This computer's link to a Personal Agent on the cloud it signed in to.
- ("device_link", "router"),
- ("local", "router"),
- ("local_site_sources", "router"),
+ ("features.desktop.api.device_link", "router"),
+ ("features.desktop.api.local", "router"),
+ ("features.sites.api.local_site_sources", "router"),
# 工具执行权限档(标准 / 只读 / 自动)——输入框工具栏那颗权限胶囊的数据源
("tool_approval", "router"),
)
@@ -91,7 +91,8 @@
def iter_edition_routers(entries):
for entry in entries:
module_name, attr, *feature_items = entry
- module = import_module(f"{__name__}.{module_name}")
+ module_path = module_name if "." in module_name else f"{__name__}.{module_name}"
+ module = import_module(module_path)
feature = feature_items[0] if feature_items else None
yield module_name, getattr(module, attr), feature
diff --git a/src/backend/api/routes/v1/agent_api_keys.py b/src/backend/api/routes/v1/agent_api_keys.py
index fcfeaefe..8790488f 100644
--- a/src/backend/api/routes/v1/agent_api_keys.py
+++ b/src/backend/api/routes/v1/agent_api_keys.py
@@ -9,9 +9,10 @@
from core.auth.backend import UserContext, get_current_user
from core.auth.capabilities import resolve_user_capabilities
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import created_response, paginated_response, success_response
-from core.services.agent_api_service import list_agent_api_calls, require_agent_manager
-from core.services.api_key_service import ApiKeyService
+from core.auth.agent_api import list_agent_api_calls, require_agent_manager
+from core.auth.api_keys import ApiKeyService
router = APIRouter(prefix="/v1/agents", tags=["Agent API Keys"])
_ALLOWED_EXPIRY_DAYS = {7, 30, 90, 180, 365}
@@ -134,15 +135,16 @@ async def toggle_key(
key_id: str,
body: ToggleAgentApiKeyRequest,
user: UserContext = Depends(_manager),
- db: Session = Depends(get_db),
):
- _find(db, user, agent_id, key_id)
- row = ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, agent_id)
- data = _dto(row)
+ def toggle(db):
+ _find(db, user, agent_id, key_id)
+ return _dto(ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, agent_id))
+
+ data = await run_in_session(toggle)
if not body.enabled:
- from core.services.api_key_runs import stop_cancelled_key_workers
+ from core.auth.api_key_runs import stop_cancelled_key_workers
- await stop_cancelled_key_workers(db, key_id, user.user_id)
+ await stop_cancelled_key_workers(key_id, user.user_id)
return success_response(data=data)
@@ -151,13 +153,15 @@ async def revoke_key(
agent_id: str,
key_id: str,
user: UserContext = Depends(_manager),
- db: Session = Depends(get_db),
):
- _find(db, user, agent_id, key_id)
- ApiKeyService(db).revoke_key(user.user_id, key_id, agent_id)
- from core.services.api_key_runs import stop_cancelled_key_workers
+ def revoke(db):
+ _find(db, user, agent_id, key_id)
+ ApiKeyService(db).revoke_key(user.user_id, key_id, agent_id)
+
+ await run_in_session(revoke)
+ from core.auth.api_key_runs import stop_cancelled_key_workers
- await stop_cancelled_key_workers(db, key_id, user.user_id)
+ await stop_cancelled_key_workers(key_id, user.user_id)
return success_response(data={"id": key_id, "revoked": True})
diff --git a/src/backend/api/routes/v1/agent_responses.py b/src/backend/api/routes/v1/agent_responses.py
index c9e70daa..c6080a25 100644
--- a/src/backend/api/routes/v1/agent_responses.py
+++ b/src/backend/api/routes/v1/agent_responses.py
@@ -1,18 +1,18 @@
"""One durable admission path for streaming and complete agent responses."""
import asyncio
-from core.services.session_operations.admission import _start_response_run
+import api.transports.run_sse as run_sse
import core.chat.context as chat_context
-import core.db.engine as db_engine
-import core.services as chat_services
-from core.chat.schemas import ChatRequest, ChatResponse
+import core.runs.runtime as runtime
+import core.chat.service as chat_services_ChatService_module
from core.auth.backend import UserContext, get_current_user
-from core.db.engine import get_db
+from core.chat.schemas import ChatRequest, ChatResponse
from core.infra.logging import get_logger
from core.infra.responses import sse_response
+from core.chat.operations.admission import start_response_run as _start_response_run
+from core.db.threads import run_in_session
from fastapi import APIRouter, Depends, HTTPException
-from orchestration import chat_run_executor
from sqlalchemy.orm import Session
logger = get_logger(__name__)
@@ -20,7 +20,7 @@
async def _wait_response(run_id: str, *, chat_id: str, message_id: str) -> ChatResponse:
- terminal = await chat_run_executor.wait_run(run_id)
+ terminal = await runtime.wait_run(run_id)
if terminal.status == "cancelled":
raise HTTPException(status_code=409, detail={"code": "run_cancelled", "run_id": run_id})
if terminal.status == "needs_attention":
@@ -34,21 +34,24 @@ async def _wait_response(run_id: str, *, chat_id: str, message_id: str) -> ChatR
RuntimeError(terminal.error_message or "chat run failed")
),
)
- with db_engine.SessionLocal() as db:
- assistant = chat_services.ChatService(db).get_message_by_id(message_id)
- if assistant is None:
- raise HTTPException(status_code=500, detail="已完成的任务缺少回复")
- meta = assistant.extra_data or {}
- return ChatResponse(
- chat_id=chat_id,
- response=assistant.content,
- timestamp=chat_context.now_iso(),
- is_markdown=bool(meta.get("is_markdown", False)),
- route=meta.get("route", "main"),
- sources=meta.get("sources", []),
- artifacts=meta.get("artifacts", []),
- warnings=meta.get("warnings", []),
- )
+ return await run_in_session(_completed_response, chat_id, message_id)
+
+
+def _completed_response(db: Session, chat_id: str, message_id: str) -> ChatResponse:
+ assistant = chat_services_ChatService_module.ChatService(db).get_message_by_id(message_id)
+ if assistant is None:
+ raise HTTPException(status_code=500, detail="已完成的任务缺少回复")
+ meta = assistant.extra_data or {}
+ return ChatResponse(
+ chat_id=chat_id,
+ response=assistant.content,
+ timestamp=chat_context.now_iso(),
+ is_markdown=bool(meta.get("is_markdown", False)),
+ route=meta.get("route", "main"),
+ sources=meta.get("sources", []),
+ artifacts=meta.get("artifacts", []),
+ warnings=meta.get("warnings", []),
+ )
@router.post(
@@ -65,16 +68,15 @@ async def _wait_response(run_id: str, *, chat_id: str, message_id: str) -> ChatR
async def agent_response(
request: ChatRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""同一请求通过 stream 选择完整 JSON 或 SSE;两者共享后台 Run 和权限校验。
子智能体请求须显式传入 agent_id;专属 API Key 只能调用绑定的目标和 API 会话。SSE 断线可通过
GET /v1/chats/stream/{run_id}?from=N 续播;HTTP 断开不会取消后台任务。
"""
- run = await _start_response_run(request, user, db)
+ run = await _start_response_run(request, user)
if request.stream:
- response = sse_response(chat_run_executor.follow_run_as_sse(run.run_id, chat_id=run.chat_id))
+ response = sse_response(run_sse.follow_run_as_sse(run.run_id, chat_id=run.chat_id))
# This is the user row committed by admission, available before any
# stream frames so clients can reconcile their optimistic message.
response.headers["x-hugagent-user-message-id"] = run.user_message_id
@@ -90,6 +92,6 @@ async def agent_response(
async def _record_http_status(user: UserContext, run_id: str, status_code: int) -> None:
if user.api_key_agent_id:
- from core.services.agent_api_service import record_agent_api_http_status
+ from core.auth.agent_api import record_agent_api_http_status
await asyncio.to_thread(record_agent_api_http_status, run_id, status_code)
diff --git a/src/backend/api/routes/v1/api_keys.py b/src/backend/api/routes/v1/api_keys.py
index 539020e2..2bc690f1 100644
--- a/src/backend/api/routes/v1/api_keys.py
+++ b/src/backend/api/routes/v1/api_keys.py
@@ -14,7 +14,7 @@
from core.db.engine import get_db
from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
from core.infra.responses import created_response, success_response
-from core.services import ApiKeyService
+from core.auth.api_keys import ApiKeyService
from fastapi import APIRouter, Depends, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -59,7 +59,7 @@ def _key_to_dict(row, *, plaintext: Optional[str] = None) -> dict:
@router.get("", summary="API-Key 列表")
-async def list_api_keys(
+def list_api_keys(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
@@ -69,7 +69,7 @@ async def list_api_keys(
@router.post("", status_code=status.HTTP_201_CREATED, summary="新建 API-Key")
-async def create_api_key(
+def create_api_key(
body: CreateApiKeyRequest,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -90,7 +90,7 @@ async def create_api_key(
@router.get("/{key_id}/reveal", summary="再次取回 API-Key 明文")
-async def reveal_api_key(
+def reveal_api_key(
key_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -117,7 +117,7 @@ async def reveal_api_key(
@router.patch("/{key_id}", summary="启用/禁用 API-Key")
-async def toggle_api_key(
+def toggle_api_key(
key_id: str,
body: ToggleApiKeyRequest,
user: UserContext = Depends(get_current_user),
@@ -131,7 +131,7 @@ async def toggle_api_key(
@router.delete("/{key_id}", summary="撤销 API-Key")
-async def revoke_api_key(
+def revoke_api_key(
key_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
diff --git a/src/backend/api/routes/v1/artifacts.py b/src/backend/api/routes/v1/artifacts.py
index 3fe2477c..44e9f5e5 100644
--- a/src/backend/api/routes/v1/artifacts.py
+++ b/src/backend/api/routes/v1/artifacts.py
@@ -14,20 +14,17 @@
from typing import Any, Dict, List, Optional, Tuple
from core.auth.backend import UserContext, get_current_user
-from core.content.kb_processing import vectorise_document_background
+from features.knowledge.processing import vectorise_document_background
from core.db.engine import SessionLocal, get_db
-from core.db.models import Artifact, ChatMessage, ChatSession, KBDocument, KBSpace, UserShadow
+from core.db.threads import run_in_session
+from core.db.models import Artifact, ChatMessage, ChatSession, UserShadow
+from features.knowledge.models import KBDocument, KBSpace
from core.db.paging import DEFAULT_PAGE_SIZE
from core.db.repository import ArtifactRepository
from core.infra.responses import error_response, success_response
-from core.services import KBService
-from core.services.artifact_edition import (
- artifact_list_scope,
- artifact_scope_fields,
- can_access_artifact,
- extend_artifact_item,
-)
-from core.storage import get_storage
+from features.knowledge.service import KBService
+from core.files.edition import artifact_list_scope, artifact_scope_fields, can_access_artifact, extend_artifact_item
+from core.files.storage import get_storage
from fastapi import APIRouter, BackgroundTasks, Depends, Query
from pydantic import BaseModel
from sqlalchemy import desc, func
@@ -60,7 +57,7 @@ def _backfill_already_done(db: Session, user_id: str) -> bool:
def _mark_backfill_done(db: Session, user_id: str) -> None:
- from core.services.user_service import UserService
+ from core.auth.users import UserService
UserService(db).update_user_metadata(
user_id, {_BACKFILL_MARKER: datetime.now(timezone.utc).isoformat()}
@@ -104,9 +101,9 @@ class AddArtifactToKBRequest(BaseModel):
# ── Shared artifact-ref helpers ───────────────────────────────────────────
-# Moved to core.content.artifact_refs so lower layers can use them without
+# Moved to core.files.artifacts.refs so lower layers can use them without
# importing this API route module. Re-exported here for existing call sites.
-from core.content.artifact_refs import extract_file_refs, infer_artifact_type # noqa: E402
+from core.files.artifacts.refs import extract_file_refs, infer_artifact_type # noqa: E402
def sanitize_chat_preview(content: Optional[str], max_len: int = 200) -> str:
@@ -140,7 +137,7 @@ def sanitize_chat_preview(content: Optional[str], max_len: int = 200) -> str:
# ── Backfill (runs once per user, ever — see the durable marker above) ────
-from core.services.artifact_backfill import _backfill_artifacts_from_messages
+from core.files.artifacts.backfill import backfill_artifacts_from_messages as _backfill_artifacts_from_messages
def _collect_artifact_kb_usage(
@@ -263,36 +260,18 @@ def list_favorite_chats(
)
-@router.get("", summary="用户文件/图片列表")
-async def list_user_artifacts(
- type: Optional[str] = Query(None, description="document | image"),
- source_kind: Optional[str] = Query(None, description="user_upload | ai_generated"),
- keyword: Optional[str] = Query(None, description="文件名搜索"),
- scope: str = Depends(artifact_list_scope),
- folder_id: Optional[str] = Query(
- None,
- description="仅 personal scope 生效:__root__=个人根目录,=该个人文件夹直接子文件,省略=全部个人文件(向后兼容)",
- ),
- page: int = Query(1, ge=1),
- page_size: int = Query(DEFAULT_PAGE_SIZE, ge=1, description="每页条数,无上限"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
+def _artifact_page(
+ db: Session,
+ uid: str,
+ *,
+ type: Optional[str],
+ source_kind: Optional[str],
+ keyword: Optional[str],
+ scope: str,
+ folder_id: Optional[str],
+ page: int,
+ page_size: int,
):
- """获取当前用户有权查看的文件与图片。"""
- uid = str(user.user_id)
-
- # One-time backfill for historical data. It scans every message the account ever
- # wrote, so it never runs inline — doing that on the event loop froze the whole
- # backend for minutes. It also must not go through ``BackgroundTasks``: those run
- # on the shared request threadpool, which every endpoint needs via ``get_db``
- # (see the postmortem at the top of core/kb/index_queue.py). Detached to the
- # default executor instead. The rows it adds are
- # historical, so the next listing picks them up.
- if uid not in _backfilled_users:
- task = asyncio.create_task(asyncio.to_thread(_run_backfill_once, uid))
- _backfill_tasks.add(task)
- task.add_done_callback(_backfill_tasks.discard)
-
repo = ArtifactRepository(db)
mime_prefix = None
if type == "image":
@@ -345,6 +324,49 @@ async def list_user_artifacts(
}
items.append(extend_artifact_item(artifact, item))
+ return items, total
+
+
+@router.get("", summary="用户文件/图片列表")
+async def list_user_artifacts(
+ type: Optional[str] = Query(None, description="document | image"),
+ source_kind: Optional[str] = Query(None, description="user_upload | ai_generated"),
+ keyword: Optional[str] = Query(None, description="文件名搜索"),
+ scope: str = Depends(artifact_list_scope),
+ folder_id: Optional[str] = Query(
+ None,
+ description="仅 personal scope 生效:__root__=个人根目录,=该个人文件夹直接子文件,省略=全部个人文件(向后兼容)",
+ ),
+ page: int = Query(1, ge=1),
+ page_size: int = Query(DEFAULT_PAGE_SIZE, ge=1, description="每页条数,无上限"),
+ user: UserContext = Depends(get_current_user),
+):
+ """获取当前用户有权查看的文件与图片。"""
+ uid = str(user.user_id)
+
+ # One-time backfill for historical data. It scans every message the account ever
+ # wrote, so it never runs inline — doing that on the event loop froze the whole
+ # backend for minutes. It also must not go through ``BackgroundTasks``: those run
+ # on the shared request threadpool, which every endpoint needs via ``get_db``
+ # (see the postmortem at the top of core/kb/index_queue.py). Detached to the
+ # default executor instead. The rows it adds are
+ # historical, so the next listing picks them up.
+ if uid not in _backfilled_users:
+ task = asyncio.create_task(asyncio.to_thread(_run_backfill_once, uid))
+ _backfill_tasks.add(task)
+ task.add_done_callback(_backfill_tasks.discard)
+
+ items, total = await run_in_session(
+ _artifact_page,
+ uid,
+ type=type,
+ source_kind=source_kind,
+ keyword=keyword,
+ scope=scope,
+ folder_id=folder_id,
+ page=page,
+ page_size=page_size,
+ )
return success_response(
data={
"items": items,
diff --git a/src/backend/api/routes/v1/audio.py b/src/backend/api/routes/v1/audio.py
index f5a36741..33a4c641 100644
--- a/src/backend/api/routes/v1/audio.py
+++ b/src/backend/api/routes/v1/audio.py
@@ -9,8 +9,9 @@
from core.auth.capabilities import resolve_user_capabilities
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.services.audio_transcription import MAX_BYTES, MAX_SECONDS, resolve_audio_model, transcribe, validate_wav
+from core.chat.audio_transcription import MAX_BYTES, MAX_SECONDS, resolve_audio_model, transcribe, validate_wav
router = APIRouter(prefix="/v1/audio", tags=["Audio"])
# Per-worker admission, no unbounded queue. Every call enters before its first await.
@@ -47,11 +48,13 @@ async def _until_disconnect(request: Request):
@router.post("/transcriptions")
-async def audio_transcription(request: Request,
- user: UserContext = Depends(require_auth()), db: Session = Depends(get_db)):
- if not _can_use_voice(user, db):
- raise HTTPException(403, "未开通语音输入权限,请联系管理员")
- model = resolve_audio_model(db)
+async def audio_transcription(request: Request, user: UserContext = Depends(require_auth())):
+ def admit(db: Session):
+ if not _can_use_voice(user, db):
+ raise HTTPException(403, "未开通语音输入权限,请联系管理员")
+ return resolve_audio_model(db)
+
+ model = await run_in_session(admit)
if model is None:
raise HTTPException(503, "尚未配置音频转写模型,请管理员在模型管理中配置")
user_key = f"{user.user_id}:{user.public_chat_visitor or ''}"
diff --git a/src/backend/api/routes/v1/auth.py b/src/backend/api/routes/v1/auth.py
index 597e0ff3..97d63acf 100644
--- a/src/backend/api/routes/v1/auth.py
+++ b/src/backend/api/routes/v1/auth.py
@@ -8,7 +8,7 @@
from typing import Optional
-from core.auth import desktop_ticket_store
+from features.desktop.auth import ticket_store as desktop_ticket_store
from core.auth.capabilities import page_admin_flags, resolve_capabilities
from core.auth.mock_ticket_store import consume_ticket
from core.auth.session import (
@@ -20,11 +20,11 @@
validate_session,
)
from core.config.settings import settings
-from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.logging import get_logger
from core.infra.responses import success_response
-from core.services import UserService
-from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from core.auth.users import UserService
+from fastapi import APIRouter, HTTPException, Request, Response
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -54,7 +54,7 @@ def _clear_session_cookie(response: Response) -> None:
def _serialize_user(db: Session, user_data: dict, ttl_seconds: Optional[int] = None) -> dict:
from core.db.models import LocalUser, UserShadow
- from core.services.local_user_service import ce_onboarding_required
+ from core.auth.local_user_policy import ce_onboarding_required
user_id = user_data["user_id"]
shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first()
@@ -80,32 +80,14 @@ def _serialize_user(db: Session, user_data: dict, ttl_seconds: Optional[int] = N
}
-@router.post("/ticket/exchange", summary="本地登录票据换取会话")
-async def ticket_exchange(
- body: TicketExchangeRequest,
- response: Response,
- db: Session = Depends(get_db),
-):
- credential = (body.code or "").strip()
- user_info = await consume_ticket(credential) if credential else None
- if user_info is None:
- raise HTTPException(
- status_code=401,
- detail={
- "code": 30002,
- "message": "Invalid or expired ticket",
- "data": {"login_url": _login_url()},
- },
- )
-
+def _bind_shadow(db: Session, user_info: dict) -> dict:
shadow = UserService(db).get_or_create_user_shadow(
user_center_id=user_info["user_center_id"],
username=user_info["username"],
email=user_info.get("email"),
avatar_url=user_info.get("avatar_url"),
)
- ttl_seconds = remember_ttl_seconds() if user_info.get("remember") else None
- session_data = {
+ return {
"user_id": shadow.user_id,
"user_center_id": shadow.user_center_id,
"username": shadow.username,
@@ -114,10 +96,28 @@ async def ticket_exchange(
"nickname": user_info.get("nickname"),
"real_name": user_info.get("real_name"),
}
+
+
+@router.post("/ticket/exchange", summary="本地登录票据换取会话")
+async def ticket_exchange(body: TicketExchangeRequest, response: Response):
+ credential = (body.code or "").strip()
+ user_info = await consume_ticket(credential) if credential else None
+ if user_info is None:
+ raise HTTPException(
+ status_code=401,
+ detail={
+ "code": 30002,
+ "message": "Invalid or expired ticket",
+ "data": {"login_url": _login_url()},
+ },
+ )
+
+ session_data = await run_in_session(_bind_shadow, user_info)
+ ttl_seconds = remember_ttl_seconds() if user_info.get("remember") else None
token = await create_session(session_data, ttl_seconds=ttl_seconds)
_set_session_cookie(response, token, ttl_seconds)
return success_response(
- data=_serialize_user(db, session_data, ttl_seconds),
+ data=await run_in_session(_serialize_user, session_data, ttl_seconds),
message="Login successful",
)
@@ -128,7 +128,7 @@ async def sso_authorize_url():
@router.get("/session/check", summary="检查当前会话状态")
-async def session_check(request: Request, db: Session = Depends(get_db)):
+async def session_check(request: Request):
token = request.cookies.get(settings.session.cookie_name)
user_data = await validate_session(token) if token else None
if user_data is None:
@@ -140,7 +140,9 @@ async def session_check(request: Request, db: Session = Depends(get_db)):
"data": {"login_url": _login_url()},
},
)
- return success_response(data=_serialize_user(db, user_data), message="Session valid")
+ return success_response(
+ data=await run_in_session(_serialize_user, user_data), message="Session valid"
+ )
@router.post("/desktop/handoff", summary="桌面端签发一次性会话交接票据")
diff --git a/src/backend/api/routes/v1/catalog.py b/src/backend/api/routes/v1/catalog.py
index e129d5f4..ed0ecbcb 100644
--- a/src/backend/api/routes/v1/catalog.py
+++ b/src/backend/api/routes/v1/catalog.py
@@ -8,19 +8,14 @@
from typing import Any, Dict, List, Optional
from core.auth.backend import UserContext, get_current_user
-from core.config.catalog_runtime import get_runtime_catalog
+from core.capabilities.catalog.runtime import get_runtime_catalog
from core.config.settings import settings
-from core.db.engine import get_db
-from core.db.repository import KBRepository
+from core.db.threads import run_in_session
+from features.knowledge.repository import KBRepository
from core.infra.exceptions import BadRequestError
from core.infra.responses import success_response
-from core.kb.external_provider import (
- get_provider_cache_identity,
- get_provider_name,
- is_enabled,
- list_collections,
-)
-from core.services import CatalogService
+from features.knowledge.edition import get_provider_cache_identity, get_provider_name, is_enabled, list_collections
+from core.capabilities.catalog.overrides import CatalogService
from fastapi import APIRouter, Depends, Path
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -162,7 +157,7 @@ async def _ensure_external_cache_ready() -> None:
def _index_modes_of(space) -> List[str]:
"""自建知识库的索引模式;解析失败按仅 RAG(历史库的语义)。"""
try:
- from core.kb.wiki.config import index_modes_of
+ from features.knowledge.wiki.config import index_modes_of
return index_modes_of(space)
except Exception: # noqa: BLE001
@@ -172,14 +167,14 @@ def _index_modes_of(space) -> List[str]:
def _kb_capabilities(space) -> Dict[str, bool]:
"""索引模式 → 能力位,字段与外接知识库的 capabilities 同构。"""
try:
- from core.kb.wiki_router import capabilities_for_local_kb
+ from features.knowledge.wiki_router import capabilities_for_local_kb
return capabilities_for_local_kb(space)
except Exception: # noqa: BLE001
return {"vector": True, "keyword": True, "wiki": False, "graph": False}
-from .catalog_ownership import _load_owned_capability_items, _plugin_component_ids, _is_owned_capability
+from features.marketplace.api.catalog_ownership import _load_owned_capability_items, _plugin_component_ids, _is_owned_capability
def _get_catalog_items_sync(user: UserContext, db: Session):
@@ -199,7 +194,7 @@ def _get_catalog_items_sync(user: UserContext, db: Session):
# invisible/unusable. The catalog itself queries the DB directly and does not use this
# cache, so this call only affects agent resolution and is harmless to display.
try:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(str(user.user_id))
except Exception as exc: # noqa: BLE001
@@ -269,7 +264,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict
# global version — otherwise the capability library would show two identically named,
# identical-content items (private id=entry-fingerprint, global id=entry, which differ
# so both entered the list). Merge by entry_name; dedupe only against enabled global items.
- from core.services.marketplace_service import base_entry_name
+ from core.capabilities.skills.installation import base_entry_name as base_entry_name
for kind, owned in (("skills", owned_skill_items), ("mcp", owned_mcp_items)):
global_ids = {it.get("id") for it in base_catalog.get(kind, []) if it.get("enabled", True)}
@@ -288,7 +283,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict
try:
external_items = _list_datasets_cached()
# Permission assignment: narrow to datasets visible to the current user (public + granted scoped; defaults to public when unset).
- from core.auth.kb_permissions import get_dataset_levels
+ from features.knowledge.edition import get_dataset_levels
ds_ids = [str(item.get("id", "")).strip() for item in external_items if item.get("id")]
ds_levels = get_dataset_levels(db, user.user_id, ds_ids)
@@ -363,8 +358,8 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict
public_local_kb_items: List[Dict[str, Any]] = []
if not is_ce:
try:
- from core.auth.kb_permissions import get_accessible_local_kb_levels, level_to_caps
- from core.db.models import KBSpace
+ from features.knowledge.edition import get_accessible_local_kb_levels, level_to_caps
+ from features.knowledge.models import KBSpace
levels = get_accessible_local_kb_levels(db, user.user_id)
owned_ids = {s.kb_id for s in spaces}
@@ -432,7 +427,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict
# 桌面双端的本机后端:云端账号的技能与连接器同步在本机登记表里,业务库中没有
# 对应行。不在这里叠加进来,能力中心就只剩本机自带的内置项——显示的和真正生效的
# 不是同一份。云端部署下叠加层为空,这几行是恒等变换。
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
overlay = device_catalog.catalog_overlay(user_id=str(user.user_id))
skill_list = device_catalog.merge_items(
@@ -464,11 +459,11 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict
@router.get("", summary="获取能力目录")
async def get_catalog_items(
- user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)
+ user: UserContext = Depends(get_current_user),
):
"""Build the synchronous catalog outside the Uvicorn event loop."""
await _ensure_external_cache_ready()
- return await run_in_threadpool(_get_catalog_items_sync, user, db)
+ return await run_in_session(lambda db: _get_catalog_items_sync(user, db))
diff --git a/src/backend/api/routes/v1/catalog_mutations.py b/src/backend/api/routes/v1/catalog_mutations.py
index dc0b37f9..4fb889c6 100644
--- a/src/backend/api/routes/v1/catalog_mutations.py
+++ b/src/backend/api/routes/v1/catalog_mutations.py
@@ -1,15 +1,15 @@
"""Capability catalog preference mutations."""
from typing import Any, Dict, Optional
from core.auth.backend import UserContext, get_current_user
-from core.config.catalog_runtime import get_runtime_catalog
+from core.capabilities.catalog.runtime import get_runtime_catalog
from core.db.engine import get_db
from core.infra.exceptions import BadRequestError
from core.infra.responses import success_response
-from core.services import CatalogService
+from core.capabilities.catalog.overrides import CatalogService
from fastapi import APIRouter, Depends, Path
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
-from .catalog_ownership import _is_owned_capability
+from features.marketplace.api.catalog_ownership import _is_owned_capability
router = APIRouter()
# Request/Response Models
@@ -76,7 +76,7 @@ def update_catalog_item(
# 云端同步来的技能 / 智能体登记在本机登记表里,业务库没有对应行,写目录覆盖
# 不会生效也读不回来。启停就写它——读写同源,运行时解析读的也是这份。
if request.enabled is not None and normalized_kind in ("skill", "agent"):
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
if device_catalog.set_enabled(normalized_kind, id, request.enabled, user_id=str(user.user_id)):
return success_response(
@@ -92,7 +92,7 @@ def update_catalog_item(
# 云端下发的连接器不在本机目录里,它是在目录解析之后按 mcp.json 的托管标志
# 追加回来的——所以启停只有写进那里才作数,写目录覆盖不会生效。
if normalized_kind == "mcp" and request.enabled is not None:
- from core.services.desktop_cloud_bridge import set_managed_connector_enabled
+ from features.desktop.bridge.managed_connectors import set_managed_connector_enabled
if set_managed_connector_enabled(id, request.enabled):
return success_response(
@@ -194,7 +194,7 @@ def update_catalog_item(
# Invalidate this user's capability cache so the toggle takes effect on the
# next message (otherwise resolve_all_runtime_enabled's 30s cache hides it).
try:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(str(user.user_id))
except Exception:
diff --git a/src/backend/api/routes/v1/chat_forks.py b/src/backend/api/routes/v1/chat_forks.py
index 6b8a4e98..e285fda9 100644
--- a/src/backend/api/routes/v1/chat_forks.py
+++ b/src/backend/api/routes/v1/chat_forks.py
@@ -5,7 +5,7 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import created_response
-from core.services.chat_fork_service import ChatForkError, ChatForkService
+from core.chat.fork import ChatForkError, ChatForkService
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, Field, field_validator
from sqlalchemy.orm import Session
diff --git a/src/backend/api/routes/v1/chat_modes.py b/src/backend/api/routes/v1/chat_modes.py
index c02cc531..c8f941e3 100644
--- a/src/backend/api/routes/v1/chat_modes.py
+++ b/src/backend/api/routes/v1/chat_modes.py
@@ -21,7 +21,7 @@
from core.db.models.chat_mode import ChatMode
from core.infra.logging import get_logger
from core.infra.responses import created_response, success_response
-from core.services.chat_mode_service import ChatModeService
+from core.chat.modes import ChatModeService
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
diff --git a/src/backend/api/routes/v1/chat_runs.py b/src/backend/api/routes/v1/chat_runs.py
index 20717039..fc272320 100644
--- a/src/backend/api/routes/v1/chat_runs.py
+++ b/src/backend/api/routes/v1/chat_runs.py
@@ -1,13 +1,12 @@
"""Chat Run management API — cancel and durable queued input."""
-from core.services.session_operations.schemas import SteerChatRunRequest
-
+import core.runs.runtime as runtime
from core.auth.backend import UserContext, get_current_user
from core.infra.logging import get_logger
from core.infra.responses import success_response
-from core.services.chat_steer_service import list_run_steers, remove_pending_steer
+from core.chat.steer import list_run_steers, remove_pending_steer
+from core.chat.operations.schemas import SteerChatRunRequest
from fastapi import APIRouter, Depends, HTTPException
-from orchestration import chat_run_executor
logger = get_logger(__name__)
@@ -20,7 +19,7 @@ async def cancel_chat_run(
user: UserContext = Depends(get_current_user),
):
"""取消正在执行的 chat run,真正杀掉后台任务;run 不存在返回 404,无权操作返回 403。"""
- from core.services.session_operations.commands import cancel_chat_run as operation
+ from core.chat.operations.commands import cancel_chat_run as operation
return await operation(run_id=run_id, user=user)
@@ -32,7 +31,7 @@ async def steer_chat_run(
user: UserContext = Depends(get_current_user),
):
"""Queue one instruction for the live ReAct loop's next safe boundary."""
- from core.services.session_operations.commands import steer_chat_run as operation
+ from core.chat.operations.commands import steer_chat_run as operation
return await operation(run_id=run_id, body=body, user=user)
@@ -42,7 +41,7 @@ async def list_chat_run_steers(
run_id: str,
user: UserContext = Depends(get_current_user),
):
- run = chat_run_executor.get_run(run_id)
+ run = runtime.get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail="run not found")
if run.user_id != user.user_id:
@@ -56,7 +55,7 @@ async def withdraw_chat_run_steer(
steer_id: str,
user: UserContext = Depends(get_current_user),
):
- run = chat_run_executor.get_run(run_id)
+ run = runtime.get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail="run not found")
if run.user_id != user.user_id:
@@ -67,7 +66,7 @@ async def withdraw_chat_run_steer(
@router.get("/{run_id}", summary="获取后台运行状态")
async def get_chat_run(run_id: str, user: UserContext = Depends(get_current_user)):
- run = chat_run_executor.get_run(run_id)
+ run = runtime.get_run(run_id)
if run is None or run.user_id != user.user_id:
raise HTTPException(404, "Run not found")
return success_response(
diff --git a/src/backend/api/routes/v1/chats/confirms.py b/src/backend/api/routes/v1/chats/confirms.py
index 96100814..d5b9714d 100644
--- a/src/backend/api/routes/v1/chats/confirms.py
+++ b/src/backend/api/routes/v1/chats/confirms.py
@@ -1,14 +1,12 @@
"""Pending tool confirmations and user decision endpoints."""
-import core.services.session_operations.schemas as chat_models
+import core.chat.operations.schemas as chat_models
import api.routes.v1.chats.pending_access as chat_pending_access
import core.auth.backend as auth_backend
-import core.db.engine as db_engine
from core.auth.backend import UserContext
from core.infra.responses import success_response
from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.orm import Session
-from starlette.concurrency import run_in_threadpool
+from core.db.threads import run_in_session
router = APIRouter()
@@ -16,7 +14,6 @@
@router.get("/pending-confirms", summary="批量查询本人会话的待确认我的空间写操作")
async def list_pending_confirms(
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""前端刷新/首屏加载时一次性拉取——用于在侧边栏对应会话上点亮蓝点。
@@ -26,12 +23,11 @@ async def list_pending_confirms(
注意:本路由必须声明在 ``GET /{chat_id}`` **之前**,否则会被 path
参数路由吞掉(FastAPI 按声明顺序匹配)。
"""
- from core.llm.tools import _tool_confirmation as _mc
+ from core.llm.tools import confirmation as _mc
items = []
for cid in await _mc.list_pending_chat_ids_shared():
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, cid, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, cid, user.user_id
):
continue
# Cannot just take "the latest one" (get_pending): when the latest is a
@@ -52,15 +48,13 @@ async def list_pending_confirms(
async def get_pending_confirm(
chat_id: str,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""前端刷新/切回该会话时恢复确认条(§13)。无待确认项时 pendings=[]。"""
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
raise HTTPException(status_code=404, detail="会话不存在或无权访问")
- from core.llm.tools import _tool_confirmation as _mc
+ from core.llm.tools import confirmation as _mc
# pendings: all outstanding items; the frontend restores the whole confirmation queue from this
# (one round of parallel tool calls can concurrently register N distinct pending confirmations).
@@ -76,7 +70,6 @@ async def file_confirm(
chat_id: str,
body: chat_models.FileConfirmBody,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""用户带外批准/拒绝一次对「我的空间」的 Write/Edit/Delete/Move(§13)。
@@ -84,12 +77,11 @@ async def file_confirm(
不重试整个操作;拒绝则模型据工具反馈调整后续行为。
"""
# Ownership check: must be the caller's own session
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
raise HTTPException(status_code=404, detail="会话不存在或无权访问")
- from core.llm.tools import _tool_confirmation as _mc
+ from core.llm.tools import confirmation as _mc
res = await _mc.set_decision_shared(
chat_id, body.confirm_id, body.decision, option_id=body.option_id
@@ -103,8 +95,7 @@ async def file_confirm(
# server restart". The latter needs a clearer message for the user (not "confirmation
# timed out" but "session was interrupted, please resend"); the frontend shows a more
# prominent notice for it instead of silently dismissing the bar.
- chat_interrupted = await run_in_threadpool(
- chat_pending_access._detect_chat_run_interrupted, db, chat_id
+ chat_interrupted = await run_in_session(chat_pending_access._detect_chat_run_interrupted, chat_id
)
return success_response(
data={
diff --git a/src/backend/api/routes/v1/chats/feedback.py b/src/backend/api/routes/v1/chats/feedback.py
index a696488d..67020df4 100644
--- a/src/backend/api/routes/v1/chats/feedback.py
+++ b/src/backend/api/routes/v1/chats/feedback.py
@@ -1,11 +1,11 @@
"""Ontology acceptance and assistant feedback endpoints."""
-import core.services.session_operations.schemas as chat_models
-import core.services.session_operations.session_context as chat_session_context
+import core.chat.operations.schemas as chat_models
+import core.chat.operations.session_context as chat_session_context
import core.auth.backend as auth_backend
import core.chat.context as chat_context
import core.db.engine as db_engine
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
from core.auth.backend import UserContext
from core.db.models import MessageFeedback
from core.infra.exceptions import ResourceNotFoundError
@@ -30,12 +30,12 @@ def accept_ontology_revision(
db: Session = Depends(db_engine.get_db),
):
"""Atomically replace one assistant message with its reviewed candidate answer."""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
target = chat_service.get_message_by_id(message_id)
if not target or target.role != "assistant":
raise ResourceNotFoundError(resource_type="chat_message", resource_id=message_id)
db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
+ db, chat_session_context.authenticated_user_id(user)
)
access = chat_service.get_session_with_access(target.chat_id, db_user_id)
if access is None:
@@ -65,7 +65,7 @@ def submit_feedback(
if body.rating not in ("like", "dislike"):
raise HTTPException(status_code=400, detail="rating must be 'like' or 'dislike'")
- db_user_id = chat_session_context._authenticated_user_id(user)
+ db_user_id = chat_session_context.authenticated_user_id(user)
existing = (
db.query(MessageFeedback)
@@ -96,7 +96,7 @@ def submit_feedback(
if body.rating == "dislike" and (body.comment or "").strip() and record.chat_id:
try:
- from core.services.ontology_evolution_service import OntologyEvolutionService
+ from core.ontology.evolution import OntologyEvolutionService
OntologyEvolutionService(db).ingest_user_correction(
user_id=db_user_id,
diff --git a/src/backend/api/routes/v1/chats/history.py b/src/backend/api/routes/v1/chats/history.py
index 61e2eb6d..34289c47 100644
--- a/src/backend/api/routes/v1/chats/history.py
+++ b/src/backend/api/routes/v1/chats/history.py
@@ -4,12 +4,12 @@
import core.auth.backend as auth_backend
import core.db.engine as db_engine
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
from core.auth.backend import UserContext
-from core.chat.display_bounds import bound_result_for_display
+from core.infra.text_bounds import bound_result_for_display
from core.infra.exceptions import ResourceNotFoundError
from core.infra.responses import success_response
-from core.services.context_usage_projection import get_chat_context_state
+from core.chat.compaction.usage_projection import get_chat_context_state
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
@@ -31,7 +31,7 @@ def list_messages(
db: Session = Depends(db_engine.get_db),
):
"""获取当前用户有权读取的会话消息列表;无权访问时返回 404。"""
- from core.services.session_operations.history import list_messages as operation
+ from core.chat.operations.history import list_messages as operation
return operation(
chat_id=chat_id,
@@ -52,11 +52,11 @@ def get_context_usage(
db: Session = Depends(db_engine.get_db),
):
"""Return the latest provider measurement and active compaction baseline."""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
if chat_service.get_session_with_access(chat_id, str(user.user_id)) is None:
raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id)
- from core.services.compaction_progress import read_progress
+ from core.chat.compaction.progress import read_progress
return success_response(
data={
@@ -84,7 +84,7 @@ def get_tool_call_result(
展开哪张卡,前端才来这里取哪一张。仍然过一遍 ``bound_result_for_display`` 的宽档
上限:单张卡也不该把 5MB 一次性塞进标签页。
"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
if chat_service.get_session_with_access(chat_id, str(user.user_id)) is None:
raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id)
@@ -119,7 +119,7 @@ def get_followups(
db: Session = Depends(db_engine.get_db),
):
"""Return follow-up questions stored in a message's extra_data."""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
session = chat_service.get_session(chat_id, user.user_id)
if not session:
diff --git a/src/backend/api/routes/v1/chats/pending_access.py b/src/backend/api/routes/v1/chats/pending_access.py
index 2eac2f36..d7d73571 100644
--- a/src/backend/api/routes/v1/chats/pending_access.py
+++ b/src/backend/api/routes/v1/chats/pending_access.py
@@ -2,8 +2,7 @@
from datetime import datetime
-import core.services.session_operations.session_context as chat_session_context
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
from sqlalchemy.orm import Session
@@ -41,7 +40,4 @@ def _detect_chat_run_interrupted(db: Session, chat_id: str) -> bool:
def _owns_pending_chat(db, chat_id, user_id):
- try:
- return chat_services.ChatService(db).get_session(chat_id, user_id) is not None
- finally:
- chat_session_context._release_request_session(db)
+ return chat_services_ChatService_module.ChatService(db).get_session(chat_id, user_id) is not None
diff --git a/src/backend/api/routes/v1/chats/questions.py b/src/backend/api/routes/v1/chats/questions.py
index 6d9842f0..1efddebd 100644
--- a/src/backend/api/routes/v1/chats/questions.py
+++ b/src/backend/api/routes/v1/chats/questions.py
@@ -1,15 +1,13 @@
"""Pending user-question recovery, answer and cancellation endpoints."""
-import core.services.session_operations.schemas as chat_models
+import core.chat.operations.schemas as chat_models
import api.routes.v1.chats.pending_access as chat_pending_access
import core.auth.backend as auth_backend
-import core.db.engine as db_engine
from core.auth.backend import UserContext
from core.infra.logging import quiet_access_log
from core.infra.responses import success_response
from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.orm import Session
-from starlette.concurrency import run_in_threadpool
+from core.db.threads import run_in_session
router = APIRouter()
@@ -18,7 +16,6 @@
@quiet_access_log
async def list_pending_user_questions(
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Restore sidebar waiting indicators without opening every conversation."""
@@ -26,8 +23,7 @@ async def list_pending_user_questions(
items = []
for chat_id in await user_questions.list_pending_chat_ids_shared():
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
continue
items.extend(
@@ -44,12 +40,10 @@ async def list_pending_user_questions(
async def get_pending_user_questions(
chat_id: str,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Return the authoritative pending queue for refresh/chat switching."""
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
raise HTTPException(status_code=404, detail="会话不存在或无权访问")
@@ -69,12 +63,10 @@ async def answer_user_question(
request_id: str,
body: chat_models.UserQuestionAnswerBody,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Validate the human answer and wake the exact suspended tool call."""
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
raise HTTPException(status_code=404, detail="会话不存在或无权访问")
@@ -88,8 +80,7 @@ async def answer_user_question(
if result.get("ok"):
return success_response(data=result)
if result.get("reason") == "stale":
- interrupted = await run_in_threadpool(
- chat_pending_access._detect_chat_run_interrupted, db, chat_id
+ interrupted = await run_in_session(chat_pending_access._detect_chat_run_interrupted, chat_id
)
return success_response(
data={
@@ -114,12 +105,10 @@ async def cancel_user_question(
chat_id: str,
request_id: str,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Cancel the pending request; the tool receives an explicit non-retry result."""
- if not await run_in_threadpool(
- chat_pending_access._owns_pending_chat, db, chat_id, user.user_id
+ if not await run_in_session(chat_pending_access._owns_pending_chat, chat_id, user.user_id
):
raise HTTPException(status_code=404, detail="会话不存在或无权访问")
@@ -128,8 +117,7 @@ async def cancel_user_question(
result = await user_questions.cancel_shared(chat_id, request_id)
if result.get("ok"):
return success_response(data=result)
- interrupted = await run_in_threadpool(
- chat_pending_access._detect_chat_run_interrupted, db, chat_id
+ interrupted = await run_in_session(chat_pending_access._detect_chat_run_interrupted, chat_id
)
return success_response(
data={
diff --git a/src/backend/api/routes/v1/chats/reruns.py b/src/backend/api/routes/v1/chats/reruns.py
index 1dd5b1e7..a514a3cd 100644
--- a/src/backend/api/routes/v1/chats/reruns.py
+++ b/src/backend/api/routes/v1/chats/reruns.py
@@ -2,26 +2,28 @@
from typing import Any, Dict
-import core.services.session_operations.admission_errors as chat_admission
-import core.services.session_operations.agent_targets as chat_agent_targets
-import core.services.session_operations.invocation as chat_invocation
-import core.services.session_operations.schemas as chat_models
-import core.services.session_operations.request_context as chat_request_context
import api.routes.v1.chats.rerun_metadata as chat_rerun_metadata
-import core.services.session_operations.session_context as chat_session_context
+import api.transports.run_sse as run_sse
import core.auth.backend as auth_backend
import core.chat.context as chat_context
-import core.db.engine as db_engine
+import core.chat.runtime.admission as admission
import core.infra.responses as responses
-import core.services as chat_services
-from core.chat.schemas import ChatRequest
+import core.chat.service as chat_services_ChatService_module
+import core.auth.users as chat_services_UserService_module
+import core.chat.operations.admission_errors as chat_admission
+import core.chat.operations.agent_targets as chat_agent_targets
+import core.chat.operations.invocation as chat_invocation
+import core.chat.operations.request_context as chat_request_context
+import core.chat.operations.schemas as chat_models
+import core.chat.operations.session_context as chat_session_context
from core.auth.backend import UserContext
+from core.chat.schemas import ChatRequest
from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner
from core.infra.exceptions import ResourceNotFoundError
-from core.services.chat_reference_service import render_reference_block
+from core.chat.references import render_reference_block
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
-from starlette.concurrency import run_in_threadpool
+from core.db.threads import run_in_session
router = APIRouter()
@@ -31,18 +33,17 @@ async def regenerate_message(
chat_id: str,
body: chat_models.RegenerateRequest,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Delete the target assistant message and all subsequent, then re-stream."""
if reserved_chat_owner(chat_id):
raise HTTPException(status_code=403, detail=reserved_chat_error(chat_id))
- def prepare():
- chat_session_context._ensure_main_model_configured()
- chat_service = chat_services.ChatService(db)
+ def prepare(db: Session):
+ chat_session_context.ensure_main_model_configured()
+ chat_service = chat_services_ChatService_module.ChatService(db)
db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
+ db, chat_session_context.authenticated_user_id(user)
)
# Shared sessions: deleting/rewriting history is only granted to the session owner / project admin (the "admin" level in the shared context).
@@ -79,7 +80,7 @@ def prepare():
**chat_rerun_metadata._restore_invocation(user_extra),
)
regen_request, _, execution_message, explicit_subagent_command = (
- chat_agent_targets._resolve_rerun_agent_targets(
+ chat_agent_targets.resolve_rerun_agent_targets(
db,
regen_request,
db_user_id,
@@ -88,30 +89,30 @@ def prepare():
assistant_message_id=target_msg.message_id,
)
)
- regen_request = chat_invocation._resolve_explicit_capability_invocation(
+ regen_request = chat_invocation.resolve_explicit_capability_invocation(
db, regen_request, db_user_id
)
- from core.services.project_init import resolve_project_init
+ from core.chat.project_init import resolve_project_init
init_message = resolve_project_init(db, regen_request, db_user_id)
- selected_model_provider_id = chat_session_context._resolve_selected_model_provider_id(
+ selected_model_provider_id = chat_session_context.resolve_selected_model_provider_id(
db, regen_request, db_user_id
)
- actual_model_name = chat_session_context._resolve_actual_chat_model_name(
+ actual_model_name = chat_session_context.resolve_actual_chat_model_name(
regen_request,
selected_model_provider_id,
)
enabled_skills, enabled_agents, enabled_mcps = chat_context.resolve_enabled_capabilities(
db, db_user_id
)
- _user_settings = chat_services.UserService(db).get_user_settings(db_user_id)
+ _user_settings = chat_services_UserService_module.UserService(db).get_user_settings(db_user_id)
effective_msg = chat_context.build_effective_user_message(
init_message or execution_message,
regen_request.quoted_follow_up,
render_reference_block(user_extra.get("referenced_chats")),
)
- context = chat_request_context._build_ctx(
+ context = chat_request_context.build_ctx(
regen_request,
db_user_id,
enabled_skills,
@@ -127,7 +128,7 @@ def prepare():
approval_mode=_user_settings.get("tool_approval_mode"),
)
- from core.services.chat_sequencer import ChatBusyError, ChatSequencer
+ from core.chat.sequencer import ChatBusyError, ChatSequencer
request_payload = regen_request.model_dump(exclude_none=True)
request_payload["operation"] = "regenerate"
@@ -163,7 +164,7 @@ def prepare():
raise chat_admission.chat_busy_http_exception(exc) from exc
try:
- session_messages = chat_session_context._load_session_messages(
+ session_messages = chat_session_context.load_session_messages(
chat_service, chat_id, db_user_id
)
kwargs = dict(
@@ -184,7 +185,7 @@ def prepare():
ChatSequencer(db).abandon_pending_run(accepted.run.run_id, reason=str(exc))
raise
- return await _start_prepared_run(prepare, db, chat_id)
+ return await _start_prepared_run(prepare, chat_id)
@router.post("/{chat_id}/edit", summary="编辑消息并重新生成 (SSE)")
@@ -192,18 +193,17 @@ async def edit_and_resend(
chat_id: str,
body: chat_models.EditAndResendRequest,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Delete the target user message and all subsequent, then re-stream with new content."""
if reserved_chat_owner(chat_id):
raise HTTPException(status_code=403, detail=reserved_chat_error(chat_id))
- def prepare():
- chat_session_context._ensure_main_model_configured()
- chat_service = chat_services.ChatService(db)
+ def prepare(db: Session):
+ chat_session_context.ensure_main_model_configured()
+ chat_service = chat_services_ChatService_module.ChatService(db)
db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
+ db, chat_session_context.authenticated_user_id(user)
)
# Shared sessions: editing history is an owner-only operation (rewriting others' / one's own earlier messages would break the collaboration context).
@@ -239,7 +239,7 @@ def prepare():
**saved_invocation,
)
edit_request, _, execution_message, explicit_subagent_command = (
- chat_agent_targets._resolve_rerun_agent_targets(
+ chat_agent_targets.resolve_rerun_agent_targets(
db,
edit_request,
db_user_id,
@@ -247,22 +247,22 @@ def prepare():
target_msg,
)
)
- from core.services.project_init import resolve_project_init
+ from core.chat.project_init import resolve_project_init
init_message = resolve_project_init(db, edit_request, db_user_id)
- edit_request = chat_invocation._resolve_explicit_capability_invocation(
+ edit_request = chat_invocation.resolve_explicit_capability_invocation(
db, edit_request, db_user_id
)
- selected_model_provider_id = chat_session_context._resolve_selected_model_provider_id(
+ selected_model_provider_id = chat_session_context.resolve_selected_model_provider_id(
db, edit_request, db_user_id
)
- actual_model_name = chat_session_context._resolve_actual_chat_model_name(
+ actual_model_name = chat_session_context.resolve_actual_chat_model_name(
edit_request, selected_model_provider_id
)
enabled_skills, enabled_agents, enabled_mcps = chat_context.resolve_enabled_capabilities(
db, db_user_id
)
- _user_settings = chat_services.UserService(db).get_user_settings(db_user_id)
+ _user_settings = chat_services_UserService_module.UserService(db).get_user_settings(db_user_id)
# Persist the edited user message
_edit_extra: Dict[str, Any] = {
@@ -285,7 +285,7 @@ def prepare():
if selected_model_provider_id:
_edit_extra["model_provider_id"] = selected_model_provider_id
- context = chat_request_context._build_ctx(
+ context = chat_request_context.build_ctx(
edit_request,
db_user_id,
enabled_skills,
@@ -301,7 +301,7 @@ def prepare():
approval_mode=_user_settings.get("tool_approval_mode"),
)
- from core.services.chat_sequencer import ChatBusyError, ChatSequencer
+ from core.chat.sequencer import ChatBusyError, ChatSequencer
request_payload = edit_request.model_dump(exclude_none=True)
request_payload["operation"] = "edit"
@@ -328,7 +328,7 @@ def prepare():
raise chat_admission.chat_busy_http_exception(exc) from exc
try:
- session_messages = chat_session_context._load_session_messages(
+ session_messages = chat_session_context.load_session_messages(
chat_service, chat_id, db_user_id
)
kwargs = dict(
@@ -353,32 +353,19 @@ def prepare():
ChatSequencer(db).abandon_pending_run(accepted.run.run_id, reason=str(exc))
raise
- return await _start_prepared_run(prepare, db, chat_id)
-
+ return await _start_prepared_run(prepare, chat_id)
-async def _start_prepared_run(prepare, db: Session, chat_id: str):
- """Release admission locks before streaming; abandon admission if startup fails."""
- from core.services.chat_sequencer import ChatSequencer
- from orchestration import chat_run_executor
- def prepare_and_release():
- try:
- return prepare()
- finally:
- chat_session_context._release_request_session(db)
+async def _start_prepared_run(prepare, chat_id: str):
+ """Admit on a session of its own, then launch; abandon admission if startup fails."""
+ from core.chat.sequencer import ChatSequencer
- kwargs = await run_in_threadpool(prepare_and_release)
+ kwargs = await run_in_session(prepare)
try:
- run = await chat_run_executor.start_run(**kwargs)
+ run = await admission.start_run(**kwargs)
except Exception as exc:
reason = str(exc)
-
- def abandon():
- try:
- ChatSequencer(db).abandon_pending_run(kwargs["accepted_run"].run_id, reason=reason)
- finally:
- chat_session_context._release_request_session(db)
-
- await run_in_threadpool(abandon)
+ run_id = kwargs["accepted_run"].run_id
+ await run_in_session(lambda db: ChatSequencer(db).abandon_pending_run(run_id, reason=reason))
raise
- return responses.sse_response(chat_run_executor.follow_run_as_sse(run.run_id, chat_id=chat_id))
+ return responses.sse_response(run_sse.follow_run_as_sse(run.run_id, chat_id=chat_id))
diff --git a/src/backend/api/routes/v1/chats/run_views.py b/src/backend/api/routes/v1/chats/run_views.py
index 8c543814..8e48f76a 100644
--- a/src/backend/api/routes/v1/chats/run_views.py
+++ b/src/backend/api/routes/v1/chats/run_views.py
@@ -2,11 +2,13 @@
from typing import Annotated
-import core.services.session_operations.session_context as chat_session_context
+import api.transports.run_sse as run_sse
import core.auth.backend as auth_backend
import core.chat.context as chat_context
import core.db.engine as db_engine
import core.infra.responses as responses
+import core.runs.runtime as runtime
+import core.chat.operations.session_context as chat_session_context
from core.auth.backend import UserContext
from core.db.reserved_chats import reserved_chat_owner
from core.infra.logging import quiet_access_log
@@ -32,7 +34,7 @@ def list_active_chat_runs(
侧边栏对没点开过的会话一无所知。这里按用户一次查完:查询只命中 pending /
running 这一小撮行,与历史 run 的数量无关。
"""
- from core.services.session_operations.runs import list_active_chat_runs as operation
+ from core.chat.operations.runs import list_active_chat_runs as operation
return operation(user=user, db=db, include_status=include_status, recent_seconds=recent_seconds)
@@ -42,27 +44,26 @@ def chat_stream_resume(
run_id: str,
from_offset: int = Query(0, alias="from", ge=0, description="从此 offset 之后继续推送"),
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""续播某个进行中的 run:刷新或断线后从指定 offset 之后重新订阅 SSE 事件。
校验 run 存在且归属当前用户(非属主 403、不存在 404),返回 text/event-stream。
"""
- from orchestration import chat_run_executor
- db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
- )
- run = chat_run_executor.get_run(run_id)
+ # A scoped session, not the request's: the stream below outlives this handler.
+ with db_engine.SessionLocal() as db:
+ db_user_id = chat_context.resolve_db_user_id(
+ db, chat_session_context.authenticated_user_id(user)
+ )
+ run = runtime.get_run(run_id)
# A module-owned chat's run is replayed by its module, not as an ordinary chat.
if run is None or reserved_chat_owner(run.chat_id):
raise HTTPException(status_code=404, detail="run not found")
if run.user_id != db_user_id:
raise HTTPException(status_code=403, detail="无权访问该 run")
resumed_chat_id = run.chat_id
- chat_session_context._release_request_session(db)
return responses.sse_response(
- chat_run_executor.follow_run_as_sse(
+ run_sse.follow_run_as_sse(
run_id, chat_id=resumed_chat_id, from_offset=from_offset
),
)
@@ -80,7 +81,7 @@ def chat_active_run(
有则返回 run_id / message_id / status / 刷新重放 offset / thinking 模式等元信息,
无则返回 data=null。需认证(有效 cookie 或 API-Key)。
"""
- from core.services.session_operations.runs import chat_active_run as operation
+ from core.chat.operations.runs import chat_active_run as operation
return operation(chat_id=chat_id, user=user, db=db)
@@ -89,16 +90,16 @@ def chat_active_run(
def chat_run_subscription(
run_id: str,
user: UserContext = Depends(auth_backend.get_current_user),
- db: Session = Depends(db_engine.get_db),
):
"""Native ordinary-chat snapshot + tail; the raw replay API is separate."""
- from orchestration import chat_run_executor
- from orchestration.run_subscription import subscribe_run
-
- db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
- )
- run = chat_run_executor.get_run(run_id)
+ from core.chat.projection.subscription import subscribe_run
+
+ # A scoped session, not the request's: the stream below outlives this handler.
+ with db_engine.SessionLocal() as db:
+ db_user_id = chat_context.resolve_db_user_id(
+ db, chat_session_context.authenticated_user_id(user)
+ )
+ run = runtime.get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail="run not found")
if run.user_id != db_user_id:
@@ -106,5 +107,4 @@ def chat_run_subscription(
payload = run.request_payload if isinstance(run.request_payload, dict) else {}
if payload.get("kind", "chat") != "chat":
raise HTTPException(status_code=400, detail="This run uses a specialized progress protocol")
- chat_session_context._release_request_session(db)
return responses.sse_response(subscribe_run(run_id))
diff --git a/src/backend/api/routes/v1/chats/sessions.py b/src/backend/api/routes/v1/chats/sessions.py
index c5eb77bd..0ade4531 100644
--- a/src/backend/api/routes/v1/chats/sessions.py
+++ b/src/backend/api/routes/v1/chats/sessions.py
@@ -2,8 +2,8 @@
from typing import Optional
-import core.services.session_operations.schemas as chat_models
-import core.services.session_operations.session_context as chat_session_context
+import core.chat.operations.schemas as chat_models
+import core.chat.operations.session_context as chat_session_context
import core.auth.backend as auth_backend
import core.chat.context as chat_context
import core.db.engine as db_engine
@@ -39,7 +39,7 @@ def list_chats(
- -created_at - Most recently created first
- created_at - Oldest created first
"""
- from core.services.session_operations.sessions import list_chats as operation
+ from core.chat.operations.sessions import list_chats as operation
return operation(
page=page,
@@ -63,7 +63,7 @@ def create_chat(
The session is automatically associated with the current authenticated user.
"""
- from core.services.session_operations.sessions import create_chat as operation
+ from core.chat.operations.sessions import create_chat as operation
return operation(request=request, user=user, db=db)
@@ -87,7 +87,7 @@ def search_chats(
Returns sessions with match_type ("title" or "content") and matched_snippet for content matches.
"""
- from core.services.session_operations.sessions import search_chats as operation
+ from core.chat.operations.sessions import search_chats as operation
return operation(q=q, scope=scope, page=page, page_size=page_size, user=user, db=db)
@@ -106,10 +106,10 @@ def list_referencable_chats(
在项目里只给同项目的会话——项目是用户自己划的话题边界。只返回标题级信息,正文由
智能体在回答时按需调 ``read_chat`` 读取。
"""
- from core.services.chat_reference_service import list_referencable_sessions, session_brief
+ from core.chat.references import list_referencable_sessions, session_brief
db_user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
+ db, chat_session_context.authenticated_user_id(user)
)
sessions = list_referencable_sessions(
db,
@@ -132,7 +132,7 @@ def get_chat(
db: Session = Depends(db_engine.get_db),
):
"""获取当前用户有权读取的会话详情。"""
- from core.services.session_operations.sessions import get_chat as operation
+ from core.chat.operations.sessions import get_chat as operation
return operation(chat_id=chat_id, user=user, db=db)
@@ -145,7 +145,7 @@ def update_chat(
db: Session = Depends(db_engine.get_db),
):
"""更新当前用户有权修改的会话元信息。"""
- from core.services.session_operations.sessions import update_chat as operation
+ from core.chat.operations.sessions import update_chat as operation
return operation(chat_id=chat_id, request=request, user=user, db=db)
@@ -157,6 +157,6 @@ def delete_chat(
db: Session = Depends(db_engine.get_db),
):
"""软删当前用户有权管理的会话。"""
- from core.services.session_operations.sessions import delete_chat as operation
+ from core.chat.operations.sessions import delete_chat as operation
return operation(chat_id=chat_id, user=user, db=db)
diff --git a/src/backend/api/routes/v1/chats/sidebar.py b/src/backend/api/routes/v1/chats/sidebar.py
index 4b4cac74..af0f29ea 100644
--- a/src/backend/api/routes/v1/chats/sidebar.py
+++ b/src/backend/api/routes/v1/chats/sidebar.py
@@ -2,11 +2,11 @@
from typing import List, Optional
-import core.services.session_operations.schemas as chat_models
-import core.services.session_operations.session_context as chat_session_context
+import core.chat.operations.schemas as chat_models
+import core.chat.operations.session_context as chat_session_context
import core.auth.backend as auth_backend
import core.db.engine as db_engine
-import core.services as chat_services
+import core.auth.users as chat_services_UserService_module
from core.auth.backend import UserContext
from core.infra.responses import success_response
from fastapi import APIRouter, Depends
@@ -25,7 +25,7 @@ def _dedup_id_list(raw: Optional[list]) -> List[str]:
"""Clean + de-duplicate an id list, preserving first-seen order."""
seen: set = set()
out: List[str] = []
- for cid in chat_session_context._clean_id_list(raw):
+ for cid in chat_session_context.clean_id_list(raw):
if cid in seen:
continue
seen.add(cid)
@@ -45,7 +45,7 @@ def get_sidebar_order(
注意:本路由必须声明在 ``GET /{chat_id}`` **之前**,否则会被 path 参数
路由吞掉(FastAPI 按声明顺序匹配)。
"""
- user_settings = chat_services.UserService(db).get_user_settings(str(user.user_id))
+ user_settings = chat_services_UserService_module.UserService(db).get_user_settings(str(user.user_id))
order = _dedup_id_list(user_settings.get(SIDEBAR_ORDER_KEY))[:SIDEBAR_ORDER_MAX]
return success_response(data={"order": order})
@@ -62,7 +62,7 @@ def update_sidebar_order(
查不到对应项而被忽略,反倒省掉一次全表校验。超出上限的尾部直接截断。
"""
order = _dedup_id_list(request.order)[:SIDEBAR_ORDER_MAX]
- chat_services.UserService(db).update_user_metadata(
+ chat_services_UserService_module.UserService(db).update_user_metadata(
user_id=str(user.user_id), patch={SIDEBAR_ORDER_KEY: order}
)
return success_response(data={"order": order})
diff --git a/src/backend/api/routes/v1/classify.py b/src/backend/api/routes/v1/classify.py
index cfccfa99..fa3c0a61 100644
--- a/src/backend/api/routes/v1/classify.py
+++ b/src/backend/api/routes/v1/classify.py
@@ -6,7 +6,7 @@
from core.auth.backend import get_current_user, UserContext
from core.infra.responses import success_response
-from core.llm.classifier import get_classifier, BUSINESS_TOPICS
+from core.models.completions.classifier import get_classifier, BUSINESS_TOPICS
from core.infra.exceptions import BadRequestError
router = APIRouter(prefix="/v1/classify", tags=["Classify"])
diff --git a/src/backend/api/routes/v1/commands.py b/src/backend/api/routes/v1/commands.py
index 1e856eeb..02aa7bea 100644
--- a/src/backend/api/routes/v1/commands.py
+++ b/src/backend/api/routes/v1/commands.py
@@ -1,17 +1,18 @@
"""Authenticated background command status, output and cancellation."""
import asyncio
-from fastapi import APIRouter, Depends, HTTPException, Request
-from starlette.responses import StreamingResponse
+
from core.auth.backend import UserContext, get_current_user
from core.infra.responses import success_response
-from core.services.commands import store
-from orchestration.commands.runtime import runtime
+from core.sandbox.commands.runtime import runtime
+from core.sandbox.commands import store
+from fastapi import APIRouter, Depends, HTTPException, Request
+from starlette.responses import StreamingResponse
router = APIRouter(prefix="/v1/commands", tags=["Commands"])
@router.get("/events")
async def events(request: Request, chat_id: str):
- from orchestration.commands.subscription import authorize, subscribe
+ from core.sandbox.commands.subscription import authorize, subscribe
user_id = await authorize(request, chat_id)
return StreamingResponse(subscribe(request, chat_id, user_id),
media_type="text/event-stream",
diff --git a/src/backend/api/routes/v1/content.py b/src/backend/api/routes/v1/content.py
index 25931239..7d22c719 100644
--- a/src/backend/api/routes/v1/content.py
+++ b/src/backend/api/routes/v1/content.py
@@ -12,11 +12,8 @@
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
-from core.content.content_blocks import (
- DEFAULT_APP_CONFIG,
- DOCS_BLOCK_MAP,
- normalize_app_config,
-)
+from core.files.content.content_blocks_constants import DEFAULT_APP_CONFIG, DOCS_BLOCK_MAP
+from core.files.content.branding import normalize_app_config
from core.db.engine import get_db
from core.db.models import ContentBlock
from core.infra.responses import success_response
@@ -25,7 +22,7 @@
@router.get("/docs", summary="获取文档内容块(前台读取)")
-async def get_docs_content(db: Session = Depends(get_db)):
+def get_docs_content(db: Session = Depends(get_db)):
"""前台读取全部可编辑内容块:功能更新时间轴、能力中心、提示词广场、页面配置与应用配置,
并附各块最近更新时间。无需鉴权,DB 缺失时对应字段返回空值。"""
updates_row = db.query(ContentBlock).filter(ContentBlock.id == "docs_updates").first()
@@ -48,7 +45,7 @@ async def get_docs_content(db: Session = Depends(get_db)):
@router.get("/docs/version", summary="获取各内容块最新更新时间(轻量轮询)")
-async def get_docs_versions(db: Session = Depends(get_db)):
+def get_docs_versions(db: Session = Depends(get_db)):
"""轻量轮询接口:仅返回各内容块的最新更新时间,供前端判断是否需要重新拉取正文。"""
rows = (
db.query(ContentBlock)
diff --git a/src/backend/api/routes/v1/desktop_capability.py b/src/backend/api/routes/v1/desktop_capability.py
deleted file mode 100644
index 63ad5e07..00000000
--- a/src/backend/api/routes/v1/desktop_capability.py
+++ /dev/null
@@ -1,824 +0,0 @@
-"""桌面双端能力桥 API(云端侧 + 本机侧共用一个路由文件,各端点自行守门)。
-
-云端侧(部署在云端 / 预发 / 生产后端):
- POST /v1/desktop/capability/token 会话换取短时 capability token
- GET /v1/desktop/capability/manifest 当前用户最终可用 MCP 清单
- POST /v1/desktop/capability/gateway/{sid}/call 动态 manifest 的 JSON 工具调用网关
- ANY /v1/desktop/capability/gateway/{sid}/mcp 已安装旧客户端使用的透明反代网关
- GET /v1/desktop/capability/models 无密钥模型拓扑
- POST /v1/desktop/capability/gateway/models/... 模型流式反代网关
- GET /v1/desktop/capability/skills/manifest 当前用户最终可用技能清单(含内容哈希)
- GET /v1/desktop/capability/skills/{id}/bundle 单个授权技能的完整 zip 包
-
-本机侧(桌面壳孵化的本机后端):
- POST /v1/desktop/capability/cloud-bridge 壳推送 {cloud_base, token}
- GET /v1/desktop/capability/cloud-bridge/status 桥诊断视图
-
-安全模型:
-- token 端点要求云端登录会话;网关/manifest 只认 capability token(HMAC 短时
- 令牌,见 core/services/desktop_capability.py),会话 cookie / 内部 token /
- 第三方密钥不出云端;
-- 网关按「该用户当前有效能力集」授权 server_id,未命中一律 404(不区分
- 不存在/无权);身份头由网关覆写,客户端伪造的 X-Current-User-Id 不生效;
-- cloud-bridge 接收端只认桌面壳 Bearer 进程秘密,不以用户管理权限替代;
- 仅在桌面桥进程(HUGAGENT_DESKTOP_BRIDGE_SECRET 已注入)下开放,
- 浏览器 Origin 请求拒绝,云端部署恒 403。
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-import time
-from typing import Optional
-
-import httpx
-from core.auth.backend import UserContext, get_current_user
-from core.infra.logging import quiet_access_log
-from core.infra.responses import success_response
-from fastapi import APIRouter, Depends, HTTPException, Request, Response
-from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
-from pydantic import BaseModel, Field
-from starlette.background import BackgroundTask
-from starlette.concurrency import run_in_threadpool
-from starlette.responses import StreamingResponse
-
-logger = logging.getLogger(__name__)
-
-from core.services.desktop_gateway_observability import ObservedGatewayRoute
-
-router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
-
-# 网关上行连接:连接短超时快速失败;读不设限(SSE 长流 / 长工具调用),
-# 上游 MCP 自身带 execution_timeout 兜底。
-_gateway_client: Optional[httpx.AsyncClient] = None
-
-
-def _client() -> httpx.AsyncClient:
- global _gateway_client
- if _gateway_client is None:
- _gateway_client = httpx.AsyncClient(
- timeout=httpx.Timeout(600.0),
- # 桌面端两条消息之间常常隔着几十秒;默认 5s 的 keepalive 意味着几乎
- # 每次模型调用都要重新 TCP + TLS 握手。
- limits=httpx.Limits(max_keepalive_connections=64, keepalive_expiry=300.0),
- )
- return _gateway_client
-
-
-# ── token 签发(云端,会话鉴权) ────────────────────────────────────────
-
-
-class CapabilityTokenBody(BaseModel):
- device_id: str = Field(
- ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
- )
-
-
-@router.post("/token", summary="签发桌面能力令牌")
-async def issue_token(body: CapabilityTokenBody, request: Request, response: Response):
- """Only an actual live cloud cookie session can mint a desktop credential."""
- import hashlib
-
- from core.auth.session import validate_session
- from core.config.settings import settings
- from core.services.desktop_capability import issue_capability_token, session_authorization_epoch
-
- cookie = request.cookies.get(settings.session.cookie_name, "")
- try:
- current = await validate_session(cookie) if cookie else None
- except Exception:
- current = None
- if (
- not current
- or not current.get("user_id")
- or not current.get("user_center_id")
- or not session_authorization_epoch(current)
- ):
- raise HTTPException(status_code=401, detail="valid cloud session required")
- data = issue_capability_token(
- str(current["user_id"]),
- device_id=body.device_id,
- issuer=str(request.base_url),
- session_hash=hashlib.sha256(cookie.encode()).hexdigest(),
- authorization_epoch=session_authorization_epoch(current),
- user_center_id=str(current["user_center_id"]),
- )
- response.headers["Cache-Control"] = "no-store"
- response.headers["Pragma"] = "no-cache"
- return success_response(data=data)
-
-
-# ── capability token 鉴权依赖 ──────────────────────────────────────────
-
-_bearer = HTTPBearer(auto_error=False)
-
-
-async def _require_capability_user(
- request: Request,
- credentials: Optional[HTTPAuthorizationCredentials] = Depends(_bearer),
-) -> str:
- from core.services.desktop_capability import CAPABILITY_DEVICE_HEADER, verify_capability_token
-
- token = credentials.credentials if credentials else ""
- user_id = await verify_capability_token(
- token,
- device_id=request.headers.get(CAPABILITY_DEVICE_HEADER, ""),
- issuer=str(request.base_url),
- )
- if not user_id:
- raise HTTPException(status_code=401, detail="invalid capability token")
- request.state.desktop_observation_user = user_id
- return user_id
-
-
-class WorkcopyCommitBody(BaseModel):
- files: dict[str, str]
- expected_revision: str | None = None
- expected_model_version: str | None = None
- request_id: str = Field(pattern=r"^[a-f0-9]{32}$")
- acknowledge_sensitive: bool = False
- create_only: bool = False
-
-
-@router.get("/workcopies/{kind}/{key}", summary="比较当前账号的能力文件版本")
-async def workcopy_snapshot(kind: str, key: str, response: Response,
- user_id: str = Depends(_require_capability_user)):
- from core.services.capability_workcopies import snapshot
- response.headers["Cache-Control"] = "no-store"
- result = await run_in_threadpool(snapshot, user_id, kind, key)
- return success_response(data=await run_in_threadpool(_public_content, user_id, result))
-
-
-@router.post("/workcopies/{kind}/{key}", summary="显式提交能力文件版本")
-async def workcopy_commit(kind: str, key: str, body: WorkcopyCommitBody, response: Response,
- user_id: str = Depends(_require_capability_user)):
- from core.services.capability_workcopies import commit
- response.headers["Cache-Control"] = "no-store"
- await run_in_threadpool(_public_content, user_id, body.files)
- result = await run_in_threadpool(commit, user_id, kind, key, body.model_dump())
- return success_response(data=await run_in_threadpool(_public_content, user_id, result))
-
-
-def _public_content(user_id: str, value, *, bundle: bool = False):
- from core.services.desktop_capability import (
- CapabilityContentRejected,
- guard_capability_bundle,
- guard_capability_content,
- )
-
- try:
- if callable(value):
- value = value()
- return (
- guard_capability_bundle(user_id, value)
- if bundle
- else guard_capability_content(user_id, value)
- )
- except CapabilityContentRejected:
- raise HTTPException(
- status_code=422,
- detail={
- "code": "integrity_failed",
- "message": "capability content blocked by credential policy",
- },
- ) from None
-
-
-def _stream_secrets(user_id: str, target: dict) -> set[str]:
- from core.services.desktop_capability import CapabilityContentRejected, gateway_stream_secrets
-
- try:
- return gateway_stream_secrets(user_id, target)
- except CapabilityContentRejected:
- raise HTTPException(
- status_code=422,
- detail={"code": "integrity_failed", "message": "credential policy unavailable"},
- ) from None
-
-
-async def _checked_upstream_bytes(upstream: httpx.Response, secrets: set[str]):
- from core.services.desktop_capability import CapabilityContentRejected, guard_capability_stream
-
- try:
- async for chunk in guard_capability_stream(upstream.aiter_raw(), secrets):
- yield chunk
- except CapabilityContentRejected:
- logger.warning("[desktop-capability] upstream content blocked code=integrity_failed")
- # Streaming headers may already be sent. Return only a fixed diagnostic;
- # the buffered bytes containing the credential are never released.
- yield b'data: {"error":{"code":"integrity_failed","message":"upstream content blocked"}}\n\n'
- finally:
- await upstream.aclose()
-
-
-# ── manifest(云端,capability token 鉴权) ─────────────────────────────
-
-
-@router.get("/manifest", summary="当前用户的云端能力清单")
-async def get_manifest(
- request: Request,
- response: Response,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import build_user_capability_manifest
-
- use_cache = request.headers.get("cache-control") != "no-cache"
- manifest = await run_in_threadpool(
- _public_content, user_id,
- lambda: build_user_capability_manifest(user_id, use_cache=use_cache),
- )
- etag = f'"{manifest["revision"]}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- response.headers.update(headers)
- return success_response(data=manifest)
-
-
-@router.get("/models", summary="桌面本机执行面的无密钥模型清单")
-async def get_model_manifest(
- request: Request,
- response: Response,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import build_user_model_manifest
-
- manifest = await run_in_threadpool(
- _public_content, user_id, lambda: build_user_model_manifest(user_id)
- )
- etag = f'"{manifest["revision"]}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- response.headers.update(headers)
- return success_response(data=manifest)
-
-
-@router.get("/skills/manifest", summary="当前用户的云端技能清单")
-async def get_skill_manifest(
- request: Request,
- response: Response,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import build_user_skill_manifest
-
- use_cache = request.headers.get("cache-control") != "no-cache"
- manifest = await run_in_threadpool(
- _public_content, user_id,
- lambda: build_user_skill_manifest(user_id, use_cache=use_cache),
- )
- etag = f'"{manifest["revision"]}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- response.headers.update(headers)
- return success_response(data=manifest)
-
-
-@router.get("/skills/{skill_id}/bundle", summary="下载一个当前授权技能的完整 zip 包")
-async def get_skill_bundle(
- skill_id: str,
- request: Request,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import resolve_skill_bundle
-
- resolved = _public_content(
- user_id, lambda: resolve_skill_bundle(user_id, skill_id), bundle=True
- )
- if resolved is None:
- raise HTTPException(status_code=404, detail="skill not available")
- data, content_hash = resolved
- etag = f'"{content_hash}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- return Response(content=data, media_type="application/zip", headers=headers)
-
-
-def _entity_manifest_response(request: Request, response: Response, manifest: dict):
- etag = f'"{manifest["revision"]}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- response.headers.update(headers)
- return success_response(data=manifest)
-
-
-def _bundle_response(request: Request, resolved):
- if resolved is None:
- raise HTTPException(status_code=404, detail="not available")
- data, content_hash = resolved
- etag = f'"{content_hash}"'
- headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
- if request.headers.get("if-none-match") == etag:
- return Response(status_code=304, headers=headers)
- return Response(content=data, media_type="application/zip", headers=headers)
-
-
-@router.get("/agents/manifest", summary="当前用户可见的智能体定义清单")
-async def get_agent_manifest(
- request: Request, response: Response, user_id: str = Depends(_require_capability_user)
-):
- from core.services.desktop_capability import build_user_agent_manifest
-
- use_cache = request.headers.get("cache-control") != "no-cache"
- manifest = await run_in_threadpool(
- _public_content, user_id,
- lambda: build_user_agent_manifest(user_id, use_cache=use_cache),
- )
- return _entity_manifest_response(request, response, manifest)
-
-
-@router.get(
- "/agents/{agent_id}/bundle", summary="下载一个智能体定义包(agent.json + instructions.md)"
-)
-async def get_agent_bundle(
- agent_id: str, request: Request, user_id: str = Depends(_require_capability_user)
-):
- from core.services.desktop_capability import resolve_agent_bundle
-
- return _bundle_response(
- request,
- _public_content(user_id, lambda: resolve_agent_bundle(user_id, agent_id), bundle=True),
- )
-
-
-@router.get("/plugins/manifest", summary="当前用户已安装插件清单")
-async def get_plugin_manifest(
- request: Request, response: Response, user_id: str = Depends(_require_capability_user)
-):
- from core.services.desktop_capability import build_user_plugin_manifest
-
- use_cache = request.headers.get("cache-control") != "no-cache"
- manifest = await run_in_threadpool(
- _public_content, user_id,
- lambda: build_user_plugin_manifest(user_id, use_cache=use_cache),
- )
- return _entity_manifest_response(request, response, manifest)
-
-
-@router.get("/plugins/{install_id}/bundle", summary="下载一个插件定义包(plugin.json)")
-async def get_plugin_bundle(
- install_id: str, request: Request, user_id: str = Depends(_require_capability_user)
-):
- from core.services.desktop_capability import resolve_plugin_bundle
-
- return _bundle_response(
- request,
- _public_content(user_id, lambda: resolve_plugin_bundle(user_id, install_id), bundle=True),
- )
-
-
-# ── MCP 网关(云端,capability token 鉴权,透明反代) ──────────────────
-
-# 请求侧按 deny-list 透传:运行时上下文头(X-Chat-Id、X-Reranker-Enabled、
-# 外接 KB 头等,见 agent_factory._inject_runtime_headers)随本机侧注入原样
-# 过桥,新增上下文头无需改网关;只拦截身份/凭据/逐跳头,身份由网关按
-# capability token 覆写。
-_DROP_REQUEST_HEADERS = {
- # 身份与凭据:capability token 不上传上游;身份头由网关重写
- "authorization",
- "cookie",
- "x-current-user-id",
- # 知识库授权按云端用户重算(头缺失时 KB MCP 自动解析可用知识库)
- "x-allowed-kb-ids",
- # 桌面桥内部头
- "x-desktop-bridge",
- "x-desktop-bridge-user",
- "x-desktop-device-id",
- "x-hugagent-target",
- # 逐跳 / 传输层头,由 httpx 按新连接重建
- "host",
- "content-length",
- "connection",
- "accept-encoding",
- "transfer-encoding",
- "keep-alive",
- "proxy-authorization",
- "te",
- "upgrade",
-}
-# 响应侧透传的头。
-_FWD_RESPONSE_HEADERS = ("content-type", "mcp-session-id", "mcp-protocol-version")
-_RUNTIME_CONTEXT_HEADERS = {
- "x-chat-id",
- "x-channel-id",
- "x-conversation-id",
- "x-allowed-dataset-ids",
- "x-reranker-enabled",
-}
-
-
-class GatewayToolCallBody(BaseModel):
- tool_name: str = Field(..., min_length=1, max_length=200)
- arguments: dict = Field(default_factory=dict)
- schema_hash: str = Field(..., min_length=64, max_length=64)
-
-
-@router.post("/gateway/{server_id}/site-publish", summary="上传桌面构建产物并云端托管")
-async def gateway_site_publish(
- server_id: str,
- request: Request,
- user_id: str = Depends(_require_capability_user),
-):
- import tarfile
-
- from core.infra.exceptions import AppException
- from core.services.desktop_capability import resolve_gateway_tool
- from core.services.desktop_capability_protocol import CapabilityManifestStaleError
- from core.services.desktop_gateway_uploads import (
- MAX_UPLOAD_OPTIONS_CHARS,
- UPLOAD_OPTIONS_HEADER,
- UPLOAD_SCHEMA_HEADER,
- endpoint_plugin,
- )
- from core.services.desktop_site_publish import (
- MAX_ARCHIVE_BYTES,
- SiteUploadOptions,
- publish_uploaded_site,
- )
- from pydantic import ValidationError
-
- try:
- resolved = await run_in_threadpool(
- resolve_gateway_tool,
- user_id,
- server_id,
- "publish_site",
- schema_hash=request.headers.get(UPLOAD_SCHEMA_HEADER, ""),
- )
- except CapabilityManifestStaleError:
- raise HTTPException(status_code=409, detail="capability manifest changed") from None
- if resolved is None or str(
- resolved["target"].get("source_plugin") or ""
- ) != endpoint_plugin("site-publish"):
- raise HTTPException(status_code=403, detail="site publishing is not authorized")
- raw_options = request.headers.get(UPLOAD_OPTIONS_HEADER, "{}")
- if len(raw_options) > MAX_UPLOAD_OPTIONS_CHARS:
- raise HTTPException(status_code=400, detail="site options too large")
- try:
- options = SiteUploadOptions.model_validate_json(raw_options)
- except ValidationError:
- raise HTTPException(status_code=400, detail="invalid site options") from None
- archive = bytearray()
- async for chunk in request.stream():
- if len(archive) + len(chunk) > MAX_ARCHIVE_BYTES:
- raise HTTPException(status_code=413, detail="site archive exceeds 40 MB")
- archive.extend(chunk)
- try:
- result = await asyncio.to_thread(publish_uploaded_site, user_id, bytes(archive), options)
- except (ValueError, tarfile.TarError):
- raise HTTPException(status_code=400, detail="invalid site build archive") from None
- except AppException as exc:
- raise HTTPException(status_code=400, detail=exc.message) from None
- return success_response(data=_public_content(user_id, result))
-
-
-@router.post(
- "/gateway/{server_id}/call",
- summary="桌面云端工具调用网关(动态 manifest schema + JSON invocation)",
-)
-async def gateway_mcp_call(
- server_id: str,
- body: GatewayToolCallBody,
- request: Request,
- user_id: str = Depends(_require_capability_user),
-):
- """Execute a currently-authorized MCP tool from the cloud network."""
- from core.services.desktop_capability import (
- CapabilityContentRejected,
- invoke_gateway_tool,
- resolve_gateway_tool,
- )
- from core.services.desktop_capability_protocol import CapabilityManifestStaleError
-
- try:
- resolved = await run_in_threadpool(
- resolve_gateway_tool,
- user_id,
- server_id,
- body.tool_name,
- schema_hash=body.schema_hash,
- )
- except CapabilityManifestStaleError as exc:
- raise HTTPException(status_code=409, detail="capability manifest changed") from exc
- if resolved is None:
- raise HTTPException(status_code=404, detail="tool not available")
-
- runtime_headers = {
- k: v for k, v in request.headers.items() if k.lower() in _RUNTIME_CONTEXT_HEADERS
- }
- from core.services.automation_remote_effect import RemoteOperationConflict
- started = time.monotonic()
- try:
- result = await invoke_gateway_tool(resolved, body.arguments, runtime_headers)
- except HTTPException:
- from extensions.mcp_servers.registry import gateway_server
- if gateway_server(resolved["server_id"]) is not None:
- raise
- raise HTTPException(502, "Cloud tool call failed") from None
- except RemoteOperationConflict:
- raise HTTPException(status_code=400, detail="operation key already bound to different arguments") from None
- except CapabilityContentRejected:
- raise HTTPException(
- status_code=422,
- detail={"code": "integrity_failed", "message": "upstream content blocked"},
- ) from None
- except asyncio.TimeoutError as exc:
- logger.warning(
- "[desktop-capability] tool call timeout user=%s server=%s tool=%s",
- user_id,
- server_id,
- body.tool_name,
- )
- raise HTTPException(status_code=504, detail="upstream mcp tool timed out") from exc
- except Exception as exc: # noqa: BLE001 - never expose cloud credentials/errors
- logger.warning(
- "[desktop-capability] tool call failed user=%s server=%s tool=%s error_type=%s",
- user_id,
- server_id,
- body.tool_name,
- type(exc).__name__,
- )
- raise HTTPException(status_code=502, detail="upstream mcp tool failed") from exc
-
- logger.info(
- "[desktop-capability] tool call user=%s server=%s tool=%s duration_ms=%.0f",
- user_id,
- server_id,
- body.tool_name,
- (time.monotonic() - started) * 1000,
- )
- return success_response(data=_public_content(user_id, result))
-
-
-class GatewayReceiptBody(BaseModel):
- tool_name: str = Field(..., min_length=1, max_length=128)
- schema_hash: str = Field(..., min_length=64, max_length=64)
- operation_id: str = Field(..., min_length=1, max_length=128)
-
-
-@router.post("/gateway/{server_id}/authorize", summary="校验本机任务工具的当前授权")
-async def gateway_authorize_local(
- server_id: str, body: GatewayReceiptBody,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import resolve_gateway_tool
- from core.services.desktop_capability_protocol import CapabilityManifestStaleError
- from core.services.automation_tool_routing import TOOLS
- try:
- resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id,
- body.tool_name, schema_hash=body.schema_hash)
- except CapabilityManifestStaleError:
- raise HTTPException(status_code=409, detail="capability manifest changed") from None
- if (resolved is None or body.tool_name not in TOOLS
- or resolved["target"].get("source_plugin") != "automation"):
- raise HTTPException(status_code=403, detail="local automation is not authorized")
- return success_response(data={"authorized": True})
-
-
-@router.post("/gateway/{server_id}/receipt", summary="查询云端操作回执")
-async def gateway_effect_receipt(
- server_id: str, body: GatewayReceiptBody,
- user_id: str = Depends(_require_capability_user),
-):
- from core.services.desktop_capability import resolve_gateway_tool
- from core.services.desktop_capability_protocol import CapabilityManifestStaleError
- from core.services.automation_remote_effect import RECEIPT_TOOLS, lookup_remote_receipt
-
- try:
- resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id,
- body.tool_name, schema_hash=body.schema_hash)
- except CapabilityManifestStaleError:
- raise HTTPException(status_code=409, detail="capability manifest changed") from None
- if resolved is None or body.tool_name not in RECEIPT_TOOLS:
- raise HTTPException(status_code=404, detail="tool not available")
- result = await run_in_threadpool(lookup_remote_receipt, user_id, server_id,
- body.tool_name, body.operation_id)
- return success_response(data=_public_content(user_id, result))
-
-
-@router.post(
- "/gateway/models/{provider_id}/{model_path:path}",
- summary="桌面模型网关(OpenAI-compatible 流式反代)",
-)
-async def gateway_model(
- provider_id: str,
- model_path: str,
- request: Request,
- user_id: str = Depends(_require_capability_user),
-):
- """把本机 Agent 的对话/向量/重排请求转发到云端内网模型。
-
- 路径、模型名和上游凭据全由云端 DB 重算;客户端提供的
- Authorization / x-api-key / model 字段都不可信。
- """
- from core.services.desktop_capability import resolve_model_gateway_target
-
- target = await run_in_threadpool(resolve_model_gateway_target, user_id, provider_id)
- if target is None or model_path.strip("/") != target["path"]:
- raise HTTPException(status_code=404, detail="model not available")
-
- try:
- payload = await request.json()
- except (json.JSONDecodeError, UnicodeDecodeError, ValueError):
- raise HTTPException(status_code=400, detail="model request must be JSON")
- if not isinstance(payload, dict):
- raise HTTPException(status_code=400, detail="model request must be a JSON object")
- payload["model"] = target["model_name"]
- request.state.desktop_observation_model = target["model_name"]
-
- headers = {
- "authorization": f"Bearer {target['api_key']}",
- "content-type": "application/json",
- "accept": request.headers.get("accept", "application/json"),
- # aiter_raw 保留原始字节,因此明确要求上游不压缩,避免在没有
- # Content-Encoding 响应头的情况下把 gzip 字节直接送给 OpenAI SDK。
- "accept-encoding": "identity",
- }
- response_secrets = await run_in_threadpool(_stream_secrets, user_id, target)
- client = _client()
- upstream_req = client.build_request(
- "POST",
- target["url"],
- headers=headers,
- content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
- )
- try:
- upstream = await client.send(upstream_req, stream=True)
- except httpx.HTTPError as exc:
- logger.warning(
- "[desktop-capability] model gateway upstream error user=%s provider=%s error_type=%s",
- user_id,
- provider_id,
- type(exc).__name__,
- )
- raise HTTPException(status_code=502, detail="upstream model unreachable")
-
- logger.info(
- "[desktop-capability] model gateway user=%s provider=%s type=%s status=%s",
- user_id,
- provider_id,
- target["provider_type"],
- upstream.status_code,
- )
- response_headers = {}
- if "content-type" in upstream.headers:
- response_headers["content-type"] = upstream.headers["content-type"]
- return StreamingResponse(
- _checked_upstream_bytes(upstream, response_secrets),
- status_code=upstream.status_code,
- headers=response_headers,
- background=BackgroundTask(upstream.aclose),
- )
-
-
-@router.api_route(
- "/gateway/{server_id}/mcp",
- methods=["POST", "GET", "DELETE"],
- summary="桌面能力网关(MCP streamable-http 反代)",
-)
-async def gateway_mcp(
- server_id: str,
- request: Request,
- user_id: str = Depends(_require_capability_user),
-):
- # 授权解析走 30s per-user 缓存、内部短会话——不经 Depends(get_db),
- # 避免流式转发全程占住连接池里的一条 DB 连接。
- from core.services.desktop_capability import resolve_gateway_target
-
- target = await run_in_threadpool(resolve_gateway_target, user_id, server_id)
- if target is None:
- raise HTTPException(status_code=404, detail="server not available")
-
- headers: dict[str, str] = {
- k: v for k, v in request.headers.items() if k.lower() not in _DROP_REQUEST_HEADERS
- }
- # 云端侧连接凭据(远程第三方 MCP 的 headers 等)只在此处物化,不出云端。
- for k, v in (target.get("headers") or {}).items():
- if isinstance(k, str) and isinstance(v, str):
- headers[k] = v
- # 身份以 capability token 为准,客户端注入的同名头已在 deny-list 拦下。
- headers["X-Current-User-Id"] = user_id
- # StreamingResponse 使用 aiter_raw 原样转发,因此不允许上游压缩后再丢失
- # Content-Encoding;这是旧客户端透明 MCP 端点的传输完整性要求。
- headers["accept-encoding"] = "identity"
-
- body = await request.body()
- response_secrets = await run_in_threadpool(_stream_secrets, user_id, target)
- client = _client()
- upstream_req = client.build_request(
- request.method, target["url"], headers=headers, content=body
- )
- try:
- upstream = await client.send(upstream_req, stream=True)
- except httpx.HTTPError as exc:
- logger.warning(
- "[desktop-capability] gateway upstream error user=%s server=%s error_type=%s",
- user_id,
- server_id,
- type(exc).__name__,
- )
- raise HTTPException(status_code=502, detail="upstream mcp unreachable")
-
- logger.info(
- "[desktop-capability] gateway %s user=%s server=%s status=%s",
- request.method,
- user_id,
- server_id,
- upstream.status_code,
- )
- resp_headers = {
- name: upstream.headers[name] for name in _FWD_RESPONSE_HEADERS if name in upstream.headers
- }
- return StreamingResponse(
- _checked_upstream_bytes(upstream, response_secrets),
- status_code=upstream.status_code,
- headers=resp_headers,
- background=BackgroundTask(upstream.aclose),
- )
-
-
-# ── 本机侧:壳推送桥配置 ────────────────────────────────────────────────
-
-
-class CloudBridgeBody(BaseModel):
- cloud_base: str = Field(..., min_length=1, description="云端后端根地址(含协议)")
- token: str = Field(..., min_length=8, description="capability token")
- expires_in: int = Field(default=600, ge=60, le=600)
- device_id: str = Field(
- ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
- )
-
-
-def _require_desktop_bridge_process() -> None:
- from core.auth.desktop_bridge import bridge_enabled
-
- if not bridge_enabled():
- raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用")
-
-
-def _require_desktop_shell_control(request: Request) -> None:
- from core.services.desktop_capability import is_desktop_shell_control
-
- _require_desktop_bridge_process()
- if not is_desktop_shell_control(
- request.headers.get("authorization", ""), request.headers.get("origin")
- ):
- raise HTTPException(status_code=401, detail="desktop shell authorization required")
-
-
-@router.post("/cloud-bridge", summary="推送云端能力桥配置(桌面壳 → 本机后端)")
-async def set_cloud_bridge(
- body: CloudBridgeBody,
- _: None = Depends(_require_desktop_shell_control),
-):
- _require_desktop_bridge_process()
- from core.services.desktop_cloud_bridge import set_state
-
- base = body.cloud_base.strip().rstrip("/")
- from urllib.parse import urlsplit
-
- try:
- parsed = urlsplit(base)
- except ValueError:
- raise HTTPException(status_code=400, detail="cloud_base 地址无效")
- if (
- parsed.scheme not in ("http", "https")
- or not parsed.hostname
- or parsed.username
- or parsed.password
- or parsed.query
- or parsed.fragment
- ):
- raise HTTPException(status_code=400, detail="cloud_base 必须是不含凭据的 http(s) 地址")
- set_state(base, body.token, body.expires_in, device_id=body.device_id)
- logger.info("[cloud-bridge] 桥配置已更新(cloud_base=%s)", base)
- return success_response(data={"ok": True})
-
-
-@router.delete("/cloud-bridge", summary="清除云端能力桥(桌面退出登录)")
-async def clear_cloud_bridge(_: None = Depends(_require_desktop_shell_control)):
- _require_desktop_bridge_process()
- from core.services.desktop_cloud_bridge import clear_state
-
- clear_state()
- return success_response(data={"ok": True})
-
-
-@router.get("/cloud-bridge/status", summary="云端能力桥状态")
-async def cloud_bridge_status(_user: UserContext = Depends(get_current_user)):
- from core.services.desktop_cloud_bridge import bridge_status
-
- return success_response(data=bridge_status())
-
-
-@router.get("/cloud-bridge/readiness", summary="桌面壳首次能力同步进度")
-@quiet_access_log
-async def cloud_bridge_readiness(_: None = Depends(_require_desktop_shell_control)):
- from core.services.desktop_cloud_bridge import initial_sync_status
-
- return success_response(data=await asyncio.to_thread(initial_sync_status))
diff --git a/src/backend/api/routes/v1/evolution.py b/src/backend/api/routes/v1/evolution.py
index 9f56c954..d9bd40c6 100644
--- a/src/backend/api/routes/v1/evolution.py
+++ b/src/backend/api/routes/v1/evolution.py
@@ -12,9 +12,9 @@
from typing import Optional
from core.auth.backend import UserContext, get_current_user
-from core.evolution.settlement import SETTLE_PENDING
+from core.evolution.settlement.settlement import SETTLE_PENDING
from core.infra.exceptions import BadRequestError
-from core.evolution.settlement_store import load_summary
+from core.evolution.settlement.settlement_store import load_summary
from core.infra.responses import success_response
from fastapi import APIRouter, Depends
from pydantic import BaseModel, Field
@@ -53,7 +53,7 @@ async def get_evolution_settings(user: UserContext = Depends(get_current_user)):
simply follows ``evolution_prefs.enabled``. This endpoint stays because
already-shipped desktop bundles still read it.
"""
- from core.evolution.user_settings import resolve_for_user
+ from core.evolution.profiles.user_settings import resolve_for_user
return success_response(data=resolve_for_user(user.user_id))
@@ -64,7 +64,7 @@ async def update_evolution_settings(
user: UserContext = Depends(get_current_user),
):
"""Legacy alias: writes the same switch as PATCH /evolution/prefs."""
- from core.evolution.user_settings import update_for_user
+ from core.evolution.profiles.user_settings import update_for_user
try:
return success_response(data=update_for_user(user.user_id, body.enabled))
@@ -80,7 +80,7 @@ async def get_contributions(user: UserContext = Depends(get_current_user)):
it actually do?" — and reports each candidate's true governance state rather
than an optimistic one.
"""
- from core.evolution.user_settings import summarise_contributions
+ from core.evolution.profiles.personal_review import summarise_contributions
return success_response(data=summarise_contributions(user.user_id))
@@ -93,7 +93,8 @@ async def list_my_candidates(user: UserContext = Depends(get_current_user)):
from other people's conversations activates for the whole fleet, and that is
not one person's call to make.
"""
- from core.evolution.user_settings import OWN_EVIDENCE_THRESHOLD, pending_for_user
+ from core.evolution.profiles.user_settings_constants import OWN_EVIDENCE_THRESHOLD
+ from core.evolution.profiles.personal_review import pending_for_user
return success_response(
data={
@@ -109,8 +110,8 @@ async def approve_my_candidate(
user: UserContext = Depends(get_current_user),
):
"""Accept a candidate as a private skill for this user."""
- from core.evolution.activation import ActivationError
- from core.evolution.user_settings import approve_for_user
+ from core.evolution.cycle.activation_types import ActivationError
+ from core.evolution.profiles.personal_review import approve_for_user
try:
return success_response(data=approve_for_user(candidate_id, user.user_id))
@@ -137,11 +138,7 @@ async def get_prefs(user: UserContext = Depends(get_current_user)):
configured" as a visible fact rather than hiding the setting — an absent
control looks like a missing feature, which is a different message.
"""
- from core.evolution.prefs import (
- ONTOLOGY_MODES,
- available_ontology_packs,
- resolve_prefs,
- )
+ from core.evolution.profiles.prefs import ONTOLOGY_MODES, available_ontology_packs, resolve_prefs
return success_response(
data={
@@ -159,7 +156,7 @@ async def update_prefs_api(
):
import asyncio
- from core.evolution.prefs import update_prefs
+ from core.evolution.profiles.prefs import update_prefs
patch = {k: v for k, v in body.model_dump().items() if v is not None}
@@ -169,7 +166,7 @@ async def update_prefs_api(
# to tell that apart from "my conversations just aren't producing patterns".
# Turning it off is never blocked.
if patch.get("enabled") is True:
- from core.evolution.readiness import check_readiness
+ from core.evolution.cycle.readiness import check_readiness
readiness = await asyncio.to_thread(check_readiness)
if not readiness.ready:
@@ -194,6 +191,6 @@ async def run_my_cycle(user: UserContext = Depends(get_current_user)):
Personal scope, so it needs no administrator: it can only ever mine this
user's evidence and produce candidates only they can accept.
"""
- from core.evolution.personal import run_personal_cycle
+ from core.evolution.profiles.personal import run_personal_cycle
return success_response(data=run_personal_cycle(user.user_id))
diff --git a/src/backend/api/routes/v1/file_parse.py b/src/backend/api/routes/v1/file_parse.py
index 38059a0b..1f7891ea 100644
--- a/src/backend/api/routes/v1/file_parse.py
+++ b/src/backend/api/routes/v1/file_parse.py
@@ -3,7 +3,8 @@
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from core.auth.backend import get_current_user, UserContext
-from core.content.file_parser import SUPPORTED_EXTENSIONS, parse_file
+from core.files.content.file_formats import SUPPORTED_EXTENSIONS
+from core.files.content.file_parser import parse_file
router = APIRouter(prefix="/v1/file", tags=["file"])
diff --git a/src/backend/api/routes/v1/file_text.py b/src/backend/api/routes/v1/file_text.py
index 4e433a30..fcffbfd3 100644
--- a/src/backend/api/routes/v1/file_text.py
+++ b/src/backend/api/routes/v1/file_text.py
@@ -7,7 +7,7 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.artifact_text_service import ArtifactTextService
+from core.files.artifacts.text import ArtifactTextService
router = APIRouter()
diff --git a/src/backend/api/routes/v1/file_upload.py b/src/backend/api/routes/v1/file_upload.py
index ef9eae86..0cdc1d3f 100644
--- a/src/backend/api/routes/v1/file_upload.py
+++ b/src/backend/api/routes/v1/file_upload.py
@@ -12,9 +12,9 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.db.models import Artifact, ChatSession, UserFolder
-from core.services.artifact_service import store_bytes_as_artifact
-from core.storage import get_storage
-from core.services.upload_retry import prepare_upload
+from core.files.artifacts.service import store_bytes_as_artifact
+from core.files.storage import get_storage
+from core.files.upload_retry import prepare_upload
from .file_text import router as text_router
logger = logging.getLogger(__name__)
@@ -59,7 +59,7 @@ def upload_user_file(
public_meta = {}
if user.public_chat_visitor:
- from core.services.public_chat_service import prepare_visitor_upload
+ from core.chat.public import prepare_visitor_upload
public_meta = prepare_visitor_upload(db, user, chat_id, folder_id)
# Verify the chat_id already exists in the database (the session may not yet be created at upload time)
@@ -121,8 +121,8 @@ def upload_user_file(
raise HTTPException(status_code=500, detail=f"文件上传失败: {e}")
if db_folder_id:
- from core.spaces.files import SpaceFiles
- from core.spaces.projects import folder_path
+ from core.files.spaces.files import SpaceFiles
+ from core.files.spaces.projects import folder_path
directory = folder_path(db, UserFolder, {"user_id": user_id}, db_folder_id)
SpaceFiles(db, user_id, get_storage()).write(
"personal", directory + "/" + file.filename, file_bytes,
@@ -138,7 +138,7 @@ def upload_user_file(
@router.put("/{file_id}", summary="覆盖已有文件内容")
-async def overwrite_file(
+def overwrite_file(
file_id: str,
file: UploadFile = File(...),
user: UserContext = Depends(get_current_user),
@@ -147,21 +147,21 @@ async def overwrite_file(
"""
用新内容覆盖已有文件,保持 file_id 和 download_url 不变。
"""
- file_bytes = await file.read()
+ file_bytes = file.file.read()
if not file_bytes:
raise HTTPException(status_code=400, detail="文件内容为空")
if len(file_bytes) > _MAX_UPLOAD_BYTES:
raise HTTPException(status_code=413, detail="文件过大,最大支持 50 MB")
- from core.artifacts.local_reference import is_local_file_ref, overwrite_reference
+ from core.files.artifacts.local_reference import is_local_file_ref, overwrite_reference
if is_local_file_ref(file_id):
return overwrite_reference(file_id, str(user.user_id), file_bytes, db)
- from core.auth.permissions_iface import resolve_artifact_access
+ from core.auth.edition import resolve_artifact_access
artifact = db.query(Artifact).filter(
Artifact.artifact_id == file_id, Artifact.deleted_at.is_(None),
).populate_existing().with_for_update().first()
- from core.services.artifact_edition import artifact_access_metadata
+ from core.files.edition import artifact_access_metadata
metadata = artifact_access_metadata(artifact) if artifact else {}
level = resolve_artifact_access(db, str(user.user_id), metadata.get("owner_id"), metadata.get("scope_id")) if artifact else "none"
if level == "none":
diff --git a/src/backend/api/routes/v1/internal_jobs.py b/src/backend/api/routes/v1/internal_jobs.py
index 3e4dc18d..5e8bb4ea 100644
--- a/src/backend/api/routes/v1/internal_jobs.py
+++ b/src/backend/api/routes/v1/internal_jobs.py
@@ -21,10 +21,11 @@
from core.db.engine import SessionLocal
from core.infra.responses import success_response
-from core.services.job_service import JobService
+from core.jobs.store import JobService
from fastapi import APIRouter, Header, HTTPException
from .job_callback_session import callback_session
+from core.db.threads import run_in_session
logger = logging.getLogger(__name__)
@@ -123,7 +124,7 @@ def _schema_hint(schema: Dict[str, Any]) -> str:
)
-from orchestration.jobs.agent import run_derived_agent, run_with_job_fence
+from core.jobs.agent import run_derived_agent, run_with_job_fence
# ── 端点 ──────────────────────────────────────────────────────────────
@@ -145,8 +146,10 @@ def validate_execution():
if remaining["seconds_left"] <= 0 or remaining["calls_left"] <= 0:
raise HTTPException(429, "作业执行预算已用尽")
- with SessionLocal() as db:
+ def _budget_left(db):
left = JobService(db).budget_left(job_id)
+ return left
+ left = await run_in_session(_budget_left)
if left["calls_left"] <= 0:
raise HTTPException(status_code=429, detail="作业子调用预算已用尽")
if left["seconds_left"] <= 0:
@@ -187,7 +190,7 @@ def validate_execution():
async with _GLOBAL_GATE, gate:
_auth(job_id, x_job_token)
try:
- from core.services.run_journal import durable_run_binding
+ from core.runs.journal import durable_run_binding
async with durable_run_binding(
user_id=ctx["user_id"],
@@ -234,7 +237,7 @@ def validate_execution():
raise HTTPException(409, "作业预算已耗尽,本项已停止,不自动重试") from exc
raise
except Exception as exc:
- from core.services.tool_effect_ledger import find_tool_outcome_unknown
+ from core.runs.tool_effects.errors import find_tool_outcome_unknown
if find_tool_outcome_unknown(exc) is not None:
raise HTTPException(
@@ -366,7 +369,7 @@ def job_log(
if msg:
logger.info("[job %s] %s", job_id, msg[:500])
try:
- from orchestration.jobs.notifications import _emit_progress
+ from core.jobs.notifications import emit_progress as _emit_progress
_emit_progress(ctx["chat_id"], msg[:200])
except Exception: # noqa: BLE001
diff --git a/src/backend/api/routes/v1/job_callback_session.py b/src/backend/api/routes/v1/job_callback_session.py
index 2455844b..51924bbf 100644
--- a/src/backend/api/routes/v1/job_callback_session.py
+++ b/src/backend/api/routes/v1/job_callback_session.py
@@ -3,7 +3,7 @@
from contextlib import contextmanager
from core.db.engine import SessionLocal
-from core.services.job_service import JobService
+from core.jobs.store import JobService
from fastapi import HTTPException
diff --git a/src/backend/api/routes/v1/jobs.py b/src/backend/api/routes/v1/jobs.py
index ecc5ca43..96cc3e51 100644
--- a/src/backend/api/routes/v1/jobs.py
+++ b/src/backend/api/routes/v1/jobs.py
@@ -16,7 +16,7 @@
from core.db.models import Job
from core.infra.logging import get_logger
from core.infra.responses import success_response
-from core.services.job_service import JobService
+from core.jobs.store import JobService
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
@@ -86,7 +86,7 @@ def get_job(
@router.post("/{job_id}/cancel", summary="用户手动取消作业")
async def cancel_job(job_id: str, user: UserContext = Depends(get_current_user)):
"""状态条上的取消按钮 —— 跑歪了的作业不该只能等它烧完预算。"""
- from orchestration import job_runtime
+ import core.jobs.runtime as job_runtime
try:
ok = await job_runtime.cancel_job(job_id, user_id=user.user_id)
diff --git a/src/backend/api/routes/v1/kb.py b/src/backend/api/routes/v1/kb.py
deleted file mode 100644
index 138d2e2b..00000000
--- a/src/backend/api/routes/v1/kb.py
+++ /dev/null
@@ -1,873 +0,0 @@
-"""Knowledge Base management API routes (v1).
-
-Route definitions only — models live in ``kb_models.py``, processing logic
-in ``core/kb_processing.py``, and CRUD in ``core/services/kb_service.py``.
-"""
-
-import hashlib
-import json
-import logging
-import os
-from typing import Optional
-
-from api.routes.v1.kb_models import (
- CreateKBSpaceRequest,
- PolishKBDescriptionRequest,
- ReindexRequest,
- UpdateChunkRequest,
- UpdateKBSpaceRequest,
-)
-from core.auth.backend import UserContext, get_current_user
-from core.config.settings import settings
-from core.content.file_validation import validate_kb_file
-from core.kb.wiki.config import index_modes_of, normalize_index_modes
-from core.db.engine import get_db
-from core.infra.exceptions import (
- AccessDeniedError,
- BadRequestError,
- FileTooLargeError,
- ResourceNotFoundError,
- StorageError,
-)
-from core.infra.responses import created_response, paginated_response, success_response
-from core.llm.chat_models import get_summarize_model
-from core.llm.message_compat import extract_text_from_chat_response
-from core.services import KBService
-from core.storage import generate_storage_key, get_storage
-from fastapi import APIRouter, Depends, File, Form, Path, Query, UploadFile, status
-from fastapi.concurrency import run_in_threadpool
-from sqlalchemy.orm import Session
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/v1/catalog/kb", tags=["Knowledge Base"])
-
-# Constants
-MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB
-MAX_PREVIEW_SIZE = 20 * 1024 * 1024 # 20MB
-
-
-def _can_read_space(db, space, user_id: str) -> bool:
- """Read permission: owner / everyone for public KBs / grantees of a scoped-visibility KB (single-source-of-truth resolution)."""
- from core.auth.kb_permissions import resolve_local_kb_level
-
- return resolve_local_kb_level(db, user_id, space.kb_id) != "none"
-
-
-def _require_kb_write(db, space, user_id: str, required: str) -> None:
- """Write permission check: required is 'edit' (upload/reindex) or 'admin' (rename/delete/edit chunks).
-
- Raises AccessDeniedError when insufficient. The owner is always admin, superadmins are always admin, granted users go by their level.
- """
- from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level
-
- level = resolve_local_kb_level(db, user_id, space.kb_id)
- if not has_kb_permission(level, required):
- raise AccessDeniedError(message="Access denied", reason=f"requires kb level >= {required}")
-
-
-def _enqueue_document_indexing(
- db,
- *,
- document_id: str,
- user_id: str,
- chunk_method: str,
- indexing_config: Optional[dict],
- index_modes: Optional[list],
-) -> None:
- """把文档排进持久化索引队列,交给常驻 worker 执行。
-
- 索引不能再跑在请求线程池里:那是 Starlette 与所有同步依赖(``get_db``)共用的
- 池子,批量上传会把它占满,导致整个后端在 handler 之前就排队。详见
- :mod:`core.kb.index_worker`。
- """
- from core.db.models import KBDocument
- from core.kb import index_queue
-
- doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first()
- if doc is None:
- logger.warning("Cannot enqueue indexing: document %s not found", document_id)
- return
- index_queue.enqueue(
- db,
- doc,
- user_id=user_id,
- chunk_method=chunk_method,
- indexing_config=indexing_config,
- index_modes=index_modes,
- )
-
-
-def _fallback_kb_description(name: str, description: str) -> str:
- base_name = (name or "").strip()
- base_desc = " ".join((description or "").split())
- if base_desc:
- return (
- f"{base_name}相关资料,适合检索概念说明、政策信息、行业动态、业务规则和常见问题。"
- f"当前范围包括:{base_desc[:120]}"
- )[:220]
- return (
- f"{base_name}相关内容,适合检索概念说明、政策解读、业务规则、行业动态、"
- f"典型案例和常见问题。"
- )[:220]
-
-
-async def _generate_kb_description(name: str, description: str) -> str:
- prompt = f"""/no_think
-你是一名知识库配置助手。请根据知识库名称和现有简介,生成一段更利于智能路由判断的“知识库简介”。
-
-目标:
-1. 帮助系统快速判断什么问题应该路由到这个知识库
-2. 文本要像知识库简介,不是宣传文案
-3. 明确主题范围、文档类型、适合回答的问题类型
-4. 用中文,1 段即可,控制在 60 到 120 字
-5. 不要使用“本知识库可以帮助你”这类口语化表达
-6. 不要使用“该知识库聚焦”“本知识库涵盖”这类空泛起手式
-7. 直接输出简介正文,不要加标题、引号、序号或解释
-
-知识库名称:{name}
-现有简介:{description or "无"}
-"""
-
- try:
- model = get_summarize_model()
- # AgentScope 2.0: model.__call__ takes list[Msg] (not list[dict])
- from agentscope.message import Msg, TextBlock
-
- result = await model(
- messages=[Msg(name="user", role="user", content=[TextBlock(type="text", text=prompt)])]
- )
- polished = extract_text_from_chat_response(result).strip()
- polished = " ".join(polished.split())
- return polished or _fallback_kb_description(name, description)
- except Exception as exc:
- logger.warning("Failed to polish KB description for %s: %s", name, exc)
- return _fallback_kb_description(name, description)
-
-
-# ── Routes ──────────────────────────────────────────────────────────────────
-
-
-@router.get("/assets/{asset_id}", summary="读取知识库媒体资产")
-def get_kb_asset(
- asset_id: str = Path(..., description="Asset ID"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """按 ID 读取知识库中的媒体资产(版面解析切出的图片等)原始字节。
-
- 资产的可见性完全跟随所属知识库空间:能读该库才能读它的图。图片链接会出现在分块正文
- 与检索结果里,故按 ID 直取,不经过库/文档路径。
- """
- from fastapi.responses import Response
-
- from core.db.models import KBAsset
- from core.db.repository import KBRepository
-
- asset = db.query(KBAsset).filter(KBAsset.asset_id == asset_id).first()
- if not asset:
- raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
-
- kb_space = KBRepository(db).get_space(asset.kb_id)
- if not kb_space or not _can_read_space(db, kb_space, user.user_id):
- # Same shape as a missing asset: an unauthorised caller must not be able to
- # probe which asset ids exist.
- raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
-
- try:
- data = get_storage().download_bytes(asset.storage_key)
- except Exception as exc:
- logger.warning("Asset %s read failed: %s", asset_id, exc)
- raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
-
- return Response(
- content=data,
- media_type=asset.mime_type or "application/octet-stream",
- headers={"Cache-Control": "private, max-age=86400"},
- )
-
-
-@router.post("/preview-chunks", summary="预览文档分块效果")
-async def preview_chunks(
- file: UploadFile = File(..., description="Document file to preview"),
- chunk_method: str = Form(
- "structured", description="Chunking method: structured|recursive|embedding_semantic|laws|qa"
- ),
- parent_chunk_size: int = Form(1024, description="Parent chunk size in tokens"),
- child_chunk_size: int = Form(128, description="Child chunk size in tokens"),
- overlap_tokens: int = Form(20, description="Overlap between child chunks"),
- parent_child_indexing: bool = Form(
- True, description="Enable parent-child indexing; False = flat chunks only"
- ),
- separators: Optional[str] = Form(
- None, description="父分块分隔符,JSON 数组字符串;为空用默认层级"
- ),
- child_separators: Optional[str] = Form(
- None, description="子分块分隔符,JSON 数组字符串;为空走定长滑窗"
- ),
- user: UserContext = Depends(get_current_user),
-):
- """预览文档的分块效果,不落库、不写入向量库。上传文件并按指定分块方法(structured/recursive/embedding_semantic/laws/qa)与父子分块参数解析,返回父块及其子块预览,用于创建知识库前调参。文件大小受 MAX_PREVIEW_SIZE(20MB)限制。"""
- content = await file.read()
- if len(content) > MAX_PREVIEW_SIZE:
- raise FileTooLargeError(max_size=MAX_PREVIEW_SIZE, actual_size=len(content))
- if len(content) == 0:
- raise BadRequestError(message="File is empty", data={"filename": file.filename})
-
- if not file.filename:
- raise BadRequestError(message="Filename is required", data={"field": "file"})
- _, mime_type = validate_kb_file(file.filename, content)
-
- from core.kb.kb_parser import _count_tokens, parse_and_chunk, parse_separators_form
-
- _separators = parse_separators_form(separators)
- _child_separators = parse_separators_form(child_separators)
-
- embed_fn = None
- if chunk_method == "embedding_semantic":
- from core.kb.kb_vector import embed_batch
-
- embed_fn = embed_batch
-
- parent_chunks = parse_and_chunk(
- content,
- mime_type,
- chunk_method=chunk_method,
- parent_size=parent_chunk_size,
- child_size=child_chunk_size,
- overlap=overlap_tokens,
- embed_fn=embed_fn,
- separators=_separators,
- child_separators=_child_separators,
- )
-
- total_children = sum(len(pc.children) for pc in parent_chunks) if parent_child_indexing else 0
-
- chunks_preview = []
- for idx, pc in enumerate(parent_chunks):
- if parent_child_indexing:
- children_preview = [
- {
- "index": c.index,
- "content": c.content[:200] + ("..." if len(c.content) > 200 else ""),
- }
- for c in pc.children[:5]
- ]
- children_count = len(pc.children)
- else:
- children_preview = []
- children_count = 0
- chunks_preview.append(
- {
- "index": idx,
- "content": pc.content,
- "token_count": _count_tokens(pc.content),
- "children_count": children_count,
- "children_preview": children_preview,
- }
- )
-
- return success_response(
- data={
- "total_chunks": len(parent_chunks),
- "total_children": total_children,
- "chunks": chunks_preview,
- },
- message="Chunk preview generated successfully",
- )
-
-
-@router.post("", status_code=status.HTTP_201_CREATED, summary="创建知识库空间")
-def create_kb_space(
- request: CreateKBSpaceRequest,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """创建归属当前登录用户的知识库空间;可见性规则由当前版本决定。"""
- visibility = (request.visibility or "private").strip()
- if visibility not in ("private", "public"):
- raise BadRequestError(
- message="visibility 仅支持 private / public", data={"field": "visibility"}
- )
- if settings.edition.edition == "ce" and visibility != "private":
- raise BadRequestError(
- message="CE 仅支持私有知识库",
- data={"field": "visibility", "allowed": ["private"]},
- )
-
- # Permission check: require the creation permission matching the chosen visibility
- from core.auth.capabilities import resolve_user_capabilities
-
- caps = resolve_user_capabilities(db, user.user_id)
- required_cap = "can_create_public_kb" if visibility == "public" else "can_create_private_kb"
- if not caps.get(required_cap, False):
- label = "公有" if visibility == "public" else "私有"
- raise AccessDeniedError(message="Access denied", reason=f"缺少「创建{label}知识库」权限")
-
- kb_service = KBService(db)
- metadata = dict(request.metadata or {})
- if request.indexing_config:
- metadata["indexing_config"] = request.indexing_config.model_dump()
-
- # 索引模式:rag(向量检索)/ wiki(LLM Wiki 图谱),两者同选即 LLM-Wiki 知识库。
- # 三种组合都会分块——wiki 省掉的是向量化,不是分块(引文标注要 chunk id、
- # 回溯原文要按 chunk 直取)。
- metadata["index_modes"] = normalize_index_modes(request.index_modes)
- if request.wiki_config:
- metadata["wiki_config"] = {
- k: v for k, v in request.wiki_config.model_dump().items() if v is not None
- }
-
- from core.services.kb_edition import initial_visibility_grants
-
- initial_grants = initial_visibility_grants(db, user.user_id, visibility)
-
- space = kb_service.create_space(
- user_id=user.user_id,
- name=request.name,
- description=request.description,
- chunk_method=request.chunk_method or "semantic",
- metadata=metadata,
- visibility=visibility,
- initial_grants=initial_grants,
- granted_by=user.user_id,
- )
- return created_response(data=space, message="Knowledge base space created successfully")
-
-
-@router.post("/polish-description", summary="AI 生成知识库简介")
-async def polish_kb_description(
- request: PolishKBDescriptionRequest,
- user: UserContext = Depends(get_current_user),
-):
- """根据知识库名称和现有简介,调用大模型生成一段更利于智能路由判断的知识库简介。名称为必填,模型失败时回退到本地模板生成。仅返回润色后的简介文本,不修改任何知识库记录。"""
- name = request.name.strip()
- description = (request.description or "").strip()
- if not name:
- raise BadRequestError(message="Knowledge base name is required", data={"field": "name"})
-
- polished = await _generate_kb_description(name, description)
- return success_response(
- data={"description": polished},
- message="Knowledge base description polished successfully",
- )
-
-
-@router.patch("/{kb_id}", summary="更新知识库空间")
-def update_kb_space(
- kb_id: str,
- request: UpdateKBSpaceRequest,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """更新指定知识库空间的名称、简介与索引模式。仅空间所有者可操作;空间不存在或无权访问时返回 404。
-
- 改索引模式不会自动补建索引:新勾选 Wiki 后要调 `POST /{kb_id}/wiki/rebuild`
- 才会把已有文档排进生成队列。
- """
- kb_service = KBService(db)
- space = kb_service.update_space(
- kb_id=kb_id,
- user_id=user.user_id,
- name=request.name,
- description=request.description,
- index_modes=request.index_modes,
- wiki_config=(request.wiki_config.model_dump() if request.wiki_config else None),
- )
- if not space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- return success_response(data=space, message="Knowledge base space updated successfully")
-
-
-@router.post("/{kb_id}/documents", status_code=status.HTTP_201_CREATED, summary="上传文档到知识库")
-async def upload_document(
- kb_id: str = Path(..., description="KB space ID"),
- file: UploadFile = File(..., description="Document file to upload"),
- title: Optional[str] = Form(None, description="Document title (defaults to filename)"),
- metadata: Optional[str] = Form(None, description="Additional metadata as JSON string"),
- indexing_config: Optional[str] = Form(None, description="Indexing config as JSON string"),
- chunk_method: Optional[str] = Form(None, description="Chunking method override"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """上传文档到指定知识库空间并异步建立索引。仅空间所有者可上传;文件大小受 MAX_FILE_SIZE(100MB)限制,可选传入标题、元数据、索引配置及分块方法覆盖。上传成功后通过后台任务完成解析、分块与向量化,立即返回 indexing_status=processing。"""
- kb_service = KBService(db)
-
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
-
- if not kb_space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- _require_kb_write(db, kb_space, user.user_id, "edit")
-
- if not file.filename:
- raise BadRequestError(message="Filename is required", data={"field": "file"})
-
- content = await file.read()
- file_size = len(content)
-
- if file_size > MAX_FILE_SIZE:
- raise FileTooLargeError(max_size=MAX_FILE_SIZE, actual_size=file_size)
- if file_size == 0:
- raise BadRequestError(message="File is empty", data={"filename": file.filename})
-
- _, mime_type = validate_kb_file(file.filename, content)
-
- checksum = hashlib.sha256(content).hexdigest()
- env = os.getenv("ENVIRONMENT", "dev")
- storage_key = generate_storage_key(
- env=env, user_id=user.user_id, category="kb_documents", filename=file.filename
- )
-
- try:
- storage = get_storage()
- # 对象存储是阻塞的网络调用,直接在 async handler 里调会卡住整个事件循环——
- # 批量上传时后端会连 SSE 和其它接口一起停摆。
- await run_in_threadpool(storage.upload_bytes, content, storage_key)
- logger.info("Document uploaded to storage: %s", storage_key)
- except StorageError as e:
- logger.error("Failed to upload document to storage: %s", e)
- raise StorageError(
- operation="upload", error=f"Failed to upload document: {e.data.get('error', str(e))}"
- )
-
- doc_title = title or file.filename
-
- if metadata:
- try:
- json.loads(metadata)
- except json.JSONDecodeError:
- raise BadRequestError(message="Invalid metadata JSON", data={"field": "metadata"})
-
- try:
- document = kb_service.upload_document(
- kb_id=kb_id,
- user_id=user.user_id,
- title=doc_title,
- filename=file.filename,
- size_bytes=file_size,
- mime_type=mime_type,
- storage_key=storage_key,
- checksum=checksum,
- )
- except PermissionError as e:
- raise AccessDeniedError(message="Access denied", reason=str(e))
-
- _effective_chunk_method = chunk_method or kb_space.chunk_method or "semantic"
- _upload_indexing_config = None
- if indexing_config:
- try:
- _upload_indexing_config = json.loads(indexing_config)
- except json.JSONDecodeError:
- raise BadRequestError(
- message="Invalid indexing_config JSON", data={"field": "indexing_config"}
- )
- _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {}
- _indexing_config = _upload_indexing_config or _space_extra.get("indexing_config")
-
- # 排进持久化索引队列,由常驻 worker 认领执行(见 core/kb/index_worker.py)。
- # 早先这里挂的是 FastAPI BackgroundTasks:同步任务落进 Starlette 的共享请求线程池,
- # 一次多选几十个文件就把池占满,后续请求连 handler 都进不去——前端一直转圈、网关
- # 报 502;而且任务是进程内的,后端一重启就全丢,文档永远停在「索引中」。
- _enqueue_document_indexing(
- db,
- document_id=document["document_id"],
- user_id=user.user_id,
- chunk_method=_effective_chunk_method,
- indexing_config=_indexing_config,
- index_modes=index_modes_of(kb_space),
- )
-
- return created_response(
- data={**document, "indexing_status": "processing"},
- message="Document uploaded successfully. Indexing in progress.",
- )
-
-
-@router.get("/{kb_id}/documents", summary="获取知识库文档列表")
-def list_documents(
- kb_id: str = Path(..., description="Local or external knowledge collection ID"),
- page: int = Query(1, ge=1, description="Page number"),
- page_size: int = Query(20, ge=1, le=100, description="Items per page"),
- keyword: Optional[str] = Query(None, description="按标题/文件名搜索(全库范围,再分页)"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """分页获取知识库文档;非本地 ID 交由当前版本的外部知识提供方处理。"""
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
-
- if not kb_space:
- from core.kb.external_provider import is_enabled, list_documents
-
- if is_enabled():
- result = list_documents(kb_id, page=page, limit=page_size, keyword=keyword or "")
- response = paginated_response(
- items=result.get("items", []),
- page=result.get("page", page),
- page_size=result.get("page_size", page_size),
- total_items=result.get("total", 0),
- message="Documents retrieved successfully",
- )
- from core.kb.external_provider import count_document_statuses
-
- status_counts = count_document_statuses(
- kb_id,
- keyword=keyword or "",
- initial_result=result,
- )
- if status_counts is not None:
- response["data"]["status_counts"] = status_counts
- return response
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
-
- # Public KBs (created in the admin console, owned by the system principal) are readable by all logged-in users
- if not _can_read_space(db, kb_space, user.user_id):
- raise AccessDeniedError(
- message="Access denied", reason="Only the KB space owner can list documents"
- )
-
- documents, total = kb_repo.list_documents(kb_id, page, page_size, keyword=keyword)
- items = [
- {
- "id": d.document_id,
- "title": d.title,
- "desc": d.filename,
- "filename": d.filename,
- "size": d.size_bytes,
- "mime_type": d.mime_type,
- "storage_key": d.storage_key,
- "uploaded_at": d.uploaded_at.isoformat(),
- "indexing_status": getattr(d, "indexing_status", "processing"),
- }
- for d in documents
- ]
- response = paginated_response(
- items=items,
- page=page,
- page_size=page_size,
- total_items=total,
- message="Documents retrieved successfully",
- )
- response["data"]["status_counts"] = kb_repo.count_document_statuses(kb_id, keyword=keyword)
- return response
-
-
-@router.get("/{kb_id}/documents/{document_id}", summary="获取知识库文档详情")
-def get_document_detail(
- kb_id: str = Path(..., description="Local or external knowledge collection ID"),
- document_id: str = Path(..., description="Document ID"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """获取文档详情;非本地 ID 交由当前版本的外部知识提供方处理。"""
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
-
- if not kb_space:
- from core.kb.external_provider import get_document_detail, is_enabled
-
- if is_enabled():
- detail = get_document_detail(kb_id, document_id)
- return success_response(data=detail, message="Document detail retrieved successfully")
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
-
- # Public KBs are readable by all logged-in users
- if not _can_read_space(db, kb_space, user.user_id):
- raise AccessDeniedError(
- message="Access denied", reason="Only the KB space owner can view document details"
- )
-
- document = kb_repo.get_document(document_id)
- if not document or document.kb_id != kb_id:
- raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
-
- from core.db.models import KBChunk
-
- chunks = (
- db.query(KBChunk)
- .filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id)
- .order_by(KBChunk.chunk_index)
- .all()
- )
- content = "\n\n".join(c.content for c in chunks) if chunks else ""
-
- return success_response(
- data={
- "id": document.document_id,
- "title": document.title,
- "desc": document.filename,
- "filename": document.filename,
- "mime_type": document.mime_type,
- "uploaded_at": document.uploaded_at.isoformat(),
- "content": content,
- },
- message="Document detail retrieved successfully",
- )
-
-
-@router.delete("/{kb_id}", summary="删除知识库空间")
-def delete_kb_space(
- kb_id: str = Path(..., description="KB space ID"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """删除指定知识库空间及其下属文档、分块与向量数据。仅空间所有者可操作;空间不存在或无权访问时返回 404。此操作不可恢复。"""
- kb_service = KBService(db)
- deleted = kb_service.delete_space(kb_id, user.user_id)
- if not deleted:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- return success_response(data=None, message="Knowledge base deleted successfully")
-
-
-@router.delete("/{kb_id}/documents/{document_id}", summary="删除知识库文档")
-def delete_kb_document(
- kb_id: str = Path(..., description="KB space ID"),
- document_id: str = Path(..., description="Document ID"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """从知识库空间删除指定文档及其分块与向量数据。仅文档所属空间的所有者可操作;文档不存在或无权访问时返回 404。此操作不可恢复。"""
- kb_service = KBService(db)
- deleted = kb_service.delete_document(document_id, user.user_id)
- if not deleted:
- raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
- return success_response(data=None, message="Document deleted successfully")
-
-
-@router.post("/{kb_id}/documents/{document_id}/reindex", summary="重新索引文档")
-def reindex_document(
- kb_id: str = Path(..., description="KB space ID"),
- document_id: str = Path(..., description="Document ID"),
- request: Optional[ReindexRequest] = None,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """对指定文档重新建立索引。会先清除该文档的旧分块及向量数据,再从存储重新下载原文,按可选的索引配置与分块方法异步重新解析、分块、向量化。仅空间所有者可操作,立即返回 indexing_status=processing。"""
- from core.db.models import KBChunk
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
- if not kb_space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- _require_kb_write(db, kb_space, user.user_id, "edit")
-
- document = kb_repo.get_document(document_id)
- if not document or document.kb_id != kb_id:
- raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
-
- db.query(KBChunk).filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id).delete()
- document.indexing_status = "processing"
- db.commit()
-
- # 资产随分块一起重建:不清会在每次重索引后留下一份孤儿图(存储与表都涨)。
- try:
- from core.kb.kb_assets import purge_document_assets
-
- purge_document_assets(db, document_id)
- except Exception as exc:
- logger.warning("Asset cleanup failed for reindex of %s: %s", document_id, exc)
-
- try:
- from core.kb.kb_vector import delete_by_document
-
- delete_by_document(document_id, user.user_id)
- except Exception as exc:
- logger.warning("Milvus cleanup failed for reindex of %s: %s", document_id, exc)
-
- _idx_cfg = None
- if request and request.indexing_config:
- _idx_cfg = request.indexing_config.model_dump()
- if not _idx_cfg:
- _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {}
- _idx_cfg = _space_extra.get("indexing_config")
-
- _reindex_chunk_method = (
- (request.chunk_method if request and request.chunk_method else None)
- or kb_space.chunk_method
- or "semantic"
- )
-
- # 与上传同一条路:排队 + 常驻 worker,原文由 worker 自己从存储取回。
- # 这里不再预先 download_bytes——几十 MB 的原文既没必要在请求里读,也不该跟着
- # 任务在内存里排队等着跑。
- _enqueue_document_indexing(
- db,
- document_id=document_id,
- user_id=user.user_id,
- chunk_method=_reindex_chunk_method,
- indexing_config=_idx_cfg,
- index_modes=index_modes_of(kb_space),
- )
-
- return success_response(
- data={"document_id": document_id, "indexing_status": "processing"},
- message="Document re-indexing started",
- )
-
-
-@router.get("/{kb_id}/chunks", summary="获取文档分块列表(含标签和问题)")
-def list_chunks(
- kb_id: str = Path(..., description="KB space ID"),
- document_id: Optional[str] = Query(None, description="Filter by document ID"),
- page: int = Query(1, ge=1),
- page_size: int = Query(20, ge=1, le=100),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """分页获取知识库分块列表,可选按 document_id 过滤。每个分块返回完整正文内容、标签和问题列表,供分块管理界面查看与编辑。仅空间所有者可访问。"""
- from core.db.models import KBChunk
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
- if not kb_space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- # Public KBs are readable by all logged-in users (chunks are read-only viewing)
- if not _can_read_space(db, kb_space, user.user_id):
- raise AccessDeniedError(message="Access denied", reason="Not the KB owner")
-
- query = db.query(KBChunk).filter(KBChunk.kb_id == kb_id)
- if document_id:
- query = query.filter(KBChunk.document_id == document_id)
- total = query.count()
- chunks = (
- query.order_by(KBChunk.chunk_index).offset((page - 1) * page_size).limit(page_size).all()
- )
-
- items = [
- {
- "chunk_id": c.chunk_id,
- "document_id": c.document_id,
- "chunk_index": c.chunk_index,
- "content": c.content,
- "tags": c.tags or [],
- "questions": c.questions or [],
- }
- for c in chunks
- ]
- return paginated_response(items=items, page=page, page_size=page_size, total_items=total)
-
-
-@router.get("/{kb_id}/chunks/{chunk_id}/children", summary="获取父块下的子块列表")
-def list_chunk_children(
- kb_id: str = Path(..., description="KB space ID"),
- chunk_id: str = Path(..., description="Parent chunk ID"),
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """返回某父块在向量库里的子块(父子分块模式)。扁平模式无子块时返回空列表。仅空间可读者可访问。"""
- from core.db.models import KBChunk
- from core.db.repository import KBRepository
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
- if not kb_space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- if not _can_read_space(db, kb_space, user.user_id):
- raise AccessDeniedError(message="Access denied", reason="Not the KB owner")
-
- parent = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first()
- if not parent:
- raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id)
-
- from core.kb.kb_vector import get_children_for_parent
-
- children = get_children_for_parent(chunk_id)
- return success_response(
- data={"children": children, "count": len(children)},
- message="Chunk children retrieved successfully",
- )
-
-
-@router.patch("/{kb_id}/chunks/{chunk_id}", summary="更新分块内容/标签/问题")
-def update_chunk(
- kb_id: str = Path(..., description="KB space ID"),
- chunk_id: str = Path(..., description="Chunk ID"),
- request: UpdateChunkRequest = ...,
- user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
-):
- """更新指定分块的正文内容、标签和问题列表,并同步刷新其在向量库中的向量与索引。
-
- 传入 ``content`` 时会对该分块重新向量化(dense + sparse);仅改标签时只刷新稀疏/标签
- 索引;问题始终单独重建问题向量行。仅空间所有者(admin 权限)可操作,系统托管知识库禁止
- 编辑分块;仅传入的字段会被更新。"""
- from core.db.models import KBChunk, KBDocument
- from core.db.repository import KBRepository
- from core.kb.kb_vector import reindex_chunk_content, reindex_chunk_tags, upsert_question_rows
-
- kb_repo = KBRepository(db)
- kb_space = kb_repo.get_space(kb_id)
- if not kb_space:
- raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
- _require_kb_write(db, kb_space, user.user_id, "admin")
- if KBService(db)._is_system_managed_space(kb_space):
- raise AccessDeniedError(
- message="Access denied", reason="System managed KB does not allow chunk edits"
- )
-
- chunk = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first()
- if not chunk:
- raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id)
-
- content_changed = request.content is not None
- if content_changed:
- new_content = request.content.strip()
- if not new_content:
- raise BadRequestError(message="分块内容不能为空", data={"field": "content"})
- chunk.content = new_content
- if request.tags is not None:
- chunk.tags = request.tags
- if request.questions is not None:
- chunk.questions = request.questions
- db.commit()
- db.refresh(chunk)
-
- try:
- doc = db.query(KBDocument).filter_by(document_id=chunk.document_id).first()
- doc_title = doc.title if doc else ""
-
- if content_changed:
- # Content changed: re-vectorize the whole chunk (current tags already included)
- reindex_chunk_content(
- chunk_id=chunk_id,
- content=chunk.content,
- tags=chunk.tags or [],
- user_id=user.user_id,
- kb_id=kb_id,
- document_id=chunk.document_id,
- title=doc_title,
- chunk_index=chunk.chunk_index,
- )
- elif request.tags is not None:
- reindex_chunk_tags(chunk_id, chunk.content, chunk.tags)
- if request.questions is not None:
- upsert_question_rows(
- parent_chunk_id=chunk_id,
- questions=chunk.questions,
- user_id=user.user_id,
- kb_id=kb_id,
- document_id=chunk.document_id,
- title=doc_title,
- chunk_index=chunk.chunk_index,
- )
- except Exception as exc:
- logger.warning("Milvus re-index failed for chunk %s: %s", chunk_id, exc)
-
- return success_response(
- data={
- "chunk_id": chunk.chunk_id,
- "content": chunk.content,
- "tags": chunk.tags,
- "questions": chunk.questions,
- },
- message="Chunk updated successfully",
- )
diff --git a/src/backend/api/routes/v1/loops.py b/src/backend/api/routes/v1/loops.py
index cc2ce7a9..ea0f2eb0 100644
--- a/src/backend/api/routes/v1/loops.py
+++ b/src/backend/api/routes/v1/loops.py
@@ -2,20 +2,23 @@
See internal design docs (§5).
"""
+
from typing import Any, Dict, List, Optional
+import api.transports.run_sse as run_sse
+import core.runs.runtime as runtime
+import orchestration.loops.admission as admission
+from core.auth.backend import UserContext, get_current_user
+from core.db.engine import get_db
+from core.db.threads import run_in_session
+from core.infra.logging import get_logger
+from core.infra.responses import sse_response, success_response
+from core.chat.service import ChatService
+from features.automation.loops import LoopService
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
-from core.auth.backend import get_current_user, UserContext
-from core.db.engine import get_db
-from core.infra.logging import get_logger
-from core.infra.responses import success_response, sse_response
-from core.services.chat_service import ChatService
-from core.services.loop_service import LoopService
-from orchestration import chat_run_executor
-
logger = get_logger(__name__)
router = APIRouter(prefix="/v1/loops", tags=["AutonomousLoop"])
@@ -157,7 +160,8 @@ def get_iterations(
])
-async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserContext, is_resume: bool = False):
+def _launch_params(db: Session, loop_id: str, req: StartLoopReq, user: UserContext, is_resume: bool) -> dict:
+ """Bind the loop's chat and resolve its launch parameters (one session)."""
_require_loop_cap(db, user.user_id)
loop = LoopService(db).get_loop(loop_id, user_id=user.user_id)
if not loop:
@@ -187,7 +191,7 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC
hitl_enabled = req.hitl_enabled or bool(saved.get("hitl_enabled"))
# 与普通聊天同一套模型选择权限闸:未开放切换/不在可选列表的 provider 一律忽略。
- from core.services.user_model_selection import resolve_user_model_provider_id
+ from core.auth.model_selection import resolve_user_model_provider_id
model_provider_id = resolve_user_model_provider_id(
db, raw_provider_id, user_id=user.user_id
@@ -204,7 +208,7 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC
# The project the loop is bound to (stored in metadata at creation) — the
# worker/reviewer scope to the project folder based on it.
project_id = (loop.extra_data or {}).get("project_id") if loop.extra_data else None
- run = await chat_run_executor.start_autonomous_loop_run(
+ return dict(
loop_id=loop_id,
chat_id=chat_id,
user_id=user.user_id,
@@ -220,8 +224,14 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC
is_resume=is_resume,
project_id=project_id,
)
+
+
+async def _launch_loop(loop_id: str, req: StartLoopReq, user: UserContext, is_resume: bool = False):
+ params = await run_in_session(_launch_params, loop_id, req, user, is_resume)
+ chat_id = params["chat_id"]
+ run = await admission.start_autonomous_loop_run(**params)
return sse_response(
- chat_run_executor.follow_run_as_sse(
+ run_sse.follow_run_as_sse(
run.run_id,
chat_id=chat_id,
error_event_factory=lambda reason: {
@@ -235,24 +245,22 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC
async def start_loop(
loop_id: str,
req: StartLoopReq,
- db: Session = Depends(get_db),
user: UserContext = Depends(get_current_user),
):
- return await _launch_loop(loop_id, req, db, user)
+ return await _launch_loop(loop_id, req, user)
@router.post("/{loop_id}/resume", summary="续跑循环(HITL 批准后 / 崩溃后,从 feature_list.json 断点续跑)")
async def resume_loop(
loop_id: str,
req: StartLoopReq,
- db: Session = Depends(get_db),
user: UserContext = Depends(get_current_user),
):
# Same entry point as start: when the driver detects a persisted
# feature_list.json it automatically resumes from the checkpoint.
# is_resume=True → don't re-record the objective as a user message again
# (it was already recorded on first start).
- return await _launch_loop(loop_id, req, db, user, is_resume=True)
+ return await _launch_loop(loop_id, req, user, is_resume=True)
class SteerLoopReq(BaseModel):
@@ -279,27 +287,34 @@ def steer_loop(
return success_response({"queued": True})
-@router.post("/{loop_id}/cancel", summary="取消循环")
-async def cancel_loop(
- loop_id: str,
- db: Session = Depends(get_db),
- user: UserContext = Depends(get_current_user),
-):
+def _cancel_target(db: Session, loop_id: str, user_id: str):
+ """The loop's active run id, or the settled response when there is none to cancel."""
# Normalize chat_id consistently with _launch_loop: prefer loop.chat_id,
# otherwise loopchat_{id}.
svc = LoopService(db)
- loop = svc.get_loop(loop_id, user_id=user.user_id)
+ loop = svc.get_loop(loop_id, user_id=user_id)
if not loop:
raise HTTPException(status_code=404, detail="loop not found")
chat_id = loop.chat_id or f"loopchat_{loop_id}"
- run = chat_run_executor.get_active_run_for_chat(chat_id)
- if not run:
- # 无活跃 run(崩溃遗留/已中断):把 loop 状态归位而不是 404——历史缺陷是
- # 僵尸 running 行永远取消不掉。
- if loop.status in ("running", "interrupted", "created", "awaiting_human"):
- loop.status = "cancelled"
- db.commit()
- return success_response({"cancelled": True, "note": "无活跃任务,已直接归位为 cancelled"})
- return success_response({"cancelled": False, "status": loop.status})
- ok = await chat_run_executor.cancel_run(run.run_id, user_id=user.user_id)
+ run = runtime.get_active_run_for_chat(chat_id, user_id=user_id)
+ if run:
+ return run.run_id, None
+ # 无活跃 run(崩溃遗留/已中断):把 loop 状态归位而不是 404——历史缺陷是
+ # 僵尸 running 行永远取消不掉。
+ if loop.status in ("running", "interrupted", "created", "awaiting_human"):
+ loop.status = "cancelled"
+ db.commit()
+ return None, {"cancelled": True, "note": "无活跃任务,已直接归位为 cancelled"}
+ return None, {"cancelled": False, "status": loop.status}
+
+
+@router.post("/{loop_id}/cancel", summary="取消循环")
+async def cancel_loop(
+ loop_id: str,
+ user: UserContext = Depends(get_current_user),
+):
+ run_id, settled = await run_in_session(_cancel_target, loop_id, user.user_id)
+ if settled is not None:
+ return success_response(settled)
+ ok = await runtime.cancel_run(run_id, user_id=user.user_id)
return success_response({"cancelled": ok})
diff --git a/src/backend/api/routes/v1/me_capabilities.py b/src/backend/api/routes/v1/me_capabilities.py
index 3e79b20e..06dd48f7 100644
--- a/src/backend/api/routes/v1/me_capabilities.py
+++ b/src/backend/api/routes/v1/me_capabilities.py
@@ -19,30 +19,18 @@
import uuid
from typing import Dict, List, Optional
-from core.capabilities.paths import capabilities_enabled
-from core.services import local_skill_editor as local_editor
+from core.harness.registry.run_capabilities import has_run_capabilities
+from features.desktop.local.skills import editor as local_editor
from core.auth.backend import UserContext, get_current_user
from core.auth.capabilities import resolve_user_capabilities
from core.config.settings import settings
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.db.models import AdminMcpServer, AdminSkill
from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
from core.infra.responses import created_response, success_response
-from core.services.mcp_management_service import (
- encrypt_mcp_headers,
- probe_mcp_connectivity,
- refresh_mcp_caches,
- validate_remote_mcp_url,
-)
-from core.services.skill_management_service import (
- build_skill_content,
- extract_instructions,
- extract_mcp_server_ids,
- parse_and_upsert_skill_zip,
- refresh_skill_caches,
- resolve_mcp_bindings,
- resolve_ontology_workflows,
-)
+from core.capabilities.mcp.management import encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url
+from core.capabilities.skills.management import build_skill_content, extract_instructions, extract_mcp_server_ids, parse_and_upsert_skill_zip, refresh_skill_caches, resolve_mcp_bindings, resolve_ontology_workflows
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from pydantic import BaseModel, Field, field_validator
from sqlalchemy.orm import Session
@@ -82,14 +70,15 @@ class CreateUserMcpRequest(BaseModel):
async def create_my_mcp_server(
body: CreateUserMcpRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""添加一个仅自己可见可用的远程 MCP 工具(HTTP/SSE)。
创建即试连:连不上返回 422,避免把不可达端点落库。出于安全,用户入口**不支持
stdio**(不能在服务器执行任意命令)。
"""
- _require_flag(str(user.user_id), db, "can_add_mcp", "自助添加 MCP")
+ await run_in_session(
+ lambda db: _require_flag(str(user.user_id), db, "can_add_mcp", "自助添加 MCP")
+ )
if not body.url.strip():
raise BadRequestError(message="url 不能为空")
@@ -113,7 +102,6 @@ async def create_my_mcp_server(
transport=body.transport,
url=body.url.strip(),
headers=body.headers or {},
- is_stable=False, # user-private MCPs don't enter the warmup connection pool
is_enabled=True,
owner_user_id=str(user.user_id),
icon=body.icon,
@@ -122,14 +110,18 @@ async def create_my_mcp_server(
created_by=str(user.user_id),
)
- ok, err = await probe_mcp_connectivity(row, db)
+ ok, err = await probe_mcp_connectivity(row, validate=True)
if not ok:
raise BadRequestError(message=f"MCP 连接失败,无法添加:{err}")
row.headers = encrypt_mcp_headers(body.headers)
- db.add(row)
- db.commit()
- db.refresh(row)
+
+ def save(db: Session) -> None:
+ db.add(row)
+ db.commit()
+ db.refresh(row)
+
+ await run_in_session(save)
refresh_mcp_caches()
return created_response(
@@ -172,7 +164,7 @@ def delete_my_mcp_server(
@router.post("/skills/upload", status_code=201, summary="自助上传私有技能")
-async def upload_my_skill(
+def upload_my_skill(
file: UploadFile = File(...),
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -182,16 +174,16 @@ async def upload_my_skill(
复用 admin 上传的解析链路,但打上 ``owner_user_id`` 并限 50MB。技能 id 取
SKILL.md frontmatter ``name``,须全局唯一(与公共/他人技能冲突会被拒绝)。
"""
- if capabilities_enabled():
+ if has_run_capabilities():
return created_response(
- data=local_editor.install_archive(str(user.user_id), await file.read())
+ data=local_editor.install_archive(str(user.user_id), file.file.read())
)
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
if not file.filename or not file.filename.endswith(".zip"):
raise BadRequestError(message="仅接受 .zip 技能包")
- data = await file.read()
+ data = file.file.read()
if len(data) > USER_SKILL_MAX_BYTES:
raise BadRequestError(
message=f"技能包过大(上限 {USER_SKILL_MAX_BYTES // (1024 * 1024)}MB)"
@@ -242,7 +234,7 @@ def create_my_skill(
技能 id 须全局唯一:不能与公共技能或他人私有技能冲突;与本人已有同名私有技能
则视为更新。
"""
- if capabilities_enabled():
+ if has_run_capabilities():
return created_response(data=local_editor.save(str(user.user_id), body))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
@@ -333,7 +325,7 @@ def create_my_skill(
)
db.commit()
if body.icon is not None:
- from core.services.skill_icon_service import set_skill_icon
+ from core.capabilities.skills.icons import set_skill_icon
set_skill_icon(db, skill_id, body.icon)
refresh_skill_caches()
@@ -375,13 +367,13 @@ def get_my_skill(
与手写新建表单的「技能正文」一一对应,编辑保存走 ``POST /v1/me/skills`` upsert。
附带 ``extra_files`` 清单(文件名/大小/是否二进制),供技能文件管理 UI 渲染。
"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.detail(str(user.user_id), skill_id))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = _get_own_skill(db, str(user.user_id), skill_id)
- from core.agent_skills.binary_files import is_binary_value
- from core.services.skill_icon_service import get_skill_icon
+ from core.capabilities.skills.binary_files import is_binary_value
+ from core.capabilities.skills.icons import get_skill_icon
extra_files = [
{
@@ -415,7 +407,7 @@ def delete_my_skill(
db: Session = Depends(get_db),
):
"""删除自己上传的私有技能。只能删自己的(owner 校验)。"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.uninstall(str(user.user_id), skill_id))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = (
@@ -431,13 +423,13 @@ def delete_my_skill(
db.delete(row)
db.commit()
- from core.services.skill_icon_service import delete_skill_icon
+ from core.capabilities.skills.icons import delete_skill_icon
delete_skill_icon(db, skill_id)
- from core.agent_skills.config import purge_skill_sandbox_files
+ from core.capabilities.skills.config import purge_skill_sandbox_files
# Desktop reconciliation retains immutable revisions and a removed tombstone.
- if not capabilities_enabled():
+ if not has_run_capabilities():
purge_skill_sandbox_files(skill_id)
refresh_skill_caches()
return success_response(data={"skill_id": skill_id, "deleted": True})
@@ -455,7 +447,7 @@ def set_my_skill_icon(
db: Session = Depends(get_db),
):
"""给自己的私有技能设置图标(owner 校验)。空串=恢复默认。"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.icon(str(user.user_id), skill_id, body.icon))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = (
@@ -466,7 +458,7 @@ def set_my_skill_icon(
if not row:
raise ResourceNotFoundError("skill", skill_id)
- from core.services.skill_icon_service import set_skill_icon
+ from core.capabilities.skills.icons import set_skill_icon
icon = set_skill_icon(db, skill_id, body.icon)
refresh_skill_caches()
diff --git a/src/backend/api/routes/v1/me_skill_files.py b/src/backend/api/routes/v1/me_skill_files.py
index c5aad0b7..23a90d08 100644
--- a/src/backend/api/routes/v1/me_skill_files.py
+++ b/src/backend/api/routes/v1/me_skill_files.py
@@ -6,8 +6,8 @@
from datetime import datetime
from typing import Dict, List, Optional
-from core.capabilities.paths import capabilities_enabled
-from core.services import local_skill_editor as local_editor
+from core.harness.registry.run_capabilities import has_run_capabilities
+from features.desktop.local.skills import editor as local_editor
from core.auth.backend import UserContext, get_current_user
from core.auth.capabilities import resolve_user_capabilities
from core.config.settings import settings
@@ -15,22 +15,8 @@
from core.db.models import AdminMcpServer, AdminSkill
from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
from core.infra.responses import created_response, success_response
-from core.services.mcp_management_service import (
- encrypt_mcp_headers,
- probe_mcp_connectivity,
- refresh_mcp_caches,
- validate_remote_mcp_url,
-)
-from core.services.skill_management_service import (
- build_skill_content,
- extract_instructions,
- extract_mcp_server_ids,
- parse_and_upsert_skill_zip,
- refresh_skill_caches,
- resolve_mcp_bindings,
- resolve_ontology_workflows,
- validate_skill_file_path,
-)
+from core.capabilities.mcp.management import encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url
+from core.capabilities.skills.management import build_skill_content, extract_instructions, extract_mcp_server_ids, parse_and_upsert_skill_zip, refresh_skill_caches, resolve_mcp_bindings, resolve_ontology_workflows, validate_skill_file_path
from fastapi import APIRouter, Depends, File, Form, HTTPException, Response, UploadFile
from pydantic import BaseModel, Field, field_validator
from sqlalchemy.orm import Session
@@ -56,12 +42,12 @@ def get_my_skill_file(
db: Session = Depends(get_db),
):
"""读取自己私有技能中的单个附加文件。二进制文件只返回 ``is_binary=true`` 不回传内容。"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.file_get(str(user.user_id), skill_id, filename))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = _get_own_skill(db, str(user.user_id), skill_id)
- from core.agent_skills.binary_files import is_binary_value
+ from core.capabilities.skills.binary_files import is_binary_value
extra = row.extra_files or {}
if filename not in extra:
@@ -84,7 +70,7 @@ def save_my_skill_file(
SKILL.md 不走本接口(正文在编辑表单里改,经 ``POST /v1/me/skills`` 重建)。
"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.file_change(str(user.user_id), skill_id, filename, body.content.encode("utf-8")))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
@@ -114,7 +100,7 @@ def delete_my_skill_file(
db: Session = Depends(get_db),
):
"""删除自己私有技能中的单个附加文件。"""
- if capabilities_enabled():
+ if has_run_capabilities():
return success_response(data=local_editor.file_change(str(user.user_id), skill_id, filename))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = _get_own_skill(db, str(user.user_id), skill_id)
@@ -132,7 +118,7 @@ def delete_my_skill_file(
@router.post("/skills/{skill_id}/files/upload", status_code=201, summary="上传我的技能文件")
-async def upload_my_skill_file(
+def upload_my_skill_file(
skill_id: str,
file: UploadFile = File(...),
path: str = Form("", description="可选:存入的相对路径(含子目录),留空取上传文件名"),
@@ -140,16 +126,16 @@ async def upload_my_skill_file(
db: Session = Depends(get_db),
):
"""以 multipart 上传单个文件到自己的私有技能,二进制按 base64 标记安全存储。"""
- if capabilities_enabled():
- return success_response(data=local_editor.file_change(str(user.user_id), skill_id, path or file.filename or "", await file.read()))
+ if has_run_capabilities():
+ return success_response(data=local_editor.file_change(str(user.user_id), skill_id, path or file.filename or "", file.file.read()))
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
- from core.agent_skills.binary_files import encode_upload
+ from core.capabilities.skills.binary_files import encode_upload
filename = validate_skill_file_path(path or file.filename or "")
if filename == "SKILL.md":
raise BadRequestError(message="SKILL.md 请在「编辑技能」表单中修改")
- raw = await file.read()
+ raw = file.file.read()
if len(raw) > USER_SKILL_FILE_MAX_BYTES:
raise BadRequestError(
message=f"文件过大(上限 {USER_SKILL_FILE_MAX_BYTES // (1024 * 1024)}MB)"
@@ -177,12 +163,12 @@ def export_my_skill(
导出布局与 zip 上传约定一致,可直接重新导入(备份/迁移/分享给管理员上架)。
"""
- if capabilities_enabled():
+ if has_run_capabilities():
return Response(content=local_editor.export(str(user.user_id), skill_id), media_type="application/zip")
_require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能")
row = _get_own_skill(db, str(user.user_id), skill_id)
- from core.services.marketplace_service import build_skill_zip
+ from features.marketplace.skill_archives import build_skill_zip
data = build_skill_zip(skill_id, row.skill_content or "", row.extra_files or {})
safe_name = "".join(c if c.isalnum() or c in "._-" else "_" for c in skill_id) or "skill"
diff --git a/src/backend/api/routes/v1/me_system.py b/src/backend/api/routes/v1/me_system.py
index 5ffebf9a..c911f31b 100644
--- a/src/backend/api/routes/v1/me_system.py
+++ b/src/backend/api/routes/v1/me_system.py
@@ -22,12 +22,10 @@
from core.config.settings import settings
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.service_probes import (
- config_row_to_dict,
- reinitialize_mcp_pool,
- test_service_group,
-)
-from core.services.system_config import SystemConfigService
+from core.infra.probes.service_probes import config_row_to_dict
+from core.capabilities.mcp.service import McpServerConfigService
+from core.infra.probes.service_probes import test_service_group
+from core.config.store.system_config import SystemConfigService
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy.orm import Session
@@ -132,7 +130,7 @@ async def update_personal_configs(
detail=f"配置项 {item.key} 不在个人可配置范围内",
)
svc.bulk_set([item.model_dump() for item in body.items], updated_by="me_system")
- asyncio.ensure_future(reinitialize_mcp_pool("me_system"))
+ McpServerConfigService.get_instance().invalidate_cache()
return success_response(data={"updated": len(body.items)})
diff --git a/src/backend/api/routes/v1/memories.py b/src/backend/api/routes/v1/memories.py
index 8cae31f7..37c2db43 100644
--- a/src/backend/api/routes/v1/memories.py
+++ b/src/backend/api/routes/v1/memories.py
@@ -24,6 +24,7 @@
from core.auth.backend import UserContext, get_current_user
from core.config.settings import settings as _jx_settings
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import error_response, success_response
from core.memory.context import MemoryContext
from core.memory.access import authorized_memory_context
@@ -31,8 +32,8 @@
from core.memory.profile import delete_field as profile_delete_field
from core.memory.profile import get as profile_get
from core.memory.service import delete_all_memories, delete_memory, get_all_memories
-from core.services import UserService
-from core.services.memory_settings_service import MemorySettingsService
+from core.auth.users import UserService
+from core.memory.settings import MemorySettingsService
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -68,7 +69,7 @@ class ProfileFieldRequest(BaseModel):
def _is_reranker_available() -> bool:
"""Check if reranker endpoint is configured at the infra level."""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
cfg = ModelConfigService.get_instance().resolve("reranker")
if cfg and cfg.base_url and cfg.model_name:
@@ -138,13 +139,37 @@ def update_memory_settings(
# ── List / clear / delete single ────────────────────────────────
+def _list_scope(db: Session, user_id: str, project_id: Optional[str]):
+ """``(read enabled, memory owner)`` for the list view; None when the project is not visible.
+
+ In a project the project's own ``memory_enabled`` applies (its owner holds the
+ memories); otherwise the user's setting does.
+ """
+ if project_id:
+ from core.auth.edition import resolve_project_access
+ from core.db.models import Project
+
+ project = db.query(Project).filter_by(project_id=project_id, deleted_at=None).first()
+ if project is None or resolve_project_access(db, user_id, project).level == "none":
+ return None
+ return _graph_scope(db, user_id, project_id)
+
+
+def _graph_scope(db: Session, user_id: str, project_id: Optional[str]):
+ if project_id:
+ from core.projects.edition import project_memory_policy
+
+ return project_memory_policy(db, project_id, user_id) or (False, user_id)
+ settings = UserService(db).get_user_settings(user_id)
+ return bool(settings.get("memory_enabled", False)), user_id
+
+
@router.get("", summary="查询长期记忆列表")
async def list_memories(
project_id: Optional[str] = Query(
None, description="若指定,只返回属于该项目 workspace 的记忆"
),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""L2 长期记忆列表(mem0/Milvus 向量)——只存"做法/口径",不存事实。
@@ -161,23 +186,10 @@ async def list_memories(
if not _jx_settings.memory.enabled:
return success_response(data={"enabled": False, "items": [], "count": 0})
- scope_user_id = str(user.user_id)
- if project_id:
- from core.services.project_scope import project_memory_policy
- from core.auth.permissions_iface import resolve_project_access
- from core.db.models import Project
-
- project = db.query(Project).filter_by(project_id=project_id, deleted_at=None).first()
- if project is None or resolve_project_access(db, scope_user_id, project).level == "none":
- return error_response(code=40401, message="项目不存在", status_code=404)
- policy = project_memory_policy(db, project_id, scope_user_id)
- if policy is None:
- return success_response(data={"enabled": False, "items": [], "count": 0})
- ws_enabled, scope_user_id = policy
- else:
- settings = UserService(db).get_user_settings(str(user.user_id))
- ws_enabled = bool(settings.get("memory_enabled", False))
-
+ scope = await run_in_session(_list_scope, str(user.user_id), project_id)
+ if scope is None:
+ return error_response(code=40401, message="项目不存在", status_code=404)
+ ws_enabled, scope_user_id = scope
if not ws_enabled:
return success_response(data={"enabled": False, "items": [], "count": 0})
@@ -291,7 +303,7 @@ async def remove_profile_field(
if not ok:
return error_response(code=50001, message="删除失败", status_code=500)
if message_id:
- from core.evolution.settlement_store import drop_entry
+ from core.evolution.settlement.settlement_store import drop_entry
drop_entry(message_id, key)
return success_response(data={"deleted": key})
@@ -311,7 +323,7 @@ async def remove_graph_relation(
if not ok:
return error_response(code=50001, message="删除失败", status_code=500)
if message_id:
- from core.evolution.settlement_store import drop_entry
+ from core.evolution.settlement.settlement_store import drop_entry
drop_entry(message_id, relation_id)
return success_response(data={"deleted": relation_id})
@@ -362,7 +374,7 @@ async def remove_memory(
if not ok:
return error_response(code=50001, message="删除失败", status_code=500)
if message_id:
- from core.evolution.settlement_store import drop_entry
+ from core.evolution.settlement.settlement_store import drop_entry
drop_entry(message_id, memory_id)
return success_response(data={"deleted": memory_id})
@@ -452,28 +464,15 @@ async def get_memory_graph(
limit: int = Query(30, ge=1, le=200, description="返回关系条数"),
project_id: Optional[str] = Query(None, description="若指定,返回该项目作用域的关系"),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""L3 图谱记忆:当前用户或项目作用域的实体关系(Neo4j)。"""
if not (_jx_settings.memory.enabled and _jx_settings.memory.graph_enabled):
return success_response(data={"enabled": False, "relations": [], "count": 0})
- scope_user_id = str(user.user_id)
- workspace_id = "default"
- if project_id:
- from core.services.project_scope import project_memory_policy
-
- policy = project_memory_policy(db, project_id, scope_user_id)
- if policy is None:
- return success_response(data={"enabled": False, "relations": [], "count": 0})
- ws_enabled, scope_user_id = policy
- if not ws_enabled:
- return success_response(data={"enabled": False, "relations": [], "count": 0})
- workspace_id = f"project:{project_id}"
- else:
- user_settings = UserService(db).get_user_settings(str(user.user_id))
- if not bool(user_settings.get("memory_enabled", False)):
- return success_response(data={"enabled": False, "relations": [], "count": 0})
+ ws_enabled, scope_user_id = await run_in_session(_graph_scope, str(user.user_id), project_id)
+ if not ws_enabled:
+ return success_response(data={"enabled": False, "relations": [], "count": 0})
+ workspace_id = f"project:{project_id}" if project_id else "default"
relations = await list_graph_relations(
scope_user_id,
diff --git a/src/backend/api/routes/v1/mock_sso.py b/src/backend/api/routes/v1/mock_sso.py
index 535270e7..3e46d3ca 100644
--- a/src/backend/api/routes/v1/mock_sso.py
+++ b/src/backend/api/routes/v1/mock_sso.py
@@ -15,7 +15,7 @@
from typing import Any, Dict, Optional
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
-from core.content.content_blocks import get_branding_info, is_register_allowed
+from core.files.content.branding import get_branding_info, is_register_allowed
from core.infra.logging import get_logger
from fastapi import APIRouter, Form, Query, Request
from fastapi.responses import HTMLResponse, RedirectResponse
@@ -1453,7 +1453,7 @@ def _try_local_login(username: str, password: str) -> Optional[Dict[str, Any]]:
try:
from core.db.engine import SessionLocal
- from core.services.local_user_service import LocalUserService
+ from core.auth.local_users import LocalUserService
except Exception as exc:
logger.warning("local auth unavailable: %s", exc)
return None
@@ -1610,30 +1610,27 @@ def _bounce(msg: str) -> RedirectResponse:
return _bounce("两次输入的密码不一致")
try:
- from core.db.engine import SessionLocal
- from core.services.local_user_service import LocalUserService
+ from core.auth.local_users import LocalUserService
+ from core.db.threads import run_in_session
except Exception as exc:
logger.warning("local auth module unavailable: %s", exc)
return _bounce("注册服务暂不可用")
- db = SessionLocal()
try:
- service = LocalUserService(db)
- result = service.register(
- code=code,
- username=username,
- nickname=nickname,
- email=email,
- password=password,
- real_name=real_name,
- phone=phone,
+ result = await run_in_session(
+ lambda db: LocalUserService(db).register(
+ code=code,
+ username=username,
+ nickname=nickname,
+ email=email,
+ password=password,
+ real_name=real_name,
+ phone=phone,
+ )
)
except Exception as exc: # noqa: BLE001
logger.exception("register failed: %s", exc)
- db.close()
return _bounce("注册失败,请稍后重试")
- finally:
- db.close()
if not result.ok or not result.user_info:
return _bounce(result.message or "注册失败")
diff --git a/src/backend/api/routes/v1/model_reasoning.py b/src/backend/api/routes/v1/model_reasoning.py
index 9ac05b2a..1dcb2cd5 100644
--- a/src/backend/api/routes/v1/model_reasoning.py
+++ b/src/backend/api/routes/v1/model_reasoning.py
@@ -2,13 +2,12 @@
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
-from sqlalchemy.orm import Session
from api.deps import require_system_settings
-from core.db.engine import get_db
-from core.db.model_repository import get_provider
+from core.db.threads import run_in_session
+from core.models.repository import get_provider
from core.infra.responses import success_response
-from core.llm.providers.reasoning_probe import discover_reasoning_efforts
+from core.models.providers.reasoning_probe import discover_reasoning_efforts
router = APIRouter()
@@ -26,11 +25,10 @@ class DetectReasoningRequest(BaseModel):
async def detect_reasoning_levels(
body: DetectReasoningRequest,
_: None = Depends(require_system_settings),
- db: Session = Depends(get_db),
):
api_key = body.api_key
if body.provider_id and not api_key:
- existing = get_provider(db, body.provider_id)
+ existing = await run_in_session(get_provider, body.provider_id)
if existing is None:
raise HTTPException(status_code=404, detail="Provider not found")
api_key = existing.api_key or ""
diff --git a/src/backend/api/routes/v1/models.py b/src/backend/api/routes/v1/models.py
index fb3ad90d..d259055d 100644
--- a/src/backend/api/routes/v1/models.py
+++ b/src/backend/api/routes/v1/models.py
@@ -27,29 +27,14 @@
from api.deps import require_config, require_system_settings
from core.auth.backend import UserContext, require_auth
-from core.auth.backend import UserContext, require_auth
from core.config.settings import settings
from core.db.engine import get_db
-from core.services.model_config import ModelConfigService
-from core.db.model_repository import (
- ROLE_DEFINITIONS,
- assign_role,
- create_provider,
- delete_provider,
- export_all,
- get_provider,
- import_all,
- list_providers,
- list_role_assignments,
- provider_has_capability,
- provider_is_referenced,
- role_required_capability,
- set_provider_test_result,
- unassign_role,
- update_provider,
-)
+from core.db.threads import run_in_session
+from core.models.config import ModelConfigService
+from core.models.repository import ROLE_DEFINITIONS, assign_role, create_provider, delete_provider, export_all, get_provider, import_all, list_providers, list_role_assignments, provider_has_capability, provider_is_referenced, role_required_capability, set_provider_test_result, unassign_role, update_provider
from core.infra.responses import success_response
-from core.services.user_model_selection import list_user_selectable_models, user_can_switch_model
+from core.auth.model_selection import user_can_switch_model
+from core.models.selection import list_user_selectable_models
router = APIRouter(prefix="/v1/models", tags=["Models"])
logger = logging.getLogger(__name__)
@@ -140,16 +125,16 @@ class ImportRequest(BaseModel):
def _invalidate_model_caches() -> None:
- """Single invalidation point for model-config changes: the resolve cache plus
- every runtime singleton built from it.
-
- The mem0 singleton captures the memory/embedding config at init time; clearing
- the resolve cache alone leaves memory running on stale credentials (a first
- init against the placeholder key means every later write fails 401). The reset
- is lazy — the next memory read/write rebuilds against the fresh config — so it
- is cheap enough to apply on every change without filtering by role.
+ """Single invalidation point for runtime singletons built from model config.
+
+ The resolve cache needs nothing here: every repository write bumps the
+ committed revision, which each lookup checks. The mem0 singleton, though,
+ captures the memory/embedding config at init time and would keep running on
+ stale credentials (a first init against the placeholder key means every
+ later write fails 401). The reset is lazy — the next memory read/write
+ rebuilds against the fresh config — so it is cheap enough to apply on every
+ change without filtering by role.
"""
- ModelConfigService.get_instance().invalidate_cache()
try:
from core.memory import service as memory_service
@@ -329,7 +314,7 @@ async def _autofill_context_length(
# cannot outlive the number it described.
extra_config.pop("context_length_source", None)
- from core.llm.providers.context_probe import discover_context_length
+ from core.models.providers.context_probe import discover_context_length
result = await discover_context_length(
provider=provider,
@@ -378,7 +363,7 @@ async def _autofill_api_protocol(
if not isinstance(extra_config, dict) or provider_type != "chat":
return
- from core.llm.providers.registry import get_spec
+ from core.models.providers.registry import get_spec
if not get_spec(provider).speaks_responses:
return
@@ -387,7 +372,7 @@ async def _autofill_api_protocol(
return
extra_config.pop("api_protocol_source", None)
- from core.llm.providers.protocol_probe import PROBE_VERSION, detect_api_protocol
+ from core.models.providers.protocol_probe import PROBE_VERSION, detect_api_protocol
result = await detect_api_protocol(
base_url=base_url, api_key=api_key, model_name=model_name
@@ -472,8 +457,8 @@ async def _ping_via_model(
"""Native/litellm vendors: actually build the model and issue one minimal non-streaming call to verify connectivity."""
from agentscope.message import Msg, TextBlock
- from core.llm.chat_models import make_chat_model
- from core.llm.providers.registry import get_spec, split_provider_extra
+ from core.models.chat_models import make_chat_model
+ from core.models.providers.registry import get_spec, split_provider_extra
spec = get_spec(provider)
provider_extra = split_provider_extra(spec, extra or {})
@@ -515,12 +500,12 @@ async def _test_connection(
extra_config: Optional[dict] = None,
) -> dict:
"""Test connectivity (dispatched by engine). Returns {success, latency_ms, error}."""
- from core.llm.providers.registry import get_spec
+ from core.models.providers.registry import get_spec
spec = get_spec(provider)
extra = extra_config or {}
if provider_type == "audio":
- from core.services.audio_transcription import test_audio_connection
+ from core.chat.audio_transcription import test_audio_connection
return await test_audio_connection(base_url, api_key, model_name)
if spec.engine == "openai":
if spec.id == "azure_openai":
@@ -540,19 +525,19 @@ def get_main_capabilities(
):
"""供前端展示模型相关入口;只暴露非敏感信息,不泄露 base_url/api_key。"""
cfg = ModelConfigService.get_instance().resolve("main_agent")
- from core.llm.reasoning_effort import reasoning_capabilities
+ from core.models.reasoning_effort import reasoning_capabilities
effort_caps = reasoning_capabilities(cfg.extra if cfg else {})
switch_enabled = bool(user and user_can_switch_model(db, user.user_id))
# Real context window (tokens) of the main chat model, so the frontend
# context-usage gauge reflects the model's true window instead of a guess.
# 0 when unconfigured — the frontend falls back to a heuristic/default.
main_context_length = int(cfg.context_length) if cfg and cfg.context_length else 0
- from core.content.artifact_reader import ATTACHMENT_PREVIEW_MAX_CHARS
+ from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS
# Image reading: either the main model is natively multimodal, or a `vision` role
# model is assigned and the vision bridge transcribes images into text evidence.
# The frontend uses `can_read_image` alone to decide whether to offer image upload;
# the two flags below only explain *how* it works.
- from core.vision import model_supports_vision, resolve_vision_config
+ from core.models.vision import model_supports_vision, resolve_vision_config
main_native_vision = model_supports_vision(cfg)
vision_cfg = resolve_vision_config()
@@ -586,7 +571,7 @@ async def list_provider_schemas(_: None = Depends(require_system_settings)):
供前端模型配置表单按所选厂商动态渲染。仅限管理员。
"""
- from core.llm.providers.registry import to_frontend_schema
+ from core.models.providers.registry import to_frontend_schema
return success_response(data=to_frontend_schema())
@@ -609,10 +594,9 @@ def list_providers_endpoint(
async def create_provider_endpoint(
body: ProviderCreateRequest,
_: None = Depends(require_system_settings),
- db: Session = Depends(get_db),
):
"""新增一个模型供应商。仅限管理员;保存前会校验厂商字段并做连通性预校验,失败则返回 400,成功后刷新模型配置缓存。"""
- from core.llm.providers.registry import validate_payload
+ from core.models.providers.registry import validate_payload
err = validate_payload(body.provider, body.provider_type, body.extra_config)
if err:
@@ -644,32 +628,41 @@ async def create_provider_endpoint(
body.extra_config,
)
- provider = create_provider(
- db,
- display_name=body.display_name,
- provider_type=body.provider_type,
- provider=body.provider,
- base_url=normalized_url,
- api_key=body.api_key,
- model_name=body.model_name,
- gateway_group=body.gateway_group,
- weight=body.weight,
- priority=body.priority,
- extra_config=body.extra_config,
- is_active=body.is_active,
- )
- given = body.model_dump(exclude_unset=True)
- if any(k in given for k in ("input_price", "output_price", "currency")):
- _upsert_pricing(
+ def save(db: Session) -> dict:
+ provider = create_provider(
db,
- model_name=provider.model_name,
- input_price=body.input_price,
- output_price=body.output_price,
- currency=body.currency,
display_name=body.display_name,
+ provider_type=body.provider_type,
+ provider=body.provider,
+ base_url=normalized_url,
+ api_key=body.api_key,
+ model_name=body.model_name,
+ gateway_group=body.gateway_group,
+ weight=body.weight,
+ priority=body.priority,
+ extra_config=body.extra_config,
+ is_active=body.is_active,
)
+ given = body.model_dump(exclude_unset=True)
+ if any(k in given for k in ("input_price", "output_price", "currency")):
+ _upsert_pricing(
+ db,
+ model_name=provider.model_name,
+ input_price=body.input_price,
+ output_price=body.output_price,
+ currency=body.currency,
+ display_name=body.display_name,
+ )
+ return _provider_to_dict(provider, _get_pricing(db, provider.model_name))
+
+ data = await run_in_session(save)
_invalidate_model_caches()
- return success_response(data=_provider_to_dict(provider, _get_pricing(db, provider.model_name)))
+ return success_response(data=data)
+
+
+def _provider_and_roles(db: Session, provider_id: str):
+ existing = get_provider(db, provider_id)
+ return existing, (provider_is_referenced(db, provider_id) if existing is not None else [])
@router.put("/providers/{provider_id}", summary="更新模型供应商")
@@ -677,7 +670,6 @@ async def update_provider_endpoint(
provider_id: str,
body: ProviderUpdateRequest,
_: None = Depends(require_system_settings),
- db: Session = Depends(get_db),
):
"""更新指定供应商的配置(仅传需修改的字段)。仅限管理员;若改动了 base_url/api_key/模型名会重新做连通性校验,成功后刷新模型配置缓存;供应商不存在返回 404。"""
fields = {k: v for k, v in body.model_dump().items() if v is not None}
@@ -687,7 +679,7 @@ async def update_provider_endpoint(
}
# If URL or credentials changed, validate the new config
- existing = get_provider(db, provider_id)
+ existing, roles = await run_in_session(_provider_and_roles, provider_id)
if existing is None:
raise HTTPException(status_code=404, detail="Provider not found")
@@ -698,11 +690,10 @@ async def update_provider_endpoint(
new_provider = fields.get("provider", getattr(existing, "provider", "openai_compatible"))
new_extra = fields.get("extra_config", existing.extra_config or {})
if new_type != existing.provider_type:
- roles = provider_is_referenced(db, provider_id)
if any(ROLE_DEFINITIONS.get(role, {}).get("type") != new_type for role in roles):
raise HTTPException(400, "请先取消该模型的角色分配,再修改模型类型")
- from core.llm.providers.registry import validate_payload
+ from core.models.providers.registry import validate_payload
err = validate_payload(new_provider, new_type, new_extra)
if err:
@@ -738,18 +729,22 @@ async def update_provider_endpoint(
fields["extra_config"],
)
- provider = update_provider(db, provider_id, **fields)
- if provider is None:
- raise HTTPException(status_code=404, detail="Provider not found")
- if price_fields:
- _upsert_pricing(
- db,
- model_name=provider.model_name,
- display_name=provider.display_name,
- **price_fields,
- )
+ def save(db: Session) -> dict:
+ provider = update_provider(db, provider_id, **fields)
+ if provider is None:
+ raise HTTPException(status_code=404, detail="Provider not found")
+ if price_fields:
+ _upsert_pricing(
+ db,
+ model_name=provider.model_name,
+ display_name=provider.display_name,
+ **price_fields,
+ )
+ return _provider_to_dict(provider, _get_pricing(db, provider.model_name))
+
+ data = await run_in_session(save)
_invalidate_model_caches()
- return success_response(data=_provider_to_dict(provider, _get_pricing(db, provider.model_name)))
+ return success_response(data=data)
@router.delete("/providers/{provider_id}", summary="删除模型供应商")
@@ -778,10 +773,9 @@ def delete_provider_endpoint(
async def test_saved_provider(
provider_id: str,
_: None = Depends(require_system_settings),
- db: Session = Depends(get_db),
):
"""对已保存的供应商发起一次实际连通性测试。仅限管理员;会把测试结果(成功/失败)回写到供应商记录,并返回耗时与错误信息;不存在返回 404。"""
- provider = get_provider(db, provider_id)
+ provider = await run_in_session(get_provider, provider_id)
if provider is None:
raise HTTPException(status_code=404, detail="Provider not found")
@@ -795,7 +789,7 @@ async def test_saved_provider(
provider.model_name,
provider.extra_config or {},
)
- set_provider_test_result(db, provider_id, result["success"])
+ await run_in_session(set_provider_test_result, provider_id, result["success"])
return success_response(data=result)
@@ -821,18 +815,17 @@ async def test_unsaved_provider(
async def detect_context_length(
body: DetectContextRequest,
_: None = Depends(require_system_settings),
- db: Session = Depends(get_db),
):
"""探测模型真实上下文窗口(token)供表单回填。仅限管理员;不落库。
返回 `{context_length, source, source_label, confidence, detail, notes}`;`context_length=0`
表示没探到,`notes` 逐级说明每个来源看到了什么,便于管理员判断该手工填多少。
"""
- from core.llm.providers.context_probe import discover_context_length
+ from core.models.providers.context_probe import discover_context_length
api_key = body.api_key
if not api_key and body.provider_id:
- existing = get_provider(db, body.provider_id)
+ existing = await run_in_session(get_provider, body.provider_id)
if existing is not None:
api_key = existing.api_key or ""
@@ -921,7 +914,6 @@ def unassign_role_endpoint(
async def _require_model_export_access(
request: Request,
authorization: Optional[str] = Header(None),
- db: Session = Depends(get_db),
user: Optional[UserContext] = Depends(require_auth(False)),
) -> None:
"""Default to config access; optionally allow authenticated desktop sync.
@@ -931,7 +923,7 @@ async def _require_model_export_access(
scope = (os.getenv("DESKTOP_MODEL_SYNC_SCOPE") or "").strip().lower()
if scope == "authenticated" and user is not None:
return
- await require_config(request, authorization=authorization, db=db)
+ await require_config(request, authorization=authorization)
@router.get("/export", summary="导出模型配置")
@@ -946,11 +938,11 @@ def export_endpoint(
return success_response(data=export_all(db))
-async def _require_import_control(request: Request, db: Session = Depends(get_db)):
- from core.services.desktop_capability import is_desktop_shell_control
+async def _require_import_control(request: Request):
+ from features.desktop.capability.service import is_desktop_shell_control
if is_desktop_shell_control(request.headers.get("authorization", ""), request.headers.get("origin")):
return
- await require_config(request=request, authorization=request.headers.get("authorization"), db=db)
+ await require_config(request=request, authorization=request.headers.get("authorization"))
@router.post("/import", summary="导入模型配置")
@@ -961,9 +953,9 @@ def import_endpoint(
db: Session = Depends(get_db),
):
"""导入模型配置(供应商 + 角色分配)。仅限管理员(CONFIG_TOKEN / can_system_config,同 export);overwrite=True 时覆盖同名条目,导入后刷新模型配置缓存。"""
- from core.services.desktop_model_credentials import sanitize_import
+ from features.desktop.bridge.model_credentials import sanitize_import
if any(str(p.get("api_key") or "").startswith(("dcap1.", "dcap2.", "desktop-capability:")) for p in body.providers):
- from core.services.desktop_capability import is_desktop_shell_control
+ from features.desktop.capability.service import is_desktop_shell_control
if not is_desktop_shell_control(request.headers.get("authorization", ""), request.headers.get("origin")):
raise HTTPException(status_code=403, detail="desktop shell control required")
try:
diff --git a/src/backend/api/routes/v1/myspace_folders.py b/src/backend/api/routes/v1/myspace_folders.py
index 0335cdac..11b62968 100644
--- a/src/backend/api/routes/v1/myspace_folders.py
+++ b/src/backend/api/routes/v1/myspace_folders.py
@@ -18,13 +18,13 @@
from typing import Optional
from api.folder_batch_schemas import FolderBatchBody
-from core.services.folder_batch import create_folder_batch
+from core.files.folder_batch import create_folder_batch
from core.db.models import UserFolder, UserShadow
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.user_folder_service import UserFolderService
+from core.projects.user_folders import UserFolderService
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
diff --git a/src/backend/api/routes/v1/ontologies.py b/src/backend/api/routes/v1/ontologies.py
index 18bd9cfb..3bd748e1 100644
--- a/src/backend/api/routes/v1/ontologies.py
+++ b/src/backend/api/routes/v1/ontologies.py
@@ -15,14 +15,9 @@
from core.db.models import OntologyDraft, OntologyEnforcementEvent, OntologyReviewRun
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from core.infra.responses import success_response
-from core.services import UserService
-from core.services.ontology_policy import (
- plugin_import_build_validation_forced,
- require_ontology_validation_permission,
- set_plugin_import_build_validation_forced,
- user_can_use_ontology_validation,
-)
-from core.services.ontology_service import OntologyService
+from core.auth.users import UserService
+from core.ontology.policy import plugin_import_build_validation_forced, require_ontology_validation_permission, set_plugin_import_build_validation_forced, user_can_use_ontology_validation
+from core.ontology.service import OntologyService
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel, Field
from sqlalchemy import func
@@ -182,7 +177,7 @@ def preview_ontology_runtime(
settings = UserService(db).get_user_settings(str(user.user_id))
if not settings.get("ontology_enabled", False):
return success_response(data={"enabled": False, "packs": [], "review_level": "none"})
- from core.services.ontology_service import build_user_ontology_runtime
+ from core.ontology.runtime_policy import build_user_ontology_runtime
_, runtime = build_user_ontology_runtime(
user_id=str(user.user_id),
@@ -699,11 +694,10 @@ def review_ontology_draft(
)
async def generate_ontology_evolution_drafts(
body: OntologyEvolutionGenerateRequest,
- db: Session = Depends(get_db),
):
- from core.services.ontology_evolution_service import OntologyEvolutionService
+ from core.ontology.evolution import generate_candidates
- result = await OntologyEvolutionService(db).generate_candidates(
+ result = await generate_candidates(
min_occurrences=body.min_occurrences,
limit=body.limit,
model_name=body.model_name,
@@ -725,7 +719,7 @@ def materialize_ontology_draft(
draft_id: str,
db: Session = Depends(get_db),
):
- from core.services.ontology_evolution_service import OntologyEvolutionService
+ from core.ontology.evolution import OntologyEvolutionService
row = OntologyEvolutionService(db).materialize_approved_draft(draft_id)
return success_response(data=_version_dict(row, include_content=True))
diff --git a/src/backend/api/routes/v1/plans.py b/src/backend/api/routes/v1/plans.py
index ae9c2e62..dca2b637 100644
--- a/src/backend/api/routes/v1/plans.py
+++ b/src/backend/api/routes/v1/plans.py
@@ -2,22 +2,22 @@
from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel, Field
-from sqlalchemy.orm import Session
-
-from core.auth.backend import get_current_user, UserContext
+import api.transports.run_sse as run_sse
+import core.runs.runtime as runtime
+import orchestration.plans.admission as admission
+from core.auth.backend import UserContext, get_current_user
from core.chat.context import generate_smart_title
from core.db.engine import get_db
-from core.infra.responses import success_response, sse_response
-from core.services.plan_service import PlanService
-from core.services.chat_service import ChatService
-from core.services.user_model_selection import (
- UserModelSelectionError,
- resolve_effective_chat_model_name,
- resolve_user_model_provider_id,
-)
+from core.db.threads import run_db, run_in_session
from core.infra.logging import get_logger
+from core.infra.responses import sse_response, success_response
+from core.chat.service import ChatService
+from core.chat.plans import PlanService
+from core.auth.model_selection import UserModelSelectionError, resolve_user_model_provider_id
+from core.models.selection import resolve_effective_chat_model_name
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
logger = get_logger(__name__)
@@ -155,7 +155,7 @@ def _load_chat_history(
# structured tool replay); long sessions get the compacted history rather than the full raw text.
if chat_id:
try:
- from core.services.compaction_service import load_session_history
+ from core.chat.compaction.service import load_session_history
history = load_session_history(ChatService(db), chat_id, user_id)
if history:
@@ -177,64 +177,66 @@ def _load_chat_history(
async def generate_plan(
req: GeneratePlanRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""SSE stream — generate a structured plan from a task description.
后台 task 跑生成工作流,HTTP 连接断开(刷新页面)任务继续;前端可通过
``GET /v1/chats/{chat_id}/active-run`` 探测,结束后从消息列表拿到完整 plan。
"""
- db_chat_id = _ensure_plan_session(
- db,
- req.chat_id,
- user.user_id,
- req.task_description,
- project_id=req.project_id,
- )
- if not db_chat_id:
- raise HTTPException(status_code=400, detail="chat_id 必填,无法创建后台生成任务")
- try:
- selected_model_provider_id = resolve_user_model_provider_id(
- db, req.model_provider_id, user_id=user.user_id
- )
- except UserModelSelectionError as exc:
- raise HTTPException(status_code=400, detail=str(exc)) from exc
- actual_model_name = resolve_effective_chat_model_name(
- selected_model_provider_id,
- fallback_model_name=req.model_name,
- )
- # On automatic entry into plan mode (enter_plan_mode), task_description is an
- # AI-expanded internal prompt and is NOT persisted as a user message — the original
- # user request and that main-agent turn are already in the same session history;
- # storing another copy would make a refresh display the internal prompt as a user
- # bubble. Manual plan mode (typed by the user) echoes normally.
- if not req.suppress_user_echo:
- _save_plan_message(
+ def prepare(db: Session):
+ db_chat_id = _ensure_plan_session(
db,
- db_chat_id,
- "user",
+ req.chat_id,
+ user.user_id,
req.task_description,
- model=actual_model_name,
- extra_data=(
- {"model_provider_id": selected_model_provider_id}
- if selected_model_provider_id
- else None
- ),
+ project_id=req.project_id,
+ )
+ if not db_chat_id:
+ raise HTTPException(status_code=400, detail="chat_id 必填,无法创建后台生成任务")
+ try:
+ selected_model_provider_id = resolve_user_model_provider_id(
+ db, req.model_provider_id, user_id=user.user_id
+ )
+ except UserModelSelectionError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ actual_model_name = resolve_effective_chat_model_name(
+ selected_model_provider_id,
+ fallback_model_name=req.model_name,
)
- session_messages = _load_chat_history(db, req.chat_id, user.user_id, req.history_messages)
- logger.warning(
- "[plan-generate] chat_id=%s, loaded %d history messages", req.chat_id, len(session_messages)
- )
+ # On automatic entry into plan mode (enter_plan_mode), task_description is an
+ # AI-expanded internal prompt and is NOT persisted as a user message — the original
+ # user request and that main-agent turn are already in the same session history;
+ # storing another copy would make a refresh display the internal prompt as a user
+ # bubble. Manual plan mode (typed by the user) echoes normally.
+ if not req.suppress_user_echo:
+ _save_plan_message(
+ db,
+ db_chat_id,
+ "user",
+ req.task_description,
+ model=actual_model_name,
+ extra_data=(
+ {"model_provider_id": selected_model_provider_id}
+ if selected_model_provider_id
+ else None
+ ),
+ )
+
+ session_messages = _load_chat_history(db, req.chat_id, user.user_id, req.history_messages)
+ logger.warning(
+ "[plan-generate] chat_id=%s, loaded %d history messages", req.chat_id, len(session_messages)
+ )
+ return db_chat_id, actual_model_name, selected_model_provider_id, session_messages
+
+ db_chat_id, actual_model_name, selected_model_provider_id, session_messages = await run_in_session(prepare)
uploaded_files = None
if req.attachments:
uploaded_files = [a.model_dump() for a in req.attachments if a.file_id]
- from orchestration import chat_run_executor
-
- run = await chat_run_executor.start_plan_generate_run(
+ run = await admission.start_plan_generate_run(
chat_id=db_chat_id,
user_id=user.user_id,
task_description=req.task_description,
@@ -249,7 +251,7 @@ async def generate_plan(
)
return sse_response(
- chat_run_executor.follow_run_as_sse(
+ run_sse.follow_run_as_sse(
run.run_id,
chat_id=db_chat_id,
error_event_factory=lambda reason: {"type": "plan_error", "error": reason},
@@ -351,7 +353,6 @@ async def execute_plan(
plan_id: str,
req: ExecutePlanRequest = ExecutePlanRequest(),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""SSE stream — execute an approved plan step by step.
@@ -359,34 +360,37 @@ async def execute_plan(
断开(用户刷新页面)不会取消任务,前端可通过 ``GET /v1/chats/{chat_id}/active-run``
+ ``GET /v1/chats/stream/{run_id}`` 续播。
"""
- svc = PlanService(db)
- plan = svc.get_plan(plan_id, user.user_id)
- if not plan:
- raise HTTPException(status_code=404, detail="计划不存在")
- if plan.status != "approved":
- raise HTTPException(status_code=400, detail=f"计划状态为 '{plan.status}',需要先审批")
-
- # Ensure chat session and save "确认执行" user message
- db_chat_id = _ensure_plan_session(
- db,
- req.chat_id,
- user.user_id,
- plan.title,
- project_id=req.project_id,
- )
- if not db_chat_id:
- raise HTTPException(status_code=400, detail="chat_id 必填,无法创建后台执行任务")
- actual_model_name = resolve_effective_chat_model_name() or "qwen"
- _save_plan_message(db, db_chat_id, "user", "确认执行", model=actual_model_name)
-
- session_messages = _load_chat_history(db, req.chat_id, user.user_id, req.history_messages)
- logger.warning(
- "[plan-execute] chat_id=%s, loaded %d history messages", req.chat_id, len(session_messages)
- )
- from orchestration import chat_run_executor
+ def prepare(db: Session):
+ svc = PlanService(db)
+ plan = svc.get_plan(plan_id, user.user_id)
+ if not plan:
+ raise HTTPException(status_code=404, detail="计划不存在")
+ if plan.status != "approved":
+ raise HTTPException(status_code=400, detail=f"计划状态为 '{plan.status}',需要先审批")
- run = await chat_run_executor.start_plan_execute_run(
+ # Ensure chat session and save "确认执行" user message
+ db_chat_id = _ensure_plan_session(
+ db,
+ req.chat_id,
+ user.user_id,
+ plan.title,
+ project_id=req.project_id,
+ )
+ if not db_chat_id:
+ raise HTTPException(status_code=400, detail="chat_id 必填,无法创建后台执行任务")
+ actual_model_name = resolve_effective_chat_model_name() or "qwen"
+ _save_plan_message(db, db_chat_id, "user", "确认执行", model=actual_model_name)
+
+ session_messages = _load_chat_history(db, req.chat_id, user.user_id, req.history_messages)
+ logger.warning(
+ "[plan-execute] chat_id=%s, loaded %d history messages", req.chat_id, len(session_messages)
+ )
+ return db_chat_id, actual_model_name, session_messages
+
+ db_chat_id, actual_model_name, session_messages = await run_in_session(prepare)
+
+ run = await admission.start_plan_execute_run(
plan_id=plan_id,
chat_id=db_chat_id,
user_id=user.user_id,
@@ -399,7 +403,7 @@ async def execute_plan(
)
return sse_response(
- chat_run_executor.follow_run_as_sse(
+ run_sse.follow_run_as_sse(
run.run_id,
chat_id=db_chat_id,
error_event_factory=lambda reason: {
@@ -415,26 +419,25 @@ async def execute_plan(
async def cancel_plan(
plan_id: str,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""Cancel a running plan — 同时杀掉关联的后台 chat_run(如果有)。"""
- svc = PlanService(db)
- plan = svc.get_plan(plan_id, user.user_id)
- if not plan:
- raise HTTPException(status_code=404, detail="计划不存在")
- if plan.status not in ("running", "approved", "draft"):
- raise HTTPException(status_code=400, detail=f"计划状态为 '{plan.status}',无法取消")
+ def cancel(db: Session) -> None:
+ svc = PlanService(db)
+ plan = svc.get_plan(plan_id, user.user_id)
+ if not plan:
+ raise HTTPException(status_code=404, detail="计划不存在")
+ if plan.status not in ("running", "approved", "draft"):
+ raise HTTPException(status_code=400, detail=f"计划状态为 '{plan.status}',无法取消")
+ svc.update_plan(plan_id, status="cancelled")
- svc.update_plan(plan_id, status="cancelled")
+ await run_in_session(cancel)
# Also cancel the background run in lockstep (only exists during the plan execute phase)
- from orchestration import chat_run_executor
-
- run = chat_run_executor.get_run_by_plan_id(plan_id)
+ run = await run_db(admission.get_run_by_plan_id, plan_id)
if run is not None:
try:
- await chat_run_executor.cancel_run(run.run_id, user_id=user.user_id)
- except chat_run_executor.ChatRunPermissionDenied:
+ await runtime.cancel_run(run.run_id, user_id=user.user_id)
+ except runtime.ChatRunPermissionDenied:
pass # the plan was already authorized; a run auth failure means user_id inference went wrong — do not block
except Exception as exc:
logger.warning(
diff --git a/src/backend/api/routes/v1/plugin_resources.py b/src/backend/api/routes/v1/plugin_resources.py
index 623a08a5..46f3a0a7 100644
--- a/src/backend/api/routes/v1/plugin_resources.py
+++ b/src/backend/api/routes/v1/plugin_resources.py
@@ -6,11 +6,12 @@
from sqlalchemy.orm import Session
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.plugins.resources import service, store
-from core.plugins.resources.stream import connect
+from core.capabilities.plugins.resources import service, store
+from core.capabilities.plugins.resources.stream import connect
-from core.plugins.resources.lifecycle import lifespan
+from core.capabilities.plugins.resources.lifecycle import lifespan
router = APIRouter(prefix="/v1", tags=["Plugin resources"], lifespan=lifespan)
class OpenRequest(BaseModel):
@@ -29,7 +30,7 @@ class ToolRequest(OpenRequest):
@router.get("/plugins/{slug}/checkpoints")
def list_checkpoints(slug: str, install_id: str | None = None, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- from core.plugins.resources.installation import resolve
+ from core.capabilities.plugins.resources.installation import resolve
selected = resolve(db, slug, str(user.user_id), install_id)
rows = db.query(store.PluginResourceCheckpoint).filter_by(user_id=str(user.user_id), install_id=selected.install_id).all()
return success_response(data={"items": [{"checkpoint_id": row.checkpoint_id, "name": row.name} for row in rows]})
@@ -44,34 +45,38 @@ def delete_checkpoint(checkpoint_id: str, user: UserContext = Depends(get_curren
return success_response(data={"deleted": True})
@router.post("/plugins/{slug}/resources/{module_id}")
-async def open_resource(slug: str, module_id: str, body: OpenRequest, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- result = await service.create(db, slug, module_id, str(user.user_id), **body.model_dump())
+async def open_resource(slug: str, module_id: str, body: OpenRequest, user: UserContext = Depends(get_current_user)):
+ result = await service.create(slug, module_id, str(user.user_id), **body.model_dump())
return success_response(data={"resource": result})
-@router.get("/chats/{chat_id}/plugin-resources")
-async def chat_resources(chat_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- import asyncio
+def _chat_resource_ids(db, chat_id, uid):
from core.db.models import ChatSession
chat = db.get(ChatSession, chat_id)
- uid = str(user.user_id)
if chat is None or chat.user_id != uid or chat.deleted_at is not None:
raise HTTPException(404, "conversation_unavailable")
- rows = db.query(store.PluginResource).filter_by(user_id=uid, chat_id=chat_id, status="active").limit(8).all()
- async def live(row):
+ rows = db.query(store.PluginResource.resource_id).filter_by(user_id=uid, chat_id=chat_id, status="active").limit(8)
+ return [resource_id for (resource_id,) in rows]
+
+@router.get("/chats/{chat_id}/plugin-resources")
+async def chat_resources(chat_id: str, user: UserContext = Depends(get_current_user)):
+ import asyncio
+ uid = str(user.user_id)
+ resource_ids = await run_in_session(_chat_resource_ids, chat_id, uid)
+ async def live(resource_id):
try:
- service.authorized(db, row.resource_id, uid)
+ row = await run_in_session(service.authorized_row, resource_id, uid)
state = await asyncio.wait_for(service.request(row, "/state"), 5)
if state.get("closed"):
return None
return store.public(row)
except (HTTPException, TimeoutError):
return None
- items = await asyncio.gather(*(live(row) for row in rows))
+ items = await asyncio.gather(*(live(resource_id) for resource_id in resource_ids))
return success_response(data={"items": [item for item in items if item is not None]})
@router.get("/plugin-resources/{resource_id}")
-async def get_resource(resource_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- row = service.authorized(db, resource_id, str(user.user_id))
+async def get_resource(resource_id: str, user: UserContext = Depends(get_current_user)):
+ row = await run_in_session(service.authorized_row, resource_id, str(user.user_id))
state = await service.request(row, "/state")
return success_response(data={"resource": store.public(row), "state": state})
@@ -83,38 +88,32 @@ def attach_resource(resource_id: str, request: Request, user: UserContext = Depe
parsed = urlsplit(origin)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise HTTPException(403, "invalid_origin")
- from core.plugins.resources.authentication import capture
+ from core.capabilities.plugins.resources.authentication import capture
return success_response(data={"ticket": store.ticket(db, row, origin, capture(request))})
@router.post("/plugin-resources/{resource_id}/asset-ticket")
def issue_asset_ticket(resource_id: str, request: Request, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
row = service.authorized(db, resource_id, str(user.user_id))
- from core.plugins.resources.installation import resolve
- from core.plugins.ui.contract import find_module
+ from core.capabilities.plugins.resources.installation import resolve
+ from features.workbench.ui.contract import find_module
if row.application_key is not None:
- from core.plugins.resources.owners import resolve_installation
+ from core.capabilities.plugins.resources.owners import resolve_installation
selected = resolve_installation(db, row, str(user.user_id))
else:
selected = resolve(db, row.slug, str(user.user_id), row.install_id)
module = find_module(selected.ui, row.module_id)
if module is None:
raise HTTPException(404, "resource_module_unavailable")
- from core.plugins.resources.authentication import capture
+ from core.capabilities.plugins.resources.authentication import capture
return success_response(data={"ticket": store.asset_ticket(db, row, capture(request)), "entry": module["entry"]})
-@router.get("/plugin-resource-assets/{execution_scope}/{ticket}/{asset_path:path}")
-async def resource_asset(execution_scope: str, ticket: str, asset_path: str, db: Session = Depends(get_db)):
- # Null-origin iframe subresources cannot carry SameSite login cookies.
- # The ticket grants static package reads only; its saved session is still revocable.
- from .plugin_ui import web_asset_response
- from core.plugins.resources.authentication import validate
- resource_id, user_id, authentication = store.resolve_asset_ticket(db, ticket)
- await validate(authentication, user_id, db)
+def _asset_response(db, execution_scope, resource_id, user_id, asset_path):
+ from features.workbench.api.plugin_ui import web_asset_response
row = service.authorized(db, resource_id, user_id)
if row.scope != execution_scope:
raise HTTPException(409, "execution_scope_mismatch")
if row.application_key is not None:
- from core.plugins.resources.owners import resolve_installation
+ from core.capabilities.plugins.resources.owners import resolve_installation
selected = resolve_installation(db, row, user_id)
response = web_asset_response(db, row.slug, user_id, asset_path, row.install_id,
_installation=selected)
@@ -124,26 +123,35 @@ async def resource_asset(execution_scope: str, ticket: str, asset_path: str, db:
response.headers["Access-Control-Allow-Origin"] = "null"
return response
+@router.get("/plugin-resource-assets/{execution_scope}/{ticket}/{asset_path:path}")
+async def resource_asset(execution_scope: str, ticket: str, asset_path: str):
+ # Null-origin iframe subresources cannot carry SameSite login cookies.
+ # The ticket grants static package reads only; its saved session is still revocable.
+ from core.capabilities.plugins.resources.authentication import validate
+ resource_id, user_id, authentication = await run_in_session(store.resolve_asset_ticket, ticket)
+ await validate(authentication, user_id)
+ return await run_in_session(_asset_response, execution_scope, resource_id, user_id, asset_path)
+
@router.websocket("/plugin-resources/{resource_id}/stream")
async def resource_stream(websocket: WebSocket, resource_id: str):
await connect(websocket, resource_id)
@router.delete("/plugin-resources/{resource_id}")
-async def close_resource(resource_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- row = service.authorized(db, resource_id, str(user.user_id))
- await service.close(db, row)
+async def close_resource(resource_id: str, user: UserContext = Depends(get_current_user)):
+ row = await run_in_session(service.authorized_row, resource_id, str(user.user_id))
+ await service.close(row)
return success_response(data={"closed": True})
@router.post("/plugin-resources/{resource_id}/downloads/{download_id}/retain")
-async def retain_download(resource_id: str, download_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- from core.plugins.resources.artifacts import retain
- row = service.authorized(db, resource_id, str(user.user_id))
+async def retain_download(resource_id: str, download_id: str, user: UserContext = Depends(get_current_user)):
+ from core.capabilities.plugins.resources.artifacts import retain
+ row = await run_in_session(service.authorized_row, resource_id, str(user.user_id))
return success_response(data=await retain(row, download_id))
@router.get("/plugin-resources/{resource_id}/downloads/{download_id}")
-async def download_resource(resource_id: str, download_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
+async def download_resource(resource_id: str, download_id: str, user: UserContext = Depends(get_current_user)):
import httpx
- row = service.authorized(db, resource_id, str(user.user_id))
+ row = await run_in_session(service.authorized_row, resource_id, str(user.user_id))
target = store.descriptor(row)
client = httpx.AsyncClient(timeout=60, headers=target["headers"], trust_env=False)
response = await client.send(client.build_request("GET", target["url"] + "/download/" + download_id), stream=True)
@@ -165,10 +173,10 @@ async def content():
return StreamingResponse(content(), media_type="application/octet-stream", headers={"Content-Disposition": response.headers.get("content-disposition", "attachment"), "Cache-Control": "no-store"})
@router.post("/internal/plugin-resources/tool")
-async def tool_resource(body: ToolRequest, request: Request, db: Session = Depends(get_db)):
- from .internal_site_auth import _check_internal_token
+async def tool_resource(body: ToolRequest, request: Request):
+ from features.sites.api.internal_site_auth import _check_internal_token
_check_internal_token(request.headers.get("x-internal-token"))
- from core.llm.mcp_invocation import verify
+ from core.llm.mcp.invocation import verify
from extensions.mcp_servers._ports import PORTS
audience = request.headers.get("x-hugagent-mcp-audience", "")
if audience not in PORTS or request.headers.get("x-chat-id") != body.chat_id:
@@ -184,21 +192,21 @@ async def tool_resource(body: ToolRequest, request: Request, db: Session = Depen
if not user_id:
raise HTTPException(401, "identity_required")
if body.action == "open":
- result = await service.create(db, body.slug, body.module_id, user_id, body.chat_id, resource_id=body.resource_id, install_id=body.install_id, checkpoint_id=body.checkpoint_id)
+ result = await service.create(body.slug, body.module_id, user_id, body.chat_id, resource_id=body.resource_id, install_id=body.install_id, checkpoint_id=body.checkpoint_id)
return success_response(data={"resource": result})
if not body.resource_id:
raise HTTPException(400, "resource_required")
- row = service.authorized(db, body.resource_id, user_id)
+ row = await run_in_session(service.authorized_row, body.resource_id, user_id)
if (row.chat_id != body.chat_id or row.slug != body.slug or row.module_id != body.module_id
or (signed_install and row.install_id != signed_install)):
raise HTTPException(403, "resource_binding_mismatch")
if body.action == "close":
await service.command(row, {"id": body.command_id, "action": "close", "params": {}}, actor="agent")
- await service.close(db, row)
+ await service.close(row)
return success_response(data={"closed": True})
if body.action == "retain_download":
await service.command(row, {"id": uuid.uuid4().hex, "action": "state", "params": {}}, actor="agent")
- from core.plugins.resources.artifacts import retain
+ from core.capabilities.plugins.resources.artifacts import retain
return success_response(data=await retain(row, str(body.params["download_id"])))
data = await service.command(row, {"id": body.command_id, "action": body.action, "params": body.params}, actor="agent")
return success_response(data=data)
diff --git a/src/backend/api/routes/v1/plugins.py b/src/backend/api/routes/v1/plugins.py
index 542515ce..24c4ff82 100644
--- a/src/backend/api/routes/v1/plugins.py
+++ b/src/backend/api/routes/v1/plugins.py
@@ -25,6 +25,7 @@
"""
from __future__ import annotations
+from features.marketplace import plugin_market
import json
import logging
@@ -40,9 +41,9 @@
from core.db.engine import get_db
from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
from core.infra.responses import created_response, success_response
-from core.services import marketplace_listing as ml
-from core.plugins import management as ps
-from core.capabilities.paths import capabilities_enabled
+from features.marketplace import edition as ml
+from core.capabilities.plugins import management as ps
+from core.harness.registry.run_capabilities import has_run_capabilities
router = APIRouter(prefix="/v1/plugins", tags=["Plugin"])
logger = logging.getLogger(__name__)
@@ -72,7 +73,7 @@ def list_plugins(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- items = ps.list_plugins(db, owner_user_id=str(user.user_id))
+ items = plugin_market.list_plugins(db, owner_user_id=str(user.user_id))
return success_response(data={"items": items})
@@ -81,7 +82,7 @@ def list_installed(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
items = ps.list_installed(db, owner_user_id=str(user.user_id), include_global=True)
# 云端账号装的插件登记在本机登记表里、业务库没有行,要投影进来才看得到。按 slug
@@ -100,7 +101,7 @@ def get_installed_detail(
):
"""已安装插件的完整详情:技能(含指令/文件清单)+ MCP(含工具列表)。"""
# 云端同步来的插件在本机业务库里查不到,走登记表那份,否则详情页直接 404。
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
projected = device_catalog.plugin_detail(install_id, user_id=str(user.user_id))
if projected is not None:
@@ -117,23 +118,23 @@ async def get_feishu_app_status(_: str = Depends(require_system_settings)):
CE does not ship the Config console, so the system-settings capability is
the shared authorization seam for both CE instance admins and EE admins.
"""
- from core.services.lark_service import LarkService
+ from integrations.lark.service import LarkService
- return success_response(data=LarkService(None).app_status())
+ return success_response(data=LarkService().app_status())
@router.post("/feishu-cli/app/init", summary="从飞书插件页发起共享应用初始化")
async def init_feishu_app(_: str = Depends(require_system_settings)):
- from core.services.lark_service import LarkService
+ from integrations.lark.service import LarkService
- return success_response(data=await LarkService(None).start_app_init())
+ return success_response(data=await LarkService().start_app_init())
@router.post("/feishu-cli/app/reset", summary="从飞书插件页重置共享应用")
async def reset_feishu_app(_: str = Depends(require_system_settings)):
- from core.services.lark_service import LarkService
+ from integrations.lark.service import LarkService
- return success_response(data=await LarkService(None).reset_app())
+ return success_response(data=await LarkService().reset_app())
@router.get("/{slug}", summary="内置插件详情")
@@ -162,8 +163,8 @@ def install_plugin(
# Both installing from and importing into the plugin marketplace require can_import_plugin
# (same as the skill marketplace's can_add_skill); granted per user via the Config backend
# "User Management -> Permission Config". Admin global install goes through admin_plugins.
- if capabilities_enabled():
- from core.services.local_market_install import install_plugin as install_local
+ if has_run_capabilities():
+ from features.desktop.local.market_install import install_plugin as install_local
return created_response(data=install_local(db, str(user.user_id), slug, body.secrets))
_require_can_import_plugin(str(user.user_id), db)
@@ -179,23 +180,23 @@ def install_plugin(
@router.post("/import", status_code=201, summary="导入外部插件(上传 zip)")
-async def import_plugin(
+def import_plugin(
file: UploadFile = File(..., description="插件包 .zip(native / Claude Code / Codex)"),
secrets: Optional[str] = Form(None, description="凭据 JSON 字符串"),
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
"""上传插件 zip 并导入(私有)。返回 import_report(imported / adapted / dropped)。"""
- if capabilities_enabled():
- from core.services.local_skill_editor import install_archive
+ if has_run_capabilities():
+ from features.desktop.local.skills.editor import install_archive
if _parse_secrets(secrets):
raise BadRequestError(message="本机插件凭据请通过连接配置设置,不写入插件包")
return created_response(
- data=install_archive(str(user.user_id), await file.read(), "plugin")
+ data=install_archive(str(user.user_id), file.file.read(), "plugin")
)
_require_can_import_plugin(str(user.user_id), db)
- raw = await file.read()
+ raw = file.file.read()
secret_map = _parse_secrets(secrets)
result = ps.import_plugin_from_zip(
db,
@@ -214,7 +215,7 @@ def uninstall_plugin(
db: Session = Depends(get_db),
):
if install_id.startswith("plugin:local:"):
- from core.plugins.local import service as local
+ from features.desktop.local.plugins import service as local
try:
current = local.get(str(user.user_id), install_id)
@@ -241,8 +242,8 @@ def set_installed_meta(
):
"""展示信息(名称/分类/图标)是界面配置:用户自己导入/安装的私有插件由用户在此
配置;管理员全局插件走 admin 接口,普通用户不可改。"""
- if capabilities_enabled() and install_id.startswith("plugin:local:"):
- from core.plugins.local.service import set_presentation
+ if has_run_capabilities() and install_id.startswith("plugin:local:"):
+ from features.desktop.local.plugins.service import set_presentation
return success_response(
data=set_presentation(str(user.user_id), install_id, body.model_dump(exclude_none=True))
@@ -271,7 +272,7 @@ def set_enabled(
db: Session = Depends(get_db),
):
# 云端同步来的插件登记在本机登记表里,业务库没有行——写用户覆盖既不生效也读不回来。
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
if device_catalog.set_plugin_enabled(install_id, body.enabled, user_id=str(user.user_id)):
return success_response(data={"install_id": install_id, "enabled": body.enabled})
diff --git a/src/backend/api/routes/v1/project_api_keys.py b/src/backend/api/routes/v1/project_api_keys.py
index d7936518..eb85ec69 100644
--- a/src/backend/api/routes/v1/project_api_keys.py
+++ b/src/backend/api/routes/v1/project_api_keys.py
@@ -9,9 +9,10 @@
from core.auth.backend import UserContext, get_current_user
from core.auth.capabilities import resolve_user_capabilities
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import created_response, success_response
-from core.services.project_api_service import require_project_key_access
-from core.services.api_key_service import ApiKeyService
+from core.projects.api import require_project_key_access
+from core.auth.api_keys import ApiKeyService
router = APIRouter(prefix="/v1/projects", tags=["Project API Keys"])
_ALLOWED_EXPIRY_DAYS = {7, 30, 90, 180, 365}
@@ -136,15 +137,16 @@ async def toggle_key(
key_id: str,
body: ToggleProjectApiKeyRequest,
user: UserContext = Depends(_manager),
- db: Session = Depends(get_db),
):
- _find(db, user, project_id, key_id)
- row = ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, project_id=project_id)
- data = _dto(row)
+ def toggle(db):
+ _find(db, user, project_id, key_id)
+ return _dto(ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, project_id=project_id))
+
+ data = await run_in_session(toggle)
if not body.enabled:
- from core.services.api_key_runs import stop_cancelled_key_workers
+ from core.auth.api_key_runs import stop_cancelled_key_workers
- await stop_cancelled_key_workers(db, key_id, user.user_id)
+ await stop_cancelled_key_workers(key_id, user.user_id)
return success_response(data=data)
@@ -153,11 +155,13 @@ async def revoke_key(
project_id: str,
key_id: str,
user: UserContext = Depends(_manager),
- db: Session = Depends(get_db),
):
- _find(db, user, project_id, key_id)
- ApiKeyService(db).revoke_key(user.user_id, key_id, project_id=project_id)
- from core.services.api_key_runs import stop_cancelled_key_workers
+ def revoke(db):
+ _find(db, user, project_id, key_id)
+ ApiKeyService(db).revoke_key(user.user_id, key_id, project_id=project_id)
+
+ await run_in_session(revoke)
+ from core.auth.api_key_runs import stop_cancelled_key_workers
- await stop_cancelled_key_workers(db, key_id, user.user_id)
+ await stop_cancelled_key_workers(key_id, user.user_id)
return success_response(data={"id": key_id, "revoked": True})
diff --git a/src/backend/api/routes/v1/project_environments.py b/src/backend/api/routes/v1/project_environments.py
index fca30b10..ff05fc95 100644
--- a/src/backend/api/routes/v1/project_environments.py
+++ b/src/backend/api/routes/v1/project_environments.py
@@ -1,12 +1,12 @@
"""Project environment configuration and authorized resource options."""
from core.auth.backend import UserContext, get_current_user
-from core.auth.permissions_iface import ProjectAccess, require_project_access
+from core.auth.edition import ProjectAccess, require_project_access
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.project_environment.config import ProjectEnvironment
-from core.services.project_environment import ProjectEnvironmentService
-from core.services.project_environment_resources import environment_resources
+from core.projects.environment.config import ProjectEnvironment
+from core.projects.environment.service import ProjectEnvironmentService
+from core.projects.environment_resources import environment_resources
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
diff --git a/src/backend/api/routes/v1/projects.py b/src/backend/api/routes/v1/projects.py
index ef1a2e58..5a483f90 100644
--- a/src/backend/api/routes/v1/projects.py
+++ b/src/backend/api/routes/v1/projects.py
@@ -26,16 +26,16 @@
from urllib.parse import quote
from core.auth.backend import UserContext, get_current_user
-from core.auth.permissions_iface import ProjectAccess, require_project_access
+from core.auth.edition import ProjectAccess, require_project_access
from core.config.local_mode import local_mode_enabled
from core.db.engine import get_db
from core.db.models import Artifact
from core.infra.responses import created_response, paginated_response, success_response
from core.infra.time import utc_now
-from core.project_environment.config import ProjectEnvironment
-from core.services.project_environment import ProjectEnvironmentService
-from core.services.project_file_service import ProjectFileService
-from core.services.project_service import ProjectService
+from core.projects.environment.config import ProjectEnvironment
+from core.projects.environment.service import ProjectEnvironmentService
+from core.projects.files import ProjectFileService
+from core.projects.edition import ProjectService
from fastapi import (
APIRouter,
BackgroundTasks,
@@ -89,7 +89,7 @@ class UpdateInstructionsBody(BaseModel):
@router.get("", summary="列出可见项目")
-async def list_projects(
+def list_projects(
q: Optional[str] = Query(None, description="按 name / description 模糊搜索"),
sort: str = Query("-last_activity_at", description="-last_activity_at | name | created"),
page: int = Query(1, ge=1),
@@ -106,7 +106,7 @@ async def list_projects(
@router.post("", status_code=status.HTTP_201_CREATED, summary="创建项目")
-async def create_project(
+def create_project(
body: CreateProjectBody,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -148,7 +148,7 @@ async def create_project(
@router.get("/{project_id}", summary="项目详情")
-async def get_project(
+def get_project(
access: ProjectAccess = Depends(require_project_access("view")),
db: Session = Depends(get_db),
):
@@ -164,7 +164,7 @@ async def get_project(
@router.patch("/{project_id}", summary="更新项目元信息 / instructions")
-async def update_project(
+def update_project(
body: UpdateProjectBody,
access: ProjectAccess = Depends(require_project_access("edit")),
db: Session = Depends(get_db),
@@ -179,7 +179,7 @@ async def update_project(
@router.delete("/{project_id}", status_code=status.HTTP_204_NO_CONTENT, summary="软删项目")
-async def delete_project(
+def delete_project(
access: ProjectAccess = Depends(require_project_access("admin")),
db: Session = Depends(get_db),
):
@@ -193,7 +193,7 @@ async def delete_project(
@router.post("/{project_id}/favorite", summary="收藏项目")
-async def favorite_project(
+def favorite_project(
access: ProjectAccess = Depends(require_project_access("view")),
db: Session = Depends(get_db),
):
@@ -204,7 +204,7 @@ async def favorite_project(
@router.delete("/{project_id}/favorite", summary="取消收藏")
-async def unfavorite_project(
+def unfavorite_project(
access: ProjectAccess = Depends(require_project_access("view")),
db: Session = Depends(get_db),
):
@@ -218,7 +218,7 @@ async def unfavorite_project(
@router.patch("/{project_id}/instructions", summary="更新项目指令(system prompt 末段注入)")
-async def update_instructions(
+def update_instructions(
body: UpdateInstructionsBody,
access: ProjectAccess = Depends(require_project_access("edit")),
db: Session = Depends(get_db),
@@ -240,7 +240,7 @@ async def update_instructions(
@router.get("/{project_id}/files", summary="项目文件列表(递归挂钩文件夹子树)")
-async def list_project_files(
+def list_project_files(
access: ProjectAccess = Depends(require_project_access("view")),
db: Session = Depends(get_db),
):
@@ -342,7 +342,7 @@ def _local_project_file_path(path: str, access: ProjectAccess) -> str:
@router.post("/{project_id}/files/upload", summary="直传文件到项目(写入挂钩文件夹)")
-async def upload_project_file(
+def upload_project_file(
file: UploadFile = File(...),
access: ProjectAccess = Depends(require_project_access("edit")),
db: Session = Depends(get_db),
@@ -350,7 +350,7 @@ async def upload_project_file(
"""直传文件到项目挂钩文件夹(filename 含路径时自动创建子文件夹);需 edit 权限。"""
if not file.filename:
raise HTTPException(status_code=400, detail="文件名不能为空")
- contents = await file.read()
+ contents = file.file.read()
if not contents:
raise HTTPException(status_code=400, detail="文件内容为空")
pf_svc = ProjectFileService(db)
@@ -366,7 +366,7 @@ async def upload_project_file(
@router.delete("/{project_id}/files/{artifact_id}", summary="软删项目文件(同步 MySpace)")
-async def remove_project_file(
+def remove_project_file(
artifact_id: str,
access: ProjectAccess = Depends(require_project_access("edit")),
db: Session = Depends(get_db),
@@ -400,7 +400,7 @@ async def remove_project_file(
@router.get("/{project_id}/chats", summary="项目内的会话列表")
-async def list_project_chats(
+def list_project_chats(
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=100),
scope: Literal["all", "mine", "shared"] = Query(
diff --git a/src/backend/api/routes/v1/prompt_management.py b/src/backend/api/routes/v1/prompt_management.py
index cb60e554..4be49bdd 100644
--- a/src/backend/api/routes/v1/prompt_management.py
+++ b/src/backend/api/routes/v1/prompt_management.py
@@ -6,6 +6,7 @@
"""
from __future__ import annotations
+from core.llm import prompt_kinds as prompt_kinds_module, prompt_rendering as prompt_rendering_module, prompt_storage as prompt_storage_module, prompt_versions as prompt_versions_module
import logging
from pathlib import Path
@@ -13,9 +14,9 @@
from api.deps import require_system_settings as require_config
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.services import prompt_version_service as pvs
-from core.services.system_config import code_capability_enabled
+from core.config.store.execution_options import code_capability_enabled
from fastapi import APIRouter, Depends, HTTPException
from prompts.prompt_config import load_prompt_config
from prompts.provider import _FILE_CONTENT_CACHE, FilesystemPromptProvider
@@ -76,8 +77,8 @@ def _load_fs_parts() -> Dict[str, Dict[str, Any]]:
def _require_active_system_version(db: Session) -> Dict[str, Any]:
"""Return the active 'system' version, seeding defaults if the pool is empty."""
- pvs.seed_from_filesystem(db=db)
- active = pvs.get_active_version("system", db=db)
+ prompt_versions_module.seed_from_filesystem(db=db)
+ active = prompt_versions_module.get_active_version("system", db=db)
if not active:
raise HTTPException(status_code=500, detail="no active system prompt version configured")
return active
@@ -85,7 +86,7 @@ def _require_active_system_version(db: Session) -> Dict[str, Any]:
def _persist_active_system(version: Dict[str, Any], db: Session) -> Dict[str, Any]:
"""Save updated parts/metadata back into the active system version."""
- saved = pvs.upsert_version(
+ saved = prompt_versions_module.upsert_version(
"system",
version["id"],
name=version.get("name"),
@@ -165,7 +166,14 @@ def _render_tool_schema_appendix(tool_schemas: List[Dict[str, Any]]) -> str:
return "\n".join(lines).strip()
-async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] = None) -> Optional[Dict[str, Any]]:
+def _active_prompt_versions(db: Session):
+ active = prompt_versions_module.get_active_version("system", db=db)
+ code_active = prompt_versions_module.get_active_version("code_exec", db=db)
+ code_seg = prompt_rendering_module.render_kind_segment("code_exec", db) if code_capability_enabled() else ""
+ return active, code_active, code_seg
+
+
+async def _runtime_prompt_preview(*, approval_mode: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""Build a runtime preview by instantiating the main agent.
If MCP/model setup is unavailable, callers fall back to static assembly.
@@ -199,8 +207,7 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] =
+ tool_appendix
)
- active = pvs.get_active_version("system", db=db)
- code_active = pvs.get_active_version("code_exec", db=db)
+ active, code_active, code_seg = await run_in_session(_active_prompt_versions)
enabled_parts = len(
[
p
@@ -208,7 +215,6 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] =
if p.get("is_enabled", True) and (p.get("content") or "").strip()
]
)
- code_seg = pvs.render_kind_segment("code_exec", db) if code_capability_enabled() else ""
part_count = (
enabled_parts
+ (1 if code_seg else 0)
@@ -243,7 +249,7 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] =
finally:
if clients:
try:
- from core.llm.mcp_manager import close_clients
+ from core.llm.mcp.manager import close_clients
await close_clients(clients)
except Exception as exc:
@@ -279,12 +285,12 @@ def list_parts(db: Session = Depends(get_db)):
@router.get("/active", dependencies=[Depends(require_config)], summary="激活版本概览")
def get_active_info(db: Session = Depends(get_db)):
"""返回各 kind 当前激活提示词版本的概要(id、名称、分段数、更新时间)。"""
- pvs.seed_from_filesystem(db=db)
- payload = pvs._load_payload(db) # noqa: SLF001
+ prompt_versions_module.seed_from_filesystem(db=db)
+ payload = prompt_storage_module.load_payload(db) # noqa: SLF001
active = payload.get("active") or {}
result = {}
for kind, vid in active.items():
- v = pvs.get_version(kind, vid, db=db) if vid else None
+ v = prompt_versions_module.get_version(kind, vid, db=db) if vid else None
result[kind] = {
"id": vid,
"name": (v or {}).get("name"),
@@ -463,22 +469,27 @@ def update_order(req: OrderUpdateRequest, db: Session = Depends(get_db)):
@router.post("/preview", dependencies=[Depends(require_config)], summary="预览运行时提示词")
-async def preview_prompt(db: Session = Depends(get_db)):
+async def preview_prompt():
"""基于当前 DB + 文件系统状态预览运行时指令面。
优先实例化主 agent,返回真实 ``sys_prompt``、AgentScope 自动追加的技能
prompt,以及工具 schema 描述的审计展开;若运行时装配失败,再退回静态
DB + 文件系统拼接。
"""
- runtime = await _runtime_prompt_preview(db)
+ runtime = await _runtime_prompt_preview()
if runtime:
return success_response(data=runtime)
+ return success_response(data=await run_in_session(_static_prompt_preview))
+
+
+def _static_prompt_preview(db: Session) -> Dict[str, Any]:
+ """The prompt assembled from the DB + filesystem, when no agent can be built."""
def _finalize(prompt: str, part_count: int, base: Dict[str, Any]):
seg = ""
if code_capability_enabled():
try:
- seg = pvs.render_kind_segment("code_exec", db) or ""
+ seg = prompt_rendering_module.render_kind_segment("code_exec", db) or ""
except Exception:
seg = ""
appended = bool(seg)
@@ -506,10 +517,10 @@ def _finalize(prompt: str, part_count: int, base: Dict[str, Any]):
"preview_mode": "static_fallback",
}
)
- return success_response(data=base)
+ return base
# Prefer the active version from the prompt_version pool
- active = pvs.get_active_version("system", db=db)
+ active = prompt_versions_module.get_active_version("system", db=db)
if active and active.get("parts"):
chunks: List[str] = []
for p in active["parts"]:
@@ -565,15 +576,15 @@ class KindCreateRequest(BaseModel):
@router.get("/kinds", dependencies=[Depends(require_config)], summary="提示词 kind(tab)清单")
def list_prompt_kinds(db: Session = Depends(get_db)):
"""内置 + 管理员自建的 kind。「模式选择」页绑定提示词时也读这份清单。"""
- pvs.seed_from_filesystem(db=db)
- return success_response(data={"kinds": pvs.all_kinds(db)})
+ prompt_versions_module.seed_from_filesystem(db=db)
+ return success_response(data={"kinds": prompt_kinds_module.all_kinds(db)})
@router.post("/kinds", dependencies=[Depends(require_config)], summary="新建提示词 tab")
def create_prompt_kind(req: KindCreateRequest, db: Session = Depends(get_db)):
"""新开一个 tab,并给它建好可直接编辑的空 default 版本。"""
try:
- created = pvs.create_custom_kind(req.key, req.label, db=db)
+ created = prompt_kinds_module.create_custom_kind(req.key, req.label, db=db)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return success_response(data=created)
@@ -586,7 +597,7 @@ def delete_prompt_kind(key: str, db: Session = Depends(get_db)):
绑了它的对话模式会退回默认提示词装配,不会让对话起不来。
"""
try:
- ok = pvs.delete_custom_kind(key, db=db)
+ ok = prompt_kinds_module.delete_custom_kind(key, db=db)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
if not ok:
@@ -598,9 +609,9 @@ def delete_prompt_kind(key: str, db: Session = Depends(get_db)):
def list_pool_versions(kind: Optional[str] = None, db: Session = Depends(get_db)):
"""列出提示词版本池中的所有版本,可按 kind 过滤;并返回各 kind 的激活映射。"""
# Auto-seed defaults on first read so the pool is never empty after boot
- pvs.seed_from_filesystem(db=db)
- items = pvs.list_versions(kind=kind, db=db)
- payload = pvs._load_payload(db) # noqa: SLF001 — access for active map
+ prompt_versions_module.seed_from_filesystem(db=db)
+ items = prompt_versions_module.list_versions(kind=kind, db=db)
+ payload = prompt_storage_module.load_payload(db) # noqa: SLF001 — access for active map
return success_response(
data={
"versions": items,
@@ -616,7 +627,7 @@ def list_pool_versions(kind: Optional[str] = None, db: Session = Depends(get_db)
)
def get_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)):
"""获取版本池中指定 kind/version_id 版本的完整内容。"""
- v = pvs.get_version(kind, version_id, db=db)
+ v = prompt_versions_module.get_version(kind, version_id, db=db)
if v is None:
raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}")
return success_response(data=v)
@@ -628,11 +639,11 @@ def create_pool_version(req: VersionUpsertRequest, db: Session = Depends(get_db)
if not req.id:
raise HTTPException(status_code=400, detail="id is required when creating a version")
# Reject duplicates explicitly
- if pvs.get_version(req.kind, req.id, db=db) is not None:
+ if prompt_versions_module.get_version(req.kind, req.id, db=db) is not None:
raise HTTPException(status_code=409, detail=f"version already exists: {req.kind}/{req.id}")
parts = [p.model_dump() for p in (req.parts or [])] if req.parts is not None else None
try:
- saved = pvs.upsert_version(
+ saved = prompt_versions_module.upsert_version(
req.kind,
req.id,
name=req.name,
@@ -659,10 +670,10 @@ def update_pool_version(
):
"""更新版本池中指定版本的元数据与分段;路径中的 kind/version_id 为准,更新后失效提示词缓存。"""
# kind/version_id in path are authoritative; body fields override metadata
- if pvs.get_version(kind, version_id, db=db) is None:
+ if prompt_versions_module.get_version(kind, version_id, db=db) is None:
raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}")
parts = [p.model_dump() for p in (req.parts or [])] if req.parts is not None else None
- saved = pvs.upsert_version(
+ saved = prompt_versions_module.upsert_version(
kind,
version_id,
name=req.name,
@@ -688,7 +699,7 @@ def update_pool_version(
def delete_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)):
"""从版本池中删除指定版本(激活中的版本不可删除)。"""
try:
- pvs.delete_version(kind, version_id, db=db)
+ prompt_versions_module.delete_version(kind, version_id, db=db)
except KeyError:
raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}")
except ValueError as exc:
@@ -704,7 +715,7 @@ def delete_pool_version(kind: str, version_id: str, db: Session = Depends(get_db
def activate_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)):
"""将指定版本设为该 kind 的激活版本,并失效提示词缓存使其立即生效。"""
try:
- pvs.activate_version(kind, version_id, db=db)
+ prompt_versions_module.activate_version(kind, version_id, db=db)
except KeyError:
raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}")
except ValueError as exc:
@@ -717,7 +728,7 @@ def activate_pool_version(kind: str, version_id: str, db: Session = Depends(get_
@router.post("/versions/seed", dependencies=[Depends(require_config)], summary="初始化版本池")
def seed_pool(db: Session = Depends(get_db)):
"""从文件系统默认提示词初始化版本池。"""
- result = pvs.seed_from_filesystem(db=db)
+ result = prompt_versions_module.seed_from_filesystem(db=db)
_invalidate_prompt_cache()
return success_response(data=result)
@@ -728,13 +739,15 @@ class DesktopPreviewRequest(BaseModel):
@router.post("/desktop-preview")
-async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_config), db: Session = Depends(get_db)):
+async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_config)):
"""Render a saved desktop version without activating it or launching an agent."""
from core.config.local_mode import local_mode_enabled
from prompts.desktop_templates import desktop_version, render_desktop_part
from prompts.desktop_workspace import SKIP_PARTS
from prompts.prompt_runtime import build_system_prompt
- version = pvs.get_version("desktop", request.version_id, db=db)
+ version = await run_in_session(
+ lambda db: prompt_versions_module.get_version("desktop", request.version_id, db=db)
+ )
if version is None:
raise HTTPException(404, "desktop prompt version not found")
if request.parts is not None:
@@ -754,13 +767,15 @@ async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_conf
)) or " "
prompt = build_system_prompt(load_prompt_config(), ctx, desktop_preview=True)
if code_capability_enabled():
- parts = pvs.effective_parts("code_exec", db=db)
+ parts = await run_in_session(
+ lambda db: prompt_rendering_module.effective_parts("code_exec", db=db)
+ )
prompt += "\n\n" + "\n\n".join(p["content"] for p in sorted(parts, key=lambda p: p.get("sort_order", 0))
if p.get("is_enabled", True) and p.get("part_id") not in SKIP_PARTS["code_exec"])
from core.sandbox.workspace.paths import WORKSPACE
from core.sandbox.workspace.desktop import workspace_directory
tool = render_desktop_part("bash_tool", cwd=workspace_directory(WORKSPACE, ctx["chat_id"]) if local else "{runtime.cwd}")
- runtime = await _runtime_prompt_preview(db, approval_mode="ask") if local else None
+ runtime = await _runtime_prompt_preview(approval_mode="ask") if local else None
if runtime:
prompt = runtime["prompt"]
conditional = {p["part_id"]: p.get("content", "") for p in version.get("parts", [])
@@ -772,17 +787,17 @@ async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_conf
@router.get("/snapshot", dependencies=[Depends(require_config)])
def export_snapshot(db: Session = Depends(get_db)):
- from core.content.content_blocks import build_prompt_snapshot
+ from core.files.content.content_blocks import build_prompt_snapshot
return success_response(build_prompt_snapshot(db))
@router.post("/snapshot", dependencies=[Depends(require_config)])
def import_snapshot(body: Dict[str, Any], db: Session = Depends(get_db)):
- from core.content.content_blocks import import_prompt_snapshot, ContentSnapshotError
+ from core.files.content.content_blocks import import_prompt_snapshot, ContentSnapshotError
try:
result = import_prompt_snapshot(db, body, overwrite=True, default_updated_by="prompt_management")
except ContentSnapshotError as exc:
raise HTTPException(400, str(exc)) from exc
- pvs.invalidate_cache()
+ prompt_storage_module.invalidate_cache()
_invalidate_prompt_cache()
return success_response(result)
diff --git a/src/backend/api/routes/v1/public_chats.py b/src/backend/api/routes/v1/public_chats.py
index f9d82f1e..9eb8662f 100644
--- a/src/backend/api/routes/v1/public_chats.py
+++ b/src/backend/api/routes/v1/public_chats.py
@@ -3,9 +3,10 @@
from core.auth.backend import UserContext, get_current_user
from core.auth.public_chat import public_session_matches
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.db.models import ChatSession, Project, UserAgent
from core.infra.responses import success_response
-from core.services import public_chat_service as service
+from core.chat import public as service
from fastapi import APIRouter, Depends, HTTPException, Response
from pydantic import BaseModel
from sqlalchemy.orm import Session
@@ -48,16 +49,22 @@ async def toggle_link(
body: LinkState,
response: Response,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
+ from core.db.models import ContentBlock
+
+ def toggle(db):
+ data = service.set_enabled(db, user.user_id, kind, resource_id, body.enabled)
+ if body.enabled:
+ return data, None
+ row = db.get(ContentBlock, service.link_id(user.user_id, kind, resource_id))
+ return data, row.payload["key_id"]
+
response.headers["Cache-Control"] = "no-store"
- data = service.set_enabled(db, user.user_id, kind, resource_id, body.enabled)
+ data, key_id = await run_in_session(toggle)
if not body.enabled:
- from core.db.models import ContentBlock
- from core.services.api_key_runs import stop_cancelled_key_workers
+ from core.auth.api_key_runs import stop_cancelled_key_workers
- row = db.get(ContentBlock, service.link_id(user.user_id, kind, resource_id))
- await stop_cancelled_key_workers(db, row.payload["key_id"], user.user_id)
+ await stop_cancelled_key_workers(key_id, user.user_id)
return success_response(data=data)
@@ -75,7 +82,7 @@ def bootstrap(
)
if not resource:
raise HTTPException(404, detail="Resource unavailable")
- from core.services.session_operations.sessions import _session_to_dict
+ from core.chat.operations.sessions import session_to_dict as _session_to_dict
scope_name = "agent_api_scope" if user.api_key_agent_id else "project_api_scope"
# JSON extraction is supported by both SQLite and Postgres. Bound the result,
diff --git a/src/backend/api/routes/v1/space_files.py b/src/backend/api/routes/v1/space_files.py
index a31e9c84..3fb69322 100644
--- a/src/backend/api/routes/v1/space_files.py
+++ b/src/backend/api/routes/v1/space_files.py
@@ -10,10 +10,11 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.spaces.access import resolve_grants
-from core.spaces.mounts import mount_revision
-from core.spaces.files import MAX_TRANSFER_BYTES, SpaceFiles
-from core.storage import get_storage
+from core.files.spaces.access import resolve_grants
+from core.files.spaces.mounts import mount_revision
+from core.files.spaces.files import MAX_TRANSFER_BYTES, SpaceFiles
+from core.files.storage import get_storage
+from core.infra.byte_range import byte_range_response
router = APIRouter(prefix="/v1/myspace", tags=["Space Files"])
@@ -43,11 +44,14 @@ def entries(space: str, path: str = "", cursor: str | None = None,
return success_response(data=files.list(space, path, cursor=cursor, limit=limit, keyword=keyword))
@router.get("/content")
-def download(space: str, path: str, inline: bool = False, files: SpaceFiles = Depends(service)):
+def download(request: Request, space: str, path: str, inline: bool = False,
+ files: SpaceFiles = Depends(service)):
data, info = files.read(space, path)
mime = mimetypes.guess_type(info.name)[0] or "application/octet-stream"
safe_inline = inline and (mime.startswith(("image/", "audio/", "video/", "text/")) or mime == "application/pdf")
- return Response(content=data, media_type=mime if safe_inline else "application/octet-stream",
+ # Range lets the canvas audio/video player seek without refetching the whole file.
+ return byte_range_response(data, request.headers.get("range"),
+ media_type=mime if safe_inline else "application/octet-stream",
headers={"Content-Disposition": ("inline" if safe_inline else "attachment") + "; filename*=UTF-8''" + quote(info.name, safe=""),
"Content-Security-Policy": "sandbox",
"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"})
@@ -80,7 +84,7 @@ def delete(space: str, path: str, directory: bool = False, files: SpaceFiles = D
@router.post("/references")
def reference(body: PathBody, files: SpaceFiles = Depends(service)):
- from core.artifacts.store import save_artifact_bytes
+ from core.files.artifacts.store import save_artifact_bytes
data, info = files.read(body.space, body.path)
mime = mimetypes.guess_type(info.name)[0] or "application/octet-stream"
extension = info.name.rpartition(".")[2] if "." in info.name else ""
@@ -95,12 +99,12 @@ def reference(body: PathBody, files: SpaceFiles = Depends(service)):
@router.get("/text")
def read_text(space: str, path: str, response: Response, files: SpaceFiles = Depends(service)):
- from core.spaces.text import SpaceText
+ from core.files.spaces.text import SpaceText
response.headers["Cache-Control"] = "no-store"
return success_response(data=SpaceText(files).read(space, path))
@router.put("/text")
def save_text(body: TextSaveRequest, space: str, path: str, files: SpaceFiles = Depends(service)):
- from core.spaces.text import SpaceText
+ from core.files.spaces.text import SpaceText
return success_response(data=SpaceText(files).save(space, path, body.content, body.revision))
diff --git a/src/backend/api/routes/v1/space_previews.py b/src/backend/api/routes/v1/space_previews.py
index af9b673e..f4750be5 100644
--- a/src/backend/api/routes/v1/space_previews.py
+++ b/src/backend/api/routes/v1/space_previews.py
@@ -6,10 +6,11 @@
import tempfile
from urllib.parse import quote
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Response
-from core.content.archive_browser import ArchiveError, inspect_archive, read_archive_member
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, Response
+from core.files.content.archive_browser import ArchiveError, inspect_archive, read_archive_member
+from core.infra.byte_range import byte_range_response
from core.infra.responses import success_response
-from core.spaces.files import SpaceFiles
+from core.files.spaces.files import SpaceFiles
from api.routes.v1.space_files import service
router = APIRouter(prefix="/v1/myspace/content")
@@ -58,7 +59,7 @@ def archive(space: str, path: str, response: Response, files: SpaceFiles = Depen
@router.get("/archive/member")
-def member(space: str, path: str, member: str, inline: bool = False,
+def member(request: Request, space: str, path: str, member: str, inline: bool = False,
files: SpaceFiles = Depends(service)):
with source(files, space, path) as (local, name):
data = read_archive_member(local, name, member)
@@ -67,7 +68,7 @@ def member(space: str, path: str, member: str, inline: bool = False,
if mime in ("text/html", "image/svg+xml"):
mime = "text/plain"
safe = inline and (mime.startswith(("image/", "audio/", "video/", "text/")) or mime == "application/pdf")
- return Response(data, media_type=mime, headers={
+ return byte_range_response(data, request.headers.get("range"), media_type=mime, headers={
"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff",
"Content-Security-Policy": "sandbox; default-src 'none'",
"Content-Disposition": ("inline" if safe else "attachment") + "; filename*=UTF-8''" + quote(filename, safe=""),
diff --git a/src/backend/api/routes/v1/summary.py b/src/backend/api/routes/v1/summary.py
index 238626d8..c898ffd9 100644
--- a/src/backend/api/routes/v1/summary.py
+++ b/src/backend/api/routes/v1/summary.py
@@ -6,7 +6,7 @@
from core.auth.backend import get_current_user, UserContext
from core.infra.responses import success_response
-from core.llm.summarizer import get_summarizer
+from core.models.completions.summarizer import get_summarizer
from core.infra.exceptions import BadRequestError
router = APIRouter(prefix="/v1/summary", tags=["Summary"])
diff --git a/src/backend/api/routes/v1/tool_approval.py b/src/backend/api/routes/v1/tool_approval.py
index 35ca66ea..4d6c15d0 100644
--- a/src/backend/api/routes/v1/tool_approval.py
+++ b/src/backend/api/routes/v1/tool_approval.py
@@ -5,7 +5,7 @@
``core.llm.tool_permissions`` 决定要不要为工具调用停下来问用户。
网页端与桌面端共用这一档——桌面端不再另存一份「本机操作权限档」,本机执行策略由
-``core.services.local_grant_service.policy_for_gate()`` 按本档位翻译。桌面端仍有的
+``features.desktop.local.grants.policy_for_gate()`` 按本档位翻译。桌面端仍有的
「授权目录 / 危险命令分类处置」(``/v1/local/grants``、``/v1/local/policy``)管的是
"本机哪些目录能动",与本档位是两件事。
"""
@@ -17,8 +17,8 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.llm.tool_permissions import normalize_approval_mode
-from core.services.user_service import UserService
+from core.sandbox.permissions.approval import normalize_approval_mode
+from core.auth.users import UserService
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy.orm import Session
@@ -39,7 +39,7 @@ def get_tool_approval_mode(
):
settings = UserService(db).get_user_settings(str(user.user_id))
if user.public_chat_visitor:
- from core.services.public_chat_service import visitor_approval_mode
+ from core.chat.public import visitor_approval_mode
mode = visitor_approval_mode(db, user, settings.get(_METADATA_KEY))
else:
mode = normalize_approval_mode(settings.get(_METADATA_KEY))
@@ -54,7 +54,7 @@ def set_tool_approval_mode(
):
mode = normalize_approval_mode(body.mode)
if user.public_chat_visitor:
- from core.services.public_chat_service import set_visitor_approval_mode
+ from core.chat.public import set_visitor_approval_mode
set_visitor_approval_mode(db, user, mode)
else:
UserService(db).update_user_metadata(user_id=str(user.user_id), patch={_METADATA_KEY: mode})
diff --git a/src/backend/api/routes/v1/users.py b/src/backend/api/routes/v1/users.py
index 2eea8dea..9947ba99 100644
--- a/src/backend/api/routes/v1/users.py
+++ b/src/backend/api/routes/v1/users.py
@@ -13,7 +13,7 @@
from core.db.repository import UserRepository
from core.infra.exceptions import AccessDeniedError, ResourceNotFoundError, ValidationError
from core.infra.responses import success_response
-from core.storage import get_storage
+from core.files.storage import get_storage
from fastapi import APIRouter, Depends, File, HTTPException, Path, UploadFile
from fastapi.responses import Response
from pydantic import BaseModel, Field
@@ -77,7 +77,7 @@ def get_current_user_info(
local = db.query(LocalUser).filter(LocalUser.user_id == user.user_id).first()
meta = dict(user_shadow.extra_data or {})
- from core.services.local_user_service import ce_onboarding_required
+ from core.auth.local_user_policy import ce_onboarding_required
data = {
"user_id": user_shadow.user_id,
@@ -96,7 +96,7 @@ def get_current_user_info(
"created_at": user_shadow.created_at.isoformat(),
}
- from core.auth.account_view import extend_current_account
+ from core.auth.edition import extend_current_account
data = extend_current_account(db, str(user.user_id), data)
@@ -125,7 +125,7 @@ def update_current_user_info(
"""更新当前用户资料(nickname / real_name / phone,仅传的字段才更新)。仅本地账号可改,外部 SSO 用户需到身份源修改。"""
from core.db.models import LocalUser
from core.infra.exceptions import ValidationError
- from core.services.local_user_service import LocalUserService
+ from core.auth.local_users import LocalUserService
local = db.query(LocalUser).filter(LocalUser.user_id == user.user_id).first()
if not local:
@@ -156,7 +156,7 @@ def change_my_password(
db: Session = Depends(get_db),
):
"""Verify the current password, save the replacement, and clear the first-login flag."""
- from core.services.local_user_service import LocalUserService
+ from core.auth.local_users import LocalUserService
result = LocalUserService(db).change_password(
user_id=user.user_id,
@@ -186,7 +186,7 @@ def complete_my_onboarding(
final, idempotent checkpoint that unlocks the normal application shell.
"""
from core.db.models import ModelProvider, ModelRoleAssignment
- from core.services.local_user_service import CE_ONBOARDING_VERSION
+ from core.auth.local_users_constants import CE_ONBOARDING_VERSION
if settings.edition.edition != "ce":
raise AccessDeniedError(
@@ -282,7 +282,7 @@ def _persist_avatar_url(
@router.post("/me/avatar", summary="上传自定义头像(multipart)")
-async def upload_my_avatar(
+def upload_my_avatar(
file: UploadFile = File(..., description="头像图片,≤2MB,支持 png/jpg/webp/gif"),
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
@@ -296,7 +296,7 @@ async def upload_my_avatar(
message="头像仅支持 PNG / JPG / WEBP / GIF",
)
- file_bytes = await file.read()
+ file_bytes = file.file.read()
if not file_bytes:
raise ValidationError(
errors=[{"field": "file", "message": "文件内容为空"}],
diff --git a/src/backend/api/startup/bootstrap.py b/src/backend/api/startup/bootstrap.py
index 886ae95a..7d3ad9ae 100644
--- a/src/backend/api/startup/bootstrap.py
+++ b/src/backend/api/startup/bootstrap.py
@@ -1,10 +1,14 @@
"""Application lifecycle operations."""
+from core.llm import prompt_versions as prompt_versions_module
import os
import sys
from core.config.settings import settings
from core.infra.logging import get_logger
logger = get_logger(__name__)
+
+from core.db.threads import run_in_session
+
async def _startup_ensure_tables():
"""Ensure database tables exist for SQLite environments.
@@ -26,16 +30,11 @@ async def _startup_seed_ce_admin():
if settings.edition.edition != "ce":
return
try:
- from core.db.engine import SessionLocal
- from core.services.local_user_service import ensure_ce_default_admin
-
- db = SessionLocal()
- try:
- user_id, created = ensure_ce_default_admin(db)
- if created:
- logger.info("[startup] CE default administrator created: %s", user_id)
- finally:
- db.close()
+ from core.auth.local_user_bootstrap import ensure_ce_default_admin
+
+ user_id, created = await run_in_session(ensure_ce_default_admin)
+ if created:
+ logger.info("[startup] CE default administrator created: %s", user_id)
except Exception as exc:
logger.warning("[startup] CE default administrator seed failed: %s", exc)
@@ -49,7 +48,7 @@ async def _startup_prepare_artifact_records():
try:
import asyncio
- from core.artifacts.store import prepare_records
+ from core.files.artifacts.store import prepare_records
written = await asyncio.to_thread(prepare_records)
if written:
@@ -60,18 +59,15 @@ async def _startup_prepare_artifact_records():
async def _startup_seed_page_config():
"""Seed default page_config content block if missing (idempotent)."""
try:
- from core.content.content_blocks import enforce_ce_branding, seed_page_config_if_missing
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- inserted = seed_page_config_if_missing(db)
- if inserted:
- logger.info("[startup] default page_config seeded")
- if enforce_ce_branding(db):
- logger.info("[startup] CE page config normalized")
- finally:
- db.close()
+ from core.files.content.branding import enforce_ce_branding, seed_page_config_if_missing
+
+ inserted, normalized = await run_in_session(
+ lambda db: (seed_page_config_if_missing(db), enforce_ce_branding(db))
+ )
+ if inserted:
+ logger.info("[startup] default page_config seeded")
+ if normalized:
+ logger.info("[startup] CE page config normalized")
except Exception as exc:
logger.warning("[startup] page_config seed failed: %s", exc)
@@ -81,17 +77,10 @@ async def _startup_seed_prompt_versions():
Idempotent: only inserts versions that don't already exist.
"""
try:
- from core.db.engine import SessionLocal
- from core.services import prompt_version_service as pvs
-
- db = SessionLocal()
- try:
- result = pvs.seed_from_filesystem(db=db)
- added = result.get("added") or []
- if added:
- logger.info("[startup] prompt versions seeded: %s", ", ".join(added))
- finally:
- db.close()
+ result = await run_in_session(lambda db: prompt_versions_module.seed_from_filesystem(db=db))
+ added = result.get("added") or []
+ if added:
+ logger.info("[startup] prompt versions seeded: %s", ", ".join(added))
except Exception as exc:
logger.warning("[startup] prompt versions seed failed: %s", exc)
@@ -102,15 +91,10 @@ async def _startup_seed_roles():
populates fresh deployments. When CE has no roles table, degrades entirely to a no-op.
"""
try:
- from core.db.engine import SessionLocal
- from core.services.edition_startup import seed_default_roles
-
- db = SessionLocal()
- try:
- added = seed_default_roles(db)
- if added:
- logger.info("[startup] default roles seeded: %s", ", ".join(added))
- finally:
- db.close()
+ from orchestration.workers.edition_startup import seed_default_roles
+
+ added = await run_in_session(seed_default_roles)
+ if added:
+ logger.info("[startup] default roles seeded: %s", ", ".join(added))
except Exception as exc:
logger.warning("[startup] roles seed failed: %s", exc)
diff --git a/src/backend/api/startup/commands.py b/src/backend/api/startup/commands.py
index c15db35d..5602925b 100644
--- a/src/backend/api/startup/commands.py
+++ b/src/backend/api/startup/commands.py
@@ -1,9 +1,10 @@
"""Background command monitoring and durable notification dispatch."""
import asyncio
import logging
+
from core.sandbox import get_sandbox_provider
-from orchestration.commands.runtime import runtime
-from orchestration.commands.notifications import dispatch
+from core.sandbox.commands.notifications import dispatch
+from core.sandbox.commands.runtime import runtime
logger = logging.getLogger(__name__)
_task = None
@@ -20,6 +21,9 @@ async def _run():
async def startup_commands():
global _task
runtime.bind_loop()
+ bind = getattr(get_sandbox_provider(), "bind_loop", None)
+ if bind is not None:
+ bind()
_task = asyncio.create_task(_run(), name="background-command-reconciliation")
async def shutdown_commands():
diff --git a/src/backend/api/startup/memory.py b/src/backend/api/startup/memory.py
index db0c65d7..35487dad 100644
--- a/src/backend/api/startup/memory.py
+++ b/src/backend/api/startup/memory.py
@@ -1,8 +1,10 @@
"""Application lifecycle operations."""
import os
import sys
+
from core.config.settings import settings
from core.infra.logging import get_logger
+
logger = get_logger(__name__)
_persona_distill_worker = None
@@ -12,7 +14,7 @@
async def _startup_evolution_scheduler():
"""Start the nightly evolution-cycle backstop."""
try:
- from orchestration.schedulers.evolution_scheduler import EvolutionCronScheduler
+ from orchestration.workers.evolution_scheduler import EvolutionCronScheduler
await EvolutionCronScheduler().start()
except Exception as exc:
@@ -24,7 +26,7 @@ async def _startup_memory_ttl_scheduler():
if not settings.memory.enabled:
return
try:
- from orchestration.schedulers.memory_ttl_scheduler import MemoryTtlScheduler
+ from orchestration.workers.memory_ttl_scheduler import MemoryTtlScheduler
await MemoryTtlScheduler().start()
except Exception as exc:
@@ -66,7 +68,7 @@ async def _startup_persona_distill_worker():
should not depend on which worker took the HTTP request.
"""
try:
- from orchestration.schedulers.persona_distill_worker import PersonaDistillWorker
+ from orchestration.workers.persona_distill_worker import PersonaDistillWorker
global _persona_distill_worker
_persona_distill_worker = PersonaDistillWorker()
@@ -91,7 +93,7 @@ async def _startup_warmup_memory():
async def _warmup() -> None:
try:
- from core.memory.service import _get_memory # type: ignore[attr-defined]
+ from core.memory.service import get_memory as _get_memory # type: ignore[attr-defined]
# _get_memory is thread-safe lazy loading; the first call establishes the Milvus + Neo4j connections
instance = _get_memory()
diff --git a/src/backend/api/startup/plugins.py b/src/backend/api/startup/plugins.py
index 276d5a90..7e324e3b 100644
--- a/src/backend/api/startup/plugins.py
+++ b/src/backend/api/startup/plugins.py
@@ -5,7 +5,9 @@
from core.infra.logging import get_logger
logger = get_logger(__name__)
-async def _startup_seed_mcp_servers():
+from core.db.threads import run_db
+
+def _seed_mcp_servers():
"""Seed the built-in global MCP catalog if the DB has none (idempotent).
On compose these rows come from alembic seed migrations; on the no-Docker
@@ -16,12 +18,9 @@ async def _startup_seed_mcp_servers():
"""
try:
from core.db.engine import SessionLocal
- from core.services.mcp_management_service import encrypt_legacy_mcp_headers
- from core.services.mcp_marketplace_service import ensure_curated_market_items
- from core.services.mcp_service import (
- prune_removed_builtin_mcp_servers,
- seed_builtin_mcp_servers_if_empty,
- )
+ from core.capabilities.mcp.management import encrypt_legacy_mcp_headers
+ from features.marketplace.mcp_curated import ensure_curated_market_items
+ from core.capabilities.mcp.service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty
db = SessionLocal()
try:
@@ -42,10 +41,14 @@ async def _startup_seed_mcp_servers():
except Exception as exc:
logger.warning("[startup] MCP catalog seed failed: %s", exc)
+
+async def _startup_seed_mcp_servers():
+ await run_db(_seed_mcp_servers)
+
async def _startup_mcp_market_monitor():
"""Start periodic remote-tool drift detection after MCP rows are available."""
try:
- from core.services.mcp_marketplace_monitor import start_monitor
+ from features.marketplace.mcp_monitor import start_monitor
start_monitor()
except Exception as exc:
@@ -53,7 +56,7 @@ async def _startup_mcp_market_monitor():
async def _shutdown_mcp_market_monitor():
try:
- from core.services.mcp_marketplace_monitor import stop_monitor
+ from features.marketplace.mcp_monitor import stop_monitor
await stop_monitor()
except Exception as exc:
@@ -64,7 +67,7 @@ async def _startup_upgrade_sites_plugin():
import asyncio
from core.db.engine import SessionLocal
- from core.plugins.local.site_upgrade import upgrade_builtin_sites
+ from features.sites.builtin_upgrade import upgrade_builtin_sites
def upgrade():
with SessionLocal() as db:
@@ -88,7 +91,7 @@ async def _startup_plugin_device_assets():
return
from core.db.engine import SessionLocal
- from core.plugins.local.assets import provision_present_plugins
+ from features.desktop.local.plugins.assets import provision_present_plugins
def provision():
with SessionLocal() as db:
@@ -100,7 +103,7 @@ def provision():
async def _startup_backfill_tool_schemas():
"""Capture real tool schemas for servers a plugin manifest only named."""
- from core.services.mcp_tool_schema_backfill import backfill_missing_tool_schemas
+ from core.capabilities.mcp.tool_schema_backfill import backfill_missing_tool_schemas
count = await backfill_missing_tool_schemas()
if count:
@@ -108,13 +111,13 @@ async def _startup_backfill_tool_schemas():
async def _startup_backfill_api_protocol():
"""Record which OpenAI wire protocol each pre-existing model endpoint speaks."""
- from core.services.api_protocol_backfill import backfill_missing_api_protocol
+ from core.models.api_protocol_backfill import backfill_missing_api_protocol
count = await backfill_missing_api_protocol()
if count:
logger.info("[startup] api_protocol recorded for %d model provider(s)", count)
-async def _startup_seed_default_plugins():
+def _seed_default_plugins():
"""Install edition-specific default plugins once per persistent database.
Local/desktop installs run the equivalent bootstrap in ``cli.py`` before
@@ -127,7 +130,7 @@ async def _startup_seed_default_plugins():
if edition not in {"ce", "ee"}:
return
if settings.deploy.is_local:
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
# Hybrid capabilities come from the authenticated cloud account. Local
# defaults also respect the installer's explicit onboarding opt-in.
@@ -135,10 +138,10 @@ async def _startup_seed_default_plugins():
return
from core.db.engine import SessionLocal
- from core.services.edition_startup import bootstrap_edition_plugins
- from core.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, ensure_default_plugins_bootstrapped
- from core.plugins.ui.contributions import refresh_builtin_ui_contributions
- from core.plugins.management.defaults import ensure_visualize_plugin_bootstrapped
+ from orchestration.workers.edition_startup import bootstrap_edition_plugins
+ from core.capabilities.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, ensure_default_plugins_bootstrapped
+ from features.workbench.ui.contributions import refresh_builtin_ui_contributions
+ from core.capabilities.plugins.management.defaults import ensure_visualize_plugin_bootstrapped
db = SessionLocal()
try:
@@ -165,3 +168,7 @@ async def _startup_seed_default_plugins():
raise
finally:
db.close()
+
+
+async def _startup_seed_default_plugins():
+ await run_db(_seed_default_plugins)
diff --git a/src/backend/api/startup/runtime.py b/src/backend/api/startup/runtime.py
index db2fac7f..82cdae46 100644
--- a/src/backend/api/startup/runtime.py
+++ b/src/backend/api/startup/runtime.py
@@ -1,13 +1,23 @@
"""Application lifecycle operations."""
-import os
-import sys
+
+import core.chat.runtime.recovery as recovery
+import orchestration.loops.recovery as loops_recovery
+import orchestration.workers.run_recovery as _run_recovery
from core.config.settings import settings
from core.infra.logging import get_logger
+
logger = get_logger(__name__)
+async def _startup_thread_budget():
+ """Size this worker's thread pools to the connection pool before serving."""
+ from core.db.threads import apply_thread_budget
+
+ apply_thread_budget()
+
+
async def _startup_watch_capability_changes():
"""智能体 / 技能 / 连接器 / 插件被增删时递增能力变更号,桌面端据此同步一次。"""
- from core.capabilities.change_signal import watch_capability_tables
+ from features.desktop.capabilities.change_signal import watch_capability_tables
from core.db.engine import engine
watch_capability_tables(engine)
@@ -15,9 +25,7 @@ async def _startup_watch_capability_changes():
async def _startup_recover_chat_runs():
"""Resume claimable chat_runs; valid leases are retried by the recovery loop."""
try:
- from orchestration import chat_run_executor
-
- count = await chat_run_executor.recover_orphan_runs()
+ count = await recovery.recover_orphan_runs()
if count:
logger.info("[startup] chat_runs orphan recovered: %d", count)
except Exception as exc:
@@ -26,9 +34,7 @@ async def _startup_recover_chat_runs():
async def _startup_resume_loops():
"""Resume interrupted autonomous loops (M4, off by default; enable with LOOP_AUTO_RESUME=true)."""
try:
- from orchestration import chat_run_executor
-
- count = await chat_run_executor.resume_running_loops()
+ count = await loops_recovery.resume_running_loops()
if count:
logger.info("[startup] autonomous loops resumed: %d", count)
except Exception as exc:
@@ -41,7 +47,7 @@ async def _startup_recover_jobs():
可直接断点续跑,已完成的项不会重做。
"""
try:
- from orchestration import job_runtime
+ import core.jobs.runtime as job_runtime
count = await job_runtime.resume_running_jobs()
if count:
@@ -55,7 +61,7 @@ async def _startup_orphan_job_reaper():
try:
import asyncio
- from orchestration import job_runtime
+ import core.jobs.runtime as job_runtime
task = asyncio.create_task(job_runtime.run_job_reaper_loop())
app.state.orphan_job_reaper_task = task
@@ -69,19 +75,20 @@ async def _startup_stale_run_reaper():
try:
import asyncio
- from orchestration import chat_run_executor
+ pass
- task = asyncio.create_task(chat_run_executor.run_stale_reaper_loop())
+ task = asyncio.create_task(_run_recovery.run_stale_reaper_loop())
app.state.stale_run_reaper_task = task
logger.info("[startup] chat_run stale reaper started")
except Exception as exc:
logger.warning("[startup] chat_run stale reaper start failed: %s", exc)
async def _shutdown_stale_run_reaper():
- from api.app import app
import asyncio
import contextlib
+ from api.app import app
+
task = getattr(app.state, "stale_run_reaper_task", None)
if task is not None:
task.cancel()
@@ -89,10 +96,11 @@ async def _shutdown_stale_run_reaper():
await task
async def _shutdown_orphan_job_reaper():
- from api.app import app
import asyncio
import contextlib
+ from api.app import app
+
task = getattr(app.state, "orphan_job_reaper_task", None)
if task is not None:
task.cancel()
@@ -101,7 +109,7 @@ async def _shutdown_orphan_job_reaper():
async def _shutdown_channel_manager():
try:
- from core.channels.manager import get_manager
+ from integrations.channels.manager import get_manager
get_manager().shutdown()
except Exception: # noqa: BLE001
@@ -123,6 +131,7 @@ async def _startup_warm_sandbox_pool():
logger.info("[startup] sandbox provider %s warmup kicked off", provider.name)
except Exception as exc: # noqa: BLE001
logger.warning("[startup] sandbox warmup failed: %s", exc)
+ raise
async def _startup_local_sidecars():
"""Spawn and verify MCP + script_runner sidecars in local profile.
@@ -133,7 +142,7 @@ async def _startup_local_sidecars():
startup behavior.
"""
try:
- from orchestration.local_subprocess import start_local_sidecars
+ from core.infra.local_runtime.startup import start_local_sidecars
await start_local_sidecars()
except Exception as exc:
@@ -144,7 +153,7 @@ async def _startup_local_sidecars():
async def _shutdown_local_sidecars():
"""Reap local-profile sidecar subprocesses on shutdown."""
try:
- from orchestration.local_subprocess import stop_local_sidecars
+ from core.infra.local_runtime.startup import stop_local_sidecars
await stop_local_sidecars()
except Exception as exc:
@@ -152,23 +161,24 @@ async def _shutdown_local_sidecars():
async def _startup_device_link():
"""Serve this computer's Personal Agent mailbox (desktop execution plane only)."""
- from core.device_link.runner import runner
+ from personal_agent.device_link.runner import runner
runner.start()
async def _shutdown_device_link():
- from core.device_link.runner import runner
+ from personal_agent.device_link.runner import runner
await runner.stop()
async def _startup_recover_datasource_sidecars():
"""Recover configured DBHub/ES MCP sidecars without blocking API startup."""
- from api.app import app
import asyncio
+ from api.app import app
+
async def _recover() -> None:
try:
- from core.services.edition_startup import recover_datasource_sidecars
+ from orchestration.workers.edition_startup import recover_datasource_sidecars
result = await recover_datasource_sidecars()
failures = [
@@ -198,10 +208,11 @@ async def _recover() -> None:
app.state.datasource_sidecar_recovery_task = asyncio.create_task(_recover())
async def _shutdown_datasource_sidecar_recovery():
- from api.app import app
import asyncio
import contextlib
+ from api.app import app
+
task = getattr(app.state, "datasource_sidecar_recovery_task", None)
if task is not None and not task.done():
task.cancel()
@@ -214,13 +225,13 @@ async def _shutdown_datasource_sidecar_recovery():
async def _startup_desktop_observability():
global _desktop_sync_worker, _desktop_capture_remove
- from core.auth.desktop_bridge import bridge_enabled
+ from features.desktop.auth.bridge import bridge_enabled
if not bridge_enabled():
return
from core.db.engine import SessionLocal
- from core.services.desktop_cloud_bridge import get_identity_state
- from core.services.desktop_observability_sync import DesktopSyncWorker, install_capture
+ from features.desktop.bridge.identity import get_identity_state
+ from features.desktop.observability.sync import DesktopSyncWorker, install_capture
if _desktop_capture_remove is None:
_desktop_capture_remove = install_capture(SessionLocal, get_identity_state)
@@ -241,15 +252,15 @@ async def _shutdown_desktop_observability():
async def _startup_channel_desktop():
global _channel_desktop_worker
- from core.auth.desktop_bridge import bridge_enabled
+ from features.desktop.auth.bridge import bridge_enabled
from core.db.engine import SessionLocal
if bridge_enabled():
- from core.services.channel_desktop_worker import ChannelDesktopWorker
+ from integrations.channels.desktop_worker import ChannelDesktopWorker
_channel_desktop_worker = ChannelDesktopWorker(SessionLocal)
else:
- from core.services.channel_relay_dispatch import ChannelRelayReaper
+ from integrations.channels.relay_dispatch import ChannelRelayReaper
_channel_desktop_worker = ChannelRelayReaper(SessionLocal)
_channel_desktop_worker.start()
@@ -261,13 +272,27 @@ async def _shutdown_channel_desktop():
_channel_desktop_worker = None
async def _startup_extensions():
- from core.extensions import lifecycle_hooks
+ from core.harness.registry import lifecycle_hooks
for start, _ in lifecycle_hooks():
await start()
async def _shutdown_extensions():
- from core.extensions import lifecycle_hooks
+ from core.harness.registry import lifecycle_hooks
for _, stop in reversed(lifecycle_hooks()):
await stop()
+
+
+async def _shutdown_sandbox_maintenance():
+ from core.sandbox import get_sandbox_provider
+ stop = getattr(get_sandbox_provider(), "stop_maintenance", None)
+ if stop is not None:
+ await stop()
+
+
+async def _shutdown_sandbox_provider():
+ from core.sandbox import get_sandbox_provider
+ shutdown = getattr(get_sandbox_provider(), "shutdown", None)
+ if shutdown is not None:
+ await shutdown()
diff --git a/src/backend/api/startup/workers.py b/src/backend/api/startup/workers.py
index e8fcc2d2..f1e06ecf 100644
--- a/src/backend/api/startup/workers.py
+++ b/src/backend/api/startup/workers.py
@@ -1,9 +1,11 @@
"""Application lifecycle operations."""
import os
-from typing import Callable
import sys
+from typing import Callable
+
from core.config.settings import settings
from core.infra.logging import get_logger
+
logger = get_logger(__name__)
def _env_flag(name: str, default: bool) -> bool:
@@ -15,9 +17,6 @@ def _env_flag(name: str, default: bool) -> bool:
def _startup_preload_enabled() -> bool:
return _env_flag("STARTUP_PRELOAD_ENABLED", True)
-def _mcp_warmup_enabled() -> bool:
- return _env_flag("MCP_WARMUP_ENABLED", True)
-
async def _shutdown_automation_scheduler():
"""Stop the automation scheduler, whether the process is exiting or this
worker just lost the leader role. Idempotent: the scheduler's own ``stop``
@@ -78,7 +77,7 @@ async def _run():
logger.warning("[startup] idle session reaper failed to start: %s", exc)
async def _startup_preload():
- """Pre-load caches and initialize MCP connection pool at startup."""
+ """Pre-load application caches at startup."""
import asyncio
if not _startup_preload_enabled():
@@ -104,7 +103,7 @@ async def _run_sync(label: str, func: Callable[[], None]) -> None:
# 2. Pre-load skill metadata
def _load_skill_metadata() -> None:
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
loader = get_skill_loader()
meta = loader.load_all_metadata()
@@ -117,13 +116,13 @@ def _load_skill_metadata() -> None:
# at the same in-sandbox path. DB skills materialize into the same
# dir on demand. Must run before sandboxes are created/used.
def _sync_builtin_skills() -> None:
- from core.agent_skills.config import sync_builtin_skills_to_sandbox_dir
- from core.capabilities.paths import capabilities_enabled
+ from core.capabilities.skills.config import sync_builtin_skills_to_sandbox_dir
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
+ if has_run_capabilities():
# Desktop store: move the pre-store layout aside first (idempotent,
# quarantines instead of deleting), then the views are link-only.
- from core.capabilities.migration import migrate_legacy_layout
+ from features.desktop.capabilities.migration import migrate_legacy_layout
report = migrate_legacy_layout()
if report is not None and (report.changed or report.skipped):
@@ -134,7 +133,7 @@ def _sync_builtin_skills() -> None:
# Pre-create the per-user skills root for the same reason as above:
# script-runner bind-mounts it, and a dockerd-created one is root-owned.
try:
- from core.agent_skills.config import get_user_skills_root
+ from core.capabilities.skills.config import get_user_skills_root
get_user_skills_root()
except Exception as exc: # noqa: BLE001
@@ -147,7 +146,7 @@ def _sync_builtin_skills() -> None:
# created by dockerd as root and the runner can't write into it →
# backend must create it first. opensandbox/cube don't use this one.
try:
- from core.sandbox.workspace.credentials import yida_shared_workspace_dir
+ from integrations.yida.storage import yida_shared_workspace_dir
p = yida_shared_workspace_dir()
p.mkdir(parents=True, exist_ok=True)
@@ -160,24 +159,13 @@ def _sync_builtin_skills() -> None:
# 3. Pre-load public DB capability overlay so first chat avoids
# scanning admin_skills/admin_mcp_servers on the request path.
def _warm_runtime_catalog() -> None:
- from core.config.catalog_runtime import warmup_runtime_catalog_cache
+ from core.capabilities.catalog.runtime import warmup_runtime_catalog_cache
warmup_runtime_catalog_cache()
logger.info("[startup] Runtime catalog DB overlay warmed")
await _run_sync("Runtime catalog", _warm_runtime_catalog)
- # 4. Initialize MCP connection pool (the big one — 1-7s savings)
- if _mcp_warmup_enabled():
- try:
- from core.llm.factory import warmup_mcp_tools
-
- await warmup_mcp_tools()
- except Exception as exc:
- logger.warning("[startup] MCP pool initialization failed: %s", exc)
- else:
- logger.info("[startup] MCP pool warmup skipped for current environment")
-
# 5. Pre-load DB prompt parts so first chat skips DB query
from prompts.prompt_runtime import warmup_prompt_cache
@@ -188,7 +176,7 @@ def _warm_runtime_catalog() -> None:
# row. See _NAV_BACKFILL_ENTRIES in content_blocks.py — append a
# new entry there each time we add a sidebar feature.
def _backfill_nav():
- from core.content.content_blocks import backfill_navigation_entries
+ from core.files.content.branding import backfill_navigation_entries
from core.db.engine import SessionLocal
db = SessionLocal()
@@ -214,7 +202,7 @@ async def _startup_automation_scheduler():
logger.info("[startup] Automation scheduler disabled")
return
try:
- from orchestration.schedulers.automation_scheduler import AutomationScheduler
+ from orchestration.automations.automation_scheduler import AutomationScheduler
global _automation_scheduler
_automation_scheduler = AutomationScheduler()
@@ -235,7 +223,7 @@ async def _startup_kb_wiki_worker():
logger.info("[startup] KB wiki worker disabled")
return
try:
- from core.kb.wiki.worker import WikiIngestWorker
+ from features.knowledge.wiki.worker import WikiIngestWorker
global _kb_wiki_worker
_kb_wiki_worker = WikiIngestWorker()
@@ -258,7 +246,7 @@ async def _startup_kb_index_worker():
logger.info("[startup] KB index worker disabled")
return
try:
- from core.kb.index_worker import KBIndexWorker
+ from features.knowledge.index_worker import KBIndexWorker
global _kb_index_worker
_kb_index_worker = KBIndexWorker()
@@ -295,7 +283,7 @@ async def _startup_channel_manager():
logger.info("[startup] Channel bots disabled")
return
try:
- from core.channels.manager import get_manager
+ from integrations.channels.manager import get_manager
await get_manager().start_all()
from core.infra import runtime_state
@@ -317,7 +305,7 @@ async def _startup_channel_manager():
async def _startup_distillation_scheduler():
"""Start the daily skill-distillation cron scheduler."""
try:
- from core.services.edition_startup import create_distillation_scheduler
+ from orchestration.workers.edition_startup import create_distillation_scheduler
global _distillation_scheduler
_distillation_scheduler = create_distillation_scheduler()
@@ -331,7 +319,9 @@ async def _startup_distillation_scheduler():
logger.warning("[startup] Distillation cron scheduler failed to start: %s", exc)
async def _shutdown_pools():
- """Close MCP connection pool, KB HTTP server, automation scheduler, and Redis on shutdown."""
+ """Close chat MCP sessions and application services on shutdown."""
+ from core.llm.mcp.chat_sessions import close_chat
+ await close_chat()
# Stop automation scheduler — same handler the leader path calls, so a
# deposed leader and an exiting process cannot stop different things.
try:
@@ -350,12 +340,13 @@ async def _shutdown_pools():
await _shutdown_idle_session_reaper()
except Exception as e:
logger.warning("idle_reaper_shutdown_error", error=str(e))
- try:
- from core.llm.mcp_pool import MCPConnectionPool
+ import asyncio
- await MCPConnectionPool.get_instance().shutdown()
- except Exception as e:
- logger.warning("mcp_pool_shutdown_error", error=str(e))
+ from core.harness.observation_writer import flush_observations
+
+ # Hook stage events and usage are written in the background; persist the
+ # tail before the database and Redis go away.
+ await asyncio.to_thread(flush_observations, 5.0)
try:
from core.infra.redis import close_redis
diff --git a/src/backend/core/content/__init__.py b/src/backend/api/transports/__init__.py
similarity index 100%
rename from src/backend/core/content/__init__.py
rename to src/backend/api/transports/__init__.py
diff --git a/src/backend/api/transports/run_sse.py b/src/backend/api/transports/run_sse.py
new file mode 100644
index 00000000..e4bc29ea
--- /dev/null
+++ b/src/backend/api/transports/run_sse.py
@@ -0,0 +1,85 @@
+"""Encode durable run events into the existing public SSE wire protocol."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+from typing import Any, AsyncIterator, Callable, Dict, Optional
+
+import core.runs.events.subscription as subscription
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def follow_run_as_sse(
+ run_id: str,
+ *,
+ chat_id: str,
+ from_offset: int = 0,
+ error_event_factory: Optional[Callable[[str], Dict[str, Any]]] = None,
+) -> AsyncIterator[str]:
+ """Wrap follow_run as SSE wire frames.
+
+ ``error_event_factory(reason)`` lets callers customize the error event shape
+ (chat protocol vs plan protocol). Defaults to a chat-style ``{type: error}``.
+ """
+
+ def _default_err(reason: str) -> Dict[str, Any]:
+ return {"type": "error", "error": reason, "chat_id": chat_id}
+
+ factory = error_event_factory or _default_err
+
+ # Decouple follow_run from the yield cadence via an intermediate queue: it
+ # lets wait_for write an SSE comment line to the wire when the stream has
+ # been silent longer than _HEARTBEAT_INTERVAL_SEC, serving as nginx /
+ # reverse-proxy keepalive. Cancelling queue.get via wait_for is safe
+ # (asyncio.Queue handles the cancel race); the underlying follow_run
+ # coroutine is never interrupted.
+ queue: asyncio.Queue = asyncio.Queue()
+
+ async def _pump() -> None:
+ try:
+ async for event in subscription.follow_run(run_id, from_offset=from_offset):
+ await queue.put(("event", event))
+ except runtime.ChatRunNotFound:
+ await queue.put(("not_found", None))
+ except Exception as exc: # noqa: BLE001
+ await queue.put(("error", exc))
+ else:
+ await queue.put(("end", None))
+
+ pump_task = asyncio.create_task(_pump(), name=f"sse_pump:{run_id}")
+ try:
+ while True:
+ try:
+ kind, payload = await asyncio.wait_for(
+ queue.get(), timeout=settings.HEARTBEAT_INTERVAL_SEC
+ )
+ except asyncio.TimeoutError:
+ yield ": heartbeat\n\n"
+ continue
+
+ if kind == "event":
+ clean = {k: v for k, v in payload.items() if not k.startswith("_")}
+ yield f"data: {json.dumps(clean, ensure_ascii=False)}\n\n"
+ elif kind == "end":
+ break
+ elif kind == "not_found":
+ yield f"data: {json.dumps(factory('run not found'), ensure_ascii=False)}\n\n"
+ break
+ elif kind == "error":
+ logger.warning(
+ "follow_run_as_sse_failed", run_id=run_id, error=str(payload), exc_info=payload
+ )
+ yield f"data: {json.dumps(factory('流式响应中断'), ensure_ascii=False)}\n\n"
+ break
+ finally:
+ pump_task.cancel()
+ with contextlib.suppress(BaseException):
+ await pump_task
+
+ yield "data: [DONE]\n\n"
diff --git a/src/backend/cli.py b/src/backend/cli.py
index 4455f89b..585df498 100644
--- a/src/backend/cli.py
+++ b/src/backend/cli.py
@@ -27,7 +27,6 @@
from pathlib import Path
from typing import Optional
-
def _configure_standard_streams() -> None:
"""Keep redirected desktop logs UTF-8 and tolerant of status symbols."""
for stream in (sys.stdout, sys.stderr):
@@ -42,7 +41,6 @@ def _configure_standard_streams() -> None:
except (LookupError, OSError, ValueError):
pass
-
def _status(message: str, *, file=None, **kwargs) -> None:
"""Print CLI status without letting a legacy code page break startup."""
stream = file or sys.stdout
@@ -56,16 +54,13 @@ def _status(message: str, *, file=None, **kwargs) -> None:
safe_message = message.encode("ascii", errors="replace").decode("ascii")
print(safe_message, file=stream, **kwargs)
-
# ── Data dir & environment ────────────────────────────────────────────────────
-
def data_dir() -> Path:
from core.config.runtime_env import local_data_dir
return local_data_dir()
-
def ensure_loopback_proxy_bypass() -> None:
"""Keep loopback traffic off any HTTP(S)/SOCKS proxy from the environment.
@@ -98,7 +93,6 @@ def ensure_loopback_proxy_bypass() -> None:
os.environ["NO_PROXY"] = merged
os.environ["no_proxy"] = merged
-
def _resolve_frontend_dist() -> Optional[str]:
"""Locate the built frontend for StaticFiles hosting."""
env = os.getenv("FRONTEND_DIST_DIR", "").strip()
@@ -109,13 +103,11 @@ def _resolve_frontend_dist() -> Optional[str]:
cand = src / "frontend" / "dist"
return str(cand) if (cand / "index.html").exists() else None
-
def _site_template_home() -> Path:
- from core.plugins.local.assets import site_template_home
+ from features.desktop.local.plugins.assets import site_template_home
return site_template_home()
-
def apply_local_env(port: int) -> dict:
"""Populate the local-profile env (idempotent; real env wins) + data dirs."""
ensure_loopback_proxy_bypass()
@@ -196,6 +188,13 @@ def apply_local_env(port: int) -> dict:
# ``defaults`` would freeze a settings object that predates
# ``DEPLOY_PROFILE=local`` — the whole process would then run as a non-local
# deployment (no ``/api`` prefix bridge, no static frontend, mock auth).
+ from orchestration.configuration import register_configuration
+ register_configuration()
+ from features import register_run_capabilities, register_model_credentials
+ register_run_capabilities()
+ register_model_credentials()
+ from integrations import register_credentials
+ register_credentials()
site_home = _site_template_home()
defaults["SITE_TEMPLATE_HOME"] = str(site_home)
defaults["SITE_TEMPLATE_DIR"] = str(site_home / "react-vite")
@@ -203,7 +202,6 @@ def apply_local_env(port: int) -> dict:
os.environ.setdefault("SITE_TEMPLATE_DIR", defaults["SITE_TEMPLATE_DIR"])
return defaults
-
def _is_initialized() -> bool:
"""True if the SQLite DB exists and has at least one local account."""
dd = data_dir()
@@ -227,24 +225,6 @@ def _is_initialized() -> bool:
except Exception:
return False
-
-# ── DB helpers (import lazily, after env is set) ─────────────────────────────
-
-
-def _ensure_schema_and_seed():
- """create_all + built-in catalog seed (mirrors the app's startup hooks)."""
- from core.db.engine import SessionLocal, init_db
- from core.services.mcp_service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty
-
- init_db()
- db = SessionLocal()
- try:
- prune_removed_builtin_mcp_servers(db)
- seed_builtin_mcp_servers_if_empty(db)
- finally:
- db.close()
-
-
# ── Onboarding ────────────────────────────────────────────────────────────────
_PRESETS = {
@@ -260,13 +240,11 @@ def _ensure_schema_and_seed():
# guess a missing context window.
DEFAULT_LOCAL_CONTEXT_LENGTH = 32768
-
def _prompt(msg: str, default: str = "") -> str:
suffix = f" [{default}]" if default else ""
val = input(f"{msg}{suffix}: ").strip()
return val or default
-
def bootstrap_admin(username: str, password: str) -> str:
"""Create the local admin account and elevate it to super_admin.
@@ -275,8 +253,8 @@ def bootstrap_admin(username: str, password: str) -> str:
step missing everywhere else in the codebase.
"""
from core.db.engine import SessionLocal
- from core.services.local_user_service import LocalUserService
- from core.services.user_service import UserService
+ from core.auth.local_users import LocalUserService
+ from core.auth.users import UserService
db = SessionLocal()
try:
@@ -288,7 +266,6 @@ def bootstrap_admin(username: str, password: str) -> str:
finally:
db.close()
-
def configure_model(
base_url: str,
api_key: str,
@@ -303,9 +280,8 @@ def configure_model(
"""
import asyncio
- from core.db import model_repository as mr
+ from core.models import repository as mr
from core.db.engine import SessionLocal
- from core.services.model_config import ModelConfigService
if test:
from api.routes.v1.models import _test_connection
@@ -348,15 +324,12 @@ def configure_model(
mr.assign_role(db, role, provider.provider_id, updated_by="onboard")
finally:
db.close()
- # Repository writes don't invalidate the read cache — do it explicitly.
- ModelConfigService.get_instance().invalidate_cache()
-
def mark_web_onboarding_complete(user_id: str) -> None:
"""Avoid repeating the browser setup after the terminal wizard succeeds."""
from core.db.engine import SessionLocal
- from core.services.local_user_service import CE_ONBOARDING_VERSION
- from core.services.user_service import UserService
+ from core.auth.local_users_constants import CE_ONBOARDING_VERSION
+ from core.auth.users import UserService
db = SessionLocal()
try:
@@ -370,7 +343,6 @@ def mark_web_onboarding_complete(user_id: str) -> None:
finally:
db.close()
-
# ── Plugins ───────────────────────────────────────────────────────────────────
# Recommended default plugin set for a personal single-machine install: task
@@ -378,41 +350,46 @@ def mark_web_onboarding_complete(user_id: str) -> None:
# (IM / email / low-code) need per-user credentials, so they're opt-in only.
_DEFAULT_PLUGIN_BOOTSTRAP_ENV = "HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS"
-
def _default_plugins() -> list:
"""推荐的默认插件集合,与库里的引导保持同一份来源。"""
- from core.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS
+ from core.capabilities.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS
return list(DEFAULT_BOOTSTRAP_PLUGIN_SLUGS)
-
def _default_plugins_marker() -> Path:
- from core.plugins.management import LOCAL_BOOTSTRAP_MARKER_NAME
+ from core.capabilities.plugins.management import LOCAL_BOOTSTRAP_MARKER_NAME
return data_dir() / LOCAL_BOOTSTRAP_MARKER_NAME
-
def list_installable_plugins() -> list:
"""Local-filesystem plugin bundles (default + marketplace) with install state."""
from core.db.engine import SessionLocal
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
+ from features.marketplace import plugin_market
+
+ from features import register_package_sources, register_resource_owners
+ register_package_sources()
+ register_resource_owners()
db = SessionLocal()
try:
- items = plugin_service.list_plugins(db, None, include_disabled=True)
+ items = plugin_market.list_plugins(db, None, include_disabled=True)
finally:
db.close()
# Only the shipped bundles are installable offline; DB-market packages need an
# admin upload that doesn't exist on a fresh local install.
return [it for it in items if it.get("source") == "builtin"]
-
def install_plugins(slugs: list) -> list:
"""Install the given plugin slugs globally (owner_user_id=None). Best-effort."""
from core.db.engine import SessionLocal
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
done = []
+ from features import register_package_sources, register_resource_owners
+ register_package_sources()
+ register_resource_owners()
+
db = SessionLocal()
try:
for slug in slugs:
@@ -427,7 +404,6 @@ def install_plugins(slugs: list) -> list:
db.close()
return done
-
def ensure_default_plugins_once() -> bool:
"""Install local-install defaults once without overriding later user choices.
@@ -453,7 +429,6 @@ def ensure_default_plugins_once() -> bool:
os.replace(temporary, marker)
return True
-
def _select_plugins_interactively(available: list) -> list:
"""Show the plugin menu, return the slugs the user picked."""
recommended = _default_plugins()
@@ -481,10 +456,8 @@ def _select_plugins_interactively(available: list) -> list:
picked.append(it["slug"])
return picked
-
# ── File parser (PDF / document parsing on upload) ────────────────────────────
-
def configure_file_parser(
api_url: str, *, backend: str = "pipeline", parse_method: str = "auto"
) -> None:
@@ -494,7 +467,7 @@ def configure_file_parser(
service at ``file_parser.api_url``. Excel/CSV/PPTX/text parse in-process and
need none of this.
"""
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc = SystemConfigService.get_instance()
# Touch the cache once so the seed rows exist before we set() them
@@ -509,10 +482,8 @@ def configure_file_parser(
updated_by="onboard",
)
-
# ── Internet search (Tavily / Baidu / LangSearch key) ────────────────────────
-
def configure_search_engine(engine: str, api_key: str) -> None:
"""Write the internet-search engine + key into system_configs.
@@ -521,7 +492,7 @@ def configure_search_engine(engine: str, api_key: str) -> None:
writing, the MCP subprocess picks up the new key via runtime_env. It can
also be changed later in the Web UI under Settings -> System Management.
"""
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc = SystemConfigService.get_instance()
svc.get_all_configs() # touch cache so seed rows exist before set()
@@ -540,13 +511,11 @@ def configure_search_engine(engine: str, api_key: str) -> None:
updated_by="onboard",
)
-
# ── Site-building template (React path) ──────────────────────────────────────
-
def report_site_template(verbose: bool = True) -> bool:
"""向导里报一句建站模板的状态;铺设本身在安装站点插件时就做了。"""
- from core.plugins.local.assets import site_template_home
+ from features.desktop.local.plugins.assets import site_template_home
home = site_template_home()
ready = (home / "init-react-site.sh").is_file()
@@ -562,12 +531,11 @@ def report_site_template(verbose: bool = True) -> bool:
_status(f" ✓ React 建站模板已铺入:{home}(未检测到 Node/npm,首次构建时需要)")
return True
-
def _probe_host_tools() -> dict:
"""Which optional host tools are present (affects local-mode capabilities)."""
import shutil
- from core.content.office import find_libreoffice_binary
+ from core.files.content.office import find_libreoffice_binary
return {
"node": shutil.which("node") or shutil.which("nodejs"),
@@ -579,7 +547,6 @@ def _probe_host_tools() -> dict:
"os_sandbox": _os_sandbox_status(),
}
-
def _os_sandbox_status() -> tuple[str, str]:
"""``(backend name, reason it is unavailable)``; the reason is empty when ready."""
from core.sandbox.permissions.os_gate import backend_name
@@ -587,7 +554,6 @@ def _os_sandbox_status() -> tuple[str, str]:
return backend_name(), confinement_unavailable_reason()
-
def _print_capability_summary() -> None:
"""End-of-onboard note on what host tools gate which capabilities."""
tools = _probe_host_tools()
@@ -616,7 +582,6 @@ def _print_capability_summary() -> None:
+ ("" if not sandbox_reason else f"({sandbox_reason}:除「完全放开」外的权限档会拒绝执行)")
)
-
def _configure_aux_model_step(
args, *, ptype: str, title: str, ok_label: str, bu, key, model, interactive: bool
) -> None:
@@ -641,11 +606,12 @@ def _configure_aux_model_step(
except Exception as exc: # noqa: BLE001
print(f"! {ok_label}失败(该能力将不可用):{exc}", file=sys.stderr)
-
def cmd_onboard(args) -> int:
apply_local_env(args.port)
print("HugAgentOS 本地初始化\n" + "─" * 40)
- _ensure_schema_and_seed()
+ from hugagent_cli.schema import ensure_schema_and_seed
+
+ ensure_schema_and_seed()
# Step 1 — admin account
if args.username and args.password:
@@ -795,10 +761,8 @@ def cmd_onboard(args) -> int:
print("正在启动服务…\n")
return cmd_serve(args)
-
# ── Serve ─────────────────────────────────────────────────────────────────────
-
def _open_browser_when_ready(port: int) -> None:
url = f"http://127.0.0.1:{port}/"
for _ in range(60):
@@ -816,7 +780,6 @@ def _open_browser_when_ready(port: int) -> None:
except Exception:
pass
-
def bootstrap_default_plugins_if_needed() -> None:
"""按形态决定要不要装本机默认插件。
@@ -824,7 +787,7 @@ def bootstrap_default_plugins_if_needed() -> None:
壳在双模式下本就不会设引导变量,UOS 壳是无条件设的——判断放在这里,两端和历史
启动脚本才是同一套规矩。
"""
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
if os.getenv(_DEFAULT_PLUGIN_BOOTSTRAP_ENV) != "1" or device_catalog.active():
return
@@ -838,10 +801,11 @@ def bootstrap_default_plugins_if_needed() -> None:
if bootstrapped:
_status("✓ 默认插件已就绪:定时任务、技能管理、站点发布")
-
def cmd_serve(args) -> int:
apply_local_env(args.port)
- _ensure_schema_and_seed()
+ from hugagent_cli.schema import ensure_schema_and_seed
+
+ ensure_schema_and_seed()
bootstrap_default_plugins_if_needed()
import uvicorn
from api.app import app
@@ -856,10 +820,8 @@ def cmd_serve(args) -> int:
uvicorn.run(app, host=host, port=port, log_level="info", access_log=False)
return 0
-
# ── Doctor ────────────────────────────────────────────────────────────────────
-
def cmd_doctor(args) -> int:
apply_local_env(args.port)
ok = True
@@ -930,10 +892,8 @@ def check(label: str, passed: bool, detail: str = "", *, required: bool = True)
print("✓ 自检通过" if ok else "✗ 存在需处理的问题(见上)")
return 0 if ok else 1
-
# ── Entry ─────────────────────────────────────────────────────────────────────
-
def build_parser() -> argparse.ArgumentParser:
# Shared options usable both before and after the subcommand
# (`hugagent --host X --port Y serve` and options after `serve` both work).
@@ -990,7 +950,6 @@ def build_parser() -> argparse.ArgumentParser:
sub.add_parser("doctor", parents=[common], help="环境自检").set_defaults(func=cmd_doctor)
return p
-
def main(argv: Optional[list] = None) -> int:
_configure_standard_streams()
parser = build_parser()
@@ -1020,6 +979,5 @@ def main(argv: Optional[list] = None) -> int:
setattr(args, attr, getattr(args, attr, val))
return args.func(args) or 0
-
if __name__ == "__main__":
sys.exit(main())
diff --git a/src/backend/core/agent_runtime/README.md b/src/backend/core/agent_runtime/README.md
new file mode 100644
index 00000000..39de8f50
--- /dev/null
+++ b/src/backend/core/agent_runtime/README.md
@@ -0,0 +1,18 @@
+# Normal agent execution
+
+[中文](README.zh-CN.md)
+
+The public entry is runner.astream_chat_workflow. It selects the main or direct
+agent route and closes the full generator chain when a consumer exits.
+
+- runner / direct: lifecycle, output review, final metadata and post-turn work.
+- prepare_main / prepare_direct: build the agent and assemble its input.
+- stream_main / stream_direct: adapt model/tool events into the existing event contract.
+- main_state / direct_state: typed state owned by one turn, never shared across runs.
+- targets, skills, budget, events: routing inputs and focused runtime helpers.
+- integrations: memory, compaction, ontology evidence/repair and evolution integration.
+- complete: the existing non-streaming entry and result shape.
+
+Durable ownership and database transactions are outside these phases.
+Chat workers in core.chat.runtime consume events and persist them under core.runs
+lease fencing. Plan and autonomous Loop composition belong to orchestration.
diff --git a/src/backend/core/agent_runtime/README.zh-CN.md b/src/backend/core/agent_runtime/README.zh-CN.md
new file mode 100644
index 00000000..298d965b
--- /dev/null
+++ b/src/backend/core/agent_runtime/README.zh-CN.md
@@ -0,0 +1,18 @@
+# 通用智能体执行
+
+[English](README.md)
+
+公开入口为 runner.astream_chat_workflow。它选择主智能体或直接智能体路径,
+并在调用方退出时关闭整条生成器链。
+
+- runner / direct:运行生命周期、输出评审、最终元数据和响应后的工作。
+- prepare_main / prepare_direct:创建智能体并组装输入。
+- stream_main / stream_direct:把模型和工具事件转换成既有事件协议。
+- main_state / direct_state:单轮持有的有类型状态,不跨任务共享。
+- targets、skills、budget、events:目标选择、技能、预算与事件辅助逻辑。
+- integrations:记忆、压缩、本体证据/修复与演化集成。
+- complete:保留原有非流式执行入口和结果结构。
+
+运行所有权与数据库事务在这些阶段之外管理。core.chat.runtime 中的聊天 worker
+消费事件,在 core.runs 租约约束下持久化输出。Plan 与自主 Loop 的流程组合
+属于外层 orchestration。
diff --git a/src/backend/core/plugins/resources/__init__.py b/src/backend/core/agent_runtime/__init__.py
similarity index 100%
rename from src/backend/core/plugins/resources/__init__.py
rename to src/backend/core/agent_runtime/__init__.py
diff --git a/src/backend/core/agent_runtime/budget.py b/src/backend/core/agent_runtime/budget.py
new file mode 100644
index 00000000..8eb040b4
--- /dev/null
+++ b/src/backend/core/agent_runtime/budget.py
@@ -0,0 +1,74 @@
+"""Read the frozen model and prompt surface a turn's compaction budget is computed from."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Tuple
+
+import core.llm.context.compacting_agent as compacting_agent
+import core.models.context_window as context_manager
+
+logger = logging.getLogger(__name__)
+
+
+def _resolve_agent_model_runtime(agent: Any, fallback_model_name: Any = "") -> Tuple[str, int]:
+ """Return the effective model name and context window baked into an agent.
+
+ AgentScope chat models expose the upstream name as ``.model``. Some provider
+ wrappers expose ``.model_name`` instead, so accept both before consulting the
+ request fallback. Channel runs intentionally omit a model selection; converting
+ that ``None`` to the literal string ``"None"`` previously made direct sub-agent
+ conversations fail context-window lookup even though the resolved main model was
+ correctly configured.
+ """
+ model = getattr(agent, "model", None)
+ state = getattr(agent, "state", None)
+ candidates = (
+ getattr(model, "model", None),
+ getattr(model, "model_name", None),
+ getattr(state, "model_name", None),
+ fallback_model_name,
+ )
+ model_name = ""
+ for candidate in candidates:
+ text = str(candidate or "").strip()
+ if text and text.lower() != "none":
+ model_name = text
+ break
+
+ context_window = int(getattr(model, "context_size", 0) or 0)
+ if context_window <= 0:
+ context_window = context_manager.resolve_model_context_window(model_name)
+ return model_name, context_window
+
+
+def _compaction_budget_inputs(agent: Any, context_window: int) -> Dict[str, Any]:
+ """Read the exact frozen prompt/tool surface cached by agent_factory."""
+ system_prompt, tool_schema = compacting_agent.frozen_prompt_surface(agent)
+ return {
+ "system_prompt": system_prompt,
+ "tool_schema": tool_schema,
+ "context_window": int(context_window or 0),
+ "model_name": str(
+ getattr(getattr(agent, "model", None), "model", None)
+ or getattr(getattr(agent, "state", None), "model_name", None)
+ or ""
+ ),
+ "model_provider_id": str(
+ getattr(getattr(agent, "model", None), "resolved_model_provider_id", "")
+ or getattr(getattr(agent, "state", None), "model_provider_id", "")
+ or ""
+ ),
+ }
+
+
+def _pre_turn_compaction_inputs(
+ agent: Any,
+ context_window: int,
+ resolved_model_name: str,
+) -> Dict[str, Any]:
+ """Adapt the shared budget snapshot to the pre-turn service signature."""
+ inputs = _compaction_budget_inputs(agent, context_window)
+ inputs.pop("model_provider_id", None)
+ inputs["model_name"] = str(inputs.get("model_name") or resolved_model_name)
+ return inputs
diff --git a/src/backend/core/agent_runtime/direct.py b/src/backend/core/agent_runtime/direct.py
new file mode 100644
index 00000000..ad1c105d
--- /dev/null
+++ b/src/backend/core/agent_runtime/direct.py
@@ -0,0 +1,458 @@
+"""Coordinate the direct agent lifecycle."""
+
+from __future__ import annotations
+from core.agent_runtime.integrations import memory
+from core.agent_runtime.integrations import memory_session
+from core.ontology import output_review_state
+from core.ontology import runtime_pack
+from core.runs.tool_effects import errors
+from core.llm.citations import allocator
+
+import asyncio
+import logging
+import typing as _typing
+from contextlib import aclosing as _aclosing
+from typing import Any, AsyncIterator, Dict, List, Optional
+
+import core.agent_runtime.budget as budget
+import core.agent_runtime.integrations.evolution as evolution
+import core.agent_runtime.integrations.memory_budget as memory_budget
+import core.agent_runtime.integrations.ontology_evidence as ontology_evidence
+import core.agent_runtime.integrations.ontology_repair as ontology_repair
+import core.agent_runtime.targets as targets
+import core.chat.message_parser as message_parser
+import core.capabilities.catalog.resolver as catalog_resolver
+
+
+from core.agent_runtime import prepare_direct, stream_direct
+from core.agent_runtime.direct_state import DirectTurn
+
+logger = logging.getLogger(__name__)
+
+
+async def _astream_subagent_direct(
+ *,
+ agent_id: str,
+ session_messages: List[Dict[str, Any]],
+ user_message: str,
+ context: Dict[str, Any],
+) -> AsyncIterator[Dict[str, Any]]:
+ """Stream a direct conversation with a user-created sub-agent.
+
+ Loads the UserAgent config from DB and uses it to build the agent
+ with custom system_prompt, MCP tools, skills, KB, and model params.
+ Shares the same streaming/memory/citation infrastructure as the main route.
+ """
+ turn = DirectTurn(context=context, session_messages=session_messages)
+ from core.llm.api_runtime.agent import apply_api_scope
+ from core.harness.registry.evaluation import apply_evaluation_scope
+
+ turn.context = apply_evaluation_scope(apply_api_scope(turn.context))
+
+ import time as _time
+
+ turn.wf_start = _time.monotonic()
+ turn.direct_read_only, turn.direct_allow_bash = targets._direct_agent_execution_permissions(
+ agent_id
+ )
+
+ turn.user_agent = targets._load_direct_user_agent(
+ agent_id,
+ str(turn.context.get("user_id", "")),
+ {
+ "enabled_skill_ids": catalog_resolver.enabled_skill_ids_from_context(turn.context)
+ or [],
+ "enabled_mcp_ids": catalog_resolver.enabled_mcp_ids_from_context(turn.context) or [],
+ "enabled_kb_ids": catalog_resolver.enabled_kb_ids_from_context(turn.context) or [],
+ "sandbox_tools_enabled": bool(turn.context.get("sandbox_tools_enabled", False)),
+ "code_capability_enabled": bool(turn.context.get("code_capability_enabled", False)),
+ },
+ )
+ from core.harness import log_writer
+
+ _direct_log_started = _time.monotonic()
+ turn.direct_tool_count = 0
+ turn.direct_skill_count = 0
+ _direct_log_finished = False
+ _is_call_subagent_dispatch = turn.context.get("direct_agent_source") == "explicit_command_tool"
+ import uuid
+
+ from core.infra.task_lifecycle import cancel_and_join, settle_task
+
+ turn.direct_log_id = uuid.uuid4().hex
+ _direct_finish_task = None
+ _direct_start_task = asyncio.create_task(
+ log_writer.start_subagent_log(
+ {
+ "id": turn.direct_log_id,
+ "user_id": str(turn.context.get("user_id", "")) or None,
+ "chat_id": turn.context.get("chat_id"),
+ "subagent_name": turn.user_agent.name,
+ "subagent_type": (
+ "user_agent" if _is_call_subagent_dispatch else "user_agent_direct"
+ ),
+ "subagent_id": agent_id,
+ "input_messages": {
+ "task": user_message,
+ "invocation": (
+ "call_subagent"
+ if _is_call_subagent_dispatch
+ else turn.context.get("direct_agent_source") or "dedicated_chat"
+ ),
+ },
+ }
+ )
+ )
+
+ async def _finish_direct_log(
+ status: str,
+ *,
+ output: str = "",
+ error: Optional[str] = None,
+ usage: Optional[Dict[str, int]] = None,
+ ) -> None:
+ nonlocal _direct_log_finished, _direct_finish_task
+
+ async def record():
+ await _direct_start_task
+ await log_writer.finish_subagent_log(
+ turn.direct_log_id,
+ status=status,
+ output_content=output or None,
+ token_usage=usage,
+ tool_calls_count=turn.direct_tool_count,
+ skill_calls_count=turn.direct_skill_count,
+ error_message=error,
+ duration_ms=int((_time.monotonic() - _direct_log_started) * 1000),
+ )
+
+ if _direct_finish_task is None:
+ _direct_finish_task = asyncio.create_task(record())
+ await asyncio.shield(_direct_finish_task)
+ _direct_log_finished = True
+
+ _direct_exit_status = "cancelled"
+ _direct_exit_error = "direct agent execution interrupted"
+ try:
+ await asyncio.shield(_direct_start_task)
+ turn.ontology_runtime = turn.context.get("ontology_runtime")
+ if not isinstance(turn.ontology_runtime, dict):
+ turn.ontology_runtime = {}
+ turn.context["ontology_runtime"] = turn.ontology_runtime
+ activations = runtime_pack.activate_runtime_for_asset(
+ turn.ontology_runtime,
+ kind="subagent",
+ asset_id=agent_id,
+ tags=list((turn.user_agent.extra_config or {}).get("ontology_tags") or []),
+ )
+ if activations:
+ await asyncio.to_thread(
+ ontology_evidence._record_ontology_activations,
+ activations,
+ turn.ontology_runtime,
+ turn.context,
+ )
+
+ # ── [memory] Non-blocking retrieval: background task, skipped on budget timeout ───
+ turn.memory_user_id = str(turn.context.get("user_id", ""))
+ turn.memory_workspace_id = str(turn.context.get("workspace_id", "") or "default")
+ _mem0_chat_id = turn.context.get("chat_id") or turn.context.get("conversation_id")
+ turn.memory_enabled = bool(turn.context.get("memory_enabled", False))
+ _mem0_write_enabled = bool(turn.context.get("memory_write_enabled", False))
+ # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id
+ _mem0_scope_user_id = str(
+ turn.context.get("memory_scope_user_id", "") or turn.memory_user_id
+ )
+ logger.info(
+ "[subagent] user=%s scope=%s ws=%s agent=%s enabled=%s",
+ turn.memory_user_id,
+ _mem0_scope_user_id,
+ turn.memory_workspace_id,
+ agent_id,
+ turn.memory_enabled,
+ )
+
+ turn.session_memory = await memory_session.open_session_memory(
+ chat_id=_mem0_chat_id,
+ scope_user_id=_mem0_scope_user_id,
+ workspace_id=turn.memory_workspace_id,
+ user_message=user_message,
+ memory_enabled=turn.memory_enabled,
+ # The retrieval budget is part of the orchestration profile: how long
+ # this task type is willing to wait for memory is an assembly decision,
+ # not a global constant. Resolved here because retrieval starts before
+ # the agent is built.
+ budget_ms=memory_budget._profile_memory_budget_ms(turn.context),
+ )
+
+ warnings: _typing.List[str] = []
+ turn.full_response = ""
+ turn.displayed_tools: set[str] = set()
+ turn.all_citations: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ turn.ontology_event_cursor = 0
+ turn.ontology_trace: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ # 证据锚点发号器:跨轮续号;创建后绑到 agent 上(见下方 attach_allocator),
+ # 中间件与本函数由此共享同一个计数器
+ turn.anchor_allocator = allocator.AnchorAllocator(
+ await asyncio.to_thread(
+ allocator.anchor_start_for_chat,
+ str(turn.context.get("chat_id") or "") or None,
+ )
+ )
+
+ try:
+ async with _aclosing(prepare_direct.prepare_direct(turn)) as phase:
+ async for event in phase:
+ yield event
+ async with _aclosing(stream_direct.stream_direct(turn)) as phase:
+ async for event in phase:
+ yield event
+
+ except Exception as e:
+ import traceback
+
+ logger.error("subagent_stream_error: %s\n%s", e, traceback.format_exc())
+ warnings.append(f"Streaming error: {str(e)[:200]}")
+
+ unknown_outcome = errors.find_tool_outcome_unknown(e)
+ if unknown_outcome is not None:
+ if turn.streaming_agent is not None:
+ await turn.streaming_agent.shutdown()
+ await _finish_direct_log("failed", error=str(unknown_outcome)[:200])
+ if unknown_outcome is e:
+ raise
+ raise unknown_outcome from e
+
+ if not turn.full_response:
+ # No answer was produced: the caller gets the failure itself. Tool
+ # results already streamed stay visible; the run is recorded as
+ # failed with a structured error instead of a fabricated reply that
+ # would read as a completed answer.
+ if turn.streaming_agent is not None:
+ await turn.streaming_agent.shutdown()
+ await _finish_direct_log("failed", error=str(e)[:200])
+ raise
+
+ pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[
+ turn.ontology_event_cursor :
+ ]
+ for ontology_event in pending_ontology_events:
+ yield dict(ontology_event)
+ turn.ontology_event_cursor += len(pending_ontology_events)
+
+ _ontology_review_owner_id = ontology_evidence._ontology_review_owner(
+ turn.ontology_runtime, turn.context
+ )
+ _ontology_review_claimed = bool(
+ turn.full_response
+ and output_review_state.claim_output_review(
+ turn.ontology_runtime, owner=_ontology_review_owner_id
+ )
+ )
+ if _ontology_review_claimed:
+ turn.review_owner = _ontology_review_owner_id
+ from core.ontology.reviewer import review_ontology_output
+
+ yield {
+ "type": "ontology_review",
+ "status": "started",
+ "level": turn.ontology_runtime.get("review_level", "checkpoint"),
+ **ontology_evidence._ontology_review_event_context(turn.ontology_runtime),
+ }
+ repair_event_queue: asyncio.Queue[_typing.Dict[str, _typing.Any]] = asyncio.Queue()
+
+ async def _remediate(payload: Dict[str, Any]) -> str:
+ primary_usage = await turn.streaming_agent.aget_usage()
+ primary_context_usage = turn.streaming_agent.get_context_usage(primary_usage)
+ try:
+ repaired, _, cursor, tool_count = (
+ await ontology_repair._run_ontology_repair_round(
+ streaming_agent=turn.streaming_agent,
+ context=turn.context,
+ payload=payload,
+ runtime=turn.ontology_runtime,
+ trace=turn.ontology_trace,
+ citations=turn.all_citations,
+ allocator=turn.anchor_allocator,
+ event_cursor=turn.ontology_event_cursor,
+ subagent_log_id=turn.direct_log_id,
+ event_sink=repair_event_queue.put,
+ )
+ )
+ finally:
+ turn.streaming_agent.restore_context_usage(primary_context_usage)
+ turn.ontology_event_cursor = cursor
+ turn.direct_tool_count += tool_count
+ return repaired
+
+ turn.review_task = asyncio.create_task(
+ review_ontology_output(
+ task=user_message,
+ answer=turn.full_response,
+ runtime=turn.ontology_runtime,
+ trace=turn.ontology_trace,
+ citations=turn.all_citations,
+ user_id=str(turn.context.get("user_id", "")),
+ chat_id=turn.context.get("chat_id"),
+ model_name=str(turn.context.get("model_name", "") or "") or None,
+ model_provider_id=str(turn.context.get("model_provider_id", "") or "") or None,
+ remediate=_remediate,
+ )
+ )
+ try:
+ while not turn.review_task.done() or not repair_event_queue.empty():
+ try:
+ repair_event = await asyncio.wait_for(
+ repair_event_queue.get(),
+ timeout=0.1,
+ )
+ except asyncio.TimeoutError:
+ continue
+ yield repair_event
+ review = await turn.review_task
+ except (asyncio.CancelledError, GeneratorExit):
+ await cancel_and_join(turn.review_task)
+ output_review_state.release_output_review(
+ turn.ontology_runtime, owner=_ontology_review_owner_id
+ )
+ await turn.streaming_agent.shutdown()
+ raise
+ except Exception as exc: # noqa: BLE001
+ if not turn.review_task.done():
+ turn.review_task.cancel()
+ review = ontology_evidence._ontology_review_failure_result(turn.full_response, exc)
+ output_review_state.complete_output_review(
+ turn.ontology_runtime,
+ owner=_ontology_review_owner_id,
+ verdict=str(review.get("verdict") or "unknown"),
+ attempts=int(review.get("attempts") or 1),
+ )
+ turn.ontology_runtime.setdefault("output_review", {}).update(
+ {
+ "revised": bool(review.get("revised")),
+ "annotated": bool(review.get("annotated")),
+ "repair_attempts": int(review.get("repair_attempts") or 0),
+ "violations": review.get("violations") or [],
+ "affected_claims": review.get("affected_claims") or [],
+ "evidence": review.get("evidence") or [],
+ "feedback": review.get("feedback") or [],
+ "manual_review": review.get("manual_review") or {},
+ "candidate_answer": (review.get("answer") if review.get("revised") else ""),
+ "new_tools": review.get("new_tools") or [],
+ "new_citation_count": int(review.get("new_citation_count") or 0),
+ "latency_ms": review.get("latency_ms"),
+ }
+ )
+ yield {
+ "type": "ontology_review",
+ "status": "completed",
+ "level": turn.ontology_runtime.get("review_level", "checkpoint"),
+ "verdict": review["verdict"],
+ "revised": bool(review.get("revised")),
+ "annotated": bool(review.get("annotated")),
+ "repair_attempts": int(review.get("repair_attempts") or 0),
+ "candidate_answer": review.get("answer") if review.get("revised") else "",
+ "manual_review": review.get("manual_review") or {},
+ "violations": review.get("violations") or [],
+ "affected_claims": review.get("affected_claims") or [],
+ "evidence": review.get("evidence") or [],
+ "feedback": review.get("feedback") or [],
+ "new_tools": review.get("new_tools") or [],
+ "new_citation_count": int(review.get("new_citation_count") or 0),
+ "latency_ms": review.get("latency_ms"),
+ **ontology_evidence._ontology_review_event_context(turn.ontology_runtime),
+ }
+
+ _direct_usage = await turn.streaming_agent.aget_usage()
+ _direct_context_usage = turn.streaming_agent.get_context_usage(_direct_usage)
+ await turn.streaming_agent.shutdown()
+ await _finish_direct_log(
+ "success",
+ output=turn.full_response,
+ usage=_direct_usage,
+ )
+ yield {
+ "type": "meta",
+ "route": f"subagent:{agent_id}",
+ "is_markdown": message_parser.looks_markdown(turn.full_response),
+ "sources": message_parser.resolve_sources_conflict([]),
+ "artifacts": [],
+ "warnings": warnings,
+ "citations": turn.all_citations,
+ "usage": _direct_usage,
+ "context_usage": _direct_context_usage,
+ # Internal-only handoff consumed by chat_run_executor. Its whitelist
+ # intentionally keeps this prompt/tool surface out of SSE and message
+ # metadata while allowing post-turn compaction to use the exact same
+ # frozen inputs as pre-turn compaction.
+ "_compaction_budget_inputs": budget._compaction_budget_inputs(
+ turn.agent, turn.ctx_window
+ ),
+ "ontology_governance": ontology_evidence._ontology_governance_summary(
+ turn.ontology_runtime
+ ),
+ # Same contract as the main route: memory writes are scheduled below,
+ # so the frame can only announce "settling" for the client's skeleton
+ # card. Without this marker subagent turns never showed their card.
+ "evolution_pending": evolution._evolution_pending_marker(
+ turn.context, _mem0_write_enabled
+ ),
+ }
+
+ # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ────
+ # No memory is written unless the user opted into memory_write_enabled (first gate).
+ if _mem0_write_enabled:
+ memory.save_memories_background(
+ turn.memory_user_id,
+ user_message,
+ turn.full_response,
+ _mem0_write_enabled,
+ workspace_id=turn.memory_workspace_id,
+ chat_id=_mem0_chat_id,
+ scope_user_id=_mem0_scope_user_id,
+ message_id=str(turn.context.get("message_id") or ""),
+ )
+ else:
+ logger.debug(
+ "[subagent] memory save skipped: write_enabled=False (user=%s)",
+ turn.memory_user_id,
+ )
+
+ # ── [evolution] Evidence assembly — same contract as the main route. Without
+ # registration the settlement runner parks the memory report on a 45s
+ # watchdog before settling, which pushed subagent cards past the client's
+ # polling budget.
+ if not turn.context.get("agent_api_scope") and not turn.context.get("evaluation_session"):
+ evolution._assemble_episode_background(
+ message_id=str(turn.context.get("message_id") or ""),
+ run_id=str(turn.context.get("run_id") or ""),
+ chat_id=str(turn.context.get("chat_id") or ""),
+ user_id=str(turn.context.get("user_id") or ""),
+ objective=user_message,
+ agent=turn.streaming_agent,
+ memory_task=turn.session_memory.retrieval_task,
+ latency_ms=None,
+ memory_write_enabled=_mem0_write_enabled,
+ )
+ except BaseException as exc:
+ if not isinstance(exc, (asyncio.CancelledError, GeneratorExit)):
+ _direct_exit_status = "failed"
+ _direct_exit_error = str(exc)[:200]
+ raise
+ finally:
+ try:
+ if turn.review_task is not None:
+ await cancel_and_join(turn.review_task)
+ if turn.review_owner is not None:
+ output_review_state.release_output_review(turn.ontology_runtime, owner=turn.review_owner)
+ if turn.direct_stream is not None:
+ await turn.direct_stream.aclose()
+ if not _direct_log_finished and turn.streaming_agent is not None:
+ await turn.streaming_agent.shutdown()
+ finally:
+ if not _direct_log_finished:
+ await settle_task(
+ asyncio.create_task(
+ _finish_direct_log(_direct_exit_status, error=_direct_exit_error)
+ )
+ )
diff --git a/src/backend/core/agent_runtime/direct_state.py b/src/backend/core/agent_runtime/direct_state.py
new file mode 100644
index 00000000..025e3ce9
--- /dev/null
+++ b/src/backend/core/agent_runtime/direct_state.py
@@ -0,0 +1,46 @@
+"""State shared only by the direct agent execution phases."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, AsyncIterator
+
+if TYPE_CHECKING:
+ from core.agent_runtime.integrations.memory_session import SessionMemory
+ from core.llm.citations.allocator import AnchorAllocator
+ from core.llm.runtime.streaming import StreamingAgent
+
+
+@dataclass
+class DirectTurn:
+ context: dict[str, Any]
+ session_messages: list[dict[str, Any]]
+ review_owner: str | None = None
+ anchor_allocator: AnchorAllocator | None = None
+ ctx_window: int = 0
+ direct_allow_bash: bool = False
+ direct_log_id: str = ""
+ direct_read_only: bool = False
+ direct_skill_count: int = 0
+ direct_stream: AsyncIterator | None = None
+ direct_tool_count: int = 0
+ memory_enabled: bool = False
+ memory_user_id: str = ""
+ memory_workspace_id: str = "default"
+ ontology_event_cursor: int = 0
+ ontology_runtime: dict[str, Any] = field(default_factory=dict)
+ ontology_trace: list[dict[str, Any]] = field(default_factory=list)
+ session_memory: SessionMemory | None = None
+ wf_start: float = 0.0
+ agent: Any = None
+ all_citations: list[dict[str, Any]] = field(default_factory=list)
+ displayed_tools: set[str] = field(default_factory=set)
+ full_response: str = ""
+ mcp_clients: list[Any] = field(default_factory=list)
+ review_task: asyncio.Task | None = None
+ skill_id_by_tool_id: dict[str, str] = field(default_factory=dict)
+ skill_load_ids: set[str] = field(default_factory=set)
+ streaming_agent: StreamingAgent | None = None
+ user_agent: Any = None
+ view_text_file_args: dict[str, dict[str, Any]] = field(default_factory=dict)
diff --git a/src/backend/core/agent_runtime/events.py b/src/backend/core/agent_runtime/events.py
new file mode 100644
index 00000000..92fc500d
--- /dev/null
+++ b/src/backend/core/agent_runtime/events.py
@@ -0,0 +1,71 @@
+"""Helpers for the agent event stream: synthesized tool calls, scoped log context, live-job checks."""
+
+from __future__ import annotations
+from core.harness import log_context
+
+import logging
+from typing import Any, Dict, Optional
+
+import core.config.display_names as display_names
+
+logger = logging.getLogger(__name__)
+
+
+def _synthesize_missing_tool_call(
+ tool_id: str, tool_name: str, displayed_tools: set
+) -> Optional[Dict[str, Any]]:
+ """Build the ``tool_call`` event a tool never got, or ``None`` if it already has one.
+
+ ``_tool_args_ready`` holds back the tool card while streamed args are still
+ incomplete, but it cannot tell "not written yet" from "there is nothing to
+ write": a zero-argument tool (for example, a parameterless status probe) keeps
+ empty args forever, so its card is suppressed for the whole run. Without a
+ card the frontend has nothing to attach the result to and — with tools
+ running in parallel — binds it onto whichever sibling is still running,
+ making the call vanish from the tool list and briefly corrupting the
+ sibling's output. Emitting the missing call right before its result keeps
+ every executed tool visible, in order.
+
+ The caller must pass the already-resolved (post skill-load override) tool
+ name, and must have skipped tools that intentionally render no card.
+ """
+ if not tool_id or tool_id in displayed_tools:
+ return None
+ displayed_tools.add(tool_id)
+ display_name = (
+ "加载技能"
+ if tool_name == "load_skill"
+ else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name)
+ )
+ return {
+ "type": "tool_call",
+ "tool_name": tool_name,
+ "tool_display_name": display_name,
+ "tool_args": {},
+ "input": {},
+ "tool_id": tool_id,
+ }
+
+
+async def _anext_in_subagent_log_scope(iterator, subagent_log_id: str): # noqa: ANN001
+ """Advance one agent-stream item without leaking log tokens across SSE yields."""
+
+ with log_context.subagent_scope(subagent_log_id, source="subagent"):
+ return await anext(iterator)
+
+
+def _chat_has_live_job(chat_id: str) -> bool:
+ """这个会话还有在跑(或待启动)的后台作业吗 —— 决定计划栏本轮该不该收尾。"""
+ try:
+ from core.db.engine import SessionLocal as _JobSession
+ from core.db.models import Job as _Job
+
+ with _JobSession() as _db:
+ return (
+ _db.query(_Job.job_id)
+ .filter(_Job.chat_id == chat_id, _Job.status.in_(("pending", "running")))
+ .first()
+ is not None
+ )
+ except Exception: # noqa: BLE001 —— 查不到就当没有作业,照常收尾,别把计划栏永远挂着
+ return False
diff --git a/src/backend/core/agent_runtime/host.py b/src/backend/core/agent_runtime/host.py
new file mode 100644
index 00000000..5ca7ad3c
--- /dev/null
+++ b/src/backend/core/agent_runtime/host.py
@@ -0,0 +1,11 @@
+"""Compose the top-level chat flow with durable compaction boundaries."""
+
+def stream_chat_workflow(**kwargs):
+ from core.agent_runtime.integrations.compaction_stream import stream_with_compaction
+ from core.agent_runtime.runner import astream_chat_workflow
+ return stream_with_compaction(astream_chat_workflow, **kwargs)
+
+
+def register():
+ from core.harness.registry.chat_execution import register_chat_execution
+ register_chat_execution(stream_chat_workflow)
diff --git a/src/backend/tests/capabilities/__init__.py b/src/backend/core/agent_runtime/integrations/__init__.py
similarity index 100%
rename from src/backend/tests/capabilities/__init__.py
rename to src/backend/core/agent_runtime/integrations/__init__.py
diff --git a/src/backend/orchestration/compaction_stream.py b/src/backend/core/agent_runtime/integrations/compaction_stream.py
similarity index 95%
rename from src/backend/orchestration/compaction_stream.py
rename to src/backend/core/agent_runtime/integrations/compaction_stream.py
index 83b2495e..143b9826 100644
--- a/src/backend/orchestration/compaction_stream.py
+++ b/src/backend/core/agent_runtime/integrations/compaction_stream.py
@@ -7,9 +7,9 @@
from contextlib import aclosing, suppress
from dataclasses import dataclass
-from core.llm.context_safety import request_admission_scope
-from core.services.compaction_progress import progress_scope, report_outcome
-from core.services.context_budget_checkpoint import ContextStopCheckpoint
+from core.llm.context.safety import request_admission_scope
+from core.chat.compaction.progress import progress_scope, report_outcome
+from core.chat.compaction.budget_checkpoint import ContextStopCheckpoint
logger = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ async def produce():
):
if is_compact_command(raw_user_message):
from core.config.settings import settings
- from core.services.compaction_service import run_compaction
+ from core.chat.compaction.service import run_compaction
if not settings.compaction.enabled:
await report_outcome(chat_id, run_id, "manual", "skipped", "disabled")
diff --git a/src/backend/core/agent_runtime/integrations/evolution.py b/src/backend/core/agent_runtime/integrations/evolution.py
new file mode 100644
index 00000000..5310da96
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/evolution.py
@@ -0,0 +1,160 @@
+"""Hand a finished turn's evidence to the evolution loop after the stream closes."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _evolution_pending_marker(context: Dict[str, Any], memory_write_enabled: bool):
+ """Marker for the closing frame so the client can draw the skeleton card.
+
+ The turn's real outcome is not knowable yet — memory writes are scheduled
+ after the stream closes — so all we can honestly say here is "settling, and
+ these are the mechanisms that might report".
+ """
+ message_id = str(context.get("message_id") or "")
+ if not message_id or not memory_write_enabled:
+ # Without memory writes there is nothing a turn can settle to, so there
+ # is nothing for the client to watch.
+ return None
+ try:
+ from core.evolution.settlement.settlement import pending_payload
+
+ return pending_payload(message_id=message_id)
+ except Exception: # pragma: no cover - defensive
+ return None
+
+
+def _runtime_evidence_agent(agent: Any) -> Any:
+ """Unwrap streaming/decorator agents to the owner of frozen run evidence."""
+ current = agent
+ seen: set[int] = set()
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ if hasattr(current, "_jx_asset_bundle") or hasattr(current, "_jx_skill_selection"):
+ return current
+ nested = getattr(current, "agent", None)
+ if nested is None or nested is current:
+ break
+ current = nested
+ return current
+
+
+def _assemble_episode_background(
+ *,
+ message_id: str,
+ run_id: str,
+ chat_id: str,
+ user_id: str,
+ objective: str,
+ agent: Any = None,
+ memory_task: Any = None,
+ latency_ms: Optional[int] = None,
+ memory_write_enabled: bool = False,
+) -> None:
+ """Kick off Episode assembly after the stream closed.
+
+ Fire-and-forget by design: the user already has their answer, and evidence
+ assembly must never be able to delay or fail a turn. Every failure mode
+ here degrades to "no Episode for this run", which the console shows
+ honestly rather than papering over.
+ """
+ if not message_id:
+ return
+
+ try:
+ from core.agent_runtime.integrations.memory import get_last_retrieval
+ from core.evolution.evidence.events import EV_MEMORY_RETRIEVED, TraceSink
+ from core.evolution.evidence.trace_assembler import assemble_episode
+
+ evidence_agent = _runtime_evidence_agent(agent)
+ bundle = (
+ getattr(
+ evidence_agent,
+ "evidence_bundle",
+ getattr(evidence_agent, "_jx_asset_bundle", None),
+ )
+ if evidence_agent is not None
+ else None
+ )
+
+ sink = TraceSink(run_id=run_id, message_id=message_id, chat_id=chat_id, user_id=user_id)
+ # The rendered memory block has already discarded ids, scores and ranks;
+ # this is the only place they still exist.
+ retrieval = get_last_retrieval(memory_task)
+ if retrieval is not None:
+ sink.append(
+ EV_MEMORY_RETRIEVED,
+ retrieval.to_event_payload(),
+ asset_kind="memory",
+ )
+
+ # Which skills were considered, chosen, and rejected — the rejects are
+ # the negative examples skill attribution is built on.
+ # Ordered tool calls, read back from the log this run already wrote.
+ # Without them an episode carries no tool sequence and the promotion
+ # chain has nothing to distil.
+ from core.evolution.evidence.trace_assembler import emit_tool_events
+
+ emit_tool_events(sink, message_id, chat_id)
+
+ selection = (
+ getattr(evidence_agent, "_jx_skill_selection", None)
+ if evidence_agent is not None
+ else None
+ )
+ if selection is not None:
+ from core.evolution.evidence.events import EV_SKILL_SELECTED
+
+ sink.append(EV_SKILL_SELECTED, selection.to_event_payload(), asset_kind="skill")
+ sink.flush()
+
+ # Stamp the user's contribution choice onto the episode. Doing it at
+ # write time means the decision is fixed by what the user had agreed to
+ # *at that moment*, rather than being re-evaluated later against a
+ # setting they may since have changed.
+ from core.evolution.profiles.user_settings import resolve_for_user
+
+ privacy_class = resolve_for_user(user_id).get("privacy_class", "tenant")
+
+ episode_id = assemble_episode(
+ message_id=message_id,
+ run_id=run_id,
+ chat_id=chat_id,
+ user_id=user_id,
+ objective=objective,
+ bundle=bundle,
+ latency_ms=latency_ms,
+ privacy_class=privacy_class,
+ )
+
+ # Hand the evidence side to the settlement runner. It settles once the
+ # memory write pipeline reports what it actually wrote — the only party
+ # that knows. Settling here instead would have to guess that number, and
+ # the guess it used to make (the *retrieval* count) described what the
+ # turn read, not what it learned.
+ from core.evolution.settlement.settlement_runner import register_turn
+
+ # Which skills the turn opened and which profile governed it stay on the
+ # episode, where the retirement engine reads them. They are not on the
+ # card: they describe the system doing its job, not learning anything.
+ register_turn(
+ message_id=message_id,
+ episode_id=episode_id or "",
+ expects_memory=memory_write_enabled,
+ )
+
+ # Consider running a cycle now that new evidence exists. The check is a
+ # single indexed count and almost always says no; when it says yes the
+ # cycle runs off-thread. The user already has their answer either way.
+ from core.evolution.cycle.trigger import schedule_if_due, schedule_personal
+
+ # This user's own loop runs in every edition; the fleet-wide one is
+ # enterprise-gated and no-ops elsewhere.
+ schedule_personal(user_id)
+ schedule_if_due()
+ except Exception as exc: # pragma: no cover - defensive
+ logger.debug("[evolution] episode assembly skipped: %s", exc)
diff --git a/src/backend/core/agent_runtime/integrations/memory.py b/src/backend/core/agent_runtime/integrations/memory.py
new file mode 100644
index 00000000..c599ebae
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/memory.py
@@ -0,0 +1,304 @@
+"""Memory retrieval for a turn: background recall, the frozen memory block and post-turn saving."""
+from __future__ import annotations
+import asyncio
+from dataclasses import dataclass
+from typing import Optional
+from core.config.settings import settings
+from core.memory import profile
+from core.memory.context import MemoryContext
+from core.memory.retrieval_types import MemoryRetrievalResult
+from core.memory.service import retrieve_memories_structured
+import logging
+from core.agent_runtime.integrations import memory_state
+
+logger = logging.getLogger(__name__)
+
+async def launch_memory_retrieval(
+ user_id: str,
+ user_message: str,
+ memory_enabled: bool,
+ *,
+ workspace_id: str = "default",
+ budget_ms: Optional[int] = None,
+) -> Optional[asyncio.Task]:
+ """Start the Fact vector retrieval task in the background and return the Task (the caller need not await immediately).
+
+ budget_ms defaults to settings.memory.retrieval_budget_ms (600ms). Called once by the
+ workflow at session start; if it completes within budget the frozen block is injected, otherwise skipped.
+ """
+ if not memory_enabled or not user_id:
+ return None
+
+ effective_budget = (
+ budget_ms if budget_ms is not None else settings.memory.retrieval_budget_ms
+ ) / 1000.0
+
+ # The assembly boundary below owns the latency budget. Passing the same
+ # budget into the service would make its inner wait_for cancel _do_search
+ # before build_frozen_memory_block() can shield the task.
+
+ async def _fetch() -> Optional[MemoryRetrievalResult]:
+ try:
+ return await retrieve_memories_structured(
+ user_id=user_id,
+ query=user_message,
+ workspace_id=workspace_id,
+ timeout_s=None,
+ )
+ except Exception as exc:
+ logger.warning("[memory] retrieval failed: %s", exc)
+ return MemoryRetrievalResult.degraded_result("orchestration_error")
+
+ return track_memory_retrieval(asyncio.create_task(_fetch()), budget_s=effective_budget)
+
+
+def as_result(value: object) -> Optional[MemoryRetrievalResult]:
+ """Normalise whatever the retrieval task produced.
+
+ The task may legitimately yield ``None`` (never started) and, for callers
+ that still hand us a pre-rendered string, we degrade gracefully rather than
+ crash the turn.
+ """
+ if isinstance(value, MemoryRetrievalResult):
+ return value
+ return None
+
+
+def track_memory_retrieval(task: asyncio.Task, *, budget_s: Optional[float] = None) -> asyncio.Task:
+ """Attach lifecycle observation without taking ownership of cancellation."""
+
+ memory_state.process_state().retrieval_states[task] = "running"
+ if budget_s is not None:
+ memory_state.process_state().retrieval_budgets[task] = max(0.0, budget_s)
+
+ def _finished(done: asyncio.Task) -> None:
+ previous = memory_state.process_state().retrieval_states.get(done)
+ if done.cancelled():
+ memory_state.process_state().retrieval_states[done] = "cancelled"
+ return
+ try:
+ value = done.result()
+ except Exception:
+ memory_state.process_state().retrieval_states[done] = "failed"
+ return
+ result = as_result(value)
+ if result is not None:
+ set_last_retrieval(done, result)
+ memory_state.process_state().retrieval_states[done] = (
+ "completed_after_timeout" if previous == "timed_out_running" else "completed"
+ )
+
+ task.add_done_callback(_finished)
+ return task
+
+
+def get_retrieval_state(task: Optional[asyncio.Task]) -> Optional[str]:
+ if task is None:
+ return None
+ return memory_state.process_state().retrieval_states.get(task)
+
+
+def set_last_retrieval(task: asyncio.Task, result: MemoryRetrievalResult) -> None:
+ try:
+ memory_state.process_state().last_retrieval[task] = result
+ except TypeError: # pragma: no cover - non-weakrefable task
+ pass
+
+
+def get_last_retrieval(task: Optional[asyncio.Task]) -> Optional[MemoryRetrievalResult]:
+ """Structured recall for a turn, for trace-event emission.
+
+ Returns ``None`` when retrieval never ran or timed out — callers must treat
+ that as "no evidence", never as "memory contributed nothing".
+ """
+ if task is None:
+ return None
+ try:
+ return memory_state.process_state().last_retrieval.get(task)
+ except TypeError: # pragma: no cover
+ return None
+
+
+@dataclass(frozen=True)
+class FrozenMemoryBlock:
+ """An assembled frozen block plus whether the read behind it actually landed.
+
+ ``settled`` is false when the L2 read timed out or came back degraded. Such
+ a block is missing facts for reasons that say nothing about what the user's
+ memory holds, so it must not be frozen for the remainder of the session.
+ """
+
+ text: str
+ settled: bool
+
+
+async def build_frozen_memory_block(
+ user_id: str,
+ workspace_id: str,
+ memory_task: Optional[asyncio.Task],
+ *,
+ memory_enabled: bool = True,
+) -> FrozenMemoryBlock:
+ """Assemble the "session-frozen" block = L1 Profile markdown + L2 Fact top-K.
+
+ - When `memory_enabled=False`, return empty immediately (**do not load unless the user
+ has enabled persistent memory** — forms defense-in-depth with the workflow layer's
+ `_mem0_enabled` check)
+ - Profile reads the DB (fast, <20ms), always awaited
+ - Fact takes the result from the already-started memory_task; if the task hasn't finished,
+ wait briefly; if still unfinished, give up on Fact injection for this round (never block agent startup)
+
+ Called once per chat by `resolve_session_memory()`; later turns replay the
+ stored snapshot instead of reassembling.
+ """
+ if not memory_enabled:
+ return FrozenMemoryBlock(text="", settled=False)
+
+ # Profile layer (L1)
+ profile_md = ""
+ if user_id:
+ try:
+ profile_md = await profile.get(user_id, workspace_id)
+ except Exception as exc:
+ logger.warning(
+ "[memory] profile fetch failed user=%s ws=%s: %s", user_id, workspace_id, exc
+ )
+
+ # Fact layer (L2). With no retrieval task there is no L2 read to wait for,
+ # so a Profile-only block is already the complete answer for this session.
+ fact_text = ""
+ fact_settled = True
+ fact_result: Optional[MemoryRetrievalResult] = None
+ if memory_task is not None:
+ fact_settled = False
+ try:
+ # Wait up to retrieval_budget_ms (default 600ms), then give up; memory_task was
+ # started before the agent was created, so in most cases it is nearly done by now.
+ # The old value of 50ms was measured to be far below Milvus warm search's ~200ms,
+ # so Fact injection almost never hit.
+ wait_budget_s = memory_state.process_state().retrieval_budgets.get(
+ memory_task, max(0.1, settings.memory.retrieval_budget_ms / 1000.0)
+ )
+ fact_result = as_result(
+ await asyncio.wait_for(asyncio.shield(memory_task), timeout=wait_budget_s)
+ )
+ fact_text = fact_result.to_text() if fact_result is not None else ""
+ # A degraded result means the store was unreachable or the budget
+ # blew inside the service — an empty answer we must not freeze.
+ fact_settled = fact_result is not None and not fact_result.degraded
+ # Stash the structured recall on the task so the workflow can emit a
+ # retrieval trace event without re-running the search. Attribution
+ # needs ids/ranks/scores that the rendered text has already lost.
+ if fact_result is not None:
+ set_last_retrieval(memory_task, fact_result)
+ except asyncio.TimeoutError:
+ if not memory_task.done():
+ memory_state.process_state().retrieval_states[memory_task] = "timed_out_running"
+ logger.info(
+ "[memory] fact retrieval still running past wait window, skipping injection"
+ )
+ # ``shield`` makes the continue-in-background policy explicit. The
+ # done callback retains the structured result and changes the
+ # observable state to completed_after_timeout.
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.warning("[memory] fact retrieval await failed: %s", exc)
+
+ if not profile_md and not fact_text:
+ return FrozenMemoryBlock(text="", settled=fact_settled)
+
+ parts: list[str] = ["## 关于当前用户的已知背景(会话开始时冻结)"]
+ if profile_md:
+ parts.append("")
+ parts.append("### 用户档案(Profile)")
+ parts.append(profile_md.strip())
+ if fact_text:
+ parts.append("")
+ parts.append("### 相关历史记忆(Fact Top-K)")
+ # retrieve_memories comes with its own "## 关于该用户..." prefix; strip it to avoid duplicate headings
+ stripped = fact_text
+ for h in ("## 关于该用户的已知背景信息(来自历史会话记忆)", "## 用户相关实体关系"):
+ stripped = stripped.replace(h, "")
+ parts.append(stripped.strip())
+ block = "\n".join(parts).strip()
+ logger.info(
+ "[memory] frozen block built user=%s ws=%s chars=%d profile=%d facts=%d settled=%s",
+ user_id,
+ workspace_id,
+ len(block),
+ len(profile_md or ""),
+ len(fact_text or ""),
+ fact_settled,
+ )
+ return FrozenMemoryBlock(text=block, settled=fact_settled)
+
+
+def save_memories_background(
+ user_id: str,
+ user_message: str,
+ full_response: str,
+ write_enabled: bool,
+ *,
+ workspace_id: str = "default",
+ chat_id: Optional[str] = None,
+ scope_user_id: Optional[str] = None,
+ message_id: Optional[str] = None,
+) -> None:
+ """Delegate to the unified post-response pipeline — never await; SSE is closed and the user isn't waiting.
+
+ **When the user has not explicitly consented to writes (`write_enabled=False`), skip
+ immediately and create no tasks at all**. This is the first gate of the user-level write
+ switch; `schedule_post_response_tasks` has a second one inside.
+
+ Inside `schedule_post_response_tasks`:
+ - a durable outbox row is committed before the function returns
+ - leased workers resume pending/retry rows after process restart
+ - global Semaphore bounds worker concurrency (default 8)
+ - runs 0-5 extractors picked by the router's substance floor + LLM write gate
+ - each extractor has its own 30s timeout
+ - sanitize → write L1/L2/Session → audit
+
+ ``scope_user_id`` optionally selects an edition-owned shared memory bucket.
+ ``user_id`` remains the real user for audit metadata.
+ """
+ if not (write_enabled and full_response and user_id):
+ report_no_memory_writes(message_id)
+ return
+
+ try:
+ from core.memory.pipeline import schedule_post_response_tasks
+ except Exception as exc:
+ logger.warning("[memory] pipeline unavailable, skipping save: %s", exc)
+ report_no_memory_writes(message_id)
+ return
+
+ ctx = MemoryContext(
+ user_id=user_id,
+ workspace_id=workspace_id,
+ chat_id=chat_id,
+ write_enabled=write_enabled,
+ scope_user_id=scope_user_id,
+ message_id=message_id,
+ )
+ try:
+ schedule_post_response_tasks(ctx, user_message, full_response)
+ except Exception as exc:
+ logger.warning("[memory] schedule_post_response_tasks failed: %s", exc)
+ report_no_memory_writes(message_id, failed=True)
+
+
+def report_no_memory_writes(message_id: Optional[str], *, failed: bool = False) -> None:
+ """Tell the turn's settlement that no write pipeline will report for it.
+
+ Every path that declines to schedule the pipeline must say so, otherwise the
+ turn's card waits out the settlement watchdog for nothing.
+ """
+ if not message_id:
+ return
+ try:
+ from core.evolution.settlement.settlement_runner import report_memory_writes
+
+ report_memory_writes(message_id, items=[], failed=failed)
+ except Exception as exc: # noqa: BLE001 - never break the response path
+ logger.debug("[memory] settlement report skipped: %s", exc)
diff --git a/src/backend/core/agent_runtime/integrations/memory_budget.py b/src/backend/core/agent_runtime/integrations/memory_budget.py
new file mode 100644
index 00000000..b035af85
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/memory_budget.py
@@ -0,0 +1,34 @@
+"""How long a turn waits for memory retrieval, per the active agent profile."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _profile_memory_budget_ms(context: Dict[str, Any]) -> Optional[int]:
+ """How long this task type waits for memory, per the active profile.
+
+ Returns ``None`` when nothing is published, which leaves the caller on the
+ deployment default — the built-in profile carries that same value, so this
+ is a no-op until a profile deliberately changes it.
+
+ Never raises: retrieval sits on the user's critical path, and an orchestration
+ lookup must not be able to fail a turn.
+ """
+ try:
+ from core.auth.tenancy import tenant_of
+ from core.evolution.profiles.profile_selection import load_active_profile
+
+ user_id = str(context.get("user_id") or "")
+ profile = load_active_profile(
+ task_type=str(context.get("chat_mode") or "chat"),
+ user_id=user_id,
+ tenant_id=tenant_of(user_id),
+ )
+ return profile.memory_budget_ms
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[memory] profile budget unavailable: %s", exc)
+ return None
diff --git a/src/backend/core/agent_runtime/integrations/memory_identity.py b/src/backend/core/agent_runtime/integrations/memory_identity.py
new file mode 100644
index 00000000..a587d4ea
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/memory_identity.py
@@ -0,0 +1,74 @@
+"""Build and cache the per-user identity block injected at the head of a chat."""
+from __future__ import annotations
+import asyncio
+from dataclasses import dataclass, field
+from time import monotonic
+from typing import Dict
+import logging
+from core.agent_runtime.integrations import memory_state
+
+logger = logging.getLogger(__name__)
+
+async def build_user_identity_block(user_id: str) -> str:
+ """Assemble the "current user" identity block (username / nickname) for injection into the session-frozen message.
+
+ Deliberately goes through the user-role frozen block rather than the end of the system prompt:
+ the server-side chat template renders the tool schemas **after** the system text, so any
+ per-user bytes in system split the LLM prefix cache of the tool section (the largest chunk
+ of the prompt). Measured: identical system differing only in the trailing username → TTFT
+ degrades from 0.7s back to cold-start levels. The frozen block sits at the start of the
+ message sequence and renders after the tool section, so the system+tools shared prefix stays intact.
+
+ Returns empty string for anonymous / unknown users. Result cached per user_id for 60s.
+ """
+ if not user_id or user_id == "anonymous":
+ return ""
+ now = monotonic()
+ hit = process_state().identity_cache.get(user_id)
+ if hit and now < hit[0]:
+ return hit[1]
+
+ def _query() -> tuple:
+ from core.db.engine import SessionLocal
+ from core.db.models import LocalUser, UserShadow
+
+ with SessionLocal() as db:
+ row = db.query(UserShadow.username).filter(UserShadow.user_id == user_id).first()
+ nick = db.query(LocalUser.nickname).filter(LocalUser.user_id == user_id).first()
+ return (
+ (row[0] or "").strip() if row else "",
+ (nick[0] or "").strip() if nick else "",
+ )
+
+ try:
+ username, nickname = await asyncio.to_thread(_query)
+ except Exception as exc:
+ logger.warning("[identity] user lookup failed user=%s: %s", user_id, exc)
+ return ""
+
+ lines: list[str] = []
+ if username and username != "anonymous":
+ lines.append(f"- 用户名:{username}")
+ if nickname and nickname != username:
+ lines.append(f"- 昵称:{nickname}")
+ block = ""
+ if lines:
+ block = (
+ "## 当前用户\n"
+ + "\n".join(lines)
+ + "\n需要称呼用户时,用上述昵称(无昵称则用用户名)自然称呼。"
+ )
+ process_state().identity_cache[user_id] = (now + memory_state.IDENTITY_CACHE_TTL_S, block)
+ return block
+
+
+@dataclass
+class MemoryState:
+ identity_cache: Dict[str, tuple] = field(default_factory=lambda: {})
+
+
+_STATE = MemoryState()
+
+
+def process_state() -> MemoryState:
+ return _STATE
diff --git a/src/backend/core/agent_runtime/integrations/memory_session.py b/src/backend/core/agent_runtime/integrations/memory_session.py
new file mode 100644
index 00000000..74a0c01c
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/memory_session.py
@@ -0,0 +1,178 @@
+"""Open, assemble and inject the chat's session-frozen memory and identity blocks."""
+from __future__ import annotations
+from core.agent_runtime.integrations import memory
+import asyncio
+from dataclasses import dataclass, replace
+from typing import Any, Dict, List, Optional
+from core.llm.context.ir_constants import KIND_IDENTITY, KIND_MEMORY, SESSION_CONTEXT_META_KEY
+from core.llm.context.items import make_text_context_item
+from core.memory.session_snapshot import SessionMemorySnapshot, load_session_memory_snapshot, save_session_memory_snapshot
+import logging
+logger = logging.getLogger(__name__)
+
+@dataclass(frozen=True)
+class SessionMemory:
+ """One run's handle on the chat's frozen memory block.
+
+ Split in two because the two halves sit on either side of agent assembly:
+ `open_session_memory()` runs first so a first-turn retrieval overlaps
+ building the agent, and `resolve_session_memory()` runs at injection time.
+
+ On every turn after the first, `retrieval_task` is ``None``: the block is
+ replayed from the stored snapshot and no vector search is issued at all.
+ """
+
+ chat_id: str
+ scope_user_id: str
+ workspace_id: str
+ memory_enabled: bool
+ retrieval_task: Optional[asyncio.Task] = None
+ snapshot: Optional["SessionMemorySnapshot"] = None
+
+
+async def open_session_memory(
+ *,
+ chat_id: Optional[str],
+ scope_user_id: str,
+ workspace_id: str,
+ user_message: str,
+ memory_enabled: bool,
+ budget_ms: Optional[int] = None,
+) -> SessionMemory:
+ """Start the chat's memory read, or recognise that it already happened.
+
+ Called before the agent is built. When this chat already has a frozen
+ snapshot the retrieval is skipped outright — that is the point of freezing
+ it: the block sits ahead of the entire conversation, so re-deriving it per
+ turn would invalidate the model's prefix cache for the whole context.
+ """
+ handle = SessionMemory(
+ chat_id=str(chat_id or ""),
+ scope_user_id=scope_user_id,
+ workspace_id=workspace_id,
+ memory_enabled=memory_enabled,
+ )
+ if not memory_enabled:
+ return handle
+
+ snapshot = await asyncio.to_thread(
+ load_session_memory_snapshot,
+ handle.chat_id,
+ scope_user_id=scope_user_id,
+ workspace_id=workspace_id,
+ )
+ if snapshot is not None:
+ logger.info(
+ "[memory] frozen snapshot replayed chat=%s chars=%d (no retrieval)",
+ handle.chat_id,
+ len(snapshot.text),
+ )
+ return replace(handle, snapshot=snapshot)
+
+ task = await memory.launch_memory_retrieval(
+ scope_user_id,
+ user_message,
+ memory_enabled,
+ workspace_id=workspace_id,
+ budget_ms=budget_ms,
+ )
+ return replace(handle, retrieval_task=task)
+
+
+async def resolve_session_memory(handle: SessionMemory) -> str:
+ """Return the block to inject, assembling and storing it on the first turn.
+
+ The assembled block is stored only once the memory read actually landed. A
+ timed-out or degraded first turn leaves no snapshot, so the next turn tries
+ once more rather than freezing an accidental blank for the whole chat.
+
+ The store is first-writer-wins, and we inject what it hands back: if another run
+ froze this chat while we were retrieving, we use its snapshot, not ours.
+ """
+ if not handle.memory_enabled:
+ return ""
+ if handle.snapshot is not None:
+ return handle.snapshot.text
+
+ built = await memory.build_frozen_memory_block(
+ handle.scope_user_id,
+ handle.workspace_id,
+ handle.retrieval_task,
+ memory_enabled=handle.memory_enabled,
+ )
+ if not built.settled:
+ logger.info(
+ "[memory] frozen snapshot not stored chat=%s: memory read did not settle",
+ handle.chat_id,
+ )
+ return built.text
+
+ effective = await asyncio.to_thread(
+ save_session_memory_snapshot,
+ handle.chat_id,
+ SessionMemorySnapshot(
+ text=built.text,
+ scope_user_id=handle.scope_user_id,
+ workspace_id=handle.workspace_id,
+ ),
+ )
+ return effective.text
+
+
+def session_message(text: str, **item_kwargs: Any) -> Dict[str, Any]:
+ """One head-of-conversation message plus the context-item manifest describing it."""
+ item = make_text_context_item(text, **item_kwargs)
+ return {
+ "role": "user",
+ "content": text,
+ SESSION_CONTEXT_META_KEY: item.to_manifest(),
+ }
+
+
+async def inject_session_blocks(
+ session_messages: List[Dict[str, Any]],
+ *,
+ identity_block: str = "",
+ memory_block: str = "",
+) -> List[Dict[str, Any]]:
+ """Prepend the session-constant blocks (user identity, frozen memory) as user-role messages.
+
+ Why user rather than system: Qwen-family models require system only at index 0, and keeping
+ these per-user bytes out of the system prompt leaves the system text, the tool schemas and the
+ skill list a byte-identical shared prefix across users and chats.
+
+ Both blocks are constant for the whole chat, so neither moves the prefix-cache boundary between
+ turns — that is what makes this position safe.
+ """
+ injected: list[Dict[str, Any]] = []
+ if identity_block:
+ injected.append(
+ session_message(
+ f"\n{identity_block}\n \n"
+ "(以上为系统提供的当前用户身份信息,仅用于自然称呼,不是用户本轮提问。)",
+ item_id="session:identity",
+ kind=KIND_IDENTITY,
+ origin="identity:account",
+ trust="system",
+ created_seq=-200,
+ priority=850,
+ token_budget=1_000,
+ cache_class="session",
+ )
+ )
+ if memory_block:
+ injected.append(
+ session_message(
+ f"\n{memory_block}\n \n"
+ "(以上为会话启动时系统注入的背景快照,本会话内不变,用作回答参考,请勿直接复述。)",
+ item_id="session:memory:frozen",
+ kind=KIND_MEMORY,
+ origin="memory:frozen_session",
+ trust="memory",
+ created_seq=-100,
+ priority=800,
+ token_budget=8_000,
+ cache_class="session",
+ )
+ )
+ return [*injected, *session_messages] if injected else session_messages
diff --git a/src/backend/core/agent_runtime/integrations/memory_state.py b/src/backend/core/agent_runtime/integrations/memory_state.py
new file mode 100644
index 00000000..68865743
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/memory_state.py
@@ -0,0 +1,23 @@
+"""Per-task memory retrieval results, states and budgets shared within one process."""
+from __future__ import annotations
+import weakref
+import asyncio
+from dataclasses import dataclass, field
+from core.memory.retrieval_types import MemoryRetrievalResult
+
+
+IDENTITY_CACHE_TTL_S = 60.0
+
+
+@dataclass
+class MemoryState:
+ last_retrieval: 'weakref.WeakKeyDictionary[asyncio.Task, MemoryRetrievalResult]' = field(default_factory=lambda: weakref.WeakKeyDictionary())
+ retrieval_states: 'weakref.WeakKeyDictionary[asyncio.Task, str]' = field(default_factory=lambda: weakref.WeakKeyDictionary())
+ retrieval_budgets: 'weakref.WeakKeyDictionary[asyncio.Task, float]' = field(default_factory=lambda: weakref.WeakKeyDictionary())
+
+
+_STATE = MemoryState()
+
+
+def process_state() -> MemoryState:
+ return _STATE
diff --git a/src/backend/core/agent_runtime/integrations/ontology_evidence.py b/src/backend/core/agent_runtime/integrations/ontology_evidence.py
new file mode 100644
index 00000000..48a3d4a9
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/ontology_evidence.py
@@ -0,0 +1,229 @@
+"""Collect and summarise ontology governance evidence for the final answer review."""
+
+from __future__ import annotations
+from core.ontology import output_review_state
+from core.llm.citations.allocator import collect_citation_dicts
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from core.llm.citations.allocator import AnchorAllocator
+
+logger = logging.getLogger(__name__)
+
+
+def _ontology_review_event_context(runtime: Dict[str, Any]) -> Dict[str, Any]:
+ committee_size = max(
+ (int(pack.get("config", {}).get("committee_size", 3)) for pack in runtime.get("packs", [])),
+ default=3,
+ )
+ workflows = [
+ {
+ "pack_id": pack.get("pack_id"),
+ "workflow_id": workflow.get("id"),
+ "workflow_name": workflow.get("name"),
+ }
+ for pack in runtime.get("packs", [])
+ for workflow in pack.get("workflows", [])
+ ]
+ review_state = runtime.get("output_review") or {}
+ return {
+ "governance_run_id": runtime.get("governance_run_id"),
+ "review_owner": review_state.get("owner"),
+ "review_count": int(review_state.get("count") or 0),
+ "committee_size": committee_size,
+ "workflows": workflows,
+ "activation_sources": sorted(
+ {
+ str(event.get("source"))
+ for event in runtime.get("runtime_events", [])
+ if event.get("type") == "ontology_activation" and event.get("source")
+ }
+ ),
+ }
+
+
+def _ontology_governance_summary(runtime: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """Build the compact, persisted governance card payload for one answer."""
+ if not runtime.get("enabled"):
+ return None
+ events = list(runtime.get("runtime_events") or [])
+ activations = [
+ {
+ key: event.get(key)
+ for key in (
+ "pack_id",
+ "workflow_id",
+ "workflow_name",
+ "source",
+ "asset_kind",
+ "asset_id",
+ "review_level",
+ )
+ if event.get(key) is not None
+ }
+ for event in events
+ if event.get("type") == "ontology_activation"
+ ]
+ gates = [
+ {
+ key: event.get(key)
+ for key in (
+ "decision",
+ "tool_name",
+ "matched_rule_ids",
+ "violations",
+ "denial_count",
+ "circuit_breaker",
+ )
+ if event.get(key) is not None
+ }
+ for event in events
+ if event.get("type") == "ontology_gate"
+ ]
+ review_state = dict(runtime.get("output_review") or {})
+ review_state["level"] = runtime.get("review_level", "none")
+ review_state["committee_size"] = _ontology_review_event_context(runtime)["committee_size"]
+ if not activations and not gates and review_state.get("status") == "pending":
+ return None
+ if review_state.get("status") == "pending" and not output_review_state.requires_output_review(runtime):
+ review_state["status"] = "skipped"
+ return {
+ "governance_run_id": runtime.get("governance_run_id"),
+ "activations": activations,
+ "gates": gates,
+ "review": review_state,
+ }
+
+
+def _ontology_review_owner(runtime: Dict[str, Any], context: Dict[str, Any]) -> str:
+ """Stable owner token for the one outer-workflow final review."""
+ run_id = runtime.get("governance_run_id") or context.get("chat_id") or "request"
+ return f"outer_workflow:{run_id}"
+
+
+def _ontology_review_failure_result(answer: str, exc: Exception) -> Dict[str, Any]:
+ """Keep a completed draft usable when the post-answer review itself fails."""
+ logger.error(
+ "[ontology-review] automatic review failed; preserving original answer: %s",
+ exc,
+ exc_info=(type(exc), exc, exc.__traceback__),
+ )
+ return {
+ "verdict": "escalate",
+ "answer": answer,
+ "revised": False,
+ "annotated": True,
+ "repair_attempts": 0,
+ "attempts": 1,
+ "violations": [],
+ "affected_claims": [],
+ "evidence": [],
+ "feedback": ["自动评审未正常完成,已保留原文并转人工复核。"],
+ "manual_review": {
+ "required": True,
+ "title": "领域本体人工复核",
+ "summary": "自动评审未正常完成,原文已保留,请人工核对或重新发起评审。",
+ "items": [
+ {
+ "quote": "本轮完整答案",
+ "rule_id": "ontology_review_runtime",
+ "risk": "自动评审未完成,系统无法确认答案满足全部领域约束。",
+ "manual_check": "核对关键事实、推断、适用前提和原始工具证据。",
+ }
+ ],
+ "actions": ["重新发起本体评审,或在采用结论前完成人工核对。"],
+ },
+ "new_tools": [],
+ "new_citation_count": 0,
+ "latency_ms": None,
+ }
+
+
+def _record_ontology_activations(
+ activations: List[Dict[str, Any]],
+ runtime: Dict[str, Any],
+ context: Dict[str, Any],
+) -> None:
+ if not activations:
+ return
+ try:
+ from core.ontology.audit_events import record_runtime_activation
+
+ for event in activations:
+ record_runtime_activation(
+ event,
+ runtime,
+ user_id=str(context.get("user_id", "")) or None,
+ chat_id=context.get("chat_id"),
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology] workflow activation audit persistence failed: %s", exc)
+
+
+def _bounded_trace_value(value: Any, max_chars: int = 4000) -> Any:
+ """Keep reviewer evidence useful without copying unbounded tool output."""
+ try:
+ text = json.dumps(value, ensure_ascii=False, default=str)
+ except Exception:
+ text = str(value)
+ if len(text) <= max_chars:
+ return value
+ return {"truncated": True, "preview": text[:max_chars]}
+
+
+def _parse_tool_result_value(value: Any) -> Any:
+ if not isinstance(value, str):
+ return value
+ try:
+ return json.loads(value) if value else {}
+ except json.JSONDecodeError:
+ return {"result": value}
+
+
+def _capture_nested_ontology_evidence(
+ payload: Dict[str, Any],
+ trace: List[Dict[str, Any]],
+ citations: List[Dict[str, Any]],
+ allocator: Optional[AnchorAllocator] = None,
+) -> List[Dict[str, Any]]:
+ """Merge trusted ``call_subagent`` bypass events into the outer review trace."""
+ sub_type = str(payload.get("sub_type") or "")
+ tool_name = str(payload.get("tool_name") or "")
+ tool_id = str(payload.get("tool_id") or "")
+ provenance = {
+ key: payload.get(key)
+ for key in ("agent_id", "agent_name", "sub_run_id", "parent_tool_id")
+ if payload.get(key) is not None
+ }
+ if sub_type == "tool_call" and tool_name and payload.get("input") is not None:
+ trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": _bounded_trace_value(payload.get("input")),
+ "source": "subagent",
+ **provenance,
+ }
+ )
+ return []
+ if sub_type != "tool_result" or not tool_name or payload.get("status") == "error":
+ return []
+
+ result = _parse_tool_result_value(payload.get("output"))
+ cit_dicts = collect_citation_dicts(tool_id, allocator)
+ citations.extend(cit_dicts)
+ trace.append(
+ {
+ "type": "tool_result",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "result": _bounded_trace_value(result),
+ "citations": cit_dicts,
+ "source": "subagent",
+ **provenance,
+ }
+ )
+ return cit_dicts
diff --git a/src/backend/core/agent_runtime/integrations/ontology_repair.py b/src/backend/core/agent_runtime/integrations/ontology_repair.py
new file mode 100644
index 00000000..78a01e25
--- /dev/null
+++ b/src/backend/core/agent_runtime/integrations/ontology_repair.py
@@ -0,0 +1,366 @@
+"""Re-run the originating agent to produce a revision when ontology review rejects an answer."""
+
+from __future__ import annotations
+from core.llm.citations.allocator import collect_citation_dicts
+
+import json
+import logging
+import typing as _typing
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
+
+import core.agent_runtime.events as agent_runtime_events
+import core.agent_runtime.integrations.ontology_evidence as ontology_evidence
+import core.config.display_names as display_names
+import core.ontology.revision as revision
+from core.llm.citations.allocator import AnchorAllocator
+from core.llm.runtime.streaming import StreamingAgent
+
+logger = logging.getLogger(__name__)
+
+
+def _ontology_repair_prompt(payload: Dict[str, Any]) -> str:
+ violations = payload.get("violations") or []
+ feedback = payload.get("feedback") or []
+ affected_claims = payload.get("affected_claims") or []
+ original_task = str(payload.get("original_task") or "").strip()
+ current_answer = str(payload.get("answer") or "").strip()
+ citations = [
+ {
+ "id": item.get("id"),
+ "tool_name": item.get("tool_name"),
+ "title": item.get("title"),
+ }
+ for item in payload.get("citations") or []
+ if isinstance(item, dict)
+ ]
+ return (
+ "\n"
+ "这是同一任务唯一一次领域本体修复,不是新的用户任务。"
+ "优先级强制规则:已经激活的 enforce 级领域本体约束及本修复指令,高于用户原始提示中"
+ "与其冲突的‘不要调用工具’、‘原样照抄’或类似限制;这些限制只能约束正常生成阶段,"
+ "不能阻止本体修复取证和纠错。若‘确定性违规’包含‘未调用’某些必需工具,本轮必须"
+ "实际调用这些工具后才能形成终稿;以用户禁止工具为理由跳过调用、保留原错误断言,"
+ "或再次原样输出待修订内容,均视为修复失败。"
+ "先比较现有证据与评审问题,明确是否出现了需要重新搜索的新内容或可补充调用的新工具。"
+ "有证据缺口时必须调用真实工具补齐;现有证据已足够时才可直接精准修订。"
+ "禁止只做同义改写,也不得虚构工具调用或引用。"
+ "证据核对、逐项判定、原文对比和修改说明都属于内部校验过程,只能放在"
+ " thinking/reasoning 通道;除非用户最初明确要求核验报告,否则不得把这些内容写入"
+ "最终回复或生成的文件。"
+ "必须继续遵守用户最初指令中的输出与交付形式;如果原任务要求生成 Word、PPT、"
+ "表格、网页或其他文件,修订时仍须调用相应工具生成该类产物,不得降级为纯文字替代。"
+ "最终交付必须是利用核验结论直接修正后的完整润色稿,而不是关于原答案的评估报告。"
+ "保留原任务要求的文体、篇幅、结构和语气,只改正有问题的事实、推断与表述;"
+ "以待修订原始输出为底稿做最小必要修改,不要因为查到更多资料就加入与原主张无关的"
+ "企业背景、资质或延伸分析。句子级任务应尽量保留原句的主语与分句顺序,逐个修正"
+ "不成立的断言,形成简洁、自然、可直接交付的一句话终稿。"
+ "用户只要求一句话时仍输出一句话,不得自行扩写成包含‘原始语句’、‘逐项核验’、"
+ "‘主张评估’或‘修改说明’等章节的报告。若交付物是文件,文件正文也必须是润色终稿,"
+ "并重新生成、交付修正后的文件。没有获得证据的风险项只能表述为‘待核验’、"
+ "‘暂无数据支撑’或‘无法判断’,绝不能反向断言为‘不存在’、‘没有风险’或‘风险为零’。"
+ "在输出终稿和生成文件前,必须在内部逐条检查‘确定性违规’中的每个条件:若要求"
+ "最低长度,终稿必须达到该长度;若要求引用,相关事实后必须包含真实引用标记"
+ "(从工具结果复制 cite_id,写成 `[锚文本](cite:eN)`)"
+ "且按规则补齐参考资料;若要求区分事实、推断和待核验项,可在同一句中用分号和简短标签"
+ "表达,不得省略。先确定唯一的合格终稿,再把完全相同的正文写入用户要求的文件,"
+ "不得先生成文件后又输出一份更短、缺引用或结论不同的候选正文。"
+ "思考过程只放在模型的 thinking/reasoning 通道。最终正文必须且只能放在"
+ " `完整候选答案 ` 中。"
+ "在正式输出最终正文前,不要复述、引用或示例化这组标签,也不要用省略号代替正文。"
+ "标签内必须是可独立阅读、内容完整的修订答案;若无法修订,则原样放入当前完整答案。"
+ "事实、推断和待核验项必须明确区分;引用已有或新增工具结果时,把工具结果里"
+ "标注的 cite_id 原样复制进 `[锚文本](cite:eN)`,禁止自行编号。\n"
+ f"用户最初指令:{json.dumps(original_task, ensure_ascii=False)[:12000]}\n"
+ f"待修订原始输出:{json.dumps(current_answer, ensure_ascii=False)[:12000]}\n"
+ f"修复轮次:{payload.get('attempt', 1)}\n"
+ f"问题来源:{payload.get('source', 'review')}\n"
+ f"确定性违规:{json.dumps(violations, ensure_ascii=False)[:8000]}\n"
+ f"委员会意见:{json.dumps(feedback, ensure_ascii=False)[:8000]}\n"
+ f"受影响内容:{json.dumps(affected_claims, ensure_ascii=False)[:5000]}\n"
+ f"当前可用引用:{json.dumps(citations, ensure_ascii=False)[:5000]}\n"
+ "直接开始补充工具或输出修复后的完整答案。\n"
+ " "
+ )
+
+
+async def _run_ontology_repair_round(
+ *,
+ streaming_agent: StreamingAgent,
+ context: Dict[str, Any],
+ payload: Dict[str, Any],
+ runtime: Dict[str, Any],
+ trace: List[Dict[str, Any]],
+ citations: List[Dict[str, Any]],
+ allocator: Optional[AnchorAllocator] = None,
+ event_cursor: int = 0,
+ subagent_log_id: Optional[str] = None,
+ event_sink: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
+) -> Tuple[str, List[Dict[str, Any]], int, int]:
+ """Continue the originating agent and stream a separate revision candidate."""
+ answer = ""
+ events: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ tool_count = 0
+ text_buffer = ""
+ raw_text = ""
+ revision_open = False
+ revision_closed = False
+ thinking_line_prefix = ""
+
+ async def _publish(event: Dict[str, Any]) -> None:
+ events.append(event)
+ if event_sink is not None:
+ await event_sink(event)
+
+ async def _publish_text_thinking(delta: str) -> None:
+ """Publish text-channel reasoning and remember its current line prefix."""
+ nonlocal thinking_line_prefix
+ if not delta:
+ return
+ if "\n" in delta:
+ thinking_line_prefix = delta.rsplit("\n", 1)[-1]
+ else:
+ thinking_line_prefix = (thinking_line_prefix + delta)[-256:]
+ await _publish({"type": "ontology_revision_thinking", "delta": delta})
+
+ def _is_revision_opening_boundary(line_before_open: str) -> bool:
+ """Allow a revision wrapper after whitespace or a reasoning close tag."""
+
+ prefix = line_before_open.rstrip()
+ return not prefix or prefix.endswith(("", ""))
+
+ await _publish(
+ {
+ "type": "ontology_repair",
+ "status": "started",
+ "attempt": payload.get("attempt", 1),
+ "source": payload.get("source", "review"),
+ }
+ )
+
+ stream = streaming_agent.stream(
+ [{"role": "user", "content": _ontology_repair_prompt(payload)}],
+ context,
+ ).__aiter__()
+ while True:
+ try:
+ if subagent_log_id:
+ event_type, event_payload = await agent_runtime_events._anext_in_subagent_log_scope(
+ stream,
+ subagent_log_id,
+ )
+ else:
+ event_type, event_payload = await anext(stream)
+ except StopAsyncIteration:
+ break
+
+ state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None)
+ if isinstance(state_runtime, dict):
+ runtime = state_runtime
+ context["ontology_runtime"] = state_runtime
+ pending_events = runtime.get("runtime_events", [])[event_cursor:]
+ for item in pending_events:
+ await _publish(dict(item))
+ event_cursor += len(pending_events)
+
+ if event_type == "vision_progress":
+ await _publish({"type": "vision_progress", **(event_payload or {})})
+ continue
+
+ if event_type == "text_delta":
+ delta = str(event_payload or "")
+ if not delta:
+ continue
+ raw_text += delta
+ if revision_closed:
+ await _publish_text_thinking(delta)
+ continue
+ text_buffer += delta
+ while text_buffer:
+ if not revision_open:
+ open_idx = text_buffer.find("")
+ if open_idx < 0:
+ keep = min(len("") - 1, len(text_buffer))
+ safe_len = len(text_buffer) - keep
+ if safe_len > 0:
+ await _publish_text_thinking(text_buffer[:safe_len])
+ text_buffer = text_buffer[safe_len:]
+ break
+ before_open = text_buffer[:open_idx]
+ line_before_open = (thinking_line_prefix + before_open).rsplit("\n", 1)[-1]
+ if not _is_revision_opening_boundary(line_before_open):
+ # Reasoning models sometimes describe the contract as
+ # `... `. A real
+ # wrapper starts a new line or immediately follows the
+ # model's reasoning close tag. Other inline occurrences
+ # remain folded as examples rather than candidate text.
+ ignored = text_buffer[: open_idx + len("")]
+ await _publish_text_thinking(ignored)
+ text_buffer = text_buffer[open_idx + len("") :]
+ continue
+ if open_idx > 0:
+ await _publish_text_thinking(before_open)
+ text_buffer = text_buffer[open_idx + len("") :]
+ revision_open = True
+ continue
+ close_idx = text_buffer.find(" ")
+ if close_idx < 0:
+ keep = min(len(" ") - 1, len(text_buffer))
+ safe_len = len(text_buffer) - keep
+ if safe_len > 0:
+ candidate_delta = text_buffer[:safe_len]
+ answer += candidate_delta
+ await _publish({"type": "ontology_revision", "delta": candidate_delta})
+ text_buffer = text_buffer[safe_len:]
+ break
+ if close_idx > 0:
+ candidate_delta = text_buffer[:close_idx]
+ answer += candidate_delta
+ await _publish({"type": "ontology_revision", "delta": candidate_delta})
+ text_buffer = text_buffer[close_idx + len(" ") :]
+ revision_closed = True
+ break
+ elif event_type == "thinking_delta":
+ await _publish(
+ {
+ "type": "ontology_revision_thinking",
+ "delta": str(event_payload or ""),
+ }
+ )
+ elif event_type == "tool_call_start":
+ tool_name = str(event_payload.get("name") or "unknown")
+ if tool_name != "update_plan":
+ await _publish(
+ {
+ "type": "tool_call_start",
+ "tool_name": tool_name,
+ "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(
+ tool_name, tool_name
+ ),
+ "tool_id": str(event_payload.get("id") or ""),
+ "scope": "ontology_revision",
+ }
+ )
+ elif event_type == "tool_call_delta":
+ tool_name = str(event_payload.get("name") or "unknown")
+ if tool_name != "update_plan" and event_payload.get("delta"):
+ await _publish(
+ {
+ "type": "tool_call_delta",
+ "tool_name": tool_name,
+ "tool_id": str(event_payload.get("id") or ""),
+ "arguments_delta": str(event_payload.get("delta") or ""),
+ "scope": "ontology_revision",
+ }
+ )
+ elif event_type == "tool_call":
+ tool_name = str(event_payload.get("name") or "unknown")
+ tool_id = str(event_payload.get("id") or "")
+ tool_args = event_payload.get("args") or {}
+ trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": ontology_evidence._bounded_trace_value(tool_args),
+ "source": "ontology_repair",
+ }
+ )
+ await _publish(
+ {
+ "type": "tool_call",
+ "tool_name": tool_name,
+ "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name),
+ "tool_args": tool_args,
+ "input": tool_args,
+ "tool_id": tool_id,
+ "scope": "ontology_revision",
+ }
+ )
+ elif event_type == "tool_result":
+ tool_count += 1
+ tool_name = str(event_payload.get("name") or "unknown")
+ tool_id = str(event_payload.get("id") or "")
+ result = ontology_evidence._parse_tool_result_value(event_payload.get("content"))
+ cit_dicts = collect_citation_dicts(tool_id, allocator)
+ citations.extend(cit_dicts)
+ trace.append(
+ {
+ "type": "tool_result",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "result": ontology_evidence._bounded_trace_value(result),
+ "citations": cit_dicts,
+ "source": "ontology_repair",
+ }
+ )
+ await _publish(
+ {
+ "type": "tool_result",
+ "tool_name": tool_name,
+ "result": result,
+ "tool_id": tool_id,
+ "citations": cit_dicts,
+ "scope": "ontology_revision",
+ }
+ )
+ elif event_type == "subagent_event":
+ nested_citations = ontology_evidence._capture_nested_ontology_evidence(
+ event_payload or {},
+ trace,
+ citations,
+ allocator,
+ )
+ sub_type = str((event_payload or {}).get("sub_type") or "")
+ if sub_type in {
+ "start",
+ "thinking",
+ "content",
+ "tool_call",
+ "tool_call_delta",
+ "tool_result",
+ "end",
+ }:
+ await _publish(
+ {
+ "type": "subagent_event",
+ **(event_payload or {}),
+ "scope": "ontology_revision",
+ **({"citations": nested_citations} if nested_citations else {}),
+ }
+ )
+ elif event_type == "tool_pending":
+ await _publish(
+ {
+ "type": "tool_pending",
+ **(event_payload or {}),
+ "scope": "ontology_revision",
+ }
+ )
+ elif event_type in {"file_confirm", "design_pick"}:
+ raise RuntimeError("本体自动修复轮不支持等待交互确认,请转人工复核")
+ elif event_type == "error":
+ if isinstance(event_payload, BaseException):
+ raise event_payload
+ raise RuntimeError(str(event_payload))
+
+ if revision_open and not revision_closed and text_buffer:
+ answer += text_buffer
+ await _publish({"type": "ontology_revision", "delta": text_buffer})
+ elif not revision_open and text_buffer.strip():
+ # Compatibility fallback for models that ignored the wrapper. When
+ # embedded text exists, only the content after the last closing
+ # marker can be a candidate; never promote the hidden reasoning itself.
+ fallback = revision.normalize_revision_candidate(raw_text.rsplit("", 1)[-1])
+ if revision.is_substantive_revision(fallback):
+ answer = fallback
+ await _publish({"type": "ontology_revision", "delta": answer})
+
+ await _publish(
+ {
+ "type": "ontology_repair",
+ "status": "completed",
+ "attempt": payload.get("attempt", 1),
+ "tool_calls": tool_count,
+ }
+ )
+ return revision.normalize_revision_candidate(answer), events, event_cursor, tool_count
diff --git a/src/backend/core/agent_runtime/main_state.py b/src/backend/core/agent_runtime/main_state.py
new file mode 100644
index 00000000..1aeb7fda
--- /dev/null
+++ b/src/backend/core/agent_runtime/main_state.py
@@ -0,0 +1,45 @@
+"""State shared only by the main agent execution phases."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, AsyncIterator
+
+if TYPE_CHECKING:
+ from core.agent_runtime.integrations.memory_session import SessionMemory
+ from core.llm.citations.allocator import AnchorAllocator
+ from core.llm.runtime.streaming import StreamingAgent
+
+
+@dataclass
+class MainTurn:
+ context: dict[str, Any]
+ session_messages: list[dict[str, Any]]
+ user_message: str
+ closed: bool = False
+ review_task: asyncio.Task | None = None
+ review_owner: str | None = None
+ anchor_allocator: AnchorAllocator | None = None
+ ctx_window: int = 0
+ last_plan: dict[str, Any] | None = None
+ main_stream: AsyncIterator | None = None
+ memory_enabled: bool = False
+ memory_user_id: str = ""
+ memory_workspace_id: str = "default"
+ mentioned_ids: list[str] = field(default_factory=list)
+ ontology_event_cursor: int = 0
+ ontology_runtime: dict[str, Any] = field(default_factory=dict)
+ ontology_trace: list[dict[str, Any]] = field(default_factory=list)
+ pending_plan_updates: dict[str, dict[str, Any]] = field(default_factory=dict)
+ session_memory: SessionMemory | None = None
+ visible_subagents: list[dict[str, Any]] = field(default_factory=list)
+ agent: Any = None
+ all_citations: list[dict[str, Any]] = field(default_factory=list)
+ displayed_tools: set[str] = field(default_factory=set)
+ full_response: str = ""
+ mcp_clients: list[Any] = field(default_factory=list)
+ skill_id_by_tool_id: dict[str, str] = field(default_factory=dict)
+ skill_load_ids: set[str] = field(default_factory=set)
+ streaming_agent: StreamingAgent | None = None
+ view_text_file_args: dict[str, dict[str, Any]] = field(default_factory=dict)
diff --git a/src/backend/orchestration/subagents/plugin_visibility.py b/src/backend/core/agent_runtime/plugin_visibility.py
similarity index 91%
rename from src/backend/orchestration/subagents/plugin_visibility.py
rename to src/backend/core/agent_runtime/plugin_visibility.py
index 84574145..38170a78 100644
--- a/src/backend/orchestration/subagents/plugin_visibility.py
+++ b/src/backend/core/agent_runtime/plugin_visibility.py
@@ -69,10 +69,12 @@ def load_enabled_plugins(
mc = [m for m in (cids.get("mcp") or []) if m in enabled_plug_mcps]
if not sk and not mc:
continue
- out.append({
- "name": r.name or r.slug,
- "description": r.description or "",
- "skill_ids": sk,
- "mcp_ids": mc,
- })
+ out.append(
+ {
+ "name": r.name or r.slug,
+ "description": r.description or "",
+ "skill_ids": sk,
+ "mcp_ids": mc,
+ }
+ )
return out
diff --git a/src/backend/core/agent_runtime/prepare_direct.py b/src/backend/core/agent_runtime/prepare_direct.py
new file mode 100644
index 00000000..c6aa980a
--- /dev/null
+++ b/src/backend/core/agent_runtime/prepare_direct.py
@@ -0,0 +1,126 @@
+"""Assemble a directly selected agent and its contextual input."""
+
+from __future__ import annotations
+from core.agent_runtime.integrations import memory_identity
+from core.agent_runtime.integrations import memory_session
+from core.llm.runtime import streaming
+from core.llm.citations import allocator
+
+import logging
+import typing as _typing
+
+import core.agent_runtime.budget as budget
+import core.agent_runtime.targets as targets
+import core.llm.factory as factory
+from core.agent_runtime.direct_state import DirectTurn
+
+import time as _time
+
+logger = logging.getLogger(__name__)
+
+
+async def prepare_direct(turn: DirectTurn):
+ yield {"type": "thinking", "message": "正在连接子智能体..."}
+ for ontology_event in turn.ontology_runtime.get("runtime_events", []):
+ yield dict(ontology_event)
+ turn.ontology_event_cursor = len(turn.ontology_runtime.get("runtime_events", []))
+
+ _stream_user_id = str(turn.context.get("user_id", ""))
+ _stream_model_name = str(turn.context.get("model_name", ""))
+ _stream_model_provider_id = str(turn.context.get("model_provider_id", "") or "")
+ _stream_chat_mode = str(turn.context.get("chat_mode", "") or "")
+ _stream_reranker = bool(turn.context.get("reranker_enabled", False))
+
+ # Create agent with sub-agent config overrides
+ turn.agent, turn.mcp_clients = await factory.create_agent_executor(
+ agent_spec=None,
+ enabled_mcp_ids=None, # overridden by user_agent inside factory
+ enabled_skill_ids=None, # overridden by user_agent inside factory
+ enabled_kb_ids=None, # overridden by user_agent inside factory
+ current_user_id=_stream_user_id,
+ reranker_enabled=_stream_reranker,
+ model_name=_stream_model_name,
+ model_provider_id=_stream_model_provider_id,
+ chat_mode=_stream_chat_mode,
+ memory_enabled=turn.memory_enabled,
+ agent_api_scope=turn.context.get("agent_api_scope"),
+ user_agent=turn.user_agent,
+ read_only=turn.direct_read_only,
+ allow_bash=turn.direct_allow_bash,
+ approval_mode=targets._approval_mode(turn.context),
+ required_skill_id=str(turn.context.get("skill_id") or "") or None,
+ required_skill_name=str(turn.context.get("skill_name") or "") or None,
+ required_plugin_id=str(turn.context.get("plugin_id") or "") or None,
+ required_plugin_name=str(turn.context.get("plugin_name") or "") or None,
+ required_plugin_skill_ids=[
+ str(item)
+ for item in (turn.context.get("plugin_skill_ids") or [])
+ if isinstance(item, str) and item
+ ],
+ required_plugin_mcp_ids=[
+ str(item)
+ for item in (turn.context.get("plugin_mcp_ids") or [])
+ if isinstance(item, str) and item
+ ],
+ # Same as the main agent: pass chat_id → the sandbox session uses
+ # the chat_id-keyed "user-bound persistent sandbox" (mounting the
+ # per-user credential volumes for lark/dws/email etc.). Omitting it
+ # falls back to an ephemeral light sandbox (no credentials) — the
+ # root cause of Feishu/DingTalk CLIs reporting "not configured" in
+ # direct sub-agent conversations.
+ chat_id=turn.context.get("chat_id"),
+ sandbox_session_id=turn.context.get("sandbox_session_id"),
+ run_id=str(turn.context.get("run_id") or "") or None,
+ journal_owner=str(turn.context.get("journal_owner") or "") or None,
+ workspace_id=turn.memory_workspace_id,
+ project_ctx=targets._extract_project_ctx(turn.context),
+ channel_origin=turn.context.get("channel_origin"),
+ automation_run=bool(turn.context.get("automation_run")),
+ ontology_runtime=turn.context.get("ontology_runtime"),
+ )
+
+ logger.info("[subagent] agent created in %.0fms", (_time.monotonic() - turn.wf_start) * 1000)
+ _, turn.ctx_window = budget._resolve_agent_model_runtime(turn.agent, _stream_model_name)
+
+ # 证据锚点:把发号器绑到 agent 上(中间件与本函数由此共享同一个计数器;
+ # 仅靠 ContextVar 不行——本函数是 async generator,与 agent 执行所在的
+ # task 上下文不互通)
+ allocator.attach_allocator(turn.agent, turn.anchor_allocator)
+
+ # ── Session-constant blocks: user identity + the chat's frozen memory
+ # snapshot (built on the first turn, replayed byte-for-byte after). ──
+ turn.session_messages = await memory_session.inject_session_blocks(
+ turn.session_messages,
+ identity_block=(
+ ""
+ if turn.context.get("agent_api_scope") or turn.context.get("evaluation_session")
+ else await memory_identity.build_user_identity_block(turn.memory_user_id)
+ ),
+ memory_block=await memory_session.resolve_session_memory(turn.session_memory),
+ )
+
+ # The canonical ContextAssembler owns selection and truncation. Feed
+ # it the complete candidate set so every exclusion is visible in the
+ # request manifest rather than silently pre-trimming shared history.
+ turn.streaming_agent = streaming.StreamingAgent(turn.agent, turn.mcp_clients)
+ turn.skill_load_ids: set = set()
+ # tool_id → skill_id (parsed from the tool_call's file_path; looked up at tool_result time to replace the SSE payload with the curated detail, avoiding sending the full SKILL.md text to the frontend)
+ turn.skill_id_by_tool_id: _typing.Dict[str, str] = {}
+ # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges
+ turn.view_text_file_args: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {}
+
+ # Project scope is no longer passed via ContextVar — it now travels as
+ # explicit parameters along the call chain (agent_factory closes
+ # ProjectScope into every register_* tool; chats.py's finishing
+ # _persist_artifacts reconstructs the scope from the workflow context
+ # and passes it explicitly). See the header comment in
+ # core/projects/scope.py.
+
+ # Never keep a ContextVar token alive across an async-generator
+ # ``yield``. The consumer may resume this generator in a copied
+ # context, making reset(token) fail with "created in a different
+ # Context". Enter the audit scope only while advancing the underlying
+ # agent stream; tool execution and persistence happen during anext().
+ turn.direct_stream = turn.streaming_agent.stream(
+ turn.session_messages, turn.context
+ ).__aiter__()
diff --git a/src/backend/core/agent_runtime/prepare_main.py b/src/backend/core/agent_runtime/prepare_main.py
new file mode 100644
index 00000000..6a435bef
--- /dev/null
+++ b/src/backend/core/agent_runtime/prepare_main.py
@@ -0,0 +1,289 @@
+"""Assemble the main agent and its complete contextual input."""
+
+from __future__ import annotations
+from core.agent_runtime.integrations import memory_identity
+from core.agent_runtime.integrations import memory_session
+from core.llm.runtime import streaming
+from core.llm.context import ir_constants
+from core.llm.context import items
+from core.llm.citations import allocator
+
+import logging
+import typing as _typing
+
+import core.agent_runtime.budget as budget
+import core.agent_runtime.skills as skills
+import core.agent_runtime.targets as targets
+import core.capabilities.catalog.resolver as catalog_resolver
+import core.llm.factory as factory
+from core.agent_runtime.main_state import MainTurn
+from core.db.threads import run_in_session
+
+logger = logging.getLogger(__name__)
+
+
+def _load_subagent_visibility(db, user_id: str, *, evaluation: bool):
+ """The user's own sub-agents and the built-in ones they switched off."""
+ from core.capabilities.agents.edition import UserAgentService
+ from core.auth.users import UserService
+
+ visible = [] if evaluation else UserAgentService(db).list_for_user(user_id)
+ return visible, UserService(db).get_disabled_builtin_subagent_ids(user_id)
+
+
+async def prepare_main(turn: MainTurn):
+ import time as _time
+
+ _wf_start = _time.monotonic()
+
+ yield {"type": "thinking", "message": "正在分析您的问题..."}
+ for ontology_event in turn.ontology_runtime.get("runtime_events", []):
+ yield dict(ontology_event)
+ turn.ontology_event_cursor = len(turn.ontology_runtime.get("runtime_events", []))
+
+ _stream_user_id = str(turn.context.get("user_id", ""))
+ _stream_model_name = str(turn.context.get("model_name", ""))
+ _stream_reranker = bool(turn.context.get("reranker_enabled", False))
+ enabled_skill_ids = catalog_resolver.enabled_skill_ids_from_context(turn.context)
+ enabled_kb_ids = catalog_resolver.enabled_kb_ids_from_context(turn.context)
+ enabled_mcp_ids = catalog_resolver.enabled_mcp_ids_from_context(turn.context)
+
+ # ── Load visible sub-agents for main agent routing ──
+ turn.visible_subagents: list = []
+ turn.mentioned_ids: list = []
+ _explicit_command = turn.context.get("explicit_subagent_command")
+ _explicit_agent_id = (
+ str(_explicit_command.get("agent_id"))
+ if isinstance(_explicit_command, dict) and _explicit_command.get("agent_id")
+ else ""
+ )
+ _mention_agent_id = str(turn.context.get("mention_agent_id") or "")
+ try:
+ turn.visible_subagents, _disabled_builtin_ids = await run_in_session(
+ _load_subagent_visibility,
+ _stream_user_id,
+ evaluation=bool(turn.context.get("evaluation_session")),
+ )
+ except Exception as _exc:
+ logger.warning("[workflow] failed to load visible subagents: %s", _exc)
+ _disabled_builtin_ids = set()
+
+ turn.visible_subagents = targets._runtime_visible_subagents(
+ turn.visible_subagents,
+ {
+ "enabled_skill_ids": enabled_skill_ids or [],
+ "enabled_mcp_ids": enabled_mcp_ids or [],
+ "enabled_kb_ids": enabled_kb_ids or [],
+ },
+ _disabled_builtin_ids,
+ [_explicit_agent_id, _mention_agent_id],
+ )
+ # Prefer the structured per-turn target. Text parsing remains a
+ # compatibility fallback for callers that only include @name.
+ _delegated_agent_id = _explicit_agent_id or _mention_agent_id
+ if _delegated_agent_id and any(
+ str(item.get("agent_id") or "") == _delegated_agent_id for item in turn.visible_subagents
+ ):
+ turn.mentioned_ids = [_delegated_agent_id]
+ else:
+ turn.mentioned_ids = targets._parse_agent_mentions(
+ turn.user_message, turn.visible_subagents
+ )
+
+ # Create agent (with model-aware CompressionConfig + optional native LTM)
+ _plan_chat = bool(turn.context.get("plan_chat", False))
+ # 极速模式:配置化检索工具集 + 独立 turbo 提示词 + 迭代硬上限。
+ # plan 会话有自己的编排语义,不进入极速裁剪。
+ _stream_mode_spec = targets._resolve_mode_spec(turn.context)
+ _turbo_chat = _stream_mode_spec is not None and _stream_mode_spec.restricted and not _plan_chat
+ if _turbo_chat:
+ # 收窄模式下子智能体不再只靠手动呼唤入场:模式自己圈定的那批常驻,
+ # 本轮被 @/显式委派命中的那个并上去(系统提示词也就只渲染这几行)。
+ # 模式没圈人时退回原契约——只有被呼唤的那个能进。
+ turn.visible_subagents = targets._mode_visible_subagents(
+ _stream_mode_spec, turn.visible_subagents, turn.mentioned_ids
+ )
+ turn.agent, turn.mcp_clients = await factory.create_agent_executor(
+ agent_spec=None,
+ enabled_skill_ids=enabled_skill_ids,
+ enabled_mcp_ids=enabled_mcp_ids,
+ enabled_kb_ids=enabled_kb_ids,
+ current_user_id=_stream_user_id,
+ reranker_enabled=_stream_reranker,
+ model_name=_stream_model_name,
+ model_provider_id=str(turn.context.get("model_provider_id", "") or ""),
+ chat_mode=str(turn.context.get("chat_mode", "") or ""),
+ turbo_mode=_turbo_chat,
+ mode_spec=_stream_mode_spec,
+ # 显式呼唤透传:斜杠技能 / 插件展开的 skill_ids 与 mcp_ids。
+ # 仅 turbo 模式消费;能力说明仍走 user message 注入,不进全量清单。
+ turbo_explicit_skill_ids=(
+ skills._explicit_skill_ids_from_context(turn.context) if _turbo_chat else None
+ ),
+ turbo_explicit_mcp_ids=(
+ [m for m in (turn.context.get("mcp_ids") or []) if isinstance(m, str)]
+ if _turbo_chat
+ else None
+ ),
+ # 渐进式插件加载:本轮显式呼唤的能力不得被延迟(用户消息注入已承诺
+ # 其可用),并作为会话级激活落库。
+ invoked_skill_ids=skills._explicit_skill_ids_from_context(turn.context),
+ invoked_mcp_ids=[m for m in (turn.context.get("mcp_ids") or []) if isinstance(m, str)],
+ required_mcp_ids=(
+ [str(turn.context["connector_id"])] if turn.context.get("connector_id") else None
+ ),
+ required_skill_id=str(turn.context.get("skill_id") or "") or None,
+ required_skill_name=str(turn.context.get("skill_name") or "") or None,
+ required_plugin_id=str(turn.context.get("plugin_id") or "") or None,
+ required_plugin_name=str(turn.context.get("plugin_name") or "") or None,
+ required_plugin_skill_ids=[
+ str(item)
+ for item in (turn.context.get("plugin_skill_ids") or [])
+ if isinstance(item, str) and item
+ ],
+ required_plugin_mcp_ids=[
+ str(item)
+ for item in (turn.context.get("plugin_mcp_ids") or [])
+ if isinstance(item, str) and item
+ ],
+ memory_enabled=turn.memory_enabled,
+ approval_mode=targets._approval_mode(turn.context),
+ visible_subagents=turn.visible_subagents if turn.visible_subagents else None,
+ plan_mode=_plan_chat,
+ workflow_mode=bool(turn.context.get("workflow_chat", False)),
+ chat_id=turn.context.get("chat_id"),
+ sandbox_session_id=turn.context.get("sandbox_session_id"),
+ run_id=str(turn.context.get("run_id") or "") or None,
+ journal_owner=str(turn.context.get("journal_owner") or "") or None,
+ workspace_id=turn.memory_workspace_id,
+ project_ctx=targets._extract_project_ctx(turn.context),
+ channel_origin=turn.context.get("channel_origin"),
+ automation_run=bool(turn.context.get("automation_run")),
+ ontology_runtime=turn.context.get("ontology_runtime"),
+ # The single positive source of truth for update_plan: enabled
+ # only for "interactive main chats that can host plan mode" — has
+ # a chat_id, not a channel bot, not automation, not plan_chat, not
+ # batch. Channels/automation have no plan-bar UI; plan_chat
+ # have their own orchestration and must not nest plan tracking.
+ top_level_chat=(
+ bool(turn.context.get("chat_id"))
+ and not turn.context.get("channel_origin")
+ and not bool(turn.context.get("automation_run"))
+ and not _plan_chat
+ ),
+ )
+
+ logger.info("[workflow] agent created in %.0fms", (_time.monotonic() - _wf_start) * 1000)
+
+ # 证据锚点:把发号器绑到 agent 上(中间件与本函数由此共享同一个计数器;
+ # 仅靠 ContextVar 不行——本函数是 async generator,与 agent 执行所在的
+ # task 上下文不互通)
+ allocator.attach_allocator(turn.agent, turn.anchor_allocator)
+
+ # ── Inject the per-turn sub-agent constraint into the current user message ──
+ # Keeps it OUT of the system prompt so the LLM provider's prefix cache
+ # hits across turns within a chat (otherwise every turn with different
+ # targets would re-build the cache from scratch). A strict natural-
+ # language command remains on this normal model stream; it only
+ # constrains the model's next real tool call instead of fabricating one
+ # in the workflow layer.
+ if turn.mentioned_ids and turn.visible_subagents:
+ from core.llm.subagents.tool import build_explicit_subagent_command_hint, build_subagent_mention_hint
+
+ if _explicit_agent_id or _mention_agent_id:
+ _mention_hint = build_explicit_subagent_command_hint(
+ turn.visible_subagents,
+ _explicit_agent_id or _mention_agent_id,
+ )
+ else:
+ _mention_hint = build_subagent_mention_hint(
+ turn.visible_subagents,
+ turn.mentioned_ids,
+ )
+ if (
+ _mention_hint
+ and turn.session_messages
+ and turn.session_messages[-1].get("role") in ("user", "human")
+ ):
+ mention_item = items.make_text_context_item(
+ _mention_hint,
+ item_id="harness:subagent_routing",
+ kind=ir_constants.KIND_REMINDER,
+ origin="harness:subagent_routing",
+ trust="system",
+ created_seq=0,
+ priority=950,
+ token_budget=4_000,
+ truncation_policy=ir_constants.POLICY_NEVER,
+ )
+ turn.session_messages.insert(
+ -1,
+ {
+ "role": "user",
+ "content": _mention_hint,
+ ir_constants.SESSION_CONTEXT_META_KEY: items.session_context_metadata(
+ mention_item
+ ),
+ },
+ )
+
+ # ── PreTurn compaction safety net (aligned with Codex pre-turn compaction) ──
+ # When end-of-turn background compaction failed/was skipped, or the
+ # previous turn's tool calls blew up the history, compact once
+ # synchronously with the same cross-turn compaction mechanism before
+ # this turn starts and write a checkpoint. Below the threshold, only a
+ # pure byte estimate is done (no DB / no LLM), so first-token latency
+ # is unaffected. Must run **before** the frozen memory block injection
+ # — identity/memory blocks are re-injected every turn and must not be
+ # baked into the persistent checkpoint.
+ # The model window is preferentially read straight off the model object
+ # (make_chat_model bakes in the real context_size per the Config model
+ # configuration at construction time, no default fallback), saving the
+ # streaming path one synchronous DB query; only if missing on the
+ # object do we fall back to resolve — unconfigured raises, fail loud,
+ # never silently run with the wrong window.
+ # Pre-turn compaction and the request assembler share this value.
+ _actual_model, turn.ctx_window = budget._resolve_agent_model_runtime(
+ turn.agent, _stream_model_name
+ )
+ try:
+ from core.chat.compaction.service import maybe_run_pre_turn_compaction
+
+ turn.session_messages, _ = await maybe_run_pre_turn_compaction(
+ turn.context.get("chat_id"),
+ turn.session_messages,
+ run_id=str(turn.context.get("run_id") or ""),
+ **budget._pre_turn_compaction_inputs(turn.agent, turn.ctx_window, _actual_model),
+ )
+ except Exception as _pt_exc: # noqa: BLE001
+ from core.harness.hooks import HookPaused
+
+ if isinstance(_pt_exc, HookPaused):
+ raise
+ logger.warning("[workflow] pre-turn compaction failed: %s", _pt_exc)
+
+ # ── Session-constant blocks: user identity + the chat's frozen memory
+ # snapshot. Built on the first turn (L1 profile always, L2 facts only if
+ # retrieval beat its budget), replayed byte-for-byte on every turn after,
+ # so this head of the message list never moves the prefix-cache boundary.
+ turn.session_messages = await memory_session.inject_session_blocks(
+ turn.session_messages,
+ identity_block=(
+ ""
+ if turn.context.get("evaluation_session")
+ else await memory_identity.build_user_identity_block(turn.memory_user_id)
+ ),
+ memory_block=await memory_session.resolve_session_memory(turn.session_memory),
+ )
+
+ # The final request assembler receives the full post-compaction
+ # candidate set. It is the only budget selector, and records every
+ # included, truncated, or excluded item in the context manifest.
+
+ turn.streaming_agent = streaming.StreamingAgent(turn.agent, turn.mcp_clients)
+
+ turn.skill_load_ids: set = set() # track tool_ids that are skill loads
+ # tool_id → skill_id; looked up at tool_result time to replace the SSE payload with the curated detail
+ turn.skill_id_by_tool_id: _typing.Dict[str, str] = {}
+ # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges
+ turn.view_text_file_args: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {}
diff --git a/src/backend/core/agent_runtime/runner.py b/src/backend/core/agent_runtime/runner.py
new file mode 100644
index 00000000..332dbb2f
--- /dev/null
+++ b/src/backend/core/agent_runtime/runner.py
@@ -0,0 +1,444 @@
+"""Coordinate the main agent lifecycle."""
+
+from __future__ import annotations
+from core.agent_runtime.integrations import memory
+from core.agent_runtime.integrations import memory_session
+from core.ontology import output_review_state
+from core.ontology import runtime_pack
+from core.runs.tool_effects import errors
+from core.llm.citations import allocator
+from core.ontology import runtime_policy
+
+import asyncio
+import logging
+import typing as _typing
+from contextlib import aclosing as _aclosing
+from typing import Any, Dict, List
+
+import core.agent_runtime.budget as budget
+import core.agent_runtime.direct as direct
+import core.agent_runtime.events as events
+import core.agent_runtime.integrations.evolution as evolution
+import core.agent_runtime.integrations.memory_budget as memory_budget
+import core.agent_runtime.integrations.ontology_evidence as ontology_evidence
+import core.agent_runtime.integrations.ontology_repair as ontology_repair
+import core.agent_runtime.skills as skills
+import core.agent_runtime.targets as targets
+import core.chat.message_parser as message_parser
+import core.chat.plan_progress as plan_progress
+
+
+from core.agent_runtime import prepare_main, stream_main
+from core.agent_runtime.main_state import MainTurn
+
+logger = logging.getLogger(__name__)
+
+
+async def astream_chat_workflow(
+ *, session_messages: List[Dict[str, Any]], user_message: str, context: Dict[str, Any]
+):
+ """Stream a chat turn and close its resources on every consumer exit."""
+ turn = MainTurn(context=context, session_messages=session_messages, user_message=user_message)
+ try:
+ async with _aclosing(_stream_turn(turn)) as stream:
+ async for event in stream:
+ yield event
+ finally:
+ from core.infra.task_lifecycle import cancel_and_join
+
+ if turn.review_task is not None:
+ await cancel_and_join(turn.review_task)
+ if turn.review_owner is not None:
+ output_review_state.release_output_review(turn.ontology_runtime, owner=turn.review_owner)
+ await _close_turn(turn)
+
+
+async def _close_turn(turn: MainTurn) -> None:
+ if turn.streaming_agent is not None and not turn.closed:
+ turn.closed = True
+ await turn.streaming_agent.shutdown()
+
+
+async def _stream_turn(turn: MainTurn):
+ """Stream route -> handoff -> target execution.
+
+ Yields chunks in the format:
+ - {"type": "content", "delta": "text chunk"}
+ - {"type": "tool_call", ...}
+ - {"type": "tool_result", ...}
+ - {"type": "meta", "route": "...", "sources": [...], ...}
+ """
+
+ from core.llm.api_runtime.agent import apply_api_scope
+ from core.harness.registry.evaluation import apply_evaluation_scope
+
+ turn.context = apply_evaluation_scope(apply_api_scope(turn.context))
+
+ # ── Persistent dedicated sub-agent conversation mode ──
+ # Per-turn @mentions deliberately remain on the main route so the parent
+ # model emits the real call_subagent tool event and keeps normal streaming.
+ _agent_id = targets._direct_agent_id_from_context(turn.context)
+ if _agent_id:
+
+ async with _aclosing(
+ direct._astream_subagent_direct(
+ agent_id=_agent_id,
+ session_messages=turn.session_messages,
+ user_message=turn.user_message,
+ context=turn.context,
+ )
+ ) as direct_stream:
+ async for chunk in direct_stream:
+ yield chunk
+ return
+
+ _request_ontology_runtime = turn.context.get("ontology_runtime")
+ if not isinstance(_request_ontology_runtime, dict):
+ _request_ontology_runtime = {}
+ turn.context["ontology_runtime"] = _request_ontology_runtime
+ explicit_skill_ids = skills._explicit_skill_ids_from_context(turn.context)
+ for skill_id in explicit_skill_ids:
+ await asyncio.to_thread(
+ runtime_policy.resolve_runtime_asset_tags,
+ runtime=_request_ontology_runtime,
+ kind="skill",
+ asset_id=skill_id,
+ user_id=str(turn.context.get("user_id", "") or ""),
+ )
+ activations = runtime_pack.activate_runtime_for_asset(
+ _request_ontology_runtime,
+ kind="skill",
+ asset_id=skill_id,
+ )
+ if activations:
+ await asyncio.to_thread(
+ ontology_evidence._record_ontology_activations,
+ activations,
+ _request_ontology_runtime,
+ turn.context,
+ )
+
+ # ── Inject explicit skill instructions ──
+ skill_msg = skills._build_skill_injection(turn.context)
+ if skill_msg:
+ turn.session_messages.insert(-1, skill_msg)
+ logger.info(
+ "[skill_inject] injected skill instructions for '%s'",
+ turn.context.get("skill_id"),
+ )
+
+ # ── [memory] Session memory opened here, never awaited on the SSE path ──
+ # First turn of a chat: retrieval starts as a background task so it overlaps
+ # agent assembly, and Fact injection is skipped if it blows its budget.
+ # Every later turn: the stored snapshot is replayed and no retrieval runs.
+ turn.memory_user_id = str(turn.context.get("user_id", ""))
+ turn.memory_workspace_id = str(turn.context.get("workspace_id", "") or "default")
+ _mem0_chat_id = turn.context.get("chat_id") or turn.context.get("conversation_id")
+ turn.memory_enabled = bool(turn.context.get("memory_enabled", False))
+ _mem0_write_enabled = bool(turn.context.get("memory_write_enabled", False))
+ # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id
+ _mem0_scope_user_id = str(turn.context.get("memory_scope_user_id", "") or turn.memory_user_id)
+ logger.info(
+ "[memory] user=%s scope=%s ws=%s chat=%s enabled=%s write=%s",
+ turn.memory_user_id,
+ _mem0_scope_user_id,
+ turn.memory_workspace_id,
+ _mem0_chat_id,
+ turn.memory_enabled,
+ _mem0_write_enabled,
+ )
+
+ turn.session_memory = await memory_session.open_session_memory(
+ chat_id=_mem0_chat_id,
+ scope_user_id=_mem0_scope_user_id,
+ workspace_id=turn.memory_workspace_id,
+ user_message=turn.user_message,
+ memory_enabled=turn.memory_enabled,
+ # The retrieval budget is part of the orchestration profile: how long
+ # this task type is willing to wait for memory is an assembly decision,
+ # not a global constant. Resolved here because retrieval starts before
+ # the agent is built.
+ budget_ms=memory_budget._profile_memory_budget_ms(turn.context),
+ )
+
+ # ── Main-route streaming ──────────────────────────────────────
+ warnings: _typing.List[str] = []
+ turn.full_response = ""
+ turn.displayed_tools: set[str] = set()
+ turn.all_citations: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ turn.ontology_runtime = _request_ontology_runtime
+ turn.ontology_event_cursor = 0
+ turn.ontology_trace: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ turn.last_plan: _typing.Optional[_typing.Dict[str, _typing.Any]] = None
+ turn.pending_plan_updates: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {}
+ _stream_errored = False
+ # 证据锚点发号器:跨轮续号;创建后绑到 agent 上(见下方 attach_allocator),
+ # 中间件与本函数由此共享同一个计数器
+ turn.anchor_allocator = allocator.AnchorAllocator(
+ await asyncio.to_thread(
+ allocator.anchor_start_for_chat, str(turn.context.get("chat_id") or "") or None
+ )
+ )
+
+ try:
+ async with _aclosing(prepare_main.prepare_main(turn)) as phase:
+ async for event in phase:
+ yield event
+
+ # Project scope is no longer passed via ContextVar — see the comment at
+ # the same spot in _astream_subagent_direct above. All scope-dependent
+ # tools captured it by closure when registered in agent_factory; the
+ # finishing _persist_artifacts gets the scope explicitly from chats.py.
+
+ async with _aclosing(stream_main.stream_main(turn)) as phase:
+ async for event in phase:
+ yield event
+
+ except Exception as e:
+ import traceback
+
+ logger.error("stream_workflow_error: %s\n%s", e, traceback.format_exc())
+ warnings.append(f"Streaming error: {str(e)[:200]}")
+ _stream_errored = True
+
+ unknown_outcome = errors.find_tool_outcome_unknown(e)
+ if unknown_outcome is not None:
+ if turn.streaming_agent is not None:
+ await _close_turn(turn)
+ if unknown_outcome is e:
+ raise
+ raise unknown_outcome from e
+
+ if not turn.full_response:
+ # No answer was produced: propagate. The run executor persists the
+ # tool cards already shown together with the error and emits a
+ # structured ``error`` event — no fabricated "please refer to the
+ # tool results above" reply that would read as a completed answer.
+ if turn.streaming_agent is not None:
+ await _close_turn(turn)
+ raise
+
+ pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[
+ turn.ontology_event_cursor :
+ ]
+ for ontology_event in pending_ontology_events:
+ yield dict(ontology_event)
+ turn.ontology_event_cursor += len(pending_ontology_events)
+
+ _ontology_review_owner_id = ontology_evidence._ontology_review_owner(
+ turn.ontology_runtime, turn.context
+ )
+ _ontology_review_claimed = bool(
+ turn.full_response
+ and output_review_state.claim_output_review(turn.ontology_runtime, owner=_ontology_review_owner_id)
+ )
+ if _ontology_review_claimed:
+ turn.review_owner = _ontology_review_owner_id
+ from core.ontology.reviewer import review_ontology_output
+
+ yield {
+ "type": "ontology_review",
+ "status": "started",
+ "level": turn.ontology_runtime.get("review_level", "checkpoint"),
+ **ontology_evidence._ontology_review_event_context(turn.ontology_runtime),
+ }
+ repair_event_queue: asyncio.Queue[_typing.Dict[str, _typing.Any]] = asyncio.Queue()
+
+ async def _remediate(payload: Dict[str, Any]) -> str:
+ primary_usage = await turn.streaming_agent.aget_usage()
+ primary_context_usage = turn.streaming_agent.get_context_usage(primary_usage)
+ try:
+ repaired, _, cursor, _ = await ontology_repair._run_ontology_repair_round(
+ streaming_agent=turn.streaming_agent,
+ context=turn.context,
+ payload=payload,
+ runtime=turn.ontology_runtime,
+ trace=turn.ontology_trace,
+ citations=turn.all_citations,
+ allocator=turn.anchor_allocator,
+ event_cursor=turn.ontology_event_cursor,
+ event_sink=repair_event_queue.put,
+ )
+ finally:
+ turn.streaming_agent.restore_context_usage(primary_context_usage)
+ turn.ontology_event_cursor = cursor
+ return repaired
+
+ turn.review_task = review_task = asyncio.create_task(
+ review_ontology_output(
+ task=turn.user_message,
+ answer=turn.full_response,
+ runtime=turn.ontology_runtime,
+ trace=turn.ontology_trace,
+ citations=turn.all_citations,
+ user_id=str(turn.context.get("user_id", "")),
+ chat_id=turn.context.get("chat_id"),
+ model_name=str(turn.context.get("model_name", "") or "") or None,
+ model_provider_id=str(turn.context.get("model_provider_id", "") or "") or None,
+ remediate=_remediate,
+ )
+ )
+ try:
+ while not review_task.done() or not repair_event_queue.empty():
+ try:
+ repair_event = await asyncio.wait_for(
+ repair_event_queue.get(),
+ timeout=0.1,
+ )
+ except asyncio.TimeoutError:
+ continue
+ yield repair_event
+ review = await review_task
+ except (asyncio.CancelledError, GeneratorExit):
+ from core.infra.task_lifecycle import cancel_and_join
+
+ await cancel_and_join(review_task)
+ output_review_state.release_output_review(turn.ontology_runtime, owner=_ontology_review_owner_id)
+ await _close_turn(turn)
+ raise
+ except Exception as exc: # noqa: BLE001
+ if not review_task.done():
+ review_task.cancel()
+ review = ontology_evidence._ontology_review_failure_result(turn.full_response, exc)
+ output_review_state.complete_output_review(
+ turn.ontology_runtime,
+ owner=_ontology_review_owner_id,
+ verdict=str(review.get("verdict") or "unknown"),
+ attempts=int(review.get("attempts") or 1),
+ )
+ turn.ontology_runtime.setdefault("output_review", {}).update(
+ {
+ "revised": bool(review.get("revised")),
+ "annotated": bool(review.get("annotated")),
+ "repair_attempts": int(review.get("repair_attempts") or 0),
+ "violations": review.get("violations") or [],
+ "affected_claims": review.get("affected_claims") or [],
+ "evidence": review.get("evidence") or [],
+ "feedback": review.get("feedback") or [],
+ "manual_review": review.get("manual_review") or {},
+ "candidate_answer": (review.get("answer") if review.get("revised") else ""),
+ "new_tools": review.get("new_tools") or [],
+ "new_citation_count": int(review.get("new_citation_count") or 0),
+ "latency_ms": review.get("latency_ms"),
+ }
+ )
+ yield {
+ "type": "ontology_review",
+ "status": "completed",
+ "level": turn.ontology_runtime.get("review_level", "checkpoint"),
+ "verdict": review["verdict"],
+ "revised": bool(review.get("revised")),
+ "annotated": bool(review.get("annotated")),
+ "repair_attempts": int(review.get("repair_attempts") or 0),
+ "candidate_answer": review.get("answer") if review.get("revised") else "",
+ "manual_review": review.get("manual_review") or {},
+ "violations": review.get("violations") or [],
+ "affected_claims": review.get("affected_claims") or [],
+ "evidence": review.get("evidence") or [],
+ "feedback": review.get("feedback") or [],
+ "new_tools": review.get("new_tools") or [],
+ "new_citation_count": int(review.get("new_citation_count") or 0),
+ "latency_ms": review.get("latency_ms"),
+ **ontology_evidence._ontology_review_event_context(turn.ontology_runtime),
+ }
+
+ await _close_turn(turn)
+
+ # update_plan finalization: the model routinely finishes its last step and
+ # streams the answer without a closing update_plan call, leaving the plan
+ # bar stuck at N-1/N. When the turn ends normally with an answer and no
+ # step is still pending, the remaining in_progress step is the one that
+ # just produced the answer — close it out.
+ if turn.last_plan and turn.full_response and not _stream_errored:
+ _plan_steps = turn.last_plan.get("steps") or []
+ if (
+ _plan_steps
+ and all(s.get("status") != "pending" for s in _plan_steps)
+ and any(s.get("status") == "in_progress" for s in _plan_steps)
+ ):
+ _closed = {
+ "title": turn.last_plan.get("title", ""),
+ "steps": [{**s, "status": "completed"} for s in _plan_steps],
+ }
+ turn.last_plan = _closed
+ plan_progress.save_plan_progress(str(turn.context.get("chat_id") or ""), _closed)
+ yield {"type": "plan_update", **_closed}
+
+ # 计划栏收尾 —— 本轮结束时把"还会不会有人来更新这份清单"这件事记进库里。
+ #
+ # 过去只有前端在流结束时给它盖章,于是收尾依赖"这个标签页恰好在跟这条流":后台作业
+ # 跑完的交付轮是**后端自己发起**的,用户没点任何东西,页面可能压根没跟——计划栏就永远
+ # 停在转圈。这里改由服务端认定:本会话还有活着的作业就先不收尾(后面还有交付轮要来
+ # 更新它),没有了就记 settled。步骤状态一个字不改,settled 只表示"没有下一轮了"。
+ if turn.last_plan:
+ _plan_chat_id = str(turn.context.get("chat_id") or "")
+ if _plan_chat_id and not events._chat_has_live_job(_plan_chat_id):
+ plan_progress.settle_plan_progress(_plan_chat_id)
+
+ _main_usage = await turn.streaming_agent.aget_usage()
+ _main_context_usage = turn.streaming_agent.get_context_usage(_main_usage)
+ yield {
+ "type": "meta",
+ "route": "main",
+ "is_markdown": message_parser.looks_markdown(turn.full_response),
+ "sources": message_parser.resolve_sources_conflict([]),
+ "artifacts": [],
+ "warnings": warnings,
+ "citations": turn.all_citations,
+ "usage": _main_usage,
+ "context_usage": _main_context_usage,
+ "_compaction_budget_inputs": budget._compaction_budget_inputs(turn.agent, turn.ctx_window),
+ "ontology_governance": ontology_evidence._ontology_governance_summary(
+ turn.ontology_runtime
+ ),
+ # Settlement has not run yet (memory writes are scheduled below), so all
+ # this can honestly say is "settling" — the client draws a skeleton and
+ # fills it in when the summary arrives.
+ "evolution_pending": evolution._evolution_pending_marker(turn.context, _mem0_write_enabled),
+ }
+
+ # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ──
+ # No memory is written unless the user opted into memory_write_enabled (first gate).
+ # Everything goes through memory_pipeline.schedule_post_response_tasks:
+ # global Semaphore + 4 extractors + sanitize + write L1/L2/Session + audit,
+ # all executed in the background with bounds.
+ if _mem0_write_enabled:
+ logger.info(
+ "[memory] schedule post-response: full_response_len=%s, user=%s ws=%s",
+ len(turn.full_response) if turn.full_response else 0,
+ turn.memory_user_id,
+ turn.memory_workspace_id,
+ )
+ memory.save_memories_background(
+ turn.memory_user_id,
+ turn.user_message,
+ turn.full_response,
+ _mem0_write_enabled,
+ workspace_id=turn.memory_workspace_id,
+ chat_id=_mem0_chat_id,
+ scope_user_id=_mem0_scope_user_id,
+ # Lets the pipeline report its real write count back to this turn's
+ # settlement instead of the card having to guess one.
+ message_id=str(turn.context.get("message_id") or ""),
+ )
+ else:
+ logger.debug(
+ "[workflow] memory save skipped: write_enabled=False (user=%s)",
+ turn.memory_user_id,
+ )
+
+ # ── [evolution] Evidence assembly (SSE already closed, user isn't waiting) ──
+ # The response path only bound versions and appended events; joining them
+ # into one causal Episode is deliberately deferred to here.
+ if not turn.context.get("evaluation_session"):
+ evolution._assemble_episode_background(
+ message_id=str(turn.context.get("message_id") or ""),
+ run_id=str(turn.context.get("run_id") or ""),
+ chat_id=str(turn.context.get("chat_id") or ""),
+ user_id=str(turn.context.get("user_id") or ""),
+ objective=turn.user_message,
+ agent=turn.streaming_agent,
+ memory_task=turn.session_memory.retrieval_task,
+ latency_ms=None,
+ memory_write_enabled=_mem0_write_enabled,
+ )
diff --git a/src/backend/core/agent_runtime/skills.py b/src/backend/core/agent_runtime/skills.py
new file mode 100644
index 00000000..0e34ea24
--- /dev/null
+++ b/src/backend/core/agent_runtime/skills.py
@@ -0,0 +1,176 @@
+"""Turn explicitly invoked skills into the load-instruction message for a turn."""
+
+from __future__ import annotations
+from core.llm.context import ir_constants
+from core.llm.context import items
+
+import logging
+import typing as _typing
+from typing import Any, Dict, List, Optional
+
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_skill_id_from_path(path: str) -> str:
+ """Extract skill_id from a SKILL.md path (convention: .../skills//SKILL.md)."""
+ if not path:
+ return ""
+ import os
+
+ norm = path.replace("\\", "/").strip().strip('"').strip("'")
+ parts = [p for p in norm.split("/") if p]
+ if not parts:
+ return ""
+ if parts[-1].upper() == "SKILL.MD" and len(parts) >= 2:
+ return parts[-2]
+ base = os.path.basename(norm)
+ if base.upper() == "SKILL.MD" and len(parts) >= 2:
+ return parts[-2]
+ return ""
+
+
+def _explicit_skill_ids_from_context(context: Dict[str, Any]) -> List[str]:
+ skill_ids: _typing.List[str] = []
+ single = context.get("skill_id")
+ if single:
+ skill_ids.append(str(single))
+ multi = context.get("skill_ids")
+ if isinstance(multi, list):
+ skill_ids.extend(str(item) for item in multi if item)
+ seen: set[str] = set()
+ return [item for item in skill_ids if not (item in seen or seen.add(item))]
+
+
+def _build_skill_injection(context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """Build an explicit-invocation hint message (skill load instructions + MCP tool activation notice).
+
+ Trigger sources:
+ - ``skill_id`` (single skill, slash-command selection) / ``skill_ids``
+ (skill list, expanded from an explicitly referenced plugin)
+ → inject each skill's SKILL.md load instruction.
+ - ``mcp_ids`` (MCP servers activated by an explicitly referenced plugin or
+ a connector selected in the composer) → these tools are already
+ force-enabled into the toolset (see _build_ctx); here we append a
+ sentence telling the model they are ready and can be called on demand.
+
+ Once enabled, ambient MCP tools stay resident and the model calls them by
+ description on its own. Explicit selections guide the model toward the selected
+ capabilities without forcing tool_choice or rejecting a reply without tool usage.
+
+ Returns a dict {"role": "user", "content": "..."} or None.
+ """
+ skill_ids = _explicit_skill_ids_from_context(context)
+
+ mcp_ids = [str(m) for m in (context.get("mcp_ids") or []) if m]
+ connector_id = str(context.get("connector_id") or "")
+ skill_id = str(context.get("skill_id") or "")
+ skill_name = context.get("skill_name")
+ plugin_name = context.get("plugin_name")
+ connector_name = context.get("connector_name")
+
+ if not skill_ids and not mcp_ids:
+ return None
+
+ sections: _typing.List[str] = []
+
+ # ── Skills: load instructions ──
+ if skill_ids:
+ try:
+ from core.capabilities.skills.loader_factory import get_skill_loader
+
+ loader = get_skill_loader()
+ metadata_all = loader.load_all_metadata()
+ entries: _typing.List[str] = []
+ for sid in skill_ids:
+ meta = metadata_all.get(sid)
+ if not meta:
+ logger.warning("[skill_inject] skill_id=%s not found", sid)
+ continue
+ # Materialize once; each deployment publishes its own usable path.
+ skill_dir = loader.get_skill_dir(sid)
+ if not skill_dir:
+ logger.warning("[skill_inject] skill_id=%s has no skill dir", sid)
+ continue
+ from core.capabilities.skills.config import model_facing_skill_dir
+
+ sandbox_dir = model_facing_skill_dir(sid, skill_dir)
+ entries.append(f'- 「{sid}」:view_text_file(file_path="{sandbox_dir}/SKILL.md")')
+ if entries:
+ sections.append("技能(使用技能时先读取对应说明文件):\n" + "\n".join(entries))
+ except Exception as e: # noqa: BLE001
+ logger.error("[skill_inject] failed to load skills %s: %s", skill_ids, e)
+
+ # ── MCP: activation notice (the tools themselves are already in the toolset; call by description) ──
+ if mcp_ids:
+ if connector_id:
+ sections.append(
+ f"MCP 工具:连接器「{connector_name or connector_id}」已被用户显式选择。"
+ "请优先按需使用该连接器完成相关任务,"
+ "也不得用其他工具冒充该连接器的结果。"
+ )
+ elif plugin_name:
+ sections.append(
+ f"MCP 工具:插件「{plugin_name}」的 {len(mcp_ids)} 个 MCP 服务已就绪。"
+ "请优先按需使用该插件的 MCP 工具或技能,"
+ "使用技能时先读取对应说明文件。"
+ )
+ else:
+ sections.append(
+ f"MCP 工具:本轮显式激活的 {len(mcp_ids)} 个 MCP 连接器服务已就绪,"
+ "可直接按需调用(无需加载文件),请优先使用它们完成相关任务。"
+ )
+
+ if not sections:
+ return None
+
+ # 极速模式警示:被呼唤技能的 SKILL.md 往往指示 Bash/沙箱/文件工具流程,
+ # 而极速模式不装配这些工具。若不在呼唤点明说,模型会照着技能说明发起
+ # 不存在的工具调用(实测会卡死在参数流上)。
+ if str(context.get("chat_mode", "") or "").lower() == "turbo" and skill_ids:
+ sections.append(
+ "注意:当前为极速模式,没有 Bash、沙箱、文件读写和文档生成工具。"
+ "技能说明中涉及这些工具的步骤一律不可执行,也不要尝试调用——"
+ "读取 SKILL.md 后,只输出你能以纯文本交付的部分(结构、正文、模板等),"
+ "并在结尾建议用户切换到「快速模式」或「思考模式」完整执行该技能。"
+ )
+
+ if plugin_name and connector_name:
+ header = (
+ f"用户已显式调用插件「{plugin_name}」并选择连接器「{connector_name}」,"
+ "请优先采用这些能力:"
+ )
+ elif plugin_name:
+ header = f"用户已显式调用插件「{plugin_name}」,请优先按需采用以下能力:"
+ elif connector_name:
+ header = f"用户已显式选择连接器「{connector_name}」,请优先采用其能力:"
+ elif skill_id:
+ header = (
+ f"用户已显式调用技能「{skill_name or skill_id}」,"
+ "请优先采用该技能,使用前读取其 SKILL.md:"
+ )
+ else:
+ header = "用户已显式指定使用以下能力,请优先采用:"
+ content = (
+ "\n"
+ + header
+ + "\n"
+ + "\n\n".join(sections)
+ + "\n "
+ )
+ item = items.make_text_context_item(
+ content,
+ item_id="harness:explicit_invocation",
+ kind=ir_constants.KIND_REMINDER,
+ origin="harness:explicit_invocation",
+ trust="system",
+ created_seq=0,
+ priority=900,
+ token_budget=8_000,
+ truncation_policy=ir_constants.POLICY_HEAD_TAIL,
+ )
+ return {
+ "role": "user",
+ "content": content,
+ ir_constants.SESSION_CONTEXT_META_KEY: items.session_context_metadata(item),
+ }
diff --git a/src/backend/orchestration/strategy.py b/src/backend/core/agent_runtime/strategy.py
similarity index 100%
rename from src/backend/orchestration/strategy.py
rename to src/backend/core/agent_runtime/strategy.py
diff --git a/src/backend/core/agent_runtime/stream_direct.py b/src/backend/core/agent_runtime/stream_direct.py
new file mode 100644
index 00000000..229a639b
--- /dev/null
+++ b/src/backend/core/agent_runtime/stream_direct.py
@@ -0,0 +1,298 @@
+"""Translate direct-agent output while preserving scoped audit context."""
+
+from __future__ import annotations
+from core.llm.citations import allocator
+
+import json
+import logging
+
+import core.agent_runtime.events as events
+import core.agent_runtime.integrations.ontology_evidence as ontology_evidence
+import core.agent_runtime.skills as skills
+import core.chat.tool_payloads as tool_payloads
+import core.config.display_names as display_names
+from core.agent_runtime.direct_state import DirectTurn
+
+logger = logging.getLogger(__name__)
+
+
+async def stream_direct(turn: DirectTurn):
+ try:
+ while True:
+ try:
+ event_type, payload = await events._anext_in_subagent_log_scope(
+ turn.direct_stream,
+ turn.direct_log_id,
+ )
+ except StopAsyncIteration:
+ break
+ state_runtime = getattr(turn.streaming_agent.agent.state, "ontology_runtime", None)
+ if isinstance(state_runtime, dict):
+ turn.ontology_runtime = state_runtime
+ turn.context["ontology_runtime"] = state_runtime
+ pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[
+ turn.ontology_event_cursor :
+ ]
+ for ontology_event in pending_ontology_events:
+ yield dict(ontology_event)
+ turn.ontology_event_cursor += len(pending_ontology_events)
+ if event_type == "text_delta":
+ turn.full_response += payload
+ yield {"type": "content", "event": "ai_message", "delta": payload}
+
+ elif event_type == "vision_progress":
+ # 识图是模型开口前的一段纯网络等待;透传给前端,让轮级状态
+ # 从「深度拥抱中」换成「图像理解中」,而不是干等一个笼统的计时器。
+ yield {"type": "vision_progress", **(payload or {})}
+
+ elif event_type == "reasoning_protocol":
+ yield {"type": "thinking", **payload}
+
+ elif event_type == "steer_applied":
+ yield {"type": "steer_applied", **(payload or {})}
+
+ elif event_type == "thinking_delta":
+ yield {"type": "thinking", "delta": payload}
+
+ elif event_type == "context_usage":
+ yield {"type": "context_usage", **(payload or {})}
+
+ elif event_type == "model_step":
+ yield {"type": "model_step", "step": payload}
+
+ elif event_type == "model_call_start":
+ yield {"type": "model_dispatch"}
+
+ elif event_type == "tool_call_start":
+ tool_name = payload.get("name", "unknown")
+ if tool_name != "update_plan":
+ yield {
+ "type": "tool_call_start",
+ "tool_name": tool_name,
+ "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(
+ tool_name, tool_name
+ ),
+ "tool_id": payload.get("id", ""),
+ }
+
+ elif event_type == "tool_call_delta":
+ tool_name = payload.get("name", "unknown")
+ if tool_name != "update_plan" and payload.get("delta"):
+ yield {
+ "type": "tool_call_delta",
+ "tool_name": tool_name,
+ "tool_id": payload.get("id", ""),
+ "arguments_delta": payload.get("delta", ""),
+ }
+
+ elif event_type == "tool_call":
+ tool_name = payload.get("name", "unknown")
+ tool_id = payload.get("id", "")
+ tool_args = payload.get("args", {})
+
+ _is_fast_emit = tool_name in tool_payloads._FAST_EMIT_TOOLS
+ if tool_id and tool_id in turn.displayed_tools:
+ if _is_fast_emit and tool_args:
+ pass # re-emit with updated args
+ else:
+ continue
+ if not tool_payloads._tool_args_ready(tool_name, tool_args):
+ continue
+ if tool_id:
+ turn.displayed_tools.add(tool_id)
+
+ is_skill_load = (
+ tool_name == "view_text_file"
+ and isinstance(tool_args, dict)
+ and "SKILL.md" in str(tool_args.get("file_path", ""))
+ )
+ if is_skill_load and tool_id:
+ turn.skill_load_ids.add(tool_id)
+ _sid = skills._extract_skill_id_from_path(str(tool_args.get("file_path", "")))
+ if _sid:
+ turn.skill_id_by_tool_id[tool_id] = _sid
+ # Non-skill view_text_file also needs trimming at the tool_result stage → record the args
+ if (
+ tool_name in ("view_text_file", "Read")
+ and not is_skill_load
+ and tool_id
+ and isinstance(tool_args, dict)
+ ):
+ turn.view_text_file_args[tool_id] = tool_args
+ emit_name = "load_skill" if is_skill_load else tool_name
+ display_name = (
+ "加载技能"
+ if is_skill_load
+ else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name)
+ )
+ safe_args = tool_args if isinstance(tool_args, dict) else {}
+ turn.ontology_trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": ontology_evidence._bounded_trace_value(safe_args),
+ }
+ )
+
+ yield {
+ "type": "tool_call",
+ "tool_name": emit_name,
+ "tool_display_name": display_name,
+ "tool_args": safe_args,
+ "input": safe_args,
+ "tool_id": tool_id,
+ }
+
+ elif event_type == "tool_progress":
+ yield {"type": "tool_progress", **payload}
+
+ elif event_type == "tool_result":
+ tool_name = payload.get("name", "unknown")
+ tool_id = payload.get("id", "")
+ is_skill_result = (tool_id and tool_id in turn.skill_load_ids) or (
+ tool_name == "view_text_file" and "SKILL.md" in str(payload.get("content", ""))
+ )
+ if is_skill_result:
+ turn.direct_skill_count += 1
+ tool_name = "load_skill"
+ else:
+ turn.direct_tool_count += 1
+
+ _missing_call = events._synthesize_missing_tool_call(
+ tool_id, tool_name, turn.displayed_tools
+ )
+ if _missing_call is not None:
+ turn.ontology_trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": {},
+ }
+ )
+ yield _missing_call
+
+ tool_content = payload.get("content", "")
+
+ try:
+ tool_result_json = json.loads(tool_content) if tool_content else {}
+ except json.JSONDecodeError:
+ tool_result_json = {"result": tool_content}
+ if isinstance(payload.get("mcp"), dict):
+ tool_result_json = payload["mcp"]
+
+ # Skill load: replace the full SKILL.md text with the same
+ # curated detail used by the capability center; affects
+ # only the SSE payload sent to the frontend — the agent's
+ # own memory still holds the full content
+ if is_skill_result:
+ _sid = turn.skill_id_by_tool_id.get(
+ tool_id, ""
+ ) or skills._extract_skill_id_from_path(str(tool_content))
+ tool_result_json = tool_payloads._build_skill_load_payload(_sid)
+ elif tool_name == "view_text_file":
+ # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview
+ tool_result_json = tool_payloads._build_view_text_file_payload(
+ turn.view_text_file_args.get(tool_id, {}), tool_content
+ )
+ elif tool_name == "Read":
+ # Claude-Code-style Read tool: JSON payload, content holds the whole file
+ tool_result_json = tool_payloads._build_read_tool_payload(
+ turn.view_text_file_args.get(tool_id, {}), tool_result_json
+ )
+ elif tool_name == "read_artifact":
+ tool_result_json = tool_payloads._build_read_artifact_payload(tool_result_json)
+
+ extracted_query = ""
+ if isinstance(tool_result_json, dict) and "result" in tool_result_json:
+ result_data = tool_result_json["result"]
+ if isinstance(result_data, dict):
+ extracted_query = result_data.get("query", result_data.get("question", ""))
+
+ cit_dicts = allocator.collect_citation_dicts(tool_id, turn.anchor_allocator)
+ turn.all_citations.extend(cit_dicts)
+ turn.ontology_trace.append(
+ {
+ "type": "tool_result",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "result": ontology_evidence._bounded_trace_value(tool_result_json),
+ "citations": cit_dicts,
+ }
+ )
+
+ yield {
+ "type": "tool_result",
+ "tool_name": tool_name,
+ "tool_args": {"query": extracted_query} if extracted_query else {},
+ "result": tool_result_json,
+ "tool_id": tool_id,
+ "citations": cit_dicts,
+ "status": payload.get("status", "success"),
+ "model_step": payload.get("model_step"),
+ }
+
+ elif event_type in ("heartbeat", "model_progress"):
+ # heartbeat = transport keep-alive; model_progress = the
+ # model is still streaming while a small argument batch or
+ # another suppressed event has nothing renderable yet —
+ # forwarded so the run watchdog counts activity (it
+ # excludes only heartbeat).
+ yield {"type": event_type}
+
+ elif event_type == "tool_pending":
+ yield {"type": "tool_pending", **(payload or {})}
+
+ elif event_type == "subagent_event":
+ # Bypass channel for the sub-agent's internal
+ # thinking/tool_call/tool_result/content — attached under
+ # the call_subagent tool card that launched it (linked via
+ # parent_tool_id).
+ if (payload or {}).get("sub_type") == "progress":
+ # Sub-agent liveness (long tool-call-arg generation with
+ # nothing renderable yet) — feed the run inactivity
+ # watchdog without writing a sub-step to the stream or
+ # the persisted tool log.
+ yield {"type": "model_progress"}
+ else:
+ nested_citations = ontology_evidence._capture_nested_ontology_evidence(
+ payload or {},
+ turn.ontology_trace,
+ turn.all_citations,
+ turn.anchor_allocator,
+ )
+ yield {
+ "type": "subagent_event",
+ **(payload or {}),
+ **({"citations": nested_citations} if nested_citations else {}),
+ }
+
+ elif event_type in (
+ "file_confirm",
+ "design_pick",
+ "user_question",
+ "user_question_resolved",
+ ):
+ # Human-interaction events (§13 MySpace write confirm /
+ # site-design pick-one-of-three / ask_user_question): a tool
+ # coroutine has suspended waiting for the user's out-of-band
+ # action. Pass through to the frontend to show the card; the
+ # agent task stays blocked in that tool and this SSE stream
+ # does not end — after the out-of-band answer POST the tool
+ # resumes in place.
+ yield {"type": event_type, **(payload or {})}
+
+ elif event_type == "error":
+ # payload may be a real exception object (kind=="err") or a
+ # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}).
+ # Raising the latter directly gives a TypeError, masking
+ # the real situation — wrap it in a RuntimeError.
+ if isinstance(payload, BaseException):
+ raise payload
+ raise RuntimeError(str(payload))
+
+ except BaseException:
+ # Keep clients alive after a normal first draft so the same agent
+ # can continue its ReAct context during ontology repair.
+ raise
diff --git a/src/backend/core/agent_runtime/stream_main.py b/src/backend/core/agent_runtime/stream_main.py
new file mode 100644
index 00000000..84916e9e
--- /dev/null
+++ b/src/backend/core/agent_runtime/stream_main.py
@@ -0,0 +1,354 @@
+"""Translate main-agent model and tool output into chat events."""
+
+from __future__ import annotations
+from core.llm.citations import allocator
+
+import json
+import logging
+
+import core.agent_runtime.events as events
+import core.agent_runtime.integrations.ontology_evidence as ontology_evidence
+import core.agent_runtime.skills as skills
+import core.chat.plan_progress as plan_progress
+import core.chat.tool_payloads as tool_payloads
+import core.config.display_names as display_names
+import core.llm.tools.plan_update as plan_update_tool
+from core.agent_runtime.main_state import MainTurn
+
+logger = logging.getLogger(__name__)
+
+
+async def stream_main(turn: MainTurn):
+ try:
+ # Project initialization is a server-expanded command. The DB row
+ # deliberately remains /init; only the live model input is expanded.
+ stream_options = (
+ {"effective_user_message": turn.user_message}
+ if turn.context.get("project_init")
+ else {}
+ )
+ turn.main_stream = turn.streaming_agent.stream(
+ turn.session_messages, turn.context, **stream_options
+ )
+ async for event_type, payload in turn.main_stream:
+ state_runtime = getattr(turn.streaming_agent.agent.state, "ontology_runtime", None)
+ if isinstance(state_runtime, dict):
+ turn.ontology_runtime = state_runtime
+ turn.context["ontology_runtime"] = state_runtime
+ pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[
+ turn.ontology_event_cursor :
+ ]
+ for ontology_event in pending_ontology_events:
+ yield dict(ontology_event)
+ turn.ontology_event_cursor += len(pending_ontology_events)
+ if event_type == "text_delta":
+ turn.full_response += payload
+ yield {"type": "content", "event": "ai_message", "delta": payload}
+
+ elif event_type == "vision_progress":
+ # 识图是模型开口前的一段纯网络等待;透传给前端,让轮级状态
+ # 从「深度拥抱中」换成「图像理解中」,而不是干等一个笼统的计时器。
+ yield {"type": "vision_progress", **(payload or {})}
+
+ elif event_type == "reasoning_protocol":
+ yield {"type": "thinking", **payload}
+
+ elif event_type == "steer_applied":
+ yield {"type": "steer_applied", **(payload or {})}
+
+ elif event_type == "thinking_delta":
+ yield {"type": "thinking", "delta": payload}
+
+ elif event_type == "context_usage":
+ yield {"type": "context_usage", **(payload or {})}
+
+ elif event_type == "model_step":
+ yield {"type": "model_step", "step": payload}
+
+ elif event_type == "model_call_start":
+ yield {"type": "model_dispatch"}
+
+ elif event_type == "tool_call_start":
+ tool_name = payload.get("name", "unknown")
+ if tool_name != "update_plan":
+ yield {
+ "type": "tool_call_start",
+ "tool_name": tool_name,
+ "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(
+ tool_name, tool_name
+ ),
+ "tool_id": payload.get("id", ""),
+ }
+
+ elif event_type == "tool_call_delta":
+ tool_name = payload.get("name", "unknown")
+ if tool_name != "update_plan" and payload.get("delta"):
+ yield {
+ "type": "tool_call_delta",
+ "tool_name": tool_name,
+ "tool_id": payload.get("id", ""),
+ "arguments_delta": payload.get("delta", ""),
+ }
+
+ elif event_type == "tool_call":
+ tool_name = payload.get("name", "unknown")
+ tool_id = payload.get("id", "")
+ tool_args = payload.get("args", {})
+
+ # Stage the full plan until its real result is saved.
+ # A tool request alone must not announce completed work.
+ if tool_name == "update_plan":
+ _pu = plan_update_tool.parse_plan_update_args(tool_args)
+ if _pu:
+ turn.pending_plan_updates[tool_id] = _pu
+ continue
+
+ # In streaming mode, the first chunk for a tool_call may
+ # arrive with empty args. For view_text_file we need args
+ # to decide if this is a skill load, so skip empty-arg
+ # duplicates until we get the complete args.
+ _is_fast_emit = tool_name in tool_payloads._FAST_EMIT_TOOLS
+ if tool_id and tool_id in turn.displayed_tools:
+ # Fast-emit tools: re-emit when args arrive so the
+ # frontend can update input display immediately.
+ if _is_fast_emit and tool_args:
+ pass # fall through to emit update
+ else:
+ continue
+ if not tool_payloads._tool_args_ready(tool_name, tool_args):
+ continue
+ if tool_id:
+ turn.displayed_tools.add(tool_id)
+
+ # Detect skill loading: view_text_file reading a SKILL.md
+ is_skill_load = (
+ tool_name == "view_text_file"
+ and isinstance(tool_args, dict)
+ and "SKILL.md" in str(tool_args.get("file_path", ""))
+ )
+ if is_skill_load and tool_id:
+ turn.skill_load_ids.add(tool_id)
+ _sid = skills._extract_skill_id_from_path(str(tool_args.get("file_path", "")))
+ if _sid:
+ turn.skill_id_by_tool_id[tool_id] = _sid
+ if (
+ tool_name in ("view_text_file", "Read")
+ and not is_skill_load
+ and tool_id
+ and isinstance(tool_args, dict)
+ ):
+ turn.view_text_file_args[tool_id] = tool_args
+ emit_name = "load_skill" if is_skill_load else tool_name
+ display_name = (
+ "加载技能"
+ if is_skill_load
+ else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name)
+ )
+ safe_args = tool_args if isinstance(tool_args, dict) else {}
+ turn.ontology_trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": ontology_evidence._bounded_trace_value(safe_args),
+ }
+ )
+
+ # Resolve sub-agent name for call_subagent tool card
+ _tc_sa_name = ""
+ if tool_name == "call_subagent" and turn.visible_subagents:
+ _tc_sa_id = safe_args.get("agent_id", "") if safe_args else ""
+ if _tc_sa_id:
+ for _sa in turn.visible_subagents:
+ if _sa.get("agent_id") == _tc_sa_id:
+ _tc_sa_name = _sa.get("name", "")
+ break
+ if not _tc_sa_name and turn.mentioned_ids:
+ for _sa in turn.visible_subagents:
+ if _sa.get("agent_id") in turn.mentioned_ids:
+ _tc_sa_name = _sa.get("name", "")
+ break
+
+ yield {
+ "type": "tool_call",
+ "tool_name": emit_name,
+ "tool_display_name": display_name,
+ "tool_args": safe_args,
+ "input": safe_args,
+ "tool_id": tool_id,
+ **({"subagent_name": _tc_sa_name} if _tc_sa_name else {}),
+ }
+
+ elif event_type == "tool_progress":
+ yield {"type": "tool_progress", **payload}
+
+ elif event_type == "tool_result":
+ tool_name = payload.get("name", "unknown")
+ tool_id = payload.get("id", "")
+ # The canonical model_step was already forwarded and
+ # persisted independently. Keep update_plan in the plan
+ # bar; only its redundant display card is suppressed.
+ if tool_name == "update_plan":
+ _pu = turn.pending_plan_updates.pop(tool_id, None)
+ if _pu and payload.get("status") == "success":
+ turn.last_plan = _pu
+ plan_progress.save_plan_progress(
+ str(turn.context.get("chat_id") or ""), _pu
+ )
+ yield {"type": "plan_update", **_pu}
+ continue
+ # Also override tool_name for skill load results
+ is_skill_result = (tool_id and tool_id in turn.skill_load_ids) or (
+ tool_name == "view_text_file" and "SKILL.md" in str(payload.get("content", ""))
+ )
+ if is_skill_result:
+ tool_name = "load_skill"
+
+ _missing_call = events._synthesize_missing_tool_call(
+ tool_id, tool_name, turn.displayed_tools
+ )
+ if _missing_call is not None:
+ turn.ontology_trace.append(
+ {
+ "type": "tool_call",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "input": {},
+ }
+ )
+ yield _missing_call
+
+ tool_content = payload.get("content", "")
+
+ # Parse tool result
+ try:
+ tool_result_json = json.loads(tool_content) if tool_content else {}
+ except json.JSONDecodeError:
+ tool_result_json = {"result": tool_content}
+ if isinstance(payload.get("mcp"), dict):
+ tool_result_json = payload["mcp"]
+
+ # Skill load: replace the full SKILL.md text with the same curated detail used by the capability center
+ if is_skill_result:
+ _sid = turn.skill_id_by_tool_id.get(
+ tool_id, ""
+ ) or skills._extract_skill_id_from_path(str(tool_content))
+ tool_result_json = tool_payloads._build_skill_load_payload(_sid)
+ elif tool_name == "view_text_file":
+ # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview
+ tool_result_json = tool_payloads._build_view_text_file_payload(
+ turn.view_text_file_args.get(tool_id, {}), tool_content
+ )
+ elif tool_name == "Read":
+ # Claude-Code-style Read tool: JSON payload, content holds the whole file
+ tool_result_json = tool_payloads._build_read_tool_payload(
+ turn.view_text_file_args.get(tool_id, {}), tool_result_json
+ )
+ elif tool_name == "read_artifact":
+ tool_result_json = tool_payloads._build_read_artifact_payload(tool_result_json)
+
+ # Extract query if present
+ extracted_query = ""
+ if isinstance(tool_result_json, dict) and "result" in tool_result_json:
+ result_data = tool_result_json["result"]
+ if isinstance(result_data, dict):
+ extracted_query = result_data.get("query", result_data.get("question", ""))
+
+ # Citations
+ cit_dicts = allocator.collect_citation_dicts(tool_id, turn.anchor_allocator)
+ turn.all_citations.extend(cit_dicts)
+ turn.ontology_trace.append(
+ {
+ "type": "tool_result",
+ "tool_id": tool_id,
+ "tool_name": tool_name,
+ "result": ontology_evidence._bounded_trace_value(tool_result_json),
+ "citations": cit_dicts,
+ }
+ )
+
+ # Resolve sub-agent name from call_subagent result text
+ _tr_sa_name = ""
+ if tool_name == "call_subagent":
+ _res_str = (
+ str(tool_result_json.get("result", ""))
+ if isinstance(tool_result_json, dict)
+ else str(tool_result_json)
+ )
+ if "【" in _res_str and "】" in _res_str:
+ _tr_sa_name = _res_str.split("【", 1)[1].split("】", 1)[0]
+
+ yield {
+ "type": "tool_result",
+ "tool_name": tool_name,
+ "tool_args": {"query": extracted_query} if extracted_query else {},
+ "result": tool_result_json,
+ "tool_id": tool_id,
+ "citations": cit_dicts,
+ "status": payload.get("status", "success"),
+ "model_step": payload.get("model_step"),
+ **({"subagent_name": _tr_sa_name} if _tr_sa_name else {}),
+ }
+
+ elif event_type in ("heartbeat", "model_progress"):
+ # heartbeat = transport keep-alive; model_progress = the
+ # model is still streaming while a small argument batch or
+ # another suppressed event has nothing renderable yet —
+ # forwarded so the run watchdog counts activity (it
+ # excludes only heartbeat).
+ yield {"type": event_type}
+
+ elif event_type == "tool_pending":
+ yield {"type": "tool_pending", **(payload or {})}
+
+ elif event_type == "subagent_event":
+ # Bypass channel for the sub-agent's internal
+ # thinking/tool_call/tool_result/content — attached under
+ # the call_subagent tool card that launched it (linked via
+ # parent_tool_id).
+ if (payload or {}).get("sub_type") == "progress":
+ # Sub-agent liveness (long tool-call-arg generation with
+ # nothing renderable yet) — feed the run inactivity
+ # watchdog without writing a sub-step to the stream or
+ # the persisted tool log.
+ yield {"type": "model_progress"}
+ else:
+ nested_citations = ontology_evidence._capture_nested_ontology_evidence(
+ payload or {},
+ turn.ontology_trace,
+ turn.all_citations,
+ turn.anchor_allocator,
+ )
+ yield {
+ "type": "subagent_event",
+ **(payload or {}),
+ **({"citations": nested_citations} if nested_citations else {}),
+ }
+
+ elif event_type in (
+ "file_confirm",
+ "design_pick",
+ "user_question",
+ "user_question_resolved",
+ ):
+ # Human-interaction events (§13 MySpace write confirm /
+ # site-design pick-one-of-three / ask_user_question): a tool
+ # coroutine has suspended waiting for the user's out-of-band
+ # action. Pass through to the frontend to show the card; the
+ # agent task stays blocked in that tool and this SSE stream
+ # does not end — after the out-of-band answer POST the tool
+ # resumes in place.
+ yield {"type": event_type, **(payload or {})}
+
+ elif event_type == "error":
+ # payload may be a real exception object (kind=="err") or a
+ # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}).
+ # Raising the latter directly gives a TypeError, masking
+ # the real situation — wrap it in a RuntimeError.
+ if isinstance(payload, BaseException):
+ raise payload
+ raise RuntimeError(str(payload))
+
+ finally:
+ if turn.main_stream is not None:
+ await turn.main_stream.aclose()
diff --git a/src/backend/core/agent_runtime/targets.py b/src/backend/core/agent_runtime/targets.py
new file mode 100644
index 00000000..cf726843
--- /dev/null
+++ b/src/backend/core/agent_runtime/targets.py
@@ -0,0 +1,208 @@
+"""Resolve which agent a turn runs: mode bindings, @mentions, direct sub-agent chats, project scope."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional, Tuple
+
+import core.projects.edition as project_scope
+
+logger = logging.getLogger(__name__)
+
+_PROJECT_CTX_KEYS = (
+ "project_id",
+ "project_api_scope",
+ "project_name",
+ "project_instructions",
+ "project_init",
+ "project_folder_name",
+ "project_folder_kind",
+ "project_folder_id",
+ # Desktop local project — needed so the prompt renders the local-mode section
+ # with the real host folder path instead of the cloud /workspace section.
+ "project_kind",
+ "project_is_local",
+ "project_local_path",
+ "project_local_slug",
+) + project_scope.edition_project_context_keys()
+
+
+def _mode_visible_subagents(spec, visible, mentioned_ids):
+ """收窄模式下最终可见的子智能体 = 模式圈定的那批 ∪ 本轮被显式 @ 的那个。
+
+ 模式没圈任何子智能体时维持原契约:只有本轮被呼唤的那个能入场(极速模式一直
+ 是这么做的)。圈了就以模式名单为准,再并上本轮呼唤的,免得用户 @ 了一个模式
+ 里没列的智能体时那次呼唤被无声吞掉。
+ """
+ allowed = {
+ a for a in (getattr(spec, "agent_ids", ()) or ()) if isinstance(a, str) and a.strip()
+ }
+ wanted = {str(m) for m in (mentioned_ids or []) if m}
+ keep = allowed | wanted
+ if not keep:
+ return []
+ return [item for item in (visible or []) if str((item or {}).get("agent_id") or "") in keep]
+
+
+def _runtime_visible_subagents(
+ user_agents,
+ parent_runtime,
+ disabled_builtin_ids,
+ explicit_agent_ids,
+):
+ """Build the default agent directory plus any agent summoned this turn.
+
+ Disabled agents stay out of the main agent's initial context. An explicit
+ ``@``/command may add exactly the selected accessible agent for this turn;
+ the route layer has already enforced ownership and platform visibility.
+ """
+ from core.capabilities.agents.builtin import merge_builtin_subagents
+
+ explicitly_selected = {str(agent_id) for agent_id in (explicit_agent_ids or []) if agent_id}
+ all_accessible = merge_builtin_subagents(
+ user_agents,
+ parent_runtime,
+ disabled_agent_ids=disabled_builtin_ids,
+ include_disabled=True,
+ )
+ return [
+ item
+ for item in all_accessible
+ if item.get("is_enabled", True) or str(item.get("agent_id") or "") in explicitly_selected
+ ]
+
+
+def _resolve_mode_spec(context: dict):
+ """从 context 解析这段对话的模式装配契约。
+
+ 线上字段是 ``mode_slug``;老客户端只发 ``chat_mode='turbo'``,按极速模式解析,
+ 所以升级期间新旧前端都能跑。解析失败返回 None(= 标准模式,不收窄)。
+ """
+ slug = str(context.get("mode_slug", "") or "").strip()
+ if not slug and str(context.get("chat_mode", "") or "").lower() == "turbo":
+ slug = "turbo"
+ if not slug:
+ return None
+ try:
+ from core.db.engine import SessionLocal
+ from core.chat.modes import ChatModeService
+
+ with SessionLocal() as db:
+ spec = ChatModeService(db).resolve(slug, str(context.get("user_id", "") or "") or None)
+ return spec
+ except Exception: # noqa: BLE001 — 模式是增强不是边界,解析炸了也要让对话跑起来
+ logger.warning("[workflow] 模式解析失败 slug=%s,按标准模式继续", slug, exc_info=True)
+ return None
+
+
+def _approval_mode(context: Dict[str, Any]) -> Optional[str]:
+ """本轮上下文里带的权限档;没带就交回 None,由工厂读用户自己存的那一档。"""
+ return context.get("approval_mode")
+
+
+def _extract_project_ctx(context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """Extract project-related fields from the workflow context. Returns None when there is no project_id."""
+ if not context.get("project_id"):
+ return None
+ return {k: context.get(k) for k in _PROJECT_CTX_KEYS}
+
+
+def _parse_agent_mentions(message: str, available_agents: list) -> list:
+ """Parse @agent_name mentions from user message.
+
+ Returns list of matched ``agent_id``s in the order they appear in
+ *message*.
+
+ When several agent names share a prefix (e.g. ``搜索`` vs ``搜索助手``)
+ a naive ``"@搜索" in message`` check matches "@搜索" *inside*
+ "@搜索助手", so typing ``@搜索助手`` would falsely also mention the
+ shorter ``搜索`` agent — and the prompt hint built downstream would
+ instruct the LLM to call both. We sort candidates by name length
+ descending and reserve consumed character ranges so the longest
+ matching name wins and prefix-shadow matches are skipped.
+ """
+ if not available_agents:
+ return []
+
+ candidates = [a for a in available_agents if a.get("name")]
+ if not candidates:
+ return []
+ candidates.sort(key=lambda a: len(a["name"]), reverse=True)
+
+ consumed: list = [] # non-overlapping (start, end) char ranges
+
+ def _overlaps(start: int, end: int) -> bool:
+ for s, e in consumed:
+ if start < e and end > s:
+ return True
+ return False
+
+ # (position, agent_id) — sorted at the end to preserve message order
+ hits: list = []
+ for agent in candidates:
+ token = "@" + agent["name"]
+ start = 0
+ while True:
+ idx = message.find(token, start)
+ if idx < 0:
+ break
+ end = idx + len(token)
+ if not _overlaps(idx, end):
+ consumed.append((idx, end))
+ hits.append((idx, agent["agent_id"]))
+ start = idx + len(token)
+
+ hits.sort(key=lambda x: x[0])
+ seen: set = set()
+ ordered: list = []
+ for _, aid in hits:
+ if aid not in seen:
+ seen.add(aid)
+ ordered.append(aid)
+ return ordered
+
+
+def _direct_agent_id_from_context(context: Dict[str, Any]) -> Optional[str]:
+ """Return the target only for a persistent dedicated sub-agent chat.
+
+ A per-turn @mention must stay on the main-model stream so it produces a
+ genuine ``call_subagent`` tool call instead of silently bypassing the
+ parent model's reasoning and streaming lifecycle.
+ """
+ value = context.get("agent_id")
+ return str(value) if value else None
+
+
+def _direct_agent_execution_permissions(agent_id: str) -> Tuple[bool, bool]:
+ """Return ``(read_only, allow_bash)`` for a dedicated sub-agent chat."""
+ from core.capabilities.agents.builtin import get_builtin_subagent
+
+ builtin_spec = get_builtin_subagent(agent_id)
+ if builtin_spec is None:
+ return False, True
+ return builtin_spec.read_only, builtin_spec.allow_bash
+
+
+def _load_direct_user_agent(
+ agent_id: str,
+ user_id: str,
+ parent_runtime: Optional[Dict[str, Any]] = None,
+):
+ """Load a built-in profile or detach a complete custom-agent configuration."""
+ from core.db.engine import SessionLocal
+ from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent
+ from core.capabilities.agents.edition import UserAgentService
+
+ builtin_spec = get_builtin_subagent(agent_id)
+ if builtin_spec is not None:
+ return build_builtin_runtime_profile(builtin_spec, parent_runtime)
+
+ with SessionLocal() as db:
+ user_agent = UserAgentService(db).get_raw_by_id(agent_id, user_id=user_id)
+ # Eagerly load every relationship/config field consumed after the DB
+ # session closes. SQLAlchemy expires lazy relationships on close.
+ _ = user_agent.mcp_server_ids, user_agent.skill_ids, user_agent.kb_ids
+ _ = user_agent.system_prompt, user_agent.model_provider_id
+ _ = user_agent.max_iters, user_agent.temperature, user_agent.max_tokens
+ _ = user_agent.timeout, user_agent.extra_config
+ return user_agent
diff --git a/src/backend/core/agent_skills/backends/__init__.py b/src/backend/core/agent_skills/backends/__init__.py
deleted file mode 100644
index 2f5626df..00000000
--- a/src/backend/core/agent_skills/backends/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""Backend abstraction for multi-source skill loading."""
-
-from .protocol import SkillBackendProtocol, SkillFileInfo
-from .filesystem import FilesystemBackend
-from .composite import CompositeBackend
-from .database import DatabaseBackend
-
-__all__ = [
- "SkillBackendProtocol",
- "SkillFileInfo",
- "FilesystemBackend",
- "CompositeBackend",
- "DatabaseBackend",
-]
diff --git a/src/backend/core/agent_skills/cache_refresh.py b/src/backend/core/agent_skills/cache_refresh.py
deleted file mode 100644
index 696dd22c..00000000
--- a/src/backend/core/agent_skills/cache_refresh.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""Shared cache invalidation after admin skill mutations.
-
-Why centralize: skill changes ripple through 4 caches (loader metadata,
-catalog JSON, per-user capability cache, system prompt). Keeping them in
-one place prevents drift between admin_skills and admin_skill_drafts routes.
-"""
-
-from __future__ import annotations
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def refresh_skill_caches() -> None:
- from core.agent_skills.loader import get_skill_loader
- from core.config.catalog_loader import invalidate_catalog_cache
- from core.capabilities.paths import capabilities_enabled
- if capabilities_enabled():
- from core.capabilities.local_projection import sync_local_skills
- sync_local_skills()
- get_skill_loader(reset=True)
- invalidate_catalog_cache()
-
- try:
- from core.config.catalog_resolver import invalidate_capability_cache
-
- invalidate_capability_cache()
- except Exception:
- pass
-
- try:
- from core.config.catalog_runtime import invalidate_runtime_catalog_cache
-
- invalidate_runtime_catalog_cache()
- except Exception:
- pass
-
- try:
- from prompts.prompt_runtime import invalidate_prompt_cache
-
- invalidate_prompt_cache()
- except Exception:
- pass
-
- try:
- from core.config.catalog_loader import load_catalog
-
- load_catalog(include_runtime_details=False)
- except Exception as exc:
- logger.warning("Failed to eagerly refresh catalog cache after skill mutation: %s", exc)
diff --git a/src/backend/core/auth/__init__.py b/src/backend/core/auth/__init__.py
index 86edd465..33d54e52 100644
--- a/src/backend/core/auth/__init__.py
+++ b/src/backend/core/auth/__init__.py
@@ -8,7 +8,7 @@
from importlib import import_module
-_BACKEND_EXPORTS = {"AuthService", "UserContext", "get_current_user", "require_auth"}
+_BACKEND_EXPORTS = {"UserContext", "get_current_user", "require_auth"}
def __getattr__(name: str):
diff --git a/src/backend/core/services/agent_api_service.py b/src/backend/core/auth/agent_api.py
similarity index 99%
rename from src/backend/core/services/agent_api_service.py
rename to src/backend/core/auth/agent_api.py
index 35ec1be6..81883fe3 100644
--- a/src/backend/core/services/agent_api_service.py
+++ b/src/backend/core/auth/agent_api.py
@@ -49,7 +49,7 @@ def _now():
def require_agent_manager(db, user_id: str, agent_id: str):
"""Only the personal owner can manage an agent credential."""
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
row = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first()
if row is None:
diff --git a/src/backend/core/auth/agent_api_scope.py b/src/backend/core/auth/agent_api_scope.py
index 50806698..5c8ec940 100644
--- a/src/backend/core/auth/agent_api_scope.py
+++ b/src/backend/core/auth/agent_api_scope.py
@@ -5,7 +5,7 @@
from fastapi import HTTPException
from core.db.models import Artifact, ChatRun, ChatSession
-from core.services.agent_api_service import make_agent_api_scope, session_matches_scope
+from core.auth.agent_api import make_agent_api_scope, session_matches_scope
def enforce_agent_api_route(request, db, user) -> None:
diff --git a/src/backend/core/services/api_key_runs.py b/src/backend/core/auth/api_key_runs.py
similarity index 85%
rename from src/backend/core/services/api_key_runs.py
rename to src/backend/core/auth/api_key_runs.py
index 98363914..be63c55b 100644
--- a/src/backend/core/services/api_key_runs.py
+++ b/src/backend/core/auth/api_key_runs.py
@@ -53,7 +53,7 @@ def require_admission_key(
project_id = payload.get("project_id")
project_scope = payload.get("project_api_scope") or {}
if key.project_id is not None:
- from core.services.project_api_service import require_project_key_access
+ from core.projects.api import require_project_key_access
require_project_key_access(db, user_id, key.project_id)
if (
@@ -79,7 +79,7 @@ def cancel_key_runs(db: Session, key_id: str, user_id: str, now: datetime) -> No
Payload bindings cover personal keys and accepted-but-not-yet-logged runs;
historical agent call records also identify pre-binding scoped runs.
"""
- from core.services.run_journal import RunJournal
+ from core.harness.registry.run_cancellation import cancel_run_in_session
rows = (
_key_runs_query(db, key_id, user_id)
@@ -90,31 +90,34 @@ def cancel_key_runs(db: Session, key_id: str, user_id: str, now: datetime) -> No
.all()
)
for row in rows:
- RunJournal.cancel_in_session(db, row, reason="API key disabled or revoked", now=now)
+ cancel_run_in_session(db, row, reason="API key disabled or revoked", now=now)
-async def stop_cancelled_key_workers(db: Session, key_id: str, user_id: str) -> None:
- """Wake SSE followers and stop local workers after the durable commit."""
- from orchestration.chat_run_executor import cancel_run
-
- runs = (
- _key_runs_query(db, key_id, user_id)
+def _cancelled_key_runs(db: Session, key_id: str, user_id: str) -> list[str]:
+ return [
+ run_id
+ for (run_id,) in _key_runs_query(db, key_id, user_id)
.with_entities(ChatRun.run_id)
.filter(
ChatRun.status == "cancelled",
ChatRun.failure_reason == "API key disabled or revoked",
)
.all()
- )
- # End the read transaction before waiting for workers using separate sessions.
- db.rollback()
- for (run_id,) in runs:
+ ]
+
+
+async def stop_cancelled_key_workers(key_id: str, user_id: str) -> None:
+ """Wake SSE followers and stop local workers after the durable commit."""
+ from core.db.threads import run_in_session
+ from core.harness.registry.run_cancellation import cancel_run
+
+ for run_id in await run_in_session(_cancelled_key_runs, key_id, user_id):
await cancel_run(run_id, user_id=user_id, stop_terminal_worker=True)
def authorize_run_execution(db: Session, run_id: str, now: datetime) -> bool:
"""Reject and durably fence work whose credential is already invalid."""
- from core.services.run_journal import RunJournal
+ from core.harness.registry.run_cancellation import cancel_run_in_session
run = db.get(ChatRun, run_id)
if run is None:
@@ -131,7 +134,7 @@ def authorize_run_execution(db: Session, run_id: str, now: datetime) -> bool:
.one()
)
if run.status in ("pending", "running", "needs_attention"):
- RunJournal.cancel_in_session(db, run, reason="API key disabled or revoked", now=now)
+ cancel_run_in_session(db, run, reason="API key disabled or revoked", now=now)
db.commit()
return False
diff --git a/src/backend/core/services/api_key_service.py b/src/backend/core/auth/api_keys.py
similarity index 95%
rename from src/backend/core/services/api_key_service.py
rename to src/backend/core/auth/api_keys.py
index e1b710d1..c39219cb 100644
--- a/src/backend/core/services/api_key_service.py
+++ b/src/backend/core/auth/api_keys.py
@@ -71,11 +71,11 @@ def create_key(
if agent_id is not None and project_id is not None:
raise ValueError("API credentials cannot have two resource scopes")
if project_id is not None:
- from core.services.project_api_service import require_project_key_access
+ from core.projects.api import require_project_key_access
require_project_key_access(self.db, user_id, project_id)
if agent_id is not None:
- from core.services.agent_api_service import require_agent_manager
+ from core.auth.agent_api import require_agent_manager
require_agent_manager(self.db, user_id, agent_id)
raw = API_KEY_PREFIX + secrets.token_urlsafe(32)
@@ -153,7 +153,7 @@ def set_enabled(
return None
row.enabled = enabled
if not enabled:
- from core.services.api_key_runs import cancel_key_runs
+ from core.auth.api_key_runs import cancel_key_runs
cancel_key_runs(self.db, key_id, user_id, _now())
self.db.commit()
@@ -186,7 +186,7 @@ def revoke_key(
return False
row.revoked_at = _now()
row.enabled = False
- from core.services.api_key_runs import cancel_key_runs
+ from core.auth.api_key_runs import cancel_key_runs
cancel_key_runs(self.db, key_id, user_id, row.revoked_at)
self.db.commit()
@@ -249,7 +249,7 @@ def resolve_api_key_identity(db: Session, raw: str) -> Optional[Tuple[UserShadow
if account is not None and account.status != "active":
return None
if row.agent_id is not None:
- from core.services.agent_api_service import require_agent_manager
+ from core.auth.agent_api import require_agent_manager
from fastapi import HTTPException
try:
@@ -258,7 +258,7 @@ def resolve_api_key_identity(db: Session, raw: str) -> Optional[Tuple[UserShadow
return None
if row.project_id is not None:
- from core.services.project_api_service import require_project_key_access
+ from core.projects.api import require_project_key_access
from fastapi import HTTPException
if row.agent_id is not None:
diff --git a/src/backend/core/auth/backend.py b/src/backend/core/auth/backend.py
index bd089bea..7ff08d95 100644
--- a/src/backend/core/auth/backend.py
+++ b/src/backend/core/auth/backend.py
@@ -8,13 +8,13 @@
from typing import Any, Dict, Optional
-import requests
-from core.auth.account_policy import AccountCapacityExceeded
+from core.auth.bearer import MOCK_TOKEN, verify_bearer
+from core.auth.edition import AccountCapacityExceeded
from core.config.settings import settings
-from core.db.engine import get_db
from core.db.repository import AuditLogRepository
+from core.db.threads import run_in_session
from core.infra.logging import get_logger
-from core.services import UserService
+from core.auth.users import UserService
from fastapi import Depends, HTTPException, Request
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from pydantic import BaseModel
@@ -46,100 +46,71 @@ def _auth_mode() -> str:
return settings.auth.mode
-class AuthService:
- """Authentication service for mock / remote Bearer token modes."""
-
- def __init__(self):
- self.auth_mode = _auth_mode()
- self.user_center_url = settings.auth.api_url
- self.timeout = settings.auth.api_timeout
- self.retry_count = settings.auth.retry_count
-
- def verify_token_remote(self, token: str) -> Optional[Dict[str, Any]]:
- """Verify token with remote user center."""
- if not self.user_center_url:
- raise HTTPException(
- status_code=500,
- detail={"code": 52002, "message": "AUTH_API_URL not configured", "data": {}},
- )
-
- headers = {"Authorization": f"Bearer {token}"}
-
- for attempt in range(self.retry_count):
- try:
- response = requests.get(
- f"{self.user_center_url}/verify", headers=headers, timeout=self.timeout
- )
+def _audit_rejected_token(db: Session) -> None:
+ # No user to attribute it to: user_id is a foreign key, so it stays NULL, and
+ # status must be one of success / failure / error.
+ try:
+ AuditLogRepository(db).create(
+ {
+ "user_id": None,
+ "action": "auth.login.failed",
+ "resource_type": "user",
+ "resource_id": "unknown",
+ "status": "failure",
+ "details": {"reason": "invalid_or_expired_token"},
+ }
+ )
+ except Exception as e:
+ logger.warning(f"Failed to log auth failure: {e}")
+
+
+def _bind_bearer_user(db: Session, identity: Dict[str, Any], *, audit_login: bool) -> UserContext:
+ """Map a verified identity onto its shadow user (database work, run off the event loop)."""
+ user_shadow = UserService(db).get_or_create_user_shadow(
+ user_center_id=identity["user_center_id"],
+ username=identity["username"],
+ email=identity.get("email"),
+ avatar_url=identity.get("avatar_url"),
+ refresh_unchanged=False,
+ )
+ if audit_login:
+ AuditLogRepository(db).create(
+ {
+ "user_id": user_shadow.user_id,
+ "action": "auth.login.success",
+ "resource_type": "user",
+ "resource_id": user_shadow.user_id,
+ "status": "success",
+ }
+ )
+ return UserContext(
+ user_id=user_shadow.user_id,
+ user_center_id=user_shadow.user_center_id,
+ username=user_shadow.username,
+ email=user_shadow.email,
+ avatar_url=user_shadow.avatar_url,
+ )
- if response.status_code == 200:
- return response.json()
- elif response.status_code == 401:
- return None
- else:
- continue
-
- except requests.RequestException as e:
- if attempt == self.retry_count - 1:
- raise HTTPException(
- status_code=502,
- detail={
- "code": 52001,
- "message": "User center unavailable",
- "data": {"error": str(e)},
- },
- )
- continue
- return None
+async def _resolve_bearer_user(token: str, *, audit: bool) -> UserContext:
+ """Verify a mock / remote Bearer token and bind it to its shadow user.
- def verify_token_mock(self, token: str) -> Dict[str, Any]:
- """Mock authentication for development."""
- if token and token != "mock_token":
- return {
- "user_center_id": token,
- "username": token,
- "email": f"{token}@mock.local",
- "avatar_url": None,
- }
- return {
- "user_center_id": settings.auth.mock_user_id,
- "username": settings.auth.mock_username,
- "email": "dev@example.com",
- "avatar_url": None,
- }
-
- def verify_token(self, token: str, db: Session = None) -> Dict[str, Any]:
- """Verify token based on configured auth mode (mock or remote)."""
- if self.auth_mode == "mock":
- return self.verify_token_mock(token)
- else:
- user_info = self.verify_token_remote(token)
- if not user_info:
- if db:
- try:
- audit_repo = AuditLogRepository(db)
- audit_repo.create(
- {
- "user_id": "unknown",
- "action": "auth.login.failed",
- "resource_type": "user",
- "resource_id": "unknown",
- "status": "failed",
- "details": {"reason": "invalid_or_expired_token"},
- }
- )
- except Exception as e:
- logger.warning(f"Failed to log auth failure: {e}")
-
- raise HTTPException(
- status_code=401,
- detail={
- "code": 30002,
- "message": "Invalid or expired token",
- "data": {"login_url": _sso_login_url()},
- },
- )
- return user_info
+ A login is audited when the user center actually verified the token, not
+ on every request that reuses that verification.
+ """
+ identity, checked = await verify_bearer(token)
+ if identity is None:
+ if audit:
+ await run_in_session(_audit_rejected_token)
+ raise HTTPException(
+ status_code=401,
+ detail={
+ "code": 30002,
+ "message": "Invalid or expired token",
+ "data": {"login_url": _sso_login_url()},
+ },
+ )
+ return await run_in_session(_bind_bearer_user, identity, audit_login=audit and checked)
# Bearer token scheme (do not auto-reject missing header so mock mode can inject dev user)
@@ -153,7 +124,7 @@ def _resolve_api_key_context(token: Optional[str], db: Session) -> Optional[User
have no Cookie). Validation (enabled/not-revoked/not-expired/user permission
bits) is handled uniformly by ``resolve_api_key_identity``.
"""
- from core.services.api_key_service import resolve_api_key_identity
+ from core.auth.api_keys import resolve_api_key_identity
identity = resolve_api_key_identity(db, token or "")
if identity is None:
@@ -172,7 +143,7 @@ def _resolve_api_key_context(token: Optional[str], db: Session) -> Optional[User
def _require_api_key_context(
- token: Optional[str], db: Session, request: Optional[Request] = None
+ db: Session, token: Optional[str], request: Optional[Request] = None
) -> UserContext:
"""Resolve an API-Key; any failure is a 401. A Bearer that looks like an API-Key
must be valid — never silently downgrade to anonymous. Shared by get_current_user
@@ -248,10 +219,23 @@ async def _resolve_session_user(request: Request) -> Optional[UserContext]:
)
+async def _resolve_host_user(request: Request) -> Optional[UserContext]:
+ """Identity injected by the host (desktop shell), when it claims this request.
+
+ Most requests are not claimed — no resolver is registered, or the request
+ carries no host credential — and pay no thread hop for asking.
+ """
+ from core.harness.registry.authentication import host_user_resolver
+
+ resolver = host_user_resolver(request)
+ if resolver is None:
+ return None
+ return await run_in_session(lambda db: resolver(request, db))
+
+
async def get_current_user(
request: Request,
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
- db: Session = Depends(get_db),
) -> UserContext:
"""Dependency to get current authenticated user.
@@ -262,29 +246,31 @@ async def get_current_user(
AUTH_MODE=session still accepts explicit API Keys; other Bearer tokens
require a session cookie.
+
+ It holds no database session: each lookup runs to completion on a session
+ of its own in a worker thread, so nothing stays checked out while the
+ request (or a stream it starts) continues. Websockets and re-validation
+ call it directly with the request and credentials they have.
"""
if request.headers.get("x-public-chat") is not None:
from core.auth.public_chat import authenticate_public_chat
- return authenticate_public_chat(request, db)
+ return await run_in_session(lambda db: authenticate_public_chat(request, db))
auth_mode = _auth_mode()
# An explicit API credential must not be widened by an accompanying cookie
# or desktop identity. Invalid/revoked scoped keys likewise fail closed.
- from core.services.api_key_service import api_key_token_from_header, is_api_key_token
+ from core.auth.api_keys import api_key_token_from_header, is_api_key_token
bearer_token = api_key_token_from_header(request.headers.get("authorization"))
if bearer_token is None and credentials is not None:
bearer_token = credentials.credentials.strip()
if is_api_key_token(bearer_token):
- return _require_api_key_context(bearer_token, db, request)
+ return await run_in_session(_require_api_key_context, bearer_token, request)
# ── 0. 桌面桥接(仅桌面壳孵化的本机后端启用;混合架构 P2)──
# 桌面壳注入的云端身份优先于本机自身的会话体系;未启用/未命中时零开销回落。
- from core.auth.desktop_bridge import resolve_bridge_user
- from starlette.concurrency import run_in_threadpool
-
- bridge_user = await run_in_threadpool(resolve_bridge_user, request, db)
+ bridge_user = await _resolve_host_user(request)
if bridge_user is not None:
return bridge_user
@@ -306,8 +292,6 @@ async def get_current_user(
)
# ── 3. mock / remote: Bearer token path ──
- current_auth = AuthService()
-
if credentials is None:
if auth_mode != "mock":
raise HTTPException(
@@ -318,39 +302,10 @@ async def get_current_user(
"data": {"login_url": _sso_login_url()},
},
)
- token = "mock_token"
+ token = MOCK_TOKEN
else:
token = credentials.credentials
-
- user_info = current_auth.verify_token(token, db)
-
- user_service = UserService(db)
- user_shadow = user_service.get_or_create_user_shadow(
- user_center_id=user_info["user_center_id"],
- username=user_info["username"],
- email=user_info.get("email"),
- avatar_url=user_info.get("avatar_url"),
- )
-
- # Audit log for successful authentication
- audit_repo = AuditLogRepository(db)
- audit_repo.create(
- {
- "user_id": user_shadow.user_id,
- "action": "auth.login.success",
- "resource_type": "user",
- "resource_id": user_shadow.user_id,
- "status": "success",
- }
- )
-
- return UserContext(
- user_id=user_shadow.user_id,
- user_center_id=user_shadow.user_center_id,
- username=user_shadow.username,
- email=user_shadow.email,
- avatar_url=user_shadow.avatar_url,
- )
+ return await _resolve_bearer_user(token, audit=True)
def require_auth(required: bool = True):
@@ -369,26 +324,22 @@ def private_endpoint(user: UserContext = Depends(require_auth(True))):
return get_current_user
else:
- async def optional_user(
- request: Request, db: Session = Depends(get_db)
- ) -> Optional[UserContext]:
+ async def optional_user(request: Request) -> Optional[UserContext]:
if request.headers.get("x-public-chat") is not None:
from core.auth.public_chat import authenticate_public_chat
- return authenticate_public_chat(request, db)
+
+ return await run_in_session(lambda db: authenticate_public_chat(request, db))
auth_mode = _auth_mode()
- from core.services.api_key_service import api_key_token_from_header
+ from core.auth.api_keys import api_key_token_from_header
auth_header = request.headers.get("Authorization", "")
raw_bearer = api_key_token_from_header(auth_header)
if raw_bearer is not None:
- return _require_api_key_context(raw_bearer, db, request)
+ return await run_in_session(_require_api_key_context, raw_bearer, request)
# Match required authentication: the desktop shell owns the local
# identity, even when a forwarded cloud cookie is absent or stale.
- from core.auth.desktop_bridge import resolve_bridge_user
- from starlette.concurrency import run_in_threadpool
-
- bridge_user = await run_in_threadpool(resolve_bridge_user, request, db)
+ bridge_user = await _resolve_host_user(request)
if bridge_user is not None:
return bridge_user
@@ -406,42 +357,18 @@ async def optional_user(
return None
# ── mock / remote: try Bearer ──
- current_auth = AuthService()
if not auth_header or not auth_header.startswith("Bearer "):
if auth_mode != "mock":
return None
- token = "mock_token"
+ token = MOCK_TOKEN
else:
token = auth_header.split(" ", 1)[1]
try:
- user_info = current_auth.verify_token(token)
- user_service = UserService(db)
- user_shadow = user_service.get_or_create_user_shadow(
- user_center_id=user_info["user_center_id"],
- username=user_info["username"],
- email=user_info.get("email"),
- avatar_url=user_info.get("avatar_url"),
- )
-
- return UserContext(
- user_id=user_shadow.user_id,
- user_center_id=user_shadow.user_center_id,
- username=user_shadow.username,
- email=user_shadow.email,
- avatar_url=user_shadow.avatar_url,
- )
+ return await _resolve_bearer_user(token, audit=False)
except AccountCapacityExceeded:
# Edition admission blocks must not degrade to anonymous access.
- try:
- db.rollback()
- except Exception:
- pass
raise
except Exception as e:
- try:
- db.rollback()
- except Exception:
- pass
logger.warning("optional_auth_resolve_failed", error=str(e))
return None
diff --git a/src/backend/core/auth/bearer.py b/src/backend/core/auth/bearer.py
new file mode 100644
index 00000000..f8b275ee
--- /dev/null
+++ b/src/backend/core/auth/bearer.py
@@ -0,0 +1,149 @@
+"""Bearer identities for AUTH_MODE=mock / remote.
+
+Resolving a Bearer token is two steps with different costs. Verifying it is a
+round trip to the user center (remote) or a pure function of the token (mock);
+binding the identity to a ``users_shadow`` row is local database work. Both used
+to run synchronously on the event loop for every request, so one slow user
+center stalled every stream the worker was serving.
+
+Verification is now asynchronous, and a verified token is remembered for
+``VERIFIED_TOKEN_TTL_SECONDS`` in the shared TTL keyspace, so every worker
+reuses it and the user center sees one request per token per window. That
+window is also how long a token revoked at the user center keeps working here.
+Rejections are never remembered: a token that failed may be valid a moment later.
+The cache is an optimization; when the store is unavailable the token is simply
+verified again.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import ssl
+import weakref
+from typing import Any, Optional
+
+import httpx
+
+from core.config.settings import settings
+from core.infra.ephemeral import get_ephemeral_state
+from core.infra.failures import record_best_effort_failure
+from fastapi import HTTPException
+
+VERIFIED_TOKEN_TTL_SECONDS = 60
+MOCK_TOKEN = "mock_token"
+_CACHE_PREFIX = "auth:bearer:"
+# The fields the backend reads. Only these are cached, so whatever else the user
+# center returns never lands in the shared store.
+_IDENTITY_FIELDS = ("user_center_id", "username", "email", "avatar_url")
+# Per event loop, the verification already in flight for each token: a burst of
+# requests carrying one uncached token shares a single call to the user center.
+_in_flight: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Task]]" = (
+ weakref.WeakKeyDictionary()
+)
+
+
+def mock_identity(token: str) -> dict[str, Any]:
+ """Mock authentication for development: the token names the user."""
+ if token and token != MOCK_TOKEN:
+ return {
+ "user_center_id": token,
+ "username": token,
+ "email": f"{token}@mock.local",
+ "avatar_url": None,
+ }
+ return {
+ "user_center_id": settings.auth.mock_user_id,
+ "username": settings.auth.mock_username,
+ "email": "dev@example.com",
+ "avatar_url": None,
+ }
+
+
+def _identity(payload: Any) -> Optional[dict[str, Any]]:
+ """The identity in a user center reply, or None when the reply names nobody."""
+ if not isinstance(payload, dict) or not payload.get("user_center_id") or "username" not in payload:
+ return None
+ return {name: payload.get(name) for name in _IDENTITY_FIELDS}
+
+
+def _tls_verification() -> ssl.SSLContext | bool:
+ """Trust the CA bundle operators point ``requests`` at, which httpx does not read."""
+ bundle = os.getenv("REQUESTS_CA_BUNDLE") or os.getenv("CURL_CA_BUNDLE")
+ return ssl.create_default_context(cafile=bundle) if bundle else True
+
+
+async def _ask_user_center(token: str) -> Optional[dict[str, Any]]:
+ """The user center's verdict: the identity it vouches for, or None when it rejects the token."""
+ url = settings.auth.api_url
+ if not url:
+ raise HTTPException(
+ status_code=500,
+ detail={"code": 52002, "message": "AUTH_API_URL not configured", "data": {}},
+ )
+ attempts = settings.auth.retry_count
+ async with httpx.AsyncClient(
+ timeout=settings.auth.api_timeout, follow_redirects=True, verify=_tls_verification()
+ ) as client:
+ for attempt in range(attempts):
+ try:
+ response = await client.get(
+ f"{url}/verify", headers={"Authorization": f"Bearer {token}"}
+ )
+ except httpx.HTTPError as error:
+ if attempt == attempts - 1:
+ raise HTTPException(
+ status_code=502,
+ detail={
+ "code": 52001,
+ "message": "User center unavailable",
+ "data": {"error": str(error)},
+ },
+ ) from None
+ continue
+ if response.status_code == 200:
+ return _identity(response.json())
+ if response.status_code == 401:
+ return None
+ return None
+
+
+async def verify_bearer(token: str) -> tuple[Optional[dict[str, Any]], bool]:
+ """Verify a Bearer token under the configured mode.
+
+ Returns ``(identity, checked)``: ``identity`` is None when the token is
+ rejected, and ``checked`` is True only for the one call that asked the user
+ center, which is what a login audit records.
+ """
+ if settings.auth.mode == "mock":
+ return mock_identity(token), False
+ key = _CACHE_PREFIX + hashlib.sha256(token.encode()).hexdigest()
+ state = get_ephemeral_state()
+ try:
+ cached = await state.get(key)
+ except Exception as error: # noqa: BLE001 - a cache outage must not fail authentication
+ record_best_effort_failure("core.auth.bearer:verify_bearer.get", error)
+ cached = None
+ if cached is not None:
+ return json.loads(cached), False
+ pending = _in_flight.setdefault(asyncio.get_running_loop(), {})
+ task = pending.get(key)
+ if task is not None:
+ # Shielded: a waiter whose client disconnects must not cancel the others' answer.
+ return await asyncio.shield(task), False
+ task = asyncio.ensure_future(_verify_and_remember(token, key, state))
+ pending[key] = task
+ task.add_done_callback(lambda _task: pending.pop(key, None))
+ return await asyncio.shield(task), True
+
+
+async def _verify_and_remember(token: str, key: str, state) -> Optional[dict[str, Any]]:
+ identity = await _ask_user_center(token)
+ if identity is not None:
+ try:
+ await state.put(key, json.dumps(identity), ttl=VERIFIED_TOKEN_TTL_SECONDS)
+ except Exception as error: # noqa: BLE001 - the next request verifies again
+ record_best_effort_failure("core.auth.bearer:verify_bearer.put", error)
+ return identity
diff --git a/src/backend/core/auth/capabilities.py b/src/backend/core/auth/capabilities.py
index a495e0b0..fa7a05f4 100644
--- a/src/backend/core/auth/capabilities.py
+++ b/src/backend/core/auth/capabilities.py
@@ -172,7 +172,7 @@ def user_has_capability(db: Session, user_id: str, flag: str) -> bool:
meta = shadow.extra_data if isinstance(shadow.extra_data, dict) else {}
if meta.get("role") == "super_admin":
return True
- from core.auth.edition_capabilities import default_capability_layers_for_user
+ from core.auth.edition import default_capability_layers_for_user
caps = resolve_capabilities(
meta,
@@ -190,7 +190,7 @@ def resolve_user_capabilities(db: Session, user_id: str) -> Dict[str, Any]:
chain is "personal → role → team → system" (both role and team degrade
defensively to ``{}``).
"""
- from core.auth.edition_capabilities import default_capability_layers_for_user
+ from core.auth.edition import default_capability_layers_for_user
from core.db.models import UserShadow
shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first()
diff --git a/src/backend/core/auth/edition.py b/src/backend/core/auth/edition.py
new file mode 100644
index 00000000..b1b2c663
--- /dev/null
+++ b/src/backend/core/auth/edition.py
@@ -0,0 +1,5 @@
+"""Select single-owner authorization for the community edition."""
+from core.auth.personal_permissions_iface import ChatAccessLevel, can_delete_session, resolve_chat_access, ProjectAccess, ProjectAccessInfo, ProjectPermissionLevel, is_project_owner, require_project_access, resolve_project_access, resolve_project_permission, PermissionLevel, has_permission, resolve_artifact_access
+from core.auth.personal_account_policy import AccountCapacityExceeded, account_capacity_block_reason, add_account_to_scope, add_invited_account_to_scope, claim_registration_credential, list_account_scopes, registration_credential_id, validate_account_scope, validate_registration_credential
+from core.auth.personal_account_view import extend_current_account
+from core.auth.personal_edition_capabilities import default_capability_layers_for_user, extend_agent_visibility_filters
diff --git a/src/backend/core/auth/local_user_bootstrap.py b/src/backend/core/auth/local_user_bootstrap.py
new file mode 100644
index 00000000..467893a0
--- /dev/null
+++ b/src/backend/core/auth/local_user_bootstrap.py
@@ -0,0 +1,83 @@
+"""Create the default CE administrator on a fresh instance."""
+from __future__ import annotations
+from core.infra.time import utc_now
+from typing import Optional
+from sqlalchemy.orm import Session
+from core.auth.password import hash_password, verify_password
+from core.config.settings import settings
+from core.db.models import LocalUser, UserShadow
+from core.auth import local_users_constants
+
+def ensure_ce_default_admin(db: Session) -> tuple[Optional[str], bool]:
+ """Ensure a fresh CE instance has an ``admin/admin`` local administrator.
+
+ The bootstrap credential is marked for mandatory password change. Existing
+ passwords are never reset. If a different local account already exists, the
+ explicit onboarding choice is preserved and no extra account is created.
+ """
+ if settings.edition.edition != "ce":
+ return None, False
+
+ shadow = db.query(UserShadow).filter(UserShadow.username == local_users_constants.CE_DEFAULT_ADMIN_USERNAME).first()
+ created = False
+
+ if shadow is None:
+ if db.query(LocalUser).count() > 0:
+ return None, False
+ user_id = "user_ce_admin"
+ shadow = UserShadow(
+ user_id=user_id,
+ user_center_id=user_id,
+ username=local_users_constants.CE_DEFAULT_ADMIN_USERNAME,
+ email=None,
+ avatar_url=None,
+ extra_data={
+ "auth_source": "local",
+ **local_users_constants.CE_DEFAULT_ADMIN_CAPABILITIES,
+ "must_change_password": True,
+ "onboarding_required": True,
+ },
+ last_sync_at=utc_now(),
+ )
+ db.add(shadow)
+ db.flush()
+ local = LocalUser(
+ user_id=user_id,
+ password_hash=hash_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD),
+ nickname="Administrator",
+ status="active",
+ password_updated_at=None,
+ )
+ db.add(local)
+ created = True
+ else:
+ local = db.query(LocalUser).filter(LocalUser.user_id == shadow.user_id).first()
+ if local is None:
+ local = LocalUser(
+ user_id=shadow.user_id,
+ password_hash=hash_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD),
+ nickname="Administrator",
+ status="active",
+ password_updated_at=None,
+ )
+ db.add(local)
+ created = True
+
+ meta = dict(shadow.extra_data or {})
+ meta.update(local_users_constants.CE_DEFAULT_ADMIN_CAPABILITIES)
+ meta.setdefault("auth_source", "local")
+ try:
+ completed_version = int(meta.get("onboarding_completed_version") or 0)
+ except (TypeError, ValueError):
+ completed_version = 0
+ if completed_version < local_users_constants.CE_ONBOARDING_VERSION:
+ meta["onboarding_required"] = True
+ else:
+ meta.pop("onboarding_required", None)
+ if verify_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD, local.password_hash):
+ meta["must_change_password"] = True
+ shadow.extra_data = meta
+ shadow.updated_at = utc_now()
+
+ db.commit()
+ return shadow.user_id, created
diff --git a/src/backend/core/auth/local_user_policy.py b/src/backend/core/auth/local_user_policy.py
new file mode 100644
index 00000000..8b5f9aa4
--- /dev/null
+++ b/src/backend/core/auth/local_user_policy.py
@@ -0,0 +1,16 @@
+"""Whether the CE bootstrap owner still has to finish web onboarding."""
+from __future__ import annotations
+from typing import Any, Dict, Optional
+from core.config.settings import settings
+from core.auth import local_users_constants
+
+def ce_onboarding_required(metadata: Optional[Dict[str, Any]]) -> bool:
+ """Return whether the CE bootstrap owner must finish the current Web setup flow."""
+ if settings.edition.edition != "ce":
+ return False
+ meta = metadata or {}
+ try:
+ completed_version = int(meta.get("onboarding_completed_version") or 0)
+ except (TypeError, ValueError):
+ completed_version = 0
+ return bool(meta.get("onboarding_required")) and completed_version < local_users_constants.CE_ONBOARDING_VERSION
diff --git a/src/backend/core/services/local_user_service.py b/src/backend/core/auth/local_users.py
similarity index 79%
rename from src/backend/core/services/local_user_service.py
rename to src/backend/core/auth/local_users.py
index ac315017..f6ed43c4 100644
--- a/src/backend/core/services/local_user_service.py
+++ b/src/backend/core/auth/local_users.py
@@ -1,71 +1,19 @@
-"""Local user system business logic: registration, login, password change, disabling."""
-
+"""Local account registration, login, password changes and profile edits."""
from __future__ import annotations
from core.infra.time import utc_now
-
-import re
import secrets
import uuid
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-
+from typing import Any, Dict, Optional
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
-
-USERNAME_PATTERN = re.compile(r"^[A-Za-z0-9_]{2,32}$")
-EMAIL_PATTERN = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$")
-
-CE_DEFAULT_ADMIN_USERNAME = "admin"
-CE_DEFAULT_ADMIN_PASSWORD = "admin"
-CE_ONBOARDING_VERSION = 1
-CE_DEFAULT_ADMIN_CAPABILITIES: Dict[str, Any] = {
- "role": "super_admin",
- "ce_default_admin": True,
- "lab_enabled": True,
- "can_use_api_key": True,
- "can_add_skill": True,
- "can_add_mcp": True,
- "can_import_plugin": True,
- "can_add_agent": True,
- "can_create_private_kb": True,
- "can_create_public_kb": False,
- "can_create_channel_bot": True,
- "can_switch_model": True,
- "can_use_ontology_validation": True,
- "can_run_autonomous_loop": True,
- "can_manage_chat_modes": False,
- "can_system_config": True,
- "can_content_manage": True,
- "allowed_apps": "*",
-}
-
-
-def ce_onboarding_required(metadata: Optional[Dict[str, Any]]) -> bool:
- """Return whether the CE bootstrap owner must finish the current Web setup flow."""
- if settings.edition.edition != "ce":
- return False
- meta = metadata or {}
- try:
- completed_version = int(meta.get("onboarding_completed_version") or 0)
- except (TypeError, ValueError):
- completed_version = 0
- return bool(meta.get("onboarding_required")) and completed_version < CE_ONBOARDING_VERSION
-
-
-from core.auth.account_policy import (
- add_account_to_scope,
- add_invited_account_to_scope,
- claim_registration_credential,
- list_account_scopes,
- registration_credential_id,
- validate_account_scope,
- validate_registration_credential,
-)
+from core.auth.edition import add_account_to_scope, add_invited_account_to_scope, claim_registration_credential, list_account_scopes, registration_credential_id, validate_account_scope, validate_registration_credential
from core.auth.password import hash_password, verify_password
from core.config.settings import settings
from core.db.models import LocalUser, UserShadow
from core.db.repository import AuditLogRepository, LocalUserRepository, UserRepository
+from core.auth import local_user_policy
+from core.auth import local_users_constants
+from dataclasses import dataclass
@dataclass
@@ -112,7 +60,7 @@ def register(
if not username:
return RegisterResult(False, "账号不能为空")
- if not USERNAME_PATTERN.match(username):
+ if not local_users_constants.USERNAME_PATTERN.match(username):
return RegisterResult(False, "账号只能包含英文、数字、下划线,长度 2-32 位")
if not nickname:
return RegisterResult(False, "用户名不能为空")
@@ -120,7 +68,7 @@ def register(
return RegisterResult(False, "用户名长度不能超过 32 位")
if not email:
return RegisterResult(False, "邮箱不能为空")
- if not EMAIL_PATTERN.match(email):
+ if not local_users_constants.EMAIL_PATTERN.match(email):
return RegisterResult(False, "邮箱格式不正确")
if len(password) < settings.auth.password_min_length:
return RegisterResult(False, f"密码长度至少 {settings.auth.password_min_length} 位")
@@ -132,7 +80,7 @@ def register(
if self.db.query(UserShadow).filter(UserShadow.email == email).first() is not None:
return RegisterResult(False, "邮箱已被使用")
- from core.auth.account_policy import account_capacity_block_reason
+ from core.auth.edition import account_capacity_block_reason
seat_block = account_capacity_block_reason(self.db)
if seat_block:
@@ -233,11 +181,11 @@ def create_by_admin(
if not username:
return RegisterResult(False, "账号不能为空")
- if not USERNAME_PATTERN.match(username):
+ if not local_users_constants.USERNAME_PATTERN.match(username):
return RegisterResult(False, "账号只能包含英文、数字、下划线,长度 2-32 位")
if len(nickname) > 32:
return RegisterResult(False, "用户名长度不能超过 32 位")
- if email and not EMAIL_PATTERN.match(email):
+ if email and not local_users_constants.EMAIL_PATTERN.match(email):
return RegisterResult(False, "邮箱格式不正确")
if len(password) < settings.auth.password_min_length:
return RegisterResult(False, f"密码长度至少 {settings.auth.password_min_length} 位")
@@ -257,7 +205,7 @@ def create_by_admin(
if not validate_account_scope(self.db, scope_id):
return RegisterResult(False, "目标范围不存在")
- from core.auth.account_policy import account_capacity_block_reason
+ from core.auth.edition import account_capacity_block_reason
seat_block = account_capacity_block_reason(self.db)
if seat_block:
@@ -362,7 +310,7 @@ def get_or_create_external_account(
if existing is not None:
return existing, False
- from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason
+ from core.auth.edition import AccountCapacityExceeded, account_capacity_block_reason
block = account_capacity_block_reason(self.db)
if block:
@@ -490,7 +438,7 @@ def _build_user_info(
"real_name": local.real_name if local else None,
"auth_source": "local",
"must_change_password": bool(meta.get("must_change_password")),
- "onboarding_required": ce_onboarding_required(meta),
+ "onboarding_required": local_user_policy.ce_onboarding_required(meta),
"teams": list_account_scopes(self.db, user_id),
}
@@ -535,78 +483,3 @@ def update_profile(
user_id, shadow.username if shadow else "", shadow=shadow, local=local
),
)
-
-
-def ensure_ce_default_admin(db: Session) -> tuple[Optional[str], bool]:
- """Ensure a fresh CE instance has an ``admin/admin`` local administrator.
-
- The bootstrap credential is marked for mandatory password change. Existing
- passwords are never reset. If a different local account already exists, the
- explicit onboarding choice is preserved and no extra account is created.
- """
- if settings.edition.edition != "ce":
- return None, False
-
- shadow = db.query(UserShadow).filter(UserShadow.username == CE_DEFAULT_ADMIN_USERNAME).first()
- created = False
-
- if shadow is None:
- if db.query(LocalUser).count() > 0:
- return None, False
- user_id = "user_ce_admin"
- shadow = UserShadow(
- user_id=user_id,
- user_center_id=user_id,
- username=CE_DEFAULT_ADMIN_USERNAME,
- email=None,
- avatar_url=None,
- extra_data={
- "auth_source": "local",
- **CE_DEFAULT_ADMIN_CAPABILITIES,
- "must_change_password": True,
- "onboarding_required": True,
- },
- last_sync_at=utc_now(),
- )
- db.add(shadow)
- db.flush()
- local = LocalUser(
- user_id=user_id,
- password_hash=hash_password(CE_DEFAULT_ADMIN_PASSWORD),
- nickname="Administrator",
- status="active",
- password_updated_at=None,
- )
- db.add(local)
- created = True
- else:
- local = db.query(LocalUser).filter(LocalUser.user_id == shadow.user_id).first()
- if local is None:
- local = LocalUser(
- user_id=shadow.user_id,
- password_hash=hash_password(CE_DEFAULT_ADMIN_PASSWORD),
- nickname="Administrator",
- status="active",
- password_updated_at=None,
- )
- db.add(local)
- created = True
-
- meta = dict(shadow.extra_data or {})
- meta.update(CE_DEFAULT_ADMIN_CAPABILITIES)
- meta.setdefault("auth_source", "local")
- try:
- completed_version = int(meta.get("onboarding_completed_version") or 0)
- except (TypeError, ValueError):
- completed_version = 0
- if completed_version < CE_ONBOARDING_VERSION:
- meta["onboarding_required"] = True
- else:
- meta.pop("onboarding_required", None)
- if verify_password(CE_DEFAULT_ADMIN_PASSWORD, local.password_hash):
- meta["must_change_password"] = True
- shadow.extra_data = meta
- shadow.updated_at = utc_now()
-
- db.commit()
- return shadow.user_id, created
diff --git a/src/backend/core/auth/local_users_constants.py b/src/backend/core/auth/local_users_constants.py
new file mode 100644
index 00000000..20170f71
--- /dev/null
+++ b/src/backend/core/auth/local_users_constants.py
@@ -0,0 +1,40 @@
+"""Validation patterns and CE bootstrap constants for local accounts."""
+from __future__ import annotations
+import re
+from typing import Any, Dict
+
+USERNAME_PATTERN = re.compile(r"^[A-Za-z0-9_]{2,32}$")
+
+
+EMAIL_PATTERN = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$")
+
+
+CE_DEFAULT_ADMIN_USERNAME = "admin"
+
+
+CE_DEFAULT_ADMIN_PASSWORD = "admin"
+
+
+CE_ONBOARDING_VERSION = 1
+
+
+CE_DEFAULT_ADMIN_CAPABILITIES: Dict[str, Any] = {
+ "role": "super_admin",
+ "ce_default_admin": True,
+ "lab_enabled": True,
+ "can_use_api_key": True,
+ "can_add_skill": True,
+ "can_add_mcp": True,
+ "can_import_plugin": True,
+ "can_add_agent": True,
+ "can_create_private_kb": True,
+ "can_create_public_kb": False,
+ "can_create_channel_bot": True,
+ "can_switch_model": True,
+ "can_use_ontology_validation": True,
+ "can_run_autonomous_loop": True,
+ "can_manage_chat_modes": False,
+ "can_system_config": True,
+ "can_content_manage": True,
+ "allowed_apps": "*",
+}
diff --git a/src/backend/core/auth/mock_ticket_store.py b/src/backend/core/auth/mock_ticket_store.py
index ada94208..823cc784 100644
--- a/src/backend/core/auth/mock_ticket_store.py
+++ b/src/backend/core/auth/mock_ticket_store.py
@@ -12,15 +12,17 @@
uvicorn worker, so the state between them cannot live in one process: with
``WEB_CONCURRENCY > 1`` an in-process dict loses roughly ``1 - 1/N`` of all
logins. It is therefore kept in Redis, with the same key/TTL discipline as
-:mod:`core.auth.oa_ticket_store` and :mod:`core.auth.desktop_ticket_store`:
+:mod:`core.auth.oa_ticket_store` and :mod:`features.desktop.auth.ticket_store`:
sha256 of the token as the key, atomic GETDEL on consume, TTL-bounded. Without
Redis the backend degrades to in-process storage, which is sound because
:mod:`core.infra.worker_count` already clamps such a deployment to one worker.
"""
+from __future__ import annotations
import hashlib
import json
import secrets
+from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
@@ -33,8 +35,6 @@
TICKET_TTL = 300 # seconds
TICKET_PREFIX = "mock_ticket_"
-_MEMORY_TICKETS: Dict[str, Dict[str, Any]] = {}
-
def _hash(token: str) -> str:
return hashlib.sha256(token.encode()).hexdigest()
@@ -46,8 +46,8 @@ def _use_memory_store() -> bool:
def _prune_expired_memory() -> None:
now = datetime.now(timezone.utc)
- for key in [k for k, v in _MEMORY_TICKETS.items() if v["expires_at"] <= now]:
- _MEMORY_TICKETS.pop(key, None)
+ for key in [k for k, v in process_state().MEMORY_TICKETS.items() if v["expires_at"] <= now]:
+ process_state().MEMORY_TICKETS.pop(key, None)
def is_local_ticket(ticket: str) -> bool:
@@ -62,7 +62,7 @@ async def generate_ticket(user_info: Dict[str, Any]) -> str:
if _use_memory_store():
_prune_expired_memory()
- _MEMORY_TICKETS[key] = {
+ process_state().MEMORY_TICKETS[key] = {
"user_info": dict(user_info),
"expires_at": datetime.now(timezone.utc) + timedelta(seconds=TICKET_TTL),
}
@@ -87,7 +87,7 @@ async def consume_ticket(ticket: str) -> Optional[Dict[str, Any]]:
if _use_memory_store():
_prune_expired_memory()
- entry = _MEMORY_TICKETS.pop(key, None)
+ entry = process_state().MEMORY_TICKETS.pop(key, None)
return dict(entry["user_info"]) if entry else None
from core.infra.redis import get_redis
@@ -107,3 +107,15 @@ async def consume_ticket(ticket: str) -> Optional[Dict[str, Any]]:
except (json.JSONDecodeError, TypeError):
logger.warning("login_ticket_corrupt", token_hash=key[:8])
return None
+
+
+@dataclass
+class MockTicketStoreState:
+ MEMORY_TICKETS: Dict[str, Dict[str, Any]] = field(default_factory=lambda: {})
+
+
+_STATE = MockTicketStoreState()
+
+
+def process_state() -> MockTicketStoreState:
+ return _STATE
diff --git a/src/backend/core/auth/model_selection.py b/src/backend/core/auth/model_selection.py
new file mode 100644
index 00000000..7cb3846f
--- /dev/null
+++ b/src/backend/core/auth/model_selection.py
@@ -0,0 +1,52 @@
+"""Authorize account model overrides before selecting an active provider."""
+from __future__ import annotations
+from typing import Optional
+from sqlalchemy.orm import Session
+from core.auth.capabilities import resolve_user_capabilities
+from core.db.models import ModelProvider
+
+class UserModelSelectionError(ValueError):
+ """Raised when a requested user-selectable model is not allowed."""
+
+
+def resolve_user_model_provider_id(
+ db: Session,
+ provider_id: Optional[str],
+ *,
+ user_id: Optional[str] = None,
+ can_switch_model: Optional[bool] = None,
+) -> Optional[str]:
+ """Return an allowed provider id for the current request.
+
+ Disabled switch means "ignore the browser-provided override" rather than
+ "fail the chat"; enabled switch validates against active chat providers.
+ """
+ pid = (provider_id or "").strip()
+ if not pid:
+ return None
+
+ enabled = (
+ can_switch_model
+ if can_switch_model is not None
+ else bool(user_id and user_can_switch_model(db, user_id))
+ )
+ if not enabled:
+ return None
+
+ provider = (
+ db.query(ModelProvider)
+ .filter(
+ ModelProvider.provider_id == pid,
+ ModelProvider.provider_type == "chat",
+ ModelProvider.is_active == True, # noqa: E712
+ )
+ .first()
+ )
+ if provider is None:
+ raise UserModelSelectionError("所选模型不可用,请刷新页面后重试。")
+ return pid
+
+
+def user_can_switch_model(db: Session, user_id: str) -> bool:
+ """Return whether the user may switch models in the chat input."""
+ return bool(resolve_user_capabilities(db, user_id).get("can_switch_model"))
diff --git a/src/backend/core/auth/oa_ticket_store.py b/src/backend/core/auth/oa_ticket_store.py
index 2bf9a48c..764b0107 100644
--- a/src/backend/core/auth/oa_ticket_store.py
+++ b/src/backend/core/auth/oa_ticket_store.py
@@ -14,10 +14,12 @@
Degrades to in-process storage when the backend has no Redis (same approach as
session.py, dev/single-instance only).
"""
+from __future__ import annotations
import hashlib
import json
import secrets
+from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
@@ -27,7 +29,6 @@
logger = get_logger(__name__)
TICKET_KEY_PREFIX = "jx:oa_ticket:"
-_MEMORY_TICKETS: dict[str, dict[str, Any]] = {}
def _ttl_seconds() -> int:
@@ -44,8 +45,8 @@ def _use_memory_store() -> bool:
def _prune_expired_memory() -> None:
now = datetime.now(timezone.utc)
- for key in [k for k, v in _MEMORY_TICKETS.items() if v.get("expires_at") and v["expires_at"] <= now]:
- _MEMORY_TICKETS.pop(key, None)
+ for key in [k for k, v in process_state().MEMORY_TICKETS.items() if v.get("expires_at") and v["expires_at"] <= now]:
+ process_state().MEMORY_TICKETS.pop(key, None)
async def issue_ticket(payload: Dict[str, Any]) -> str:
@@ -62,7 +63,7 @@ async def issue_ticket(payload: Dict[str, Any]) -> str:
if _use_memory_store():
_prune_expired_memory()
- _MEMORY_TICKETS[key] = {
+ process_state().MEMORY_TICKETS[key] = {
"payload": body,
"expires_at": datetime.now(timezone.utc) + timedelta(seconds=ttl),
}
@@ -84,7 +85,7 @@ async def consume_ticket(token: str) -> Optional[Dict[str, Any]]:
if _use_memory_store():
_prune_expired_memory()
- entry = _MEMORY_TICKETS.pop(key, None)
+ entry = process_state().MEMORY_TICKETS.pop(key, None)
if entry is None:
return None
return dict(entry["payload"])
@@ -106,3 +107,15 @@ async def consume_ticket(token: str) -> Optional[Dict[str, Any]]:
except (json.JSONDecodeError, TypeError):
logger.warning("oa_ticket_corrupt", token_hash=key[:8])
return None
+
+
+@dataclass
+class OaTicketStoreState:
+ MEMORY_TICKETS: dict[str, dict[str, Any]] = field(default_factory=lambda: {})
+
+
+_STATE = OaTicketStoreState()
+
+
+def process_state() -> OaTicketStoreState:
+ return _STATE
diff --git a/src/backend/core/auth/account_policy.py b/src/backend/core/auth/personal_account_policy.py
similarity index 100%
rename from src/backend/core/auth/account_policy.py
rename to src/backend/core/auth/personal_account_policy.py
diff --git a/src/backend/core/auth/account_view.py b/src/backend/core/auth/personal_account_view.py
similarity index 100%
rename from src/backend/core/auth/account_view.py
rename to src/backend/core/auth/personal_account_view.py
diff --git a/src/backend/core/auth/edition_capabilities.py b/src/backend/core/auth/personal_edition_capabilities.py
similarity index 100%
rename from src/backend/core/auth/edition_capabilities.py
rename to src/backend/core/auth/personal_edition_capabilities.py
diff --git a/src/backend/core/auth/permissions_iface.py b/src/backend/core/auth/personal_permissions_iface.py
similarity index 99%
rename from src/backend/core/auth/permissions_iface.py
rename to src/backend/core/auth/personal_permissions_iface.py
index 6e03d405..ba5f27d0 100644
--- a/src/backend/core/auth/permissions_iface.py
+++ b/src/backend/core/auth/personal_permissions_iface.py
@@ -77,7 +77,7 @@ def require_project_access(
from core.auth.backend import UserContext, get_current_user # 避免循环导入
- async def _dep(
+ def _dep(
project_id: str = Path(..., description="项目 ID"),
request: Request = None,
user: "UserContext" = Depends(get_current_user),
diff --git a/src/backend/core/auth/project_api_scope.py b/src/backend/core/auth/project_api_scope.py
index 0d05eb7e..a6b2c2d6 100644
--- a/src/backend/core/auth/project_api_scope.py
+++ b/src/backend/core/auth/project_api_scope.py
@@ -5,7 +5,7 @@
from fastapi import HTTPException
from core.db.models import Artifact, ChatRun, ChatSession
-from core.services.project_api_service import make_project_api_scope, project_session_matches
+from core.projects.api import make_project_api_scope, project_session_matches
def enforce_project_api_route(request, db, user) -> None:
diff --git a/src/backend/core/auth/public_chat.py b/src/backend/core/auth/public_chat.py
index a17742ad..ec23aac3 100644
--- a/src/backend/core/auth/public_chat.py
+++ b/src/backend/core/auth/public_chat.py
@@ -3,13 +3,13 @@
import re
from core.db.models import Artifact, ChatRun, ChatSession
-from core.services.public_chat_service import resolve_public_user
+from core.auth.public_identity import resolve_public_user
from fastapi import HTTPException
def public_session_matches(session, user):
- from core.services.agent_api_service import make_agent_api_scope, session_matches_scope
- from core.services.project_api_service import make_project_api_scope, project_session_matches
+ from core.auth.agent_api import make_agent_api_scope, session_matches_scope
+ from core.projects.api import make_project_api_scope, project_session_matches
if not session:
return False
diff --git a/src/backend/core/auth/public_identity.py b/src/backend/core/auth/public_identity.py
new file mode 100644
index 00000000..c3b22678
--- /dev/null
+++ b/src/backend/core/auth/public_identity.py
@@ -0,0 +1,36 @@
+"""Resolve a visitor credential without granting the owner's account identity."""
+import hashlib
+import re
+from fastapi import HTTPException
+from core.db.models import ContentBlock, UserApiKey
+from core.infra.crypto import decrypt_secret
+from core.auth.api_keys import resolve_api_key_identity
+
+def digest(value):
+ return hashlib.sha256(value.encode()).hexdigest()
+
+
+def resolve_public_user(db, token, visitor):
+ from core.auth.backend import UserContext
+
+ if not re.fullmatch(r"[A-Za-z0-9_-]{43}", token) or not re.fullmatch(
+ r"[A-Za-z0-9_-]{32,64}", visitor
+ ):
+ raise HTTPException(401, detail="Invalid public chat link")
+ row = db.get(ContentBlock, "publicchat:token:" + digest(token))
+ key = db.get(UserApiKey, row.payload["key_id"]) if row else None
+ identity = (
+ resolve_api_key_identity(db, decrypt_secret(key.key_enc)) if key and key.key_enc else None
+ )
+ if not identity:
+ raise HTTPException(401, detail="Public chat link is unavailable")
+ owner, key = identity
+ return UserContext(
+ user_id=owner.user_id,
+ user_center_id=owner.user_center_id or "",
+ username=owner.username,
+ api_key_id=key.id,
+ api_key_agent_id=key.agent_id,
+ api_key_project_id=key.project_id,
+ public_chat_visitor=digest(visitor),
+ )
diff --git a/src/backend/core/auth/session.py b/src/backend/core/auth/session.py
index 93b9b32b..2df60e4a 100644
--- a/src/backend/core/auth/session.py
+++ b/src/backend/core/auth/session.py
@@ -4,10 +4,12 @@
Value: JSON-encoded user data
TTL: SESSION_TTL_HOURS (default 8h)
"""
+from __future__ import annotations
import hashlib
import json
import secrets
+from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
@@ -18,7 +20,6 @@
logger = get_logger(__name__)
SESSION_KEY_PREFIX = "jx:session:"
-_MEMORY_SESSIONS: dict[str, dict[str, Any]] = {}
def _ttl_seconds() -> int:
@@ -33,7 +34,7 @@ def remember_ttl_seconds() -> int:
"""
days = 30.0
try:
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
raw = SystemConfigService.get_instance().get("auth.remember_me_days", "30")
if raw:
@@ -66,11 +67,11 @@ def _prune_expired_memory_sessions() -> None:
now = datetime.now(timezone.utc)
expired = [
key
- for key, value in _MEMORY_SESSIONS.items()
+ for key, value in process_state().MEMORY_SESSIONS.items()
if value.get("expires_at") and value["expires_at"] <= now
]
for key in expired:
- _MEMORY_SESSIONS.pop(key, None)
+ process_state().MEMORY_SESSIONS.pop(key, None)
def generate_token() -> str:
@@ -112,7 +113,7 @@ async def create_session(
if _use_memory_store():
_prune_expired_memory_sessions()
- _MEMORY_SESSIONS[token_hash] = {
+ process_state().MEMORY_SESSIONS[token_hash] = {
"payload": payload,
"expires_at": datetime.now(timezone.utc) + timedelta(seconds=ttl),
}
@@ -141,7 +142,7 @@ async def validate_session(token: str) -> Optional[Dict[str, Any]]:
if _use_memory_store():
_prune_expired_memory_sessions()
- entry = _MEMORY_SESSIONS.get(token_hash)
+ entry = process_state().MEMORY_SESSIONS.get(token_hash)
if entry is None:
return None
# Sliding renewal uses the session's own TTL (a long "remember me" session is not truncated to the default duration)
@@ -178,7 +179,7 @@ async def find_session_by_hash(token_hash: str) -> Optional[Dict[str, Any]]:
return None
if _use_memory_store():
_prune_expired_memory_sessions()
- entry = _MEMORY_SESSIONS.get(token_hash)
+ entry = process_state().MEMORY_SESSIONS.get(token_hash)
return dict(entry["payload"]) if entry else None
raw = await get_redis().get(f"{SESSION_KEY_PREFIX}{token_hash}")
if raw is None:
@@ -199,18 +200,15 @@ async def revoke_session(token: str) -> bool:
token_hash = _hash_token(token)
data = await find_session_by_hash(token_hash)
if _use_memory_store():
- deleted = _MEMORY_SESSIONS.pop(token_hash, None) is not None
+ deleted = process_state().MEMORY_SESSIONS.pop(token_hash, None) is not None
else:
deleted = bool(await get_redis().delete(f"{SESSION_KEY_PREFIX}{token_hash}"))
if deleted:
logger.info("session_revoked", token_hash=token_hash[:8])
if data and data.get("user_id"):
- from core.applications.revocation import revoke_user_workbench
- from core.db.engine import SessionLocal
-
+ from core.harness.registry.resource_owners import resource_owner
try:
- with SessionLocal() as db:
- await revoke_user_workbench(db, data["user_id"], token_hash)
+ await resource_owner().revoke_session(data["user_id"], token_hash)
except Exception as exc:
# The credential is already revoked. Preserve successful logout;
# expired workers stay denied while the resource janitor retries.
@@ -244,3 +242,15 @@ def expires_at_iso(ttl_seconds: Optional[int] = None) -> str:
"""Calculate and return the expiration time as an ISO string."""
ttl = _resolve_ttl(ttl_seconds)
return (datetime.now(timezone.utc) + timedelta(seconds=ttl)).isoformat()
+
+
+@dataclass
+class SessionState:
+ MEMORY_SESSIONS: dict[str, dict[str, Any]] = field(default_factory=lambda: {})
+
+
+_STATE = SessionState()
+
+
+def process_state() -> SessionState:
+ return _STATE
diff --git a/src/backend/core/services/user_service.py b/src/backend/core/auth/users.py
similarity index 92%
rename from src/backend/core/services/user_service.py
rename to src/backend/core/auth/users.py
index d490fb0d..2800e655 100644
--- a/src/backend/core/services/user_service.py
+++ b/src/backend/core/auth/users.py
@@ -5,7 +5,7 @@
from datetime import datetime
from typing import Any, Dict, Optional, Set
-from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason
+from core.auth.edition import AccountCapacityExceeded, account_capacity_block_reason
from core.db.models import UserShadow
from core.db.repository import AuditLogRepository, UserRepository
from sqlalchemy.orm import Session
@@ -100,9 +100,8 @@ def get_user_settings(self, user_id: str) -> Dict[str, Any]:
return {}
metadata = dict(user.extra_data) if user.extra_data else {}
- # Imported lazily to keep the services package's compatibility exports
- # from creating an import cycle during application startup.
- from core.services.memory_settings_service import MemorySettingsService
+ # Imported lazily to avoid an import cycle during application startup.
+ from core.memory.settings import MemorySettingsService
return MemorySettingsService(self.db).apply_effective_defaults(metadata)
@@ -127,7 +126,7 @@ def get_disabled_builtin_subagent_ids(self, user_id: str) -> Set[str]:
if not isinstance(raw_ids, list):
return set()
- from core.llm.builtin_subagents import get_builtin_subagent
+ from core.capabilities.agents.builtin import get_builtin_subagent
return {
agent_id
@@ -139,7 +138,7 @@ def get_disabled_builtin_subagent_ids(self, user_id: str) -> Set[str]:
def set_builtin_subagent_enabled(self, user_id: str, agent_id: str, enabled: bool) -> None:
"""Persist one user's platform sub-agent switch; defaults stay enabled."""
- from core.llm.builtin_subagents import get_builtin_subagent
+ from core.capabilities.agents.builtin import get_builtin_subagent
if get_builtin_subagent(agent_id) is None:
raise LookupError(f"Builtin sub-agent {agent_id} not found")
diff --git a/src/backend/core/capabilities/__init__.py b/src/backend/core/capabilities/__init__.py
index b98922f8..37c1c8cb 100644
--- a/src/backend/core/capabilities/__init__.py
+++ b/src/backend/core/capabilities/__init__.py
@@ -1,16 +1 @@
-"""Desktop capability runtime: partitioned storage, name resolution, flat runtime views.
-
-Three layers, deliberately separated (capability plan v2.0, ch. 04):
-
-- **store** — ``/s////``: immutable versions, one
- sub-tree per source profile, same-named keys from different profiles coexist.
-- **resolver** — the single place that decides which candidate a runtime name points
- to for this device/user. Explicit user choice → explicit request → the current
- account's usable candidate → the only usable candidate → otherwise a conflict.
-- **view** — a flat directory of directory links (``/``) the
- execution plane and the model see. The model-facing path contract
- ``/workspace/skills/`` never changes.
-
-The package is inert unless :func:`paths.capability_root` is set — only the desktop
-local runtime keeps a file store; cloud deployments keep their database truth.
-"""
+"""Capabilities an agent can use: skills, plugins, MCP servers, sub-agents and their catalog."""
diff --git a/src/backend/core/capabilities/agents/__init__.py b/src/backend/core/capabilities/agents/__init__.py
new file mode 100644
index 00000000..d47444f5
--- /dev/null
+++ b/src/backend/core/capabilities/agents/__init__.py
@@ -0,0 +1 @@
+"""User-defined agents: storage, versioning and the resources they may bind."""
diff --git a/src/backend/core/capabilities/agents/available_resources.py b/src/backend/core/capabilities/agents/available_resources.py
new file mode 100644
index 00000000..c71ac626
--- /dev/null
+++ b/src/backend/core/capabilities/agents/available_resources.py
@@ -0,0 +1,273 @@
+"""List the MCP servers, skills, plugins and knowledge spaces an agent may bind."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+import logging
+logger = logging.getLogger(__name__)
+
+def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]:
+ """Return MCP servers, skills, plugins, and KB spaces bindable to agents.
+
+ Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists
+ and instead bound as a whole via the ``plugins`` list (plugin =
+ installable/removable unit, expanded at runtime into its skills+tools).
+ owner_user_id is used to include that user's private plugins and MCPs.
+
+ The MCP list intentionally contains capabilities the user has personally
+ switched off. A sub-agent binding is an explicit, narrower capability
+ grant and therefore may opt into one of those tools without turning it
+ on for the user's main agent. Deployment-global MCPs disabled by an
+ administrator remain unavailable.
+ """
+ from core.db.models import AdminMcpServer, InstalledPlugin
+ from sqlalchemy import or_
+ # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ──
+ # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins.
+ try:
+ pq = self.db.query(InstalledPlugin)
+ if owner_user_id:
+ pq = pq.filter(
+ or_(
+ InstalledPlugin.owner_user_id == owner_user_id,
+ InstalledPlugin.owner_user_id.is_(None),
+ )
+ )
+ else:
+ pq = pq.filter(InstalledPlugin.owner_user_id.is_(None))
+ plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all()
+ except Exception as exc: # noqa: BLE001
+ logger.debug("Failed to list installed plugins: %s", exc)
+ plugin_rows = []
+
+ plugin_skill_ids: set = set()
+ plugin_mcp_ids: set = set()
+ # The same plugin may exist both as a global version (install_id=slug@global)
+ # and as the user's private version (slug@) — same name → duplicate
+ # display (the user sees two "定时任务管理" entries). Dedupe by slug and
+ # show only one: prefer the user's private version (its components carry
+ # the user's fingerprint and match their sandbox credentials); fall back
+ # to the global version when there is no private one.
+ # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all**
+ # rows (including the deduped-away one), so the loose skill/MCP lists
+ # can fully exclude both versions' components.
+ _plugin_by_slug: Dict[str, Dict[str, Any]] = {}
+ for p in plugin_rows:
+ cids = p.component_ids or {}
+ s_ids = list(cids.get("skills") or [])
+ m_ids = list(cids.get("mcp") or [])
+ plugin_skill_ids.update(s_ids)
+ plugin_mcp_ids.update(m_ids)
+ slug = (p.install_id or "").rsplit("@", 1)[0]
+ is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id
+ existing = _plugin_by_slug.get(slug)
+ if existing is None or (is_owned and not existing["_owned"]):
+ _plugin_by_slug[slug] = {
+ "id": p.install_id,
+ "name": p.name,
+ "description": p.description or "",
+ "skill_count": len(s_ids),
+ "mcp_count": len(m_ids),
+ "_owned": is_owned,
+ }
+ plugin_list: List[Dict[str, Any]] = [
+ {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values()
+ ]
+
+ # Built-in plugin MCPs can also be present in the static catalog without
+ # a source_plugin DB row. They still belong under the plugin selector,
+ # not the loose MCP selector.
+ try:
+ from core.capabilities.plugins.management import builtin_plugin_component_ids
+
+ _, builtin_plugin_mcp_ids = builtin_plugin_component_ids()
+ plugin_mcp_ids.update(builtin_plugin_mcp_ids)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("Failed to load built-in plugin MCP ids: %s", exc)
+
+ # Resolve the user's personal on/off layer once so each MCP option can
+ # explain whether it is already enabled for the main agent. This flag
+ # is display metadata only; disabled options remain bindable here.
+ enabled_mcp_ids: Optional[set[str]] = None
+ if owner_user_id:
+ try:
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+
+ _skills, _agents, resolved_mcps = resolve_all_runtime_enabled(
+ self.db, owner_user_id
+ )
+ if resolved_mcps is not None:
+ enabled_mcp_ids = set(resolved_mcps)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("Failed to resolve user MCP enablement: %s", exc)
+
+ # ── MCP tools (exclude plugin-sourced + owner isolation) ──
+ # Owner isolation: private entries (owner_user_id non-null) are visible
+ # only to their owner. Otherwise private copies produced by other users
+ # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list →
+ # duplicates of "定时任务" etc. Empty owner = globally shared entry,
+ # visible to everyone.
+ _mcp_owner_ok = (
+ or_(
+ AdminMcpServer.owner_user_id.is_(None),
+ AdminMcpServer.owner_user_id == owner_user_id,
+ )
+ if owner_user_id
+ else AdminMcpServer.owner_user_id.is_(None)
+ )
+ mcp_servers = (
+ self.db.query(AdminMcpServer)
+ .filter(
+ _mcp_owner_ok,
+ # Authoritative exclusion: any plugin-sourced MCP (source_plugin
+ # nonnull) never enters the loose list — it is bound as a whole via
+ # the plugins list instead. More robust than checking only
+ # plugin_mcp_ids (doesn't depend on the plugin row still
+ # existing/being visible), and also blocks orphaned plugin MCP rows.
+ AdminMcpServer.source_plugin.is_(None),
+ )
+ .order_by(AdminMcpServer.sort_order)
+ .all()
+ )
+ mcp_list: List[Dict[str, Any]] = []
+ seen_mcp_ids: set = set()
+ for server in mcp_servers:
+ # A global false is an administrator lock. A private false is the
+ # owner's personal off state and remains eligible for an explicit
+ # sub-agent binding.
+ if server.owner_user_id is None and not server.is_enabled:
+ continue
+ if server.server_id in plugin_mcp_ids:
+ continue
+ mcp_list.append(
+ {
+ "id": server.server_id,
+ "name": server.display_name,
+ "description": server.description,
+ "enabled": (
+ server.server_id in enabled_mcp_ids
+ if enabled_mcp_ids is not None
+ else bool(server.is_enabled)
+ ),
+ }
+ )
+ seen_mcp_ids.add(server.server_id)
+
+ # Some umbrella/built-in MCP entries are catalog-defined and do not
+ # necessarily have a same-named AdminMcpServer row. Include every
+ # administrator-enabled catalog item so the selector is complete.
+ try:
+ from core.capabilities.catalog.runtime import get_runtime_catalog
+
+ runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False)
+ for item in runtime_catalog.get("mcp") or []:
+ item_id = str(item.get("id") or "").strip()
+ if (
+ not item_id
+ or item_id in seen_mcp_ids
+ or item_id in plugin_mcp_ids
+ or not bool(item.get("enabled", True))
+ ):
+ continue
+ mcp_list.append(
+ {
+ "id": item_id,
+ "name": item.get("name") or item_id,
+ "description": item.get("description") or item.get("desc") or "",
+ "enabled": (
+ item_id in enabled_mcp_ids
+ if enabled_mcp_ids is not None
+ else bool(item.get("enabled", True))
+ ),
+ }
+ )
+ seen_mcp_ids.add(item_id)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("Failed to load catalog MCP resources: %s", exc)
+
+ # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ──
+ # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all**
+ # materialized skills on disk — including other users' private skills
+ # and plugin skills materialized by other users' plugin installs.
+ # Excluding via plugin_skill_ids alone (components of plugins visible to
+ # the current user only) would miss some, letting other users' plugin
+ # skills sneak into "bindable skills". So compute two **authoritative**
+ # exclusion sets directly from AdminSkill and filter both sources
+ # uniformly:
+ # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner);
+ # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user).
+ from core.db.models import AdminSkill
+
+ all_plugin_skill_ids: set = set()
+ foreign_private_skill_ids: set = set()
+ try:
+ for row in self.db.query(
+ AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id
+ ).all():
+ if row.source_plugin:
+ all_plugin_skill_ids.add(row.skill_id)
+ if row.owner_user_id and row.owner_user_id != owner_user_id:
+ foreign_private_skill_ids.add(row.skill_id)
+ except Exception as exc:
+ logger.debug("Failed to precompute skill exclusion sets: %s", exc)
+
+ _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids
+
+ skill_list: List[Dict[str, Any]] = []
+ try:
+ _skill_owner_ok = (
+ or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id)
+ if owner_user_id
+ else AdminSkill.owner_user_id.is_(None)
+ )
+ db_skills = (
+ self.db.query(AdminSkill)
+ .filter(
+ AdminSkill.is_enabled == True,
+ _skill_owner_ok,
+ )
+ .order_by(AdminSkill.updated_at.desc())
+ .all()
+ )
+ seen_ids = set()
+ for s in db_skills:
+ if s.skill_id in _excluded_skill_ids:
+ continue
+ skill_list.append(
+ {"id": s.skill_id, "name": s.display_name, "description": s.description or ""}
+ )
+ seen_ids.add(s.skill_id)
+ except Exception:
+ seen_ids = set()
+
+ try:
+ from core.capabilities.skills.loader_factory import get_skill_loader
+
+ loader = get_skill_loader()
+ for sid, meta in loader.load_all_metadata().items():
+ if sid not in seen_ids and sid not in _excluded_skill_ids:
+ skill_list.append(
+ {
+ "id": sid,
+ "name": getattr(meta, "name", sid),
+ "description": getattr(meta, "description", ""),
+ }
+ )
+ except Exception as exc:
+ logger.debug("Failed to load filesystem skills: %s", exc)
+
+ # KB spaces
+ kb_list: List[Dict[str, Any]] = []
+ try:
+ from core.harness.registry.catalog_options import catalog_options
+ kb_list = catalog_options("kb_spaces", self.db)
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.agents.available_resources:list_available_resources', fallback_error)
+
+ return {
+ "mcp_servers": mcp_list,
+ "skills": skill_list,
+ "plugins": plugin_list,
+ "kb_spaces": kb_list,
+ "ontology_tags": self._ontology_tag_options(),
+ }
diff --git a/src/backend/core/capabilities/agents/base.py b/src/backend/core/capabilities/agents/base.py
new file mode 100644
index 00000000..79da15d5
--- /dev/null
+++ b/src/backend/core/capabilities/agents/base.py
@@ -0,0 +1,435 @@
+"""CRUD, versioning and permission checks for user-defined agents."""
+from __future__ import annotations
+from core.capabilities.agents import available_resources
+from core.capabilities.agents import version_history
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models import UserAgent
+from core.db.repository import AuditLogRepository, UserAgentRepository
+from core.harness.registry.build_governance import ensure_ontology_build_valid
+from sqlalchemy.orm import Session
+import logging
+from core.capabilities.agents import base_constants
+
+logger = logging.getLogger(__name__)
+
+
+def desktop_store() -> bool:
+ """Whether this process keeps a desktop capability file store (agents projected there)."""
+ from core.harness.registry.run_capabilities import has_run_capabilities
+
+ return has_run_capabilities()
+
+
+class UserAgentBaseService:
+ """Service for user agent CRUD and permission checks."""
+
+ def __init__(self, db: Session):
+ self.db = db
+ self.repo = UserAgentRepository(db)
+
+ # ── Queries ──────────────────────────────────────────────────────
+
+ def list_for_user(self, user_id: str) -> List[Dict[str, Any]]:
+ agents = self.repo.list_for_user(user_id)
+ rows = [self._serialize(a) for a in agents]
+ if desktop_store():
+ # Desktop: the bridged account's stored agents join the device's own;
+ # a display-name clash is decided by the resolver, never by order.
+ from core.harness.registry.local_publication import local_publication
+
+ return local_publication().merge_visible_agents(user_id, rows)
+ return rows
+
+ def list_admin(self) -> List[Dict[str, Any]]:
+ agents = self.repo.list_admin()
+ return [self._serialize(a) for a in agents]
+
+ def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]:
+ agent = self.repo.get_by_id(agent_id)
+ if not agent:
+ stored = self._stored_account_agent(agent_id, user_id)
+ if stored is not None:
+ return stored.to_serialized()
+ raise LookupError(f"Agent {agent_id} not found")
+ if user_id and not self._is_accessible(agent, user_id):
+ raise PermissionError("No access to this agent")
+ return self._serialize(agent)
+
+ def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent:
+ """Return the ORM object (for direct use by workflow/factory).
+
+ On a desktop the bridged account's stored agents are a second, equally
+ authoritative source: an ``AgentDefinition`` with the same attribute
+ surface is returned for them.
+ """
+ agent = self.repo.get_by_id(agent_id)
+ if not agent:
+ stored = self._stored_account_agent(agent_id, user_id)
+ if stored is not None:
+ return stored # type: ignore[return-value]
+ raise LookupError(f"Agent {agent_id} not found")
+ if user_id and not self._is_accessible(agent, user_id):
+ raise PermissionError("No access to this agent")
+ return agent
+
+ @staticmethod
+ def _stored_account_agent(agent_id: str, user_id: Optional[str] = None):
+ if not desktop_store():
+ return None
+ from core.harness.registry.local_publication import local_publication
+ from core.harness.registry.execution_account import account_authorized_for
+
+ if not account_authorized_for(user_id):
+ return None
+ return local_publication().account_agent_definition(agent_id)
+
+ def _project_to_store(self, agent: UserAgent) -> None:
+ if not desktop_store():
+ return
+ from core.harness.registry.local_publication import local_publication
+
+ local_publication().publish_local_agent(self._serialize(agent))
+
+ @staticmethod
+ def _remove_from_store(agent_id: str) -> None:
+ if not desktop_store():
+ return
+ from core.harness.registry.local_publication import local_publication
+
+ local_publication().remove_local_agent(agent_id)
+
+ # ── Mutations ────────────────────────────────────────────────────
+
+ def create(
+ self,
+ user_id: Optional[str],
+ operator_name: Optional[str],
+ owner_type: str,
+ data: Dict[str, Any],
+ scope_id: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ data = dict(data)
+ incoming_extra = dict(data.get("extra_config") or {})
+ ontology_tags = list(
+ data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or []
+ )
+ incoming_extra["ontology_tags"] = ontology_tags
+ data["extra_config"] = incoming_extra
+ ensure_ontology_build_valid(
+ self.db,
+ asset_type="subagent",
+ name=str(data.get("name") or ""),
+ description=str(data.get("description") or ""),
+ instructions=str(data.get("system_prompt") or ""),
+ mcp_server_ids=list(data.get("mcp_server_ids") or []),
+ skill_ids=list(data.get("skill_ids") or []),
+ plugin_ids=list(data.get("plugin_ids") or []),
+ ontology_tags=ontology_tags,
+ )
+ self._validate_create_scope(user_id, owner_type, scope_id)
+
+ agent_id = f"ua_{uuid.uuid4().hex[:16]}"
+ created_at = self._now_iso()
+ creation_history = [
+ {
+ "version": base_constants.DEFAULT_AGENT_VERSION,
+ "timestamp": created_at,
+ "content": "创建了子智能体",
+ "operator_name": operator_name or user_id or "未知用户",
+ "details": [],
+ }
+ ]
+ extra_config = self._merge_extra_config(
+ current_extra=None,
+ incoming_extra=incoming_extra,
+ version=base_constants.DEFAULT_AGENT_VERSION,
+ change_history=creation_history,
+ )
+
+ record = {
+ "agent_id": agent_id,
+ "owner_type": owner_type,
+ **self._owner_fields(user_id, owner_type, scope_id),
+ "created_by": user_id,
+ **data,
+ "extra_config": extra_config,
+ }
+ agent = self.repo.create(record)
+ self._project_to_store(agent)
+ self._audit(
+ user_id,
+ "agent.create",
+ agent_id,
+ {"owner_type": owner_type, "name": data.get("name")},
+ )
+ return self._serialize(agent)
+
+ def update(
+ self,
+ agent_id: str,
+ user_id: Optional[str],
+ operator_name: Optional[str],
+ owner_type: str,
+ data: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ agent = self.repo.get_by_id(agent_id)
+ if not agent:
+ raise LookupError(f"Agent {agent_id} not found")
+ self._check_ownership(agent, user_id, owner_type)
+
+ data = dict(data)
+ current_extra = dict(agent.extra_config or {})
+ incoming_extra = dict(data.get("extra_config") or {})
+ ontology_tags = list(
+ data.pop(
+ "ontology_tags",
+ incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []),
+ )
+ or []
+ )
+ incoming_extra["ontology_tags"] = ontology_tags
+ if "extra_config" in data or ontology_tags != list(
+ current_extra.get("ontology_tags") or []
+ ):
+ data["extra_config"] = incoming_extra
+ ensure_ontology_build_valid(
+ self.db,
+ asset_type="subagent",
+ name=str(data.get("name", agent.name) or ""),
+ description=str(data.get("description", agent.description) or ""),
+ instructions=str(data.get("system_prompt", agent.system_prompt) or ""),
+ mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []),
+ skill_ids=list(data.get("skill_ids", agent.skill_ids) or []),
+ plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []),
+ ontology_tags=ontology_tags,
+ )
+
+ changed_fields = self._collect_changed_fields(agent, data)
+ versioned_fields = [field for field in changed_fields if field not in base_constants.NON_VERSIONED_FIELDS]
+ changed_labels = [base_constants.VERSIONED_FIELDS[field] for field in changed_fields]
+ next_version = self._read_version(current_extra)
+ change_history = self._read_change_history(current_extra)
+
+ if changed_labels:
+ change_summary = self._build_change_summary(changed_fields, data)
+ change_details = self._build_change_details(agent, changed_fields, data)
+ entry_version = next_version
+ if versioned_fields:
+ next_version = self._increment_version(next_version)
+ entry_version = next_version
+ change_history.append(
+ {
+ "version": entry_version,
+ "timestamp": self._now_iso(),
+ "content": change_summary,
+ "operator_name": operator_name or user_id or "未知用户",
+ "details": change_details,
+ }
+ )
+ change_history = change_history[-base_constants.MAX_CHANGE_HISTORY:]
+
+ payload = dict(data)
+ payload["extra_config"] = self._merge_extra_config(
+ current_extra=current_extra,
+ incoming_extra=incoming_extra,
+ version=next_version,
+ change_history=change_history,
+ )
+
+ agent = self.repo.update(agent_id, payload)
+ self._project_to_store(agent)
+ audit_details = {"fields": list(data.keys())}
+ if changed_labels:
+ audit_details["change_summary"] = change_summary
+ audit_details["version"] = next_version
+ self._audit(user_id, "agent.update", agent_id, audit_details)
+ return self._serialize(agent)
+
+ def delete(
+ self,
+ agent_id: str,
+ user_id: Optional[str],
+ owner_type: str,
+ ) -> bool:
+ agent = self.repo.get_by_id(agent_id)
+ if not agent:
+ raise LookupError(f"Agent {agent_id} not found")
+ self._check_ownership(agent, user_id, owner_type)
+
+ ok = self.repo.delete(agent_id)
+ if ok:
+ self._remove_from_store(agent_id)
+ self._audit(user_id, "agent.delete", agent_id)
+ return ok
+
+ def toggle_enabled(self, agent_id: str) -> Dict[str, Any]:
+ agent = self.repo.get_by_id(agent_id)
+ if not agent:
+ raise LookupError(f"Agent {agent_id} not found")
+ new_val = not agent.is_enabled
+ agent = self.repo.update(agent_id, {"is_enabled": new_val})
+ self._project_to_store(agent)
+ return self._serialize(agent)
+
+ # ── Available resources ──────────────────────────────────────────
+
+ def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]:
+ return available_resources.list_available_resources(self, owner_user_id)
+
+ def _ontology_tag_options(self) -> List[Dict[str, Any]]:
+ """Controlled labels that activate a sub-agent ontology workflow at runtime."""
+ from core.harness.registry.build_governance import list_asset_tag_options
+
+ return list_asset_tag_options(self.db, "subagent")
+
+ # ── Helpers ───────────────────────────────────────────────────────
+
+ def _validate_create_scope(
+ self, user_id: Optional[str], owner_type: str, scope_id: Optional[str]
+ ) -> None:
+ if owner_type == "user":
+ if not user_id:
+ raise ValueError("user_id required for user agents")
+ if self.repo.count_user_agents(user_id) >= base_constants.MAX_USER_AGENTS:
+ raise ValueError(f"Maximum {base_constants.MAX_USER_AGENTS} agents per user reached")
+ return
+ if owner_type == "admin":
+ return
+ raise ValueError(f"Unsupported agent owner type: {owner_type}")
+
+ @staticmethod
+ def _owner_fields(
+ user_id: Optional[str], owner_type: str, scope_id: Optional[str]
+ ) -> Dict[str, Any]:
+ return {"user_id": user_id if owner_type == "user" else None}
+
+ def _is_accessible(self, agent: UserAgent, user_id: str) -> bool:
+ if agent.owner_type == "admin" and agent.is_enabled:
+ return True
+ if agent.owner_type == "user" and agent.user_id == user_id:
+ return True
+ return False
+
+ def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None:
+ # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes
+ if owner_type == "admin":
+ if agent.owner_type != "admin":
+ raise PermissionError("Admin can only modify admin agents")
+ return
+ if agent.owner_type == "user" and agent.user_id == user_id:
+ return
+ raise PermissionError("You can only modify your own agents")
+
+ @classmethod
+ def _serialize(cls, agent: UserAgent) -> Dict[str, Any]:
+ extra_config = agent.extra_config or {}
+ return {
+ "agent_id": agent.agent_id,
+ "owner_type": agent.owner_type,
+ "user_id": agent.user_id,
+ **cls._serialize_scope(agent),
+ "name": agent.name,
+ "avatar": agent.avatar,
+ "description": agent.description,
+ "system_prompt": agent.system_prompt,
+ "welcome_message": agent.welcome_message,
+ "suggested_questions": agent.suggested_questions or [],
+ "mcp_server_ids": agent.mcp_server_ids or [],
+ "skill_ids": agent.skill_ids or [],
+ "plugin_ids": agent.plugin_ids or [],
+ "kb_ids": agent.kb_ids or [],
+ "model_provider_id": agent.model_provider_id,
+ "temperature": float(agent.temperature) if agent.temperature is not None else None,
+ "max_tokens": agent.max_tokens,
+ "max_iters": agent.max_iters,
+ "timeout": agent.timeout,
+ "is_enabled": agent.is_enabled,
+ "sort_order": agent.sort_order,
+ "source_market_slug": agent.source_market_slug,
+ "ontology_tags": list(extra_config.get("ontology_tags") or []),
+ "extra_config": extra_config,
+ "version": cls._read_version(extra_config),
+ "change_history": cls._read_change_history(extra_config),
+ "created_at": agent.created_at.isoformat() if agent.created_at else None,
+ "updated_at": agent.updated_at.isoformat() if agent.updated_at else None,
+ "created_by": agent.created_by,
+ }
+
+ @staticmethod
+ def _serialize_scope(agent: UserAgent) -> Dict[str, Any]:
+ return {}
+
+ def _audit(
+ self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None
+ ) -> None:
+ try:
+ audit_repo = AuditLogRepository(self.db)
+ audit_repo.create(
+ {
+ "user_id": user_id,
+ "action": action,
+ "resource_type": "user_agent",
+ "resource_id": resource_id,
+ "details": details or {},
+ "status": "success",
+ }
+ )
+ except Exception as exc:
+ logger.warning("Audit log failed: %s", exc)
+
+ @staticmethod
+ def _normalize_value(value: Any) -> Any:
+ return version_history.normalize_value(value)
+
+ @staticmethod
+ def _now_iso() -> str:
+ return version_history.now_iso()
+
+ @classmethod
+ def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]:
+ return version_history.collect_changed_fields(cls, agent, data)
+
+ @staticmethod
+ def _read_version(extra_config: Optional[Dict[str, Any]]) -> str:
+ return version_history.read_version(UserAgentBaseService, extra_config)
+
+ @staticmethod
+ def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]:
+ return version_history.read_change_history(UserAgentBaseService, extra_config)
+
+ @staticmethod
+ def _increment_version(version: str) -> str:
+ return version_history.increment_version(UserAgentBaseService, version)
+
+ @staticmethod
+ def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str:
+ return version_history.build_change_summary(changed_fields, data)
+
+ @classmethod
+ def _build_change_details(
+ cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any]
+ ) -> List[Dict[str, str]]:
+ return version_history.build_change_details(cls, agent, changed_fields, data)
+
+ @staticmethod
+ def _stringify_detail_value(field: str, value: Any) -> str:
+ return version_history.stringify_detail_value(field, value)
+
+ @staticmethod
+ def _normalize_change_details(details: Any) -> List[Dict[str, str]]:
+ return version_history.normalize_change_details(details)
+
+ @staticmethod
+ def _normalize_version(version: str) -> str:
+ return version_history.normalize_version(version)
+
+ @staticmethod
+ def _merge_extra_config(
+ current_extra: Optional[Dict[str, Any]],
+ incoming_extra: Optional[Dict[str, Any]],
+ *,
+ version: str,
+ change_history: List[Dict[str, str]],
+ ) -> Dict[str, Any]:
+ return version_history.merge_extra_config(current_extra, incoming_extra, version=version, change_history=change_history)
diff --git a/src/backend/core/capabilities/agents/base_constants.py b/src/backend/core/capabilities/agents/base_constants.py
new file mode 100644
index 00000000..e49125f6
--- /dev/null
+++ b/src/backend/core/capabilities/agents/base_constants.py
@@ -0,0 +1,33 @@
+"""Limits and versioned-field lists for user-defined agents."""
+from __future__ import annotations
+
+
+MAX_USER_AGENTS = 20
+
+
+DEFAULT_AGENT_VERSION = "V1.0"
+
+
+MAX_CHANGE_HISTORY = 30
+
+
+NON_VERSIONED_FIELDS = {"is_enabled"}
+
+
+VERSIONED_FIELDS = {
+ "name": "名称",
+ "description": "简介",
+ "system_prompt": "角色设定",
+ "welcome_message": "开场白",
+ "suggested_questions": "推荐问题",
+ "mcp_server_ids": "绑定工具",
+ "skill_ids": "绑定技能",
+ "plugin_ids": "绑定插件",
+ "kb_ids": "绑定知识库",
+ "model_provider_id": "模型",
+ "temperature": "温度",
+ "max_tokens": "最大输出长度",
+ "max_iters": "最大推理轮次",
+ "timeout": "超时时间",
+ "is_enabled": "启用状态",
+}
diff --git a/src/backend/core/llm/builtin_subagents.py b/src/backend/core/capabilities/agents/builtin.py
similarity index 98%
rename from src/backend/core/llm/builtin_subagents.py
rename to src/backend/core/capabilities/agents/builtin.py
index ba03ab01..ce43c2a4 100644
--- a/src/backend/core/llm/builtin_subagents.py
+++ b/src/backend/core/capabilities/agents/builtin.py
@@ -6,6 +6,7 @@
"""
from __future__ import annotations
+from core.harness.registry.model_services import render_prompt_part
import json
import logging
@@ -223,9 +224,8 @@ def refresh_builtin_subagents(
def load_builtin_subagent_prompt(spec: BuiltinSubagentSpec) -> str:
"""Load the active DB prompt part, falling back to the versioned file."""
try:
- from core.services import prompt_version_service
- prompt = prompt_version_service.render_active_prompt_part("subagents", spec.role)
+ prompt = render_prompt_part("subagents", spec.role)
if prompt:
return prompt
except Exception:
diff --git a/src/backend/core/services/user_agent_service.py b/src/backend/core/capabilities/agents/edition.py
similarity index 64%
rename from src/backend/core/services/user_agent_service.py
rename to src/backend/core/capabilities/agents/edition.py
index 4c95be63..ffab9e59 100644
--- a/src/backend/core/services/user_agent_service.py
+++ b/src/backend/core/capabilities/agents/edition.py
@@ -1,6 +1,6 @@
"""Community custom sub-agent service."""
-from core.services.user_agent_base import UserAgentBaseService
+from core.capabilities.agents.base import UserAgentBaseService
UserAgentService = UserAgentBaseService
diff --git a/src/backend/core/services/agent_markdown.py b/src/backend/core/capabilities/agents/markdown.py
similarity index 100%
rename from src/backend/core/services/agent_markdown.py
rename to src/backend/core/capabilities/agents/markdown.py
diff --git a/src/backend/core/capabilities/agents/version_history.py b/src/backend/core/capabilities/agents/version_history.py
new file mode 100644
index 00000000..a9e99ecb
--- /dev/null
+++ b/src/backend/core/capabilities/agents/version_history.py
@@ -0,0 +1,180 @@
+"""Version numbers and change history for user-defined agents."""
+from __future__ import annotations
+from core.capabilities.agents.base_constants import DEFAULT_AGENT_VERSION, VERSIONED_FIELDS
+from core.infra.time import utc_now
+import re
+from decimal import Decimal
+from typing import Any, Dict, List, Optional
+from core.db.models import UserAgent
+
+def normalize_value(value: Any) -> Any:
+ if isinstance(value, Decimal):
+ return float(value)
+ if isinstance(value, list):
+ return list(value)
+ return value
+
+
+def now_iso() -> str:
+ return utc_now().replace(microsecond=0).isoformat()
+
+
+def collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]:
+ fields: List[str] = []
+ for field in VERSIONED_FIELDS:
+ if field not in data:
+ continue
+ old_value = cls._normalize_value(getattr(agent, field, None))
+ new_value = cls._normalize_value(data.get(field))
+ if old_value != new_value:
+ fields.append(field)
+ return fields
+
+
+def read_version(base_service_type, extra_config: Optional[Dict[str, Any]]) -> str:
+ if not isinstance(extra_config, dict):
+ return DEFAULT_AGENT_VERSION
+ raw = extra_config.get("version")
+ return base_service_type._normalize_version(raw if isinstance(raw, str) else "")
+
+
+def read_change_history(base_service_type, extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]:
+ if not isinstance(extra_config, dict):
+ return []
+ raw_history = extra_config.get("change_history")
+ if not isinstance(raw_history, list):
+ return []
+
+ history: List[Dict[str, str]] = []
+ for item in raw_history:
+ if not isinstance(item, dict):
+ continue
+ timestamp = item.get("timestamp")
+ content = item.get("content")
+ version = item.get("version")
+ operator_name = item.get("operator_name")
+ details = item.get("details")
+ if not isinstance(timestamp, str) or not isinstance(content, str):
+ continue
+ history.append(
+ {
+ "timestamp": timestamp,
+ "content": content,
+ "version": base_service_type._normalize_version(
+ version if isinstance(version, str) else ""
+ ),
+ "operator_name": (
+ operator_name
+ if isinstance(operator_name, str) and operator_name.strip()
+ else "未知用户"
+ ),
+ "details": base_service_type._normalize_change_details(details),
+ }
+ )
+ return history
+
+
+def increment_version(base_service_type, version: str) -> str:
+ normalized = base_service_type._normalize_version(version)
+ match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized)
+ if not match:
+ return "V1.1"
+ major, minor = (int(part) for part in match.groups())
+ return f"V{major}.{minor + 1}"
+
+
+def build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str:
+ if changed_fields == ["is_enabled"]:
+ return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体"
+
+ changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields]
+ if not changed_labels:
+ return "更新了智能体配置"
+ if len(changed_labels) <= 3:
+ return f"修改了{'、'.join(changed_labels)}"
+ preview = "、".join(changed_labels[:3])
+ return f"修改了{preview}等{len(changed_labels)}项"
+
+
+def build_change_details(
+ cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any]
+) -> List[Dict[str, str]]:
+ details: List[Dict[str, str]] = []
+ for field in changed_fields:
+ old_value = cls._stringify_detail_value(field, getattr(agent, field, None))
+ new_value = cls._stringify_detail_value(field, data.get(field))
+ details.append(
+ {
+ "field": VERSIONED_FIELDS[field],
+ "before": old_value,
+ "after": new_value,
+ }
+ )
+ return details
+
+
+def stringify_detail_value(field: str, value: Any) -> str:
+ if field == "is_enabled":
+ return "启用" if bool(value) else "关闭"
+ if value is None:
+ return "未填写"
+ if isinstance(value, list):
+ return "、".join(str(item) for item in value) if value else "未填写"
+ if isinstance(value, bool):
+ return "是" if value else "否"
+ text = str(value).strip()
+ return text if text else "未填写"
+
+
+def normalize_change_details(details: Any) -> List[Dict[str, str]]:
+ if not isinstance(details, list):
+ return []
+ normalized: List[Dict[str, str]] = []
+ for item in details:
+ if not isinstance(item, dict):
+ continue
+ field = item.get("field")
+ before = item.get("before")
+ after = item.get("after")
+ if not isinstance(field, str):
+ continue
+ normalized.append(
+ {
+ "field": field,
+ "before": before if isinstance(before, str) else "未填写",
+ "after": after if isinstance(after, str) else "未填写",
+ }
+ )
+ return normalized
+
+
+def normalize_version(version: str) -> str:
+ if not isinstance(version, str) or not version.strip():
+ return DEFAULT_AGENT_VERSION
+ raw = version.strip()
+ if re.match(r"^[Vv]\d+\.\d+$", raw):
+ return f"V{raw[1:]}"
+ legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw)
+ if legacy_patch:
+ major, minor, patch = (int(part) for part in legacy_patch.groups())
+ return f"V{major}.{minor + patch}"
+ legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw)
+ if legacy_minor:
+ major, minor = (int(part) for part in legacy_minor.groups())
+ return f"V{major}.{minor}"
+ return DEFAULT_AGENT_VERSION
+
+
+def merge_extra_config(
+ current_extra: Optional[Dict[str, Any]],
+ incoming_extra: Optional[Dict[str, Any]],
+ *,
+ version: str,
+ change_history: List[Dict[str, str]],
+) -> Dict[str, Any]:
+ merged = dict(current_extra or {})
+ if isinstance(incoming_extra, dict):
+ merged.update(incoming_extra)
+ merged["version"] = version
+ merged["change_history"] = change_history
+ return merged
diff --git a/src/backend/core/capabilities/archive.py b/src/backend/core/capabilities/archive.py
index 1eb93539..a69a95d9 100644
--- a/src/backend/core/capabilities/archive.py
+++ b/src/backend/core/capabilities/archive.py
@@ -9,8 +9,8 @@
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Dict, Iterable, List, Tuple
-from .errors import IntegrityFailed
-from .paths import _WINDOWS_RESERVED
+from core.harness.capability_errors import IntegrityFailed
+from core.capabilities.identity_rules import _WINDOWS_RESERVED
MAX_MEMBERS = 5000
MAX_DEPTH = 24
@@ -28,7 +28,7 @@ def _reject(name: str, why: str) -> None:
raise IntegrityFailed(f"unsafe archive member {name!r}: {why}")
-def _normalize_member(name: str) -> str:
+def normalize_member(name: str) -> str:
if not name:
_reject(name, "empty name")
if name.startswith("/") or "\\" in name:
@@ -48,7 +48,7 @@ def _normalize_member(name: str) -> str:
return "/".join(parts)
-def _validate_names(names: List[str]) -> None:
+def validate_names(names: List[str]) -> None:
spellings: Dict[str, str] = {}
files = set()
directories = set()
@@ -91,7 +91,7 @@ def inspect_zip(data: bytes) -> Tuple[zipfile.ZipFile, List[ArchiveEntry], str]:
if info.flag_bits & 1:
_reject(info.filename, "encrypted member")
name = info.filename.rstrip("/") if info.is_dir() else info.filename
- norm = _normalize_member(name)
+ norm = normalize_member(name)
if info.is_dir():
continue
if info.file_size > MAX_MEMBER_BYTES:
@@ -109,7 +109,7 @@ def inspect_zip(data: bytes) -> Tuple[zipfile.ZipFile, List[ArchiveEntry], str]:
entries = [
ArchiveEntry(n[len(strip) + 1 :] if strip else n, size) for n, size in normalized
]
- _validate_names([entry.relative_path for entry in entries])
+ validate_names([entry.relative_path for entry in entries])
return zf, entries, strip
except Exception:
zf.close()
@@ -145,13 +145,13 @@ def write_files(dest: Path, files: Dict[str, bytes | str]) -> List[str]:
prepared = []
total = 0
for rel, body in files.items():
- norm = _normalize_member(rel)
+ norm = normalize_member(rel)
data = body if isinstance(body, bytes) else body.encode("utf-8")
total += len(data)
if len(data) > MAX_MEMBER_BYTES or total > MAX_TOTAL_BYTES:
raise IntegrityFailed("package exceeds the allowed size")
prepared.append((norm, data))
- _validate_names([name for name, _ in prepared])
+ validate_names([name for name, _ in prepared])
dest.mkdir(parents=True, exist_ok=False)
for name, data in prepared:
target = dest.joinpath(*PurePosixPath(name).parts)
@@ -166,7 +166,7 @@ def iter_file_stats(root: Path) -> Iterable[Tuple[str, Path, os.stat_result]]:
Never descend into links or reparse points. Returning each entry's lstat
avoids repeating filesystem calls for type checks, signatures and sizes.
"""
- from .junction import _native, is_directory_link
+ from core.infra.directory_links import native_path as _native, is_directory_link
if is_directory_link(root):
raise IntegrityFailed("package root must be a real directory")
diff --git a/src/backend/core/capabilities/catalog/__init__.py b/src/backend/core/capabilities/catalog/__init__.py
new file mode 100644
index 00000000..0ef93965
--- /dev/null
+++ b/src/backend/core/capabilities/catalog/__init__.py
@@ -0,0 +1 @@
+"""The capability catalog: what is installed, overridden and visible to each user."""
diff --git a/src/backend/core/config/catalog.py b/src/backend/core/capabilities/catalog/api.py
similarity index 92%
rename from src/backend/core/config/catalog.py
rename to src/backend/core/capabilities/catalog/api.py
index bdbd69d6..e425de07 100644
--- a/src/backend/core/config/catalog.py
+++ b/src/backend/core/capabilities/catalog/api.py
@@ -26,23 +26,15 @@
from typing import Any, Dict, List
-from core.config.catalog_common import _item, _normalize_kind
-from core.config.catalog_loader import (
- _write_catalog,
- ensure_default_catalog,
- invalidate_catalog_cache,
- load_catalog,
-)
-
-# Re-export so existing ``from core.config.catalog import invalidate_catalog_cache``
-# and ``from core.config.catalog import ensure_default_catalog`` keep working.
+from core.capabilities.catalog.common import _item, _normalize_kind
+from core.capabilities.catalog.loader import _write_catalog, ensure_default_catalog, load_catalog
+
__all__ = [
"get_catalog",
"is_enabled",
"get_enabled_ids",
"set_enabled",
"reorder_items",
- "invalidate_catalog_cache",
"ensure_default_catalog",
]
diff --git a/src/backend/core/config/catalog_common.py b/src/backend/core/capabilities/catalog/common.py
similarity index 90%
rename from src/backend/core/config/catalog_common.py
rename to src/backend/core/capabilities/catalog/common.py
index 26b7fd41..594ccba4 100644
--- a/src/backend/core/config/catalog_common.py
+++ b/src/backend/core/capabilities/catalog/common.py
@@ -11,7 +11,7 @@
from pathlib import Path
from typing import Any, Dict, Optional
-_CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", str(Path(__file__).with_name("catalog.json"))))
+CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", str(Path(__file__).resolve().parents[2] / "config" / "catalog.json")))
_LOGGER = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ def _item(
def _read_raw_catalog() -> Dict[str, Any]:
- raw = _CATALOG_PATH.read_text(encoding="utf-8")
+ raw = CATALOG_PATH.read_text(encoding="utf-8")
data = json.loads(raw) if raw.strip() else {}
if not isinstance(data, dict):
raise ValueError("catalog must be a JSON object")
diff --git a/src/backend/core/config/catalog_details.py b/src/backend/core/capabilities/catalog/details.py
similarity index 96%
rename from src/backend/core/config/catalog_details.py
rename to src/backend/core/capabilities/catalog/details.py
index c34f31b3..fc3df3fc 100644
--- a/src/backend/core/config/catalog_details.py
+++ b/src/backend/core/capabilities/catalog/details.py
@@ -5,13 +5,7 @@
import logging
from typing import Any, Dict, Optional
-from core.config.catalog_loader import (
- DB_HIDDEN_SERVERS,
- DB_UMBRELLA_ID,
- DB_UMBRELLA_NAME,
- DB_UMBRELLA_DESC,
- _database_query_capability_available,
-)
+from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID, DB_UMBRELLA_NAME, DB_UMBRELLA_DESC, _database_query_capability_available
_LOGGER = logging.getLogger(__name__)
@@ -51,10 +45,10 @@ def skill_body_from_raw(raw: str) -> str:
if not raw:
return ""
try:
- from core.agent_skills.registry import _split_frontmatter
+ from core.capabilities.skills.registry import split_frontmatter
try:
- _, body = _split_frontmatter(raw)
+ _, body = split_frontmatter(raw)
except Exception:
# Without valid frontmatter, use the full text directly
return raw.strip()
@@ -98,7 +92,7 @@ def _load_dynamic_skill_specs() -> Dict[str, Dict[str, Any]]:
4. ``""`` (when even SKILL.md is unavailable, frontend shows "暂无详情")
"""
try:
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
loader = get_skill_loader()
# Refresh metadata cache to support hot-reload for bind-mounted skill files.
@@ -114,7 +108,7 @@ def _load_dynamic_skill_specs() -> Dict[str, Dict[str, Any]]:
try:
from core.db.engine import SessionLocal
from core.db.models import AdminSkill
- from core.services.skill_icon_service import get_skill_icons
+ from core.capabilities.skills.icons import get_skill_icons
with SessionLocal() as db:
for sid, intro in db.query(AdminSkill.skill_id, AdminSkill.user_intro).all():
@@ -178,7 +172,7 @@ def _curated_detail_for_owned(sid: str) -> Optional[Dict[str, Any]]:
detail prefers AdminSkill.user_intro, otherwise falls back to the SKILL.md body. Returns None if the skill is not found.
"""
try:
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
loader = get_skill_loader()
metadata = loader.load_all_metadata().get(sid)
@@ -193,7 +187,7 @@ def _curated_detail_for_owned(sid: str) -> Optional[Dict[str, Any]]:
try:
from core.db.engine import SessionLocal
from core.db.models import AdminSkill
- from core.services.skill_icon_service import get_skill_icons
+ from core.capabilities.skills.icons import get_skill_icons
with SessionLocal() as db:
row = db.query(AdminSkill.user_intro).filter(AdminSkill.skill_id == sid).first()
diff --git a/src/backend/core/config/catalog_loader.py b/src/backend/core/capabilities/catalog/loader.py
similarity index 85%
rename from src/backend/core/config/catalog_loader.py
rename to src/backend/core/capabilities/catalog/loader.py
index 3c812109..42a3d628 100644
--- a/src/backend/core/config/catalog_loader.py
+++ b/src/backend/core/capabilities/catalog/loader.py
@@ -11,10 +11,12 @@
import copy
import json
import logging
+from dataclasses import dataclass, field
from time import monotonic
from typing import Any, Dict, List
-from core.config.catalog_common import _CATALOG_PATH, _item, _read_raw_catalog
+from core.capabilities.catalog import common as catalog_common
+from core.capabilities.catalog.common import _item, _read_raw_catalog
_LOGGER = logging.getLogger(__name__)
@@ -47,19 +49,19 @@ def _database_query_capability_available() -> bool:
# ── In-memory catalog cache (TTL-based) ────────────────────────────────────
-_CATALOG_CACHE: Dict[bool, Dict[str, Any]] = {} # key = include_runtime_details
-_CATALOG_CACHE_TIME: Dict[bool, float] = {}
+ # key = include_runtime_details
+
_CATALOG_CACHE_TTL: float = 10.0 # seconds
def invalidate_catalog_cache() -> None:
"""Clear the in-memory catalog cache (call after writes)."""
- _CATALOG_CACHE.clear()
- _CATALOG_CACHE_TIME.clear()
+ process_state().CATALOG_CACHE.clear()
+ process_state().CATALOG_CACHE_TIME.clear()
def _write_catalog(data: Dict[str, Any]) -> None:
- _CATALOG_PATH.write_text(
+ catalog_common.CATALOG_PATH.write_text(
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
invalidate_catalog_cache()
@@ -75,9 +77,9 @@ def _load_builtin_skill_metadata() -> List[Any]:
are runtime capabilities and must not become repository catalog defaults.
"""
try:
- from core.agent_skills.backends.filesystem import FilesystemBackend
- from core.agent_skills.config import get_default_skill_sources
- from core.agent_skills.registry import _load_skill_metadata_from_file
+ from core.capabilities.skills.backends.filesystem import FilesystemBackend
+ from core.capabilities.skills.config import get_default_skill_sources
+ from core.capabilities.skills.registry import _load_skill_metadata_from_file
except Exception as e:
_LOGGER.warning("Failed to import built-in skill loader: %s", e)
return []
@@ -248,9 +250,9 @@ def _filter_to_static_defaults(data: Dict[str, Any], defaults: Dict[str, Any]) -
def ensure_default_catalog() -> Dict[str, Any]:
"""Create catalog.json with repository defaults if missing; return loaded catalog."""
- if not _CATALOG_PATH.exists():
+ if not catalog_common.CATALOG_PATH.exists():
cat = _default_catalog()
- _CATALOG_PATH.write_text(
+ catalog_common.CATALOG_PATH.write_text(
json.dumps(cat, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
invalidate_catalog_cache()
@@ -265,17 +267,17 @@ def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]:
Results are cached in-memory for up to ``_CATALOG_CACHE_TTL`` seconds to
avoid repeated disk I/O and dynamic source loading on every request.
"""
- from core.config.catalog_migration import _migrate_legacy_shape
+ from core.capabilities.catalog.migration import _migrate_legacy_shape
now = monotonic()
- cached_time = _CATALOG_CACHE_TIME.get(include_runtime_details, 0.0)
- if include_runtime_details in _CATALOG_CACHE and (now - cached_time) < _CATALOG_CACHE_TTL:
- return copy.deepcopy(_CATALOG_CACHE[include_runtime_details])
+ cached_time = process_state().CATALOG_CACHE_TIME.get(include_runtime_details, 0.0)
+ if include_runtime_details in process_state().CATALOG_CACHE and (now - cached_time) < _CATALOG_CACHE_TTL:
+ return copy.deepcopy(process_state().CATALOG_CACHE[include_runtime_details])
- if not _CATALOG_PATH.exists():
+ if not catalog_common.CATALOG_PATH.exists():
result = _default_catalog()
- _CATALOG_CACHE[include_runtime_details] = copy.deepcopy(result)
- _CATALOG_CACHE_TIME[include_runtime_details] = monotonic()
+ process_state().CATALOG_CACHE[include_runtime_details] = copy.deepcopy(result)
+ process_state().CATALOG_CACHE_TIME[include_runtime_details] = monotonic()
return result
try:
@@ -300,10 +302,23 @@ def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]:
_strip_static_detail_fields(data)
if include_runtime_details:
- from core.config.catalog_details import _attach_runtime_details
+ from core.capabilities.catalog.details import _attach_runtime_details
_attach_runtime_details(data)
- _CATALOG_CACHE[include_runtime_details] = copy.deepcopy(data)
- _CATALOG_CACHE_TIME[include_runtime_details] = monotonic()
+ process_state().CATALOG_CACHE[include_runtime_details] = copy.deepcopy(data)
+ process_state().CATALOG_CACHE_TIME[include_runtime_details] = monotonic()
return data
+
+
+@dataclass
+class LoaderState:
+ CATALOG_CACHE: Dict[bool, Dict[str, Any]] = field(default_factory=lambda: {})
+ CATALOG_CACHE_TIME: Dict[bool, float] = field(default_factory=lambda: {})
+
+
+_STATE = LoaderState()
+
+
+def process_state() -> LoaderState:
+ return _STATE
diff --git a/src/backend/core/config/catalog_migration.py b/src/backend/core/capabilities/catalog/migration.py
similarity index 97%
rename from src/backend/core/config/catalog_migration.py
rename to src/backend/core/capabilities/catalog/migration.py
index 1611b421..46c0266c 100644
--- a/src/backend/core/config/catalog_migration.py
+++ b/src/backend/core/capabilities/catalog/migration.py
@@ -9,7 +9,7 @@
import logging
from typing import Any, Dict, List
-from core.config.catalog_common import _item
+from core.capabilities.catalog.common import _item
_LOGGER = logging.getLogger(__name__)
@@ -59,7 +59,7 @@ def _migrate_legacy_shape(raw: Dict[str, Any]) -> Dict[str, Any]:
If the data already has {skills, agents, mcp, kb} it is normalised
in-place. Otherwise it is rebuilt from legacy key conventions.
"""
- from core.config.catalog_loader import _default_catalog
+ from core.capabilities.catalog.loader import _default_catalog
# Already new shape.
if all(k in raw for k in ("skills", "agents", "mcp", "kb")):
diff --git a/src/backend/core/services/catalog_service.py b/src/backend/core/capabilities/catalog/overrides.py
similarity index 100%
rename from src/backend/core/services/catalog_service.py
rename to src/backend/core/capabilities/catalog/overrides.py
diff --git a/src/backend/core/config/catalog_resolver.py b/src/backend/core/capabilities/catalog/resolver.py
similarity index 87%
rename from src/backend/core/config/catalog_resolver.py
rename to src/backend/core/capabilities/catalog/resolver.py
index 956343cc..c87de751 100644
--- a/src/backend/core/config/catalog_resolver.py
+++ b/src/backend/core/capabilities/catalog/resolver.py
@@ -8,24 +8,22 @@
from __future__ import annotations
import logging
+from dataclasses import dataclass, field
from threading import Lock
from time import monotonic
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
-from core.config.catalog import get_enabled_ids, is_enabled
-from core.config.catalog_runtime import get_runtime_catalog
+from core.capabilities.catalog.api import get_enabled_ids, is_enabled
+from core.capabilities.catalog.runtime import get_runtime_catalog
logger = logging.getLogger(__name__)
# ── Per-user capability cache ────────────────────────────────────────────
_CAPABILITY_CACHE_TTL = 30.0 # seconds
-_capability_cache_lock = Lock()
+
# user_id -> (expires_at, (skills, agents, mcps))
-_capability_cache: Dict[
- str, Tuple[float, Tuple[Optional[List[str]], Optional[List[str]], Optional[List[str]]]]
-] = {}
# ── Context helpers (extract typed lists from a runtime context dict) ────────
@@ -151,7 +149,7 @@ def _collect(model, id_col, label: str, *, dep_gated: bool = False) -> List[str]
out.append(rid)
return out
- from core.capabilities.local_plugin_runtime import configs
+ from core.harness.registry.execution_catalog import local_plugin_configs as configs
owned_mcps = set(_collect(AdminMcpServer, AdminMcpServer.server_id, "mcps"))
mcp_overrides = _ov_map("mcps")
@@ -231,29 +229,21 @@ def _clean(raw: Optional[List[str]]) -> List[str]:
else:
allowed_mcp_ids.discard(mid)
- from core.capabilities.local_plugin_runtime import configs
+ from core.harness.registry.execution_catalog import local_plugin_configs as configs
allowed_mcp_ids.update(configs(user_id))
# A suspended marketplace installation is no longer an active grant.
# Its admin row remains for recovery, so existence alone must not revive it.
if requested_mcps:
- from core.db.models import McpMarketInstallation
-
- suspended_ids = {
- row[0]
- for row in db.query(McpMarketInstallation.server_id)
- .filter(
- McpMarketInstallation.server_id.in_(requested_mcps),
- McpMarketInstallation.status == "suspended",
- )
- .all()
- }
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
+
+ suspended_ids = marketplace_catalog().suspended_mcp_ids(db, requested_mcps)
allowed_mcp_ids.difference_update(suspended_ids)
- from core.capabilities.paths import capabilities_enabled
- if capabilities_enabled():
- from core.capabilities.invocation import resolve_explicit_ids
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ if has_run_capabilities():
+ from core.harness.registry.run_capabilities import resolve_explicit_ids
allowed_skill_ids, allowed_mcp_ids = resolve_explicit_ids(
user_id, allowed_skill_ids, allowed_mcp_ids
)
@@ -275,18 +265,15 @@ def _apply_desktop_cloud_bridge(result, *, user_id=None):
"""
skills, agents, mcps = result
try:
- from core.services.desktop_cloud_bridge import (
- apply_to_enabled_mcp_ids,
- apply_to_enabled_skill_ids,
- )
+ from core.harness.registry.execution_catalog import apply_to_enabled_mcp_ids, apply_to_enabled_skill_ids
bridged_mcps = apply_to_enabled_mcp_ids(list(mcps)) if mcps is not None else None
bridged_skills = apply_to_enabled_skill_ids(list(skills)) if skills is not None else None
except Exception: # noqa: BLE001 - 保留本机默认值,再独立检查本机可用性
bridged_skills, _, bridged_mcps = result
- from core.capabilities.paths import capabilities_enabled
- if capabilities_enabled() and bridged_skills is not None:
- from core.capabilities.skills import filter_available_names
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ if has_run_capabilities() and bridged_skills is not None:
+ from core.harness.registry.run_capabilities import filter_available_names
bridged_skills = filter_available_names(bridged_skills, user_id=user_id)
return (bridged_skills, agents, bridged_mcps)
@@ -309,18 +296,18 @@ def resolve_all_runtime_enabled(
"""
# Check cache first
now = monotonic()
- with _capability_cache_lock:
- cached = _capability_cache.get(user_id)
+ with process_state().capability_cache_lock:
+ cached = process_state().capability_cache.get(user_id)
if cached is not None:
expires_at, result = cached
if now < expires_at:
return _apply_desktop_cloud_bridge(result, user_id=user_id)
else:
- _capability_cache.pop(user_id, None)
+ process_state().capability_cache.pop(user_id, None)
try:
# Lazy import to avoid circular dependency at module level
- from core.services import CatalogService
+ from core.capabilities.catalog.overrides import CatalogService
base_catalog = get_runtime_catalog(db, include_runtime_details=False)
svc = CatalogService(db)
@@ -344,7 +331,7 @@ def resolve_all_runtime_enabled(
# Dedup: if a privately-installed skill / MCP has already been set global by an admin (same underlying entry_name), drop the private
# copy — otherwise the same capability would be loaded twice (global id=entry, private id=entry-fingerprint), which is wasteful and also lets
# the model see two identically-named, identical-content tools/skills. Same semantics as the dedup on the /v1/catalog display side.
- from core.services.marketplace_service import base_entry_name
+ from core.capabilities.skills.installation import base_entry_name
if owned_skills:
_global_skills = set(skills)
@@ -363,8 +350,8 @@ def resolve_all_runtime_enabled(
result = (skills, agents, mcps)
# Store in cache
- with _capability_cache_lock:
- _capability_cache[user_id] = (now + _CAPABILITY_CACHE_TTL, result)
+ with process_state().capability_cache_lock:
+ process_state().capability_cache[user_id] = (now + _CAPABILITY_CACHE_TTL, result)
return _apply_desktop_cloud_bridge(result, user_id=user_id)
except Exception as exc:
@@ -374,8 +361,21 @@ def resolve_all_runtime_enabled(
def invalidate_capability_cache(user_id: Optional[str] = None) -> None:
"""Invalidate capability cache. Pass user_id to clear a specific user, or None for all."""
- with _capability_cache_lock:
+ with process_state().capability_cache_lock:
if user_id is None:
- _capability_cache.clear()
+ process_state().capability_cache.clear()
else:
- _capability_cache.pop(user_id, None)
+ process_state().capability_cache.pop(user_id, None)
+
+
+@dataclass
+class ResolverState:
+ capability_cache_lock: object = field(default_factory=lambda: Lock())
+ capability_cache: Dict[str, Tuple[float, Tuple[Optional[List[str]], Optional[List[str]], Optional[List[str]]]]] = field(default_factory=lambda: {})
+
+
+_STATE = ResolverState()
+
+
+def process_state() -> ResolverState:
+ return _STATE
diff --git a/src/backend/core/config/catalog_runtime.py b/src/backend/core/capabilities/catalog/runtime.py
similarity index 87%
rename from src/backend/core/config/catalog_runtime.py
rename to src/backend/core/capabilities/catalog/runtime.py
index 3bfb5166..b26d32cd 100644
--- a/src/backend/core/config/catalog_runtime.py
+++ b/src/backend/core/capabilities/catalog/runtime.py
@@ -10,21 +10,22 @@
import copy
import logging
+from dataclasses import dataclass, field
from threading import Lock
from time import monotonic
from typing import Any, Dict, List, Optional
-from core.config.catalog import get_catalog
-from core.config.catalog_common import _item
-from core.config.catalog_details import resolve_skill_detail
-from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, _database_query_capability_available
-from core.config.edition_display_names import edition_mcp_icons
+from core.capabilities.catalog.api import get_catalog
+from core.capabilities.catalog.common import _item
+from core.capabilities.catalog.details import resolve_skill_detail
+from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, _database_query_capability_available
+from core.config.edition import edition_mcp_icons
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
-_DEFAULT_MCP_ICONS: Dict[str, str] = {
+DEFAULT_MCP_ICONS: Dict[str, str] = {
"query_database": "/home/mcp/database.svg",
"retrieve_dataset_content": "/home/mcp/knowledge.svg",
"internet_search": "/home/mcp/internet.svg",
@@ -33,14 +34,12 @@
}
_DATABASE_QUERY_ENABLED_CONFIG = "database_query.capability_enabled"
_RUNTIME_DB_CACHE_TTL = 30.0
-_runtime_db_cache: Dict[bool, tuple[float, bool, List[Dict[str, Any]], List[Dict[str, Any]]]] = {}
-_runtime_db_cache_lock = Lock()
def invalidate_runtime_catalog_cache() -> None:
"""Clear cached public DB capability overlays."""
- with _runtime_db_cache_lock:
- _runtime_db_cache.clear()
+ with process_state().runtime_db_cache_lock:
+ process_state().runtime_db_cache.clear()
def merge_items_by_id(
@@ -70,13 +69,12 @@ def merge_items_by_id(
return merged
-
def _public_db_skill_items(db: Session, *, include_runtime_details: bool) -> List[Dict[str, Any]]:
from core.db.models import AdminSkill
icons: Dict[str, str] = {}
if include_runtime_details:
- from core.services.skill_icon_service import get_skill_icons
+ from core.capabilities.skills.icons import get_skill_icons
icons = get_skill_icons(db)
rows = (
@@ -112,7 +110,7 @@ def _public_db_skill_items(db: Session, *, include_runtime_details: bool) -> Lis
def _public_db_mcp_items(db: Session, *, include_runtime_details: bool) -> List[Dict[str, Any]]:
from core.config.user_intros import MCP_SERVER_USER_INTROS
from core.db.models import AdminMcpServer
- from core.services.mcp_service import is_removed_builtin_mcp_server
+ from core.capabilities.mcp.service import is_removed_builtin_mcp_server
rows = (
db.query(AdminMcpServer)
@@ -142,7 +140,7 @@ def _public_db_mcp_items(db: Session, *, include_runtime_details: bool) -> List[
enabled=bool(row.is_enabled),
version="1",
config={"server": row.server_id},
- icon=row.icon or _DEFAULT_MCP_ICONS.get(row.server_id, ""),
+ icon=row.icon or DEFAULT_MCP_ICONS.get(row.server_id, ""),
)
item["created_at"] = row.created_at.isoformat() if row.created_at else None
item["tools"] = [
@@ -175,8 +173,6 @@ def _config_bool(db: Session, key: str, default: bool) -> bool:
return default
-
-
def _set_database_query_state(catalog: Dict[str, Any], enabled: bool) -> None:
for item in catalog.get("mcp") or []:
if not isinstance(item, dict) or item.get("id") != DB_UMBRELLA_ID:
@@ -194,13 +190,13 @@ def _public_db_overlay(
overrides and private capabilities remain resolved in catalog_resolver.
"""
now = monotonic()
- with _runtime_db_cache_lock:
- cached = _runtime_db_cache.get(include_runtime_details)
+ with process_state().runtime_db_cache_lock:
+ cached = process_state().runtime_db_cache.get(include_runtime_details)
if cached is not None:
expires_at, db_query_enabled, skills, mcps = cached
if now < expires_at:
return db_query_enabled, copy.deepcopy(skills), copy.deepcopy(mcps)
- _runtime_db_cache.pop(include_runtime_details, None)
+ process_state().runtime_db_cache.pop(include_runtime_details, None)
db_query_enabled = _config_bool(db, _DATABASE_QUERY_ENABLED_CONFIG, True)
try:
@@ -214,8 +210,8 @@ def _public_db_overlay(
logger.warning("runtime catalog DB MCP overlay failed: %s", exc)
mcps = []
- with _runtime_db_cache_lock:
- _runtime_db_cache[include_runtime_details] = (
+ with process_state().runtime_db_cache_lock:
+ process_state().runtime_db_cache[include_runtime_details] = (
now + _RUNTIME_DB_CACHE_TTL,
db_query_enabled,
copy.deepcopy(skills),
@@ -252,7 +248,7 @@ def get_runtime_catalog(
if not isinstance(catalog.get(key), list):
catalog[key] = []
- from core.capabilities import device_catalog
+ from core.harness.registry import device_catalog
if device_catalog.active():
# 混合模式:能力一律以云端账号同步下来的那份为准。本机自带的内置技能 / 连接器
@@ -290,3 +286,16 @@ def get_runtime_catalog(
logger.warning("runtime catalog DB overlay merge failed: %s", exc)
return catalog
+
+
+@dataclass
+class RuntimeState:
+ runtime_db_cache: Dict[bool, tuple[float, bool, List[Dict[str, Any]], List[Dict[str, Any]]]] = field(default_factory=lambda: {})
+ runtime_db_cache_lock: object = field(default_factory=lambda: Lock())
+
+
+_STATE = RuntimeState()
+
+
+def process_state() -> RuntimeState:
+ return _STATE
diff --git a/src/backend/core/capabilities/identity_rules.py b/src/backend/core/capabilities/identity_rules.py
new file mode 100644
index 00000000..d1c5093c
--- /dev/null
+++ b/src/backend/core/capabilities/identity_rules.py
@@ -0,0 +1,36 @@
+"""Capability kinds and portable identity segments, independent of host storage."""
+import re
+
+KIND_SKILL = "skill"
+
+KIND_PLUGIN = "plugin"
+
+KIND_AGENT = "agent"
+
+KIND_MCP = "mcp"
+
+KINDS = (KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP)
+
+LOCAL_PROFILE = "local"
+
+BUILTIN_PROFILE = "builtin"
+
+_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,120}$")
+
+_WINDOWS_RESERVED = {
+ "CON",
+ "PRN",
+ "AUX",
+ "NUL",
+ *(f"COM{i}" for i in range(1, 10)),
+ *(f"LPT{i}" for i in range(1, 10)),
+}
+
+def safe_segment(value: str) -> str:
+ """A single path segment safe on every platform, or ``ValueError``."""
+ seg = (value or "").strip()
+ if seg != value or seg.endswith((".", " ")) or not _SEGMENT_RE.match(seg) or seg in (".", ".."):
+ raise ValueError(f"unsafe path segment: {value!r}")
+ if seg.split(".")[0].upper() in _WINDOWS_RESERVED:
+ raise ValueError(f"reserved name: {value!r}")
+ return seg
diff --git a/src/backend/core/services/desktop_capability_protocol.py b/src/backend/core/capabilities/manifest_protocol.py
similarity index 99%
rename from src/backend/core/services/desktop_capability_protocol.py
rename to src/backend/core/capabilities/manifest_protocol.py
index 5bccf3c7..070d73c6 100644
--- a/src/backend/core/services/desktop_capability_protocol.py
+++ b/src/backend/core/capabilities/manifest_protocol.py
@@ -49,7 +49,6 @@ def canonical_hash(value: Any) -> str:
def token_claims(token: str) -> Dict[str, Any]:
"""Read routing labels only; server-side signature/session checks authorize use."""
try:
- import base64
raw = str(token).split(".")
if len(raw) != 3 or raw[0] not in ("dcap1", "dcap2"):
return {}
@@ -82,7 +81,7 @@ def public_tool_schema(raw: Any) -> Optional[Dict[str, Any]]:
"inputSchema": schema,
}
# Only the versioned execution declaration crosses the boundary; arbitrary metadata does not.
- from core.services.management_contract import META_KEY
+ from core.capabilities.plugins.management.contract import META_KEY
meta = raw.get("_meta") or raw.get("meta") or {}
if isinstance(meta, dict) and META_KEY in meta:
public["_meta"] = {META_KEY: copy.deepcopy(meta[META_KEY])}
diff --git a/src/backend/core/capabilities/mcp/__init__.py b/src/backend/core/capabilities/mcp/__init__.py
new file mode 100644
index 00000000..0509af9f
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/__init__.py
@@ -0,0 +1 @@
+"""MCP server installation, management and OAuth connections."""
diff --git a/src/backend/core/services/mcp_builtin_specs.py b/src/backend/core/capabilities/mcp/builtin_specs.py
similarity index 94%
rename from src/backend/core/services/mcp_builtin_specs.py
rename to src/backend/core/capabilities/mcp/builtin_specs.py
index 802cfb79..62a91d01 100644
--- a/src/backend/core/services/mcp_builtin_specs.py
+++ b/src/backend/core/capabilities/mcp/builtin_specs.py
@@ -12,7 +12,6 @@
'- "各区今年 1–9 月固投同比"\n\n## 输出示例\n- 精确数值 + 数据口径标注\n'
"- 时间 / 地区 / 行业等多维度数据切片\n- 同比、环比、累计值自动计算\n"
),
- "is_stable": True,
"is_enabled": True,
"sort_order": 0,
"icon": "/home/mcp/knowledge.svg",
@@ -22,7 +21,6 @@
"display_name": "知识库检索",
"description": "从公有/私有知识库中语义检索政策文件、产业报告及用户上传文档,支持混合检索与重排序。",
"user_intro": None,
- "is_stable": False,
"is_enabled": True,
"sort_order": 1,
"icon": "/home/mcp/learning.svg",
@@ -32,7 +30,6 @@
"display_name": "互联网搜索",
"description": "检索公开网页、官方文档、新闻及财经资讯,支持多查询并发、来源去重和原文核验。",
"user_intro": None,
- "is_stable": True,
"is_enabled": True,
"sort_order": 2,
"icon": "/home/mcp/internet.svg",
@@ -42,7 +39,6 @@
"display_name": "网站信息抓取",
"description": "抓取指定网页 URL 的内容,提取正文文本或 Markdown,支持搜索引擎结果页解析。",
"user_intro": None,
- "is_stable": True,
"is_enabled": True,
"sort_order": 6,
"icon": "/home/mcp/source.svg",
diff --git a/src/backend/core/services/edition_mcp.py b/src/backend/core/capabilities/mcp/edition.py
similarity index 100%
rename from src/backend/core/services/edition_mcp.py
rename to src/backend/core/capabilities/mcp/edition.py
diff --git a/src/backend/core/services/mcp_management_service.py b/src/backend/core/capabilities/mcp/management.py
similarity index 89%
rename from src/backend/core/services/mcp_management_service.py
rename to src/backend/core/capabilities/mcp/management.py
index 0adab34e..dfb0aaa8 100644
--- a/src/backend/core/services/mcp_management_service.py
+++ b/src/backend/core/capabilities/mcp/management.py
@@ -3,15 +3,14 @@
from __future__ import annotations
import asyncio
-import hashlib
import ipaddress
-import json
import logging
import socket
-from typing import Any, Dict, Iterable, List
+from typing import Any, Dict, List
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
import httpx
+import httpx2
from core.db.models import AdminMcpServer
from core.infra.crypto import decrypt_secret, encrypt_secret
from core.infra.exceptions import BadRequestError
@@ -167,7 +166,7 @@ def auth_schema_from_headers(headers: Dict[str, str] | None) -> List[Dict[str, A
]
-from core.services.mcp_tool_metadata import tool_snapshot_hash, snapshot_tool
+from core.capabilities.mcp.tool_metadata import tool_snapshot_hash, snapshot_tool
def _is_forbidden_ip(address: str) -> bool:
@@ -317,16 +316,16 @@ def refresh_mcp_caches() -> None:
"""Invalidate MCP, catalog, capability, and prompt caches after a mutation."""
invalidators = []
try:
- from core.services.mcp_service import McpServerConfigService
+ from core.capabilities.mcp.service import McpServerConfigService
invalidators.append(McpServerConfigService.get_instance().invalidate_cache)
except Exception as exc: # noqa: BLE001
logger.warning("Unable to load the MCP cache invalidator: %s", exc)
for module_name, function_name in (
- ("core.config.catalog_loader", "invalidate_catalog_cache"),
- ("core.config.catalog_runtime", "invalidate_runtime_catalog_cache"),
- ("core.config.catalog_resolver", "invalidate_capability_cache"),
+ ("core.capabilities.catalog.loader", "invalidate_catalog_cache"),
+ ("core.capabilities.catalog.runtime", "invalidate_runtime_catalog_cache"),
+ ("core.capabilities.catalog.resolver", "invalidate_capability_cache"),
("prompts.prompt_runtime", "invalidate_prompt_cache"),
):
try:
@@ -346,12 +345,18 @@ def refresh_mcp_caches() -> None:
async def probe_mcp_connectivity(
row: AdminMcpServer,
- db: Session | None = None,
+ *,
+ validate: bool = False,
oauth_provider: Any = None,
timeout_seconds: float | None = None,
) -> tuple[bool, str]:
- """Return whether an MCP server can connect and enumerate its tools."""
- from core.services.mcp_service import McpServerConfigService
+ """Return whether an MCP server can connect and enumerate its tools.
+
+ ``row`` only supplies the connection config and receives ``tools_json``;
+ ``validate`` also runs the Domain Pack build check on the discovered tools,
+ on a session of its own.
+ """
+ from core.capabilities.mcp.service import McpServerConfigService
if timeout_seconds is None:
timeout_seconds = (
@@ -361,30 +366,23 @@ async def probe_mcp_connectivity(
)
cfg = McpServerConfigService.get_instance()._row_to_config(row)
+ cfg["discovery_timeout"] = timeout_seconds
if oauth_provider is not None:
cfg["oauth_provider"] = oauth_provider
async def _do_probe() -> None:
- from core.llm.mcp_pool import make_client
+ from core.harness.registry.mcp_clients import discover_tools
if row.transport not in ("stdio", "streamable_http", "sse"):
raise RuntimeError(f"Unknown transport: {row.transport}")
- is_http = row.transport in ("streamable_http", "sse")
- # AgentScope's stateful HTTP MCP client owns an AnyIO cancel scope that
- # is bound to the task which opened it. Failed handshakes (notably 401)
- # can finalize the underlying async generator in another task, masking
- # the real HTTP error with "Attempted to exit cancel scope...". A
- # stateless client keeps the complete list_tools lifecycle in this task.
- client = make_client(row.server_id, cfg, is_stateful=not is_http)
+ resource = await discover_tools(row.server_id, cfg)
try:
- if not is_http:
- await client.connect()
- discovered = await client.list_tools()
- tools_meta = []
+ discovered = resource.descriptors
tools_meta = [snapshot_tool(tool) for tool in discovered or []]
row.tools_json = tools_meta
- if db is not None and tools_meta:
- from core.ontology.build_validator import OntologyBuildValidator
+ if validate and tools_meta:
+ from core.db.threads import run_in_session
+ from core.harness.registry.build_governance import validate_build
extra_config = row.extra_config or {}
ontology_tags = {
@@ -401,7 +399,8 @@ async def _do_probe() -> None:
for tag in tags
if str(tag).strip()
)
- report = OntologyBuildValidator(db).validate(
+ report = await run_in_session(
+ validate_build,
asset_type="tool",
name=row.display_name,
description=row.description or "",
@@ -413,14 +412,7 @@ async def _do_probe() -> None:
messages = "; ".join(item.message for item in report.errors)
raise RuntimeError(f"Domain Pack 构建校验失败:{messages}")
finally:
- # Stateless HTTP list_tools opens and closes its own connection.
- # Calling close() is unnecessary and can re-enter the SDK's
- # task-bound cancel scope. Stateful stdio still requires cleanup.
- if not is_http:
- try:
- await client.close()
- except BaseException as exc: # noqa: BLE001
- logger.debug("MCP probe client close failed: %s", exc)
+ await resource.close()
try:
await asyncio.wait_for(_do_probe(), timeout=timeout_seconds)
@@ -463,13 +455,13 @@ def _format_mcp_probe_exception(exc: BaseException) -> str:
if is_cancel_noise:
continue
- if isinstance(current, httpx.HTTPStatusError) and current.response is not None:
+ if isinstance(current, httpx2.HTTPStatusError) and current.response is not None:
status = current.response.status_code
reason = current.response.reason_phrase
if status in (401, 403):
return f"远端 MCP 返回 {status} {reason},请检查鉴权设置和 Token"
return f"远端 MCP 返回 HTTP {status} {reason}"
- if isinstance(current, httpx.ConnectError):
+ if isinstance(current, httpx2.ConnectError):
return "无法连接远端 MCP,请检查地址、网络和服务状态"
if message:
return f"{type(current).__name__}: {message}"
diff --git a/src/backend/core/capabilities/mcp/oauth.py b/src/backend/core/capabilities/mcp/oauth.py
new file mode 100644
index 00000000..5fe43e84
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/oauth.py
@@ -0,0 +1,385 @@
+"""Browser OAuth install flow for remote MCP servers."""
+from __future__ import annotations
+from core.capabilities.mcp.management import materialize_mcp_http_connection as market_dependency_materialize_mcp_http_connection
+import asyncio
+import contextlib
+import json
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode, urlparse
+from core.config.settings import settings
+from core.db.models import AdminMcpServer
+from core.harness.registry.marketplace_catalog import marketplace_catalog
+from core.infra.crypto import decrypt_secret, encrypt_secret
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.ephemeral import get_ephemeral_state
+from core.capabilities.mcp.management import probe_mcp_connectivity, validate_remote_mcp_url
+from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata
+from sqlalchemy.orm import Session
+from core.capabilities.mcp import oauth_storage
+
+
+FLOW_TTL_SECONDS = 10 * 60
+
+
+DISCOVERY_GRACE_SECONDS = 5.0
+
+
+@dataclass
+class OAuthInstallFlow:
+ flow_id: str
+ slug: str
+ owner_user_id: Optional[str]
+ installed_by: Optional[str]
+ auth_method: str
+ credentials: Dict[str, str]
+ callback_url: str
+ client_id: str = ""
+ client_secret: str = ""
+ created_at: float = field(default_factory=time.time)
+ status: str = "starting"
+ authorization_url: str = ""
+ error: str = ""
+ result: Dict[str, Any] = field(default_factory=dict)
+ callback_code: str = ""
+ callback_state: Optional[str] = None
+ callback_issuer: Optional[str] = None
+ ready: asyncio.Event = field(default_factory=asyncio.Event)
+ callback_received: asyncio.Event = field(default_factory=asyncio.Event)
+
+
+def flow_key(flow_id: str) -> str:
+ return f"mcp:oauth:flow:{flow_id}"
+
+
+def callback_key(flow_id: str) -> str:
+ return f"mcp:oauth:callback:{flow_id}"
+
+
+async def store_flow_status(flow: OAuthInstallFlow) -> None:
+ payload = {
+ "flow_id": flow.flow_id,
+ "slug": flow.slug,
+ "owner_user_id": flow.owner_user_id,
+ "status": flow.status,
+ "authorization_url": flow.authorization_url,
+ "error": flow.error,
+ "result": dict(flow.result or {}),
+ }
+ await get_ephemeral_state().put(
+ flow_key(flow.flow_id),
+ json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
+ ttl=FLOW_TTL_SECONDS,
+ )
+
+
+def public_oauth_callback_url(request: Any) -> str:
+ """Build a trusted browser callback without reflecting an arbitrary Host header."""
+ configured = settings.server.mcp_oauth_public_base_url
+ if configured:
+ parsed = urlparse(configured)
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not parsed.netloc
+ or parsed.username
+ or parsed.password
+ or parsed.query
+ or parsed.fragment
+ ):
+ raise BadRequestError(message="MCP_OAUTH_PUBLIC_BASE_URL 配置无效")
+ return f"{configured}/v1/mcp-market/oauth/callback"
+
+ origin = str(request.headers.get("origin") or "").rstrip("/")
+ parsed_origin = urlparse(origin)
+ if not origin or parsed_origin.scheme not in {"http", "https"} or not parsed_origin.netloc:
+ raise BadRequestError(message="OAuth 登录必须从浏览器发起")
+ request_host = str(request.headers.get("host") or request.url.netloc).lower()
+ if settings.server.is_prod and parsed_origin.netloc.lower() != request_host:
+ raise BadRequestError(
+ message="跨域部署必须配置 MCP_OAUTH_PUBLIC_BASE_URL(例如 http(s)://app.example.com/api)"
+ )
+ return f"{origin}/api/v1/mcp-market/oauth/callback"
+
+
+def purge_expired_flows() -> None:
+ cutoff = time.time() - FLOW_TTL_SECONDS
+ for flow_id, flow in list(process_state().flows.items()):
+ if flow.created_at < cutoff:
+ process_state().flows.pop(flow_id, None)
+
+
+def get_version(db: Session, slug: str) -> tuple[Any, Any]:
+ return marketplace_catalog().mcp_version(db, slug)
+
+
+def _oauth_method(db: Session, slug: str, auth_method: str):
+ _, version = get_version(db, slug)
+ auth_config = marketplace_catalog().normalize_auth_config(
+ version.auth_config, list(version.auth_schema or [])
+ )
+ return next((row for row in auth_config["methods"] if row["id"] == auth_method), None)
+
+
+async def start_oauth_install(
+ *,
+ slug: str,
+ owner_user_id: Optional[str],
+ installed_by: Optional[str],
+ auth_method: str,
+ credentials: Dict[str, str],
+ callback_url_base: str,
+ client_id: str = "",
+ client_secret: str = "",
+) -> Dict[str, str]:
+ """Start a browser OAuth install and wait only until the authorize URL is known."""
+
+
+ from core.db.threads import run_in_session
+
+ purge_expired_flows()
+ method = await run_in_session(_oauth_method, slug, auth_method)
+ if not method or method["type"] != "oauth2":
+ raise BadRequestError(message="该 MCP 不支持所选 OAuth 认证方式")
+ if method.get("client_id_required") and not client_id.strip():
+ raise BadRequestError(message="该 OAuth 服务需要 Client ID")
+ if method.get("client_secret_required") and not client_secret.strip():
+ raise BadRequestError(message="该 OAuth 服务需要 Client Secret")
+ flow_id = f"mcpoauth_{uuid.uuid4().hex}"
+ callback_url = f"{callback_url_base}?{urlencode({'flow_id': flow_id})}"
+ flow = OAuthInstallFlow(
+ flow_id=flow_id,
+ slug=slug,
+ owner_user_id=owner_user_id,
+ installed_by=installed_by,
+ auth_method=auth_method,
+ credentials=dict(credentials or {}),
+ callback_url=callback_url,
+ client_id=client_id.strip(),
+ client_secret=client_secret.strip(),
+ )
+ process_state().flows[flow_id] = flow
+ await store_flow_status(flow)
+ asyncio.create_task(run_flow(flow), name=f"mcp-oauth-{flow_id}")
+ with contextlib.suppress(asyncio.TimeoutError):
+ await asyncio.wait_for(flow.ready.wait(), timeout=DISCOVERY_GRACE_SECONDS)
+ if flow.status == "failed":
+ raise BadRequestError(message=flow.error or "无法启动 OAuth 登录")
+ return {
+ "flow_id": flow.flow_id,
+ "authorization_url": flow.authorization_url,
+ "status": flow.status,
+ }
+
+
+def _flow_inputs(db, flow: "OAuthInstallFlow"):
+ """What the OAuth probe needs from the market listing, read in one session."""
+ item, version = get_version(db, flow.slug)
+ auth_config = marketplace_catalog().normalize_auth_config(
+ version.auth_config, list(version.auth_schema or [])
+ )
+ method = next(row for row in auth_config["methods"] if row["id"] == flow.auth_method)
+ headers = marketplace_catalog().installation_secrets(
+ version,
+ flow.credentials,
+ {},
+ auth_method=flow.auth_method,
+ )
+ candidate = AdminMcpServer(
+ server_id=f"oauth_probe_{uuid.uuid4().hex[:16]}",
+ display_name=item.display_name,
+ description=item.description or "",
+ transport=version.transport,
+ url=version.url,
+ headers=headers,
+ args=[],
+ env_vars={},
+ env_inherit=[],
+ is_enabled=False,
+ extra_config={},
+ )
+ runtime_url, _ = market_dependency_materialize_mcp_http_connection(version.url, headers)
+ return method, candidate, runtime_url
+
+
+async def run_flow(flow: OAuthInstallFlow) -> None:
+ """Drive one OAuth install; it may wait minutes for the user and holds no session."""
+ from core.db.threads import run_in_session
+
+ try:
+ method, candidate, runtime_url = await run_in_session(_flow_inputs, flow)
+ await validate_remote_mcp_url(runtime_url, require_https=False)
+
+ metadata = OAuthClientMetadata(
+ application_type=("native" if urlparse(flow.callback_url).hostname in {"localhost", "127.0.0.1", "::1"} else "web"),
+ redirect_uris=[flow.callback_url],
+ token_endpoint_auth_method=("client_secret_post" if flow.client_secret else "none"),
+ grant_types=["authorization_code", "refresh_token"],
+ response_types=["code"],
+ scope=" ".join(method.get("scopes") or []) or None,
+ client_name="HugAgentOS MCP Client",
+ )
+ storage = oauth_storage.OAuthBundleStorage(
+ {
+ "server_url": runtime_url,
+ "client_metadata": metadata.model_dump(mode="json", exclude_none=True),
+ }
+ )
+ if flow.client_id:
+ await storage.set_client_info(
+ OAuthClientInformationFull(
+ **metadata.model_dump(mode="json", exclude_none=True),
+ client_id=flow.client_id,
+ client_secret=flow.client_secret or None,
+ )
+ )
+
+ async def redirect_handler(url: str) -> None:
+ flow.authorization_url = url
+ flow.status = "waiting_for_user"
+ await store_flow_status(flow)
+ flow.ready.set()
+
+ async def callback_handler() -> AuthorizationCodeResult:
+ deadline = time.monotonic() + 300.0
+ while time.monotonic() < deadline:
+ if flow.callback_received.is_set():
+ if flow.error:
+ raise RuntimeError(flow.error)
+ return AuthorizationCodeResult(code=flow.callback_code or "", state=flow.callback_state, iss=flow.callback_issuer)
+ raw = await get_ephemeral_state().get(callback_key(flow.flow_id))
+ if raw:
+ payload = json.loads(raw)
+ callback_error = str(payload.get("error") or "")
+ if callback_error:
+ raise RuntimeError(callback_error)
+ encrypted_code = str(payload.get("code") or "")
+ callback_code = decrypt_secret(encrypted_code) if encrypted_code else ""
+ return AuthorizationCodeResult(code=callback_code or "", state=payload.get("state"), iss=payload.get("iss"))
+ await asyncio.sleep(0.5)
+ raise TimeoutError("OAuth 登录回调超时")
+
+ provider = oauth_storage.build_oauth_provider(
+ runtime_url,
+ storage.bundle,
+ storage=storage,
+ redirect_handler=redirect_handler,
+ callback_handler=callback_handler,
+ )
+ ok, error = await probe_mcp_connectivity(
+ candidate,
+ validate=True,
+ oauth_provider=provider,
+ timeout_seconds=300.0,
+ )
+ if not ok:
+ raise RuntimeError(f"OAuth MCP 连接失败:{error}")
+ if flow.error:
+ raise RuntimeError(flow.error)
+ persisted = await get_ephemeral_state().get(flow_key(flow.flow_id))
+ if persisted:
+ persisted_status = json.loads(persisted)
+ if persisted_status.get("status") == "failed":
+ raise RuntimeError(str(persisted_status.get("error") or "OAuth 登录已取消"))
+ bundle = storage.capture_context(provider)
+ flow.status = "installing"
+ await store_flow_status(flow)
+ flow.result = await marketplace_catalog().install_market_item(
+ flow.slug,
+ owner_user_id=flow.owner_user_id,
+ installed_by=flow.installed_by,
+ credentials=flow.credentials,
+ auth_method=flow.auth_method,
+ oauth_bundle=bundle,
+ )
+ flow.status = "completed"
+ await store_flow_status(flow)
+ except BaseException as exc: # noqa: BLE001
+ flow.status = "failed"
+ flow.error = str(exc)
+ await store_flow_status(flow)
+ finally:
+ flow.ready.set()
+
+
+async def complete_callback(
+ flow_id: str,
+ *,
+ code: str = "",
+ state: Optional[str] = None,
+ iss: Optional[str] = None,
+ error: str = "",
+) -> None:
+ flow = process_state().flows.get(flow_id)
+ public_raw = await get_ephemeral_state().get(flow_key(flow_id))
+ if not flow and not public_raw:
+ raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
+ await get_ephemeral_state().put(
+ callback_key(flow_id),
+ json.dumps(
+ {
+ "code": encrypt_secret(code) if code else "",
+ "state": state,
+ "iss": iss,
+ "error": error,
+ },
+ ensure_ascii=False,
+ separators=(",", ":"),
+ ),
+ ttl=FLOW_TTL_SECONDS,
+ )
+ if flow:
+ flow.callback_code = code
+ flow.callback_state = state
+ flow.callback_issuer = iss
+ flow.error = error
+ flow.status = "failed" if error else "processing_callback"
+ flow.callback_received.set()
+ await store_flow_status(flow)
+ else:
+ payload = json.loads(public_raw)
+ payload["status"] = "failed" if error else "processing_callback"
+ payload["error"] = error
+ await get_ephemeral_state().put(
+ flow_key(flow_id),
+ json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
+ ttl=FLOW_TTL_SECONDS,
+ )
+
+
+async def cancel_flow(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]:
+ """Idempotently stop an OAuth flow owned by the current installer."""
+ status = await get_flow_status(flow_id, owner_user_id=owner_user_id)
+ if status["status"] in {"completed", "failed"}:
+ return status
+ await complete_callback(flow_id, error="OAuth 登录已取消")
+ return await get_flow_status(flow_id, owner_user_id=owner_user_id)
+
+
+async def get_flow_status(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]:
+ raw = await get_ephemeral_state().get(flow_key(flow_id))
+ if not raw:
+ raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
+ payload = json.loads(raw)
+ if payload.get("owner_user_id") != owner_user_id:
+ raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
+ return {
+ "flow_id": flow_id,
+ "status": str(payload.get("status") or "starting"),
+ "authorization_url": str(payload.get("authorization_url") or ""),
+ "error": str(payload.get("error") or ""),
+ "result": dict(payload.get("result") or {}),
+ }
+
+
+@dataclass
+class OauthState:
+ flows: Dict[str, OAuthInstallFlow] = field(default_factory=lambda: {})
+
+
+_STATE = OauthState()
+
+
+def process_state() -> OauthState:
+ return _STATE
diff --git a/src/backend/core/capabilities/mcp/oauth_storage.py b/src/backend/core/capabilities/mcp/oauth_storage.py
new file mode 100644
index 00000000..5bf6b3f3
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/oauth_storage.py
@@ -0,0 +1,124 @@
+"""Encrypted token storage that backs the MCP SDK's OAuth provider."""
+from __future__ import annotations
+import json
+import time
+from typing import Any, Dict, Optional
+from core.db.engine import SessionLocal
+from core.db.models import AdminMcpServer
+from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, mcp_oauth_bundle_storage_key, validate_remote_mcp_url
+from mcp.client.auth import OAuthClientProvider, TokenStorage
+from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAuthMetadata, OAuthToken, ProtectedResourceMetadata
+
+class OAuthBundleStorage(TokenStorage):
+ """SDK token storage backed by an encrypted MCP installation row."""
+
+ def __init__(self, bundle: Optional[Dict[str, Any]] = None, *, server_id: Optional[str] = None):
+ self.bundle: Dict[str, Any] = dict(bundle or {})
+ self.server_id = server_id
+
+ async def get_tokens(self) -> OAuthToken | None:
+ raw = self.bundle.get("tokens")
+ return OAuthToken.model_validate(raw) if isinstance(raw, dict) else None
+
+ async def set_tokens(self, tokens: OAuthToken) -> None:
+ previous = self.bundle.get("tokens") if isinstance(self.bundle.get("tokens"), dict) else {}
+ if not tokens.refresh_token and previous.get("refresh_token"):
+ tokens = tokens.model_copy(update={"refresh_token": previous["refresh_token"]})
+ self.bundle["tokens"] = tokens.model_dump(mode="json", exclude_none=True)
+ self.bundle["expires_at"] = (
+ time.time() + int(tokens.expires_in) if tokens.expires_in is not None else None
+ )
+ self._persist()
+
+ async def get_client_info(self) -> OAuthClientInformationFull | None:
+ raw = self.bundle.get("client_info")
+ return OAuthClientInformationFull.model_validate(raw) if isinstance(raw, dict) else None
+
+ async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
+ self.bundle["client_info"] = client_info.model_dump(mode="json", exclude_none=True)
+ self._persist()
+
+ def capture_context(self, provider: OAuthClientProvider) -> Dict[str, Any]:
+ context = provider.context
+ self.bundle["server_url"] = context.server_url
+ self.bundle["client_metadata"] = context.client_metadata.model_dump(
+ mode="json", exclude_none=True
+ )
+ self.bundle["oauth_metadata"] = (
+ context.oauth_metadata.model_dump(mode="json", exclude_none=True)
+ if context.oauth_metadata
+ else None
+ )
+ self.bundle["protected_resource_metadata"] = (
+ context.protected_resource_metadata.model_dump(mode="json", exclude_none=True)
+ if context.protected_resource_metadata
+ else None
+ )
+ self.bundle["auth_server_url"] = context.auth_server_url
+ self.bundle["protocol_version"] = context.protocol_version
+ self._persist()
+ return dict(self.bundle)
+
+ def _persist(self) -> None:
+ if not self.server_id:
+ return
+ db = SessionLocal()
+ try:
+ row = (
+ db.query(AdminMcpServer).filter(AdminMcpServer.server_id == self.server_id).first()
+ )
+ if not row:
+ return
+ headers = decrypt_mcp_headers(row.headers)
+ headers[mcp_oauth_bundle_storage_key()] = json.dumps(
+ self.bundle, ensure_ascii=False, separators=(",", ":")
+ )
+ row.headers = encrypt_mcp_headers(headers)
+ db.commit()
+ finally:
+ db.close()
+
+
+def build_oauth_provider(
+ server_url: str,
+ bundle: Dict[str, Any],
+ *,
+ storage: Optional[OAuthBundleStorage] = None,
+ redirect_handler=None,
+ callback_handler=None,
+) -> OAuthClientProvider:
+ """Rehydrate an SDK provider for first login or automatic refresh."""
+ actual_storage = storage or OAuthBundleStorage(bundle)
+ metadata_raw = bundle.get("client_metadata") or {
+ "redirect_uris": ["http://localhost/oauth/callback"],
+ "client_name": "HugAgentOS MCP Client",
+ "grant_types": ["authorization_code", "refresh_token"],
+ "response_types": ["code"],
+ }
+ provider = OAuthClientProvider(
+ server_url=server_url,
+ client_metadata=OAuthClientMetadata.model_validate(metadata_raw),
+ storage=actual_storage,
+ redirect_handler=redirect_handler,
+ callback_handler=callback_handler,
+ )
+
+ async def _validate_oauth_request(request) -> None:
+ # OAuth discovery metadata is controlled by the remote resource. Apply
+ # the same DNS/IP SSRF boundary to every SDK-generated request, not only
+ # to the original MCP URL.
+ await validate_remote_mcp_url(str(request.url), require_https=False)
+
+ provider.mcp_request_hook = _validate_oauth_request
+ context = provider.context
+ if isinstance(bundle.get("oauth_metadata"), dict):
+ context.oauth_metadata = OAuthMetadata.model_validate(bundle["oauth_metadata"])
+ if isinstance(bundle.get("protected_resource_metadata"), dict):
+ context.protected_resource_metadata = ProtectedResourceMetadata.model_validate(
+ bundle["protected_resource_metadata"]
+ )
+ context.auth_server_url = bundle.get("auth_server_url")
+ context.protocol_version = bundle.get("protocol_version")
+ expires_at = bundle.get("expires_at")
+ context.token_expiry_time = float(expires_at) if expires_at else None
+ return provider
diff --git a/src/backend/core/services/mcp_service.py b/src/backend/core/capabilities/mcp/service.py
similarity index 91%
rename from src/backend/core/services/mcp_service.py
rename to src/backend/core/capabilities/mcp/service.py
index 9359b975..25ced2fd 100644
--- a/src/backend/core/services/mcp_service.py
+++ b/src/backend/core/capabilities/mcp/service.py
@@ -1,7 +1,7 @@
"""MCP server configuration service (DB-driven, cached).
Reads MCP server configs from the admin_mcp_servers table and provides
-them in the same dict format that MCPConnectionPool and agent_factory
+them in the same dict format that run-owned MCP discovery and agent_factory
expect (compatible with the old MCP_SERVERS dict from mcp_config.py).
Thread-safe singleton with a 30s TTL cache.
@@ -17,19 +17,19 @@
from core.db.engine import SessionLocal
from core.db.models import AdminMcpServer
-from core.services.edition_mcp import edition_retired_builtin_mcp_server_ids
+from core.capabilities.mcp.edition import edition_retired_builtin_mcp_server_ids
logger = logging.getLogger(__name__)
_CACHE_TTL = 30.0
-def _rewrite_builtin_mcp_host(url: str) -> str:
+def rewrite_builtin_mcp_host(url: str) -> str:
"""Point built-in MCP URLs at the configured host.
Plugin-provided MCP servers (automation_task / skill_manager / site_publish)
hardcode the Docker service name in their manifests
- (``http://mcp:/mcp/``). That host only resolves inside the compose
+ (``http://mcp:/mcp``). That host only resolves inside the compose
network, so in the no-Docker local profile (``MCP_HOST=127.0.0.1``) the
plugin's tool calls would be unreachable. Rewrite **only** the literal ``mcp``
host to ``settings.server.mcp_host``: a no-op on compose (mcp_host == "mcp"),
@@ -55,7 +55,8 @@ def _rewrite_builtin_mcp_host(url: str) -> str:
port = {base: PORTS[sid] for sid, base in _BASE_PORTS.items()}.get(port, port)
netloc = f"{host}:{port}" if port else host
- return urlunparse(parts._replace(netloc=netloc))
+ path = "/mcp" if parts.path == "/mcp/" else parts.path
+ return urlunparse(parts._replace(netloc=netloc, path=path))
except Exception:
return url
@@ -93,7 +94,6 @@ def get_all_servers(
"env": {...}, # merged: env_inherit from OS + env_vars
"url": "...", # for HTTP/SSE
"headers": {...},
- "is_stable": True,
}
"""
if not use_cache:
@@ -170,7 +170,6 @@ def _row_to_config(self, row: AdminMcpServer) -> dict:
"""Convert a DB row to the config dict format."""
cfg: Dict[str, Any] = {
"transport": row.transport,
- "is_stable": row.is_stable,
# 来源元数据(桌面双端桥按组件基名做 logical 去重时消费;
# 连接层不读这两个键)。
"source_plugin": row.source_plugin,
@@ -184,22 +183,19 @@ def _row_to_config(self, row: AdminMcpServer) -> dict:
plaintext_headers: Dict[str, str] = {}
if row.headers:
- from core.services.mcp_management_service import decrypt_mcp_headers
+ from core.capabilities.mcp.management import decrypt_mcp_headers
plaintext_headers = decrypt_mcp_headers(row.headers)
# HTTP/SSE fields. Marketplace credentials that belong in the URL are
# decrypted and materialized only here, immediately before connecting.
if row.transport in ("streamable_http", "sse"):
- from core.services.mcp_management_service import (
- materialize_mcp_http_connection,
- mcp_oauth_bundle_storage_key,
- )
+ from core.capabilities.mcp.management import materialize_mcp_http_connection, mcp_oauth_bundle_storage_key
runtime_url, runtime_headers = materialize_mcp_http_connection(
row.url or "", plaintext_headers
)
- cfg["url"] = _rewrite_builtin_mcp_host(runtime_url)
+ cfg["url"] = rewrite_builtin_mcp_host(runtime_url)
if runtime_headers:
cfg["headers"] = runtime_headers
oauth_raw = plaintext_headers.get(mcp_oauth_bundle_storage_key())
@@ -207,10 +203,7 @@ def _row_to_config(self, row: AdminMcpServer) -> dict:
try:
import json
- from core.services.mcp_oauth_service import (
- OAuthBundleStorage,
- build_oauth_provider,
- )
+ from core.capabilities.mcp.oauth_storage import OAuthBundleStorage, build_oauth_provider
oauth_bundle = json.loads(oauth_raw)
storage = OAuthBundleStorage(oauth_bundle, server_id=row.server_id)
@@ -265,22 +258,24 @@ def _build_env(self, row: AdminMcpServer) -> Dict[str, str]:
# Phase 3: apply DB-driven overlays (model config, system config)
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
overlay = ModelConfigService.get_instance().get_mcp_env_overlay()
if overlay:
env.update(overlay)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.mcp.service:_build_env', fallback_error)
try:
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc_overlay = SystemConfigService.get_instance().get_service_env_overlay()
if svc_overlay:
env.update(svc_overlay)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.mcp.service:_build_env', fallback_error)
return env
@@ -306,7 +301,7 @@ def get_owned_servers(
if strict:
raise
logger.warning("[mcp_service] Failed to load owned servers for %s: %s", user_id, exc)
- from core.capabilities.local_plugin_runtime import configs
+ from core.harness.registry.execution_catalog import local_plugin_configs as configs
owned.update(configs(user_id, enabled_only=enabled_only))
return owned
@@ -315,11 +310,6 @@ def get_server(self, server_id: str) -> Optional[dict]:
servers = self.get_all_servers(enabled_only=False)
return servers.get(server_id)
- def get_stable_server_ids(self) -> Set[str]:
- """Return set of server_ids where is_stable=True."""
- servers = self.get_all_servers(enabled_only=True)
- return {k for k, v in servers.items() if v.get("is_stable")}
-
def invalidate_cache(self) -> None:
"""Clear cache so next call re-reads from DB."""
with self._lock:
@@ -348,13 +338,13 @@ def get_all_rows(self) -> List[AdminMcpServer]:
# has since deleted, and it is a no-op on any DB that already has the catalog.
#
# The URL is templated per-server from ``extensions.mcp_servers._ports`` + ``settings.server.
-# mcp_host`` at seed time, so compose gets ``http://mcp:/mcp/`` and local
-# gets ``http://127.0.0.1:/mcp/`` from the same definition. Only the
+# mcp_host`` at seed time, so compose gets ``http://mcp:/mcp`` and local
+# gets ``http://127.0.0.1:/mcp`` from the same definition. Only the
# built-in servers the ``mcp`` container / ``_launcher`` actually serves are
# included; plugin-provided servers (automation_task / skill_manager /
# site_publish) arrive via plugin install, and external-service ones (db_query /
# es_query) need their own containers, so neither belongs in the base seed.
-from core.services.mcp_builtin_specs import BUILTIN_MCP_SERVERS
+from core.capabilities.mcp.builtin_specs import BUILTIN_MCP_SERVERS
# Built-ins retired from the packaged runtime. Keep this explicit tombstone
# set separate from ``BUILTIN_MCP_SERVERS`` so upgraded create_all/SQLite
@@ -444,7 +434,7 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]:
it bootstraps a fresh ``create_all`` DB (local profile) without touching a DB
that alembic already seeded and without resurrecting admin-deleted rows.
"""
- from core.config.settings import settings
+ from core.config.mcp_config import mcp_http_url
from extensions.mcp_servers._ports import PORTS
existing = (
@@ -458,7 +448,6 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]:
if existing is not None:
return []
- host = settings.server.mcp_host
seeded: List[str] = []
for spec in BUILTIN_MCP_SERVERS:
sid = spec["server_id"]
@@ -472,8 +461,7 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]:
description=spec.get("description", ""),
user_intro=spec.get("user_intro"),
transport="streamable_http",
- url=f"http://{host}:{port}/mcp/",
- is_stable=spec.get("is_stable", True),
+ url=mcp_http_url(sid),
is_enabled=spec.get("is_enabled", True),
sort_order=spec.get("sort_order", 0),
icon=spec.get("icon"),
diff --git a/src/backend/core/capabilities/mcp/tool_metadata.py b/src/backend/core/capabilities/mcp/tool_metadata.py
new file mode 100644
index 00000000..0c1a220a
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/tool_metadata.py
@@ -0,0 +1,13 @@
+"""Capture the same public MCP schema for discovery, hashing and desktop delivery."""
+import hashlib
+import json
+from core.capabilities.manifest_protocol import public_tool_schema
+
+
+def snapshot_tool(tool):
+ return public_tool_schema(tool.model_dump(by_alias=True, mode="json"))
+
+
+def tool_snapshot_hash(tools):
+ normalized = sorted([public_tool_schema(t) for t in tools or []], key=lambda t: t["name"])
+ return hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
diff --git a/src/backend/core/capabilities/mcp/tool_schema.py b/src/backend/core/capabilities/mcp/tool_schema.py
new file mode 100644
index 00000000..b55f05fd
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/tool_schema.py
@@ -0,0 +1,20 @@
+"""Interpret server tool schema metadata without loading the SDK."""
+from typing import Any
+
+def has_usable_schema(tool: Any) -> bool:
+ """Whether a manifest tool entry carries the server's real ``inputSchema``.
+
+ A tool the platform only knows by name and description (a plugin manifest
+ lists its tools for display, without schemas) must not be handed to a model:
+ the parameter object would be empty, so the model has nowhere to put its
+ arguments and every call arrives blank.
+
+ The marker is whether a schema was captured at all, not whether it declares
+ parameters: a server is free to describe an argument-less tool as
+ ``{"type": "object"}``, while an entry that never reached the server carries
+ no ``inputSchema`` key, which normalizes to an empty dict.
+ """
+ if not isinstance(tool, dict):
+ return False
+ schema = tool.get("inputSchema") or tool.get("input_schema")
+ return isinstance(schema, dict) and bool(schema)
diff --git a/src/backend/core/capabilities/mcp/tool_schema_backfill.py b/src/backend/core/capabilities/mcp/tool_schema_backfill.py
new file mode 100644
index 00000000..375f7bf9
--- /dev/null
+++ b/src/backend/core/capabilities/mcp/tool_schema_backfill.py
@@ -0,0 +1,120 @@
+"""Capture real tool schemas for MCP servers that only know their tools by name.
+
+A plugin manifest declares its tools for display — ``{name, description}`` with no
+parameters. Probing the running server is what fills ``AdminMcpServer.tools_json``
+with each tool's ``inputSchema``, and that column is the desktop's only source of
+tool parameters: the cloud ships those entries verbatim in the capability manifest
+and the desktop builds the model-facing tool from them. A server left with
+display-only entries therefore offers the model parameter-less tools, so arguments
+have nowhere to go and every call arrives blank.
+
+Backfill runs where the servers are reachable (cloud startup, and after a plugin
+install or upgrade rewrites the manifest), never on the per-message path.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Probing is a network round trip per server; keep startup bounded while still
+# repairing a whole installation in one pass.
+_MAX_CONCURRENCY = 4
+_PROBE_TIMEOUT_S = 20.0
+
+
+def _needs_backfill(tools_json) -> bool:
+ from core.capabilities.mcp.tool_schema import has_usable_schema
+
+ if not isinstance(tools_json, list) or not tools_json:
+ return False
+ return any(not has_usable_schema(item) for item in tools_json)
+
+
+async def backfill_missing_tool_schemas(server_ids: Optional[List[str]] = None) -> int:
+ """Probe enabled MCP servers whose stored tools carry no schema.
+
+ Returns the number of servers whose ``tools_json`` was refreshed. Failures are
+ logged and skipped: an unreachable server keeps its display-only entries, and
+ the manifest client withholds those tools rather than offering broken ones.
+ """
+ from core.capabilities.mcp.management import probe_mcp_connectivity
+ from core.db.threads import run_in_session
+
+ from core.harness.registry import device_catalog
+
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ if has_run_capabilities() and not server_ids:
+ from core.harness.registry.local_publication import migrate_local_plugins as migrate_plugins
+ from core.harness.registry.local_publication import migrate_local_skills as migrate_skills
+ await asyncio.to_thread(migrate_plugins)
+ await asyncio.to_thread(migrate_skills)
+
+ if device_catalog.active() and not server_ids:
+ # 混合模式下工具 schema 随云端能力清单下发,不靠探活取。业务库里留下的
+ # 本机插件行指向 compose 服务名,桌面端根本连不上——照探只会每次启动都
+ # 刷一轮「无法连接远端 MCP」,修不好任何东西。
+ logger.debug("[mcp-schema] hybrid desktop: schemas come from the cloud manifest, skipping probe")
+ return 0
+
+ rows = await run_in_session(_servers_to_backfill, server_ids)
+ if not rows:
+ return 0
+
+ semaphore = asyncio.Semaphore(_MAX_CONCURRENCY)
+ repaired: Dict[str, list] = {}
+
+ async def _probe(row) -> None:
+ async with semaphore:
+ try:
+ ok, detail = await probe_mcp_connectivity(
+ row, validate=True, timeout_seconds=_PROBE_TIMEOUT_S
+ )
+ except Exception as exc: # noqa: BLE001 - one server must not stop the pass
+ logger.warning(
+ "[mcp-schema] probe raised for '%s': %s", row.server_id, exc
+ )
+ return
+ if ok and not _needs_backfill(row.tools_json):
+ repaired[row.server_id] = row.tools_json
+ else:
+ logger.warning(
+ "[mcp-schema] '%s' still has tools without a schema: %s",
+ row.server_id,
+ detail,
+ )
+
+ await asyncio.gather(*(_probe(row) for row in rows))
+ if repaired:
+ await run_in_session(_store_schemas, repaired)
+ logger.info(
+ "[mcp-schema] tool schemas captured for %d server(s): %s",
+ len(repaired),
+ ", ".join(sorted(repaired)),
+ )
+ return len(repaired)
+
+
+def _servers_to_backfill(db, server_ids):
+ """Enabled servers whose tools lack schemas, detached for probing."""
+ from core.capabilities.plugins.management.bundle_upgrade import refresh
+ from core.db.models import AdminMcpServer
+
+ refresh(db)
+ query = db.query(AdminMcpServer).filter(AdminMcpServer.is_enabled.is_(True))
+ if server_ids:
+ query = query.filter(AdminMcpServer.server_id.in_(list(server_ids)))
+ rows = [row for row in query.all() if _needs_backfill(row.tools_json)]
+ db.expunge_all()
+ return rows
+
+
+def _store_schemas(db, repaired) -> None:
+ from core.db.models import AdminMcpServer
+
+ for row in db.query(AdminMcpServer).filter(AdminMcpServer.server_id.in_(list(repaired))):
+ row.tools_json = repaired[row.server_id]
+ db.commit()
diff --git a/src/backend/core/capabilities/plugins/README.md b/src/backend/core/capabilities/plugins/README.md
new file mode 100644
index 00000000..266e230f
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/README.md
@@ -0,0 +1,23 @@
+# Plugin capabilities
+
+This package owns plugin manifests, installation records, capability selection and UI contracts.
+
+| Package | Responsibility |
+| --- | --- |
+| `management/` | Install, enable, uninstall, inspect and publish database-backed plugins |
+| `packaging/` | Discover files, extract archives and normalize manifests |
+| `runtime/` | Resolve deferred capabilities and retain per-chat activation state |
+| `ui/` | Validate UI declarations, project browser-safe data and authorize data queries |
+
+Framework-specific activation belongs to `core.llm.tools.plugin_activation`.
+Device-local installation and migration belong to `features.desktop.local.plugins`.
+Marketplace publication policy belongs to `features.marketplace`.
+
+Installation and packaging must not import the agent factory or SDK activation.
+Packaging does not publish database records: archive extraction belongs to
+`packaging/archive.py`, while installation persistence belongs to `management/`.
+
+Use explicit owner imports and patch the consuming interface in tests. The old
+`core.plugins` and `core.services` modules are removed; no compatibility exports
+are provided. Bundled resource paths are relative to the backend root in local
+and Docker installations.
diff --git a/src/backend/core/plugins/__init__.py b/src/backend/core/capabilities/plugins/__init__.py
similarity index 100%
rename from src/backend/core/plugins/__init__.py
rename to src/backend/core/capabilities/plugins/__init__.py
diff --git a/src/backend/core/plugins/connections.py b/src/backend/core/capabilities/plugins/connections.py
similarity index 83%
rename from src/backend/core/plugins/connections.py
rename to src/backend/core/capabilities/plugins/connections.py
index 6641ab0b..984c7820 100644
--- a/src/backend/core/plugins/connections.py
+++ b/src/backend/core/capabilities/plugins/connections.py
@@ -10,6 +10,6 @@ def connection_type(value: Any) -> str | None:
def manifest_connection(manifest: dict) -> str | None:
- from core.plugins.packaging.importer import _ext_or_top, manifest_extensions
+ from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions
return connection_type(_ext_or_top(manifest, manifest_extensions(manifest), "connection"))
diff --git a/src/backend/core/capabilities/plugins/management/__init__.py b/src/backend/core/capabilities/plugins/management/__init__.py
new file mode 100644
index 00000000..656d1cb6
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/management/__init__.py
@@ -0,0 +1,43 @@
+"""Public plugin lifecycle API; implementations follow domain ownership."""
+
+from core.capabilities.plugins.management.admin_config import get_plugin_admin_config, set_plugin_admin_config
+from core.capabilities.plugins.management.details import exclude_market_skill, get_market_skill_exclusions, get_plugin_detail
+from core.capabilities.plugins.management.installation import ensure_default_plugins_bootstrapped, import_plugin, import_plugin_from_zip, install_plugin
+from core.capabilities.plugins.management.lifecycle import set_installed_plugin_meta, set_plugin_component_enabled, set_plugin_component_enabled_for_user, set_plugin_enabled, set_plugin_enabled_for_user, uninstall_plugin
+from core.capabilities.plugins.management.packages import delete_market_package, publish_plugin_zip_to_market
+from core.capabilities.plugins.management.queries import get_installed_detail, list_installed
+from core.capabilities.plugins.packaging.definitions import BUILTIN_PLUGIN_MARKET_META, DEFAULT_BOOTSTRAP_MARKER_ID, DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, LOCAL_BOOTSTRAP_MARKER_NAME, MAX_ICON_LEN, MAX_ICON_URL_LEN, MAX_ZIP_BYTES, PLUGIN_BUNDLES_DIR, PLUGIN_MARKET_META_BLOCK_ID, PLUGIN_SOURCE_DIRS
+from core.capabilities.plugins.packaging.sources import builtin_plugin_component_ids
+
+__all__ = [
+ "builtin_plugin_component_ids",
+ "list_installed",
+ "get_installed_detail",
+ "get_plugin_admin_config",
+ "set_plugin_admin_config",
+ "get_market_skill_exclusions",
+ "get_plugin_detail",
+ "exclude_market_skill",
+ "install_plugin",
+ "ensure_default_plugins_bootstrapped",
+ "import_plugin",
+ "import_plugin_from_zip",
+ "publish_plugin_zip_to_market",
+ "delete_market_package",
+ "uninstall_plugin",
+ "set_plugin_enabled",
+ "set_plugin_component_enabled",
+ "set_plugin_enabled_for_user",
+ "set_plugin_component_enabled_for_user",
+ "set_installed_plugin_meta",
+ "PLUGIN_BUNDLES_DIR",
+ "PLUGIN_SOURCE_DIRS",
+ "MAX_ZIP_BYTES",
+ "DEFAULT_BOOTSTRAP_PLUGIN_SLUGS",
+ "DEFAULT_BOOTSTRAP_MARKER_ID",
+ "LOCAL_BOOTSTRAP_MARKER_NAME",
+ "PLUGIN_MARKET_META_BLOCK_ID",
+ "BUILTIN_PLUGIN_MARKET_META",
+ "MAX_ICON_LEN",
+ "MAX_ICON_URL_LEN",
+]
diff --git a/src/backend/core/plugins/management/admin_config.py b/src/backend/core/capabilities/plugins/management/admin_config.py
similarity index 88%
rename from src/backend/core/plugins/management/admin_config.py
rename to src/backend/core/capabilities/plugins/management/admin_config.py
index 4b01deb8..68facf01 100644
--- a/src/backend/core/plugins/management/admin_config.py
+++ b/src/backend/core/capabilities/plugins/management/admin_config.py
@@ -1,4 +1,4 @@
-"""Plugin admin config responsibilities."""
+"""Read and write a plugin's admin-level configuration."""
from __future__ import annotations
@@ -6,15 +6,15 @@
from typing import Any, Dict, List, Optional
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.plugins.packaging import sources as plugin_sources
-from core.plugins.packaging.importer import _ext_or_top, manifest_extensions, normalize_plugin_dir
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions, normalize_plugin_dir
logger = logging.getLogger(__name__)
def _admin_config_for_slug(slug: str) -> Optional[Dict[str, Any]]:
"""Locate the plugin bundle by slug and read its admin_config declaration; None if absent."""
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
if plugin_dir is None:
return None
try:
@@ -23,14 +23,14 @@ def _admin_config_for_slug(slug: str) -> Optional[Dict[str, Any]]:
return None
-def _connection_for_slug(slug: str) -> Optional[str]:
+def connection_for_slug(slug: str) -> Optional[str]:
"""Read a plugin's account connection type by slug (e.g. dingtalk / lark); None if absent.
Like admin_config, read live from the bundle at detail-view time, not
stored in a DB column — imported plugins (whose bundle no longer exists)
naturally return None without affecting display.
"""
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
if plugin_dir is None:
return None
try:
@@ -45,7 +45,7 @@ def _connection_for_slug(slug: str) -> Optional[str]:
def _has_admin_config_for_slug(slug: str) -> bool:
"""Lightweight check for whether a plugin declares admin_config (reads only plugin.json, no full normalize)."""
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
if plugin_dir is None:
return False
try:
@@ -82,7 +82,7 @@ def _admin_config_view(
"""
if not admin_config:
return None
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc = SystemConfigService.get_instance()
fields: List[Dict[str, Any]] = []
@@ -133,7 +133,7 @@ def set_plugin_admin_config(
items = [{"key": str(k), "value": v} for k, v in (values or {}).items() if k in allowed]
if not items:
raise BadRequestError(message="没有可写入的配置项(key 不属于该插件)")
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
SystemConfigService.get_instance().bulk_set(items, updated_by=updated_by)
return _admin_config_view(ac, with_values=True)
diff --git a/src/backend/core/services/manager_bundle_upgrade.py b/src/backend/core/capabilities/plugins/management/bundle_upgrade.py
similarity index 92%
rename from src/backend/core/services/manager_bundle_upgrade.py
rename to src/backend/core/capabilities/plugins/management/bundle_upgrade.py
index 56aed7a6..f75bc2c3 100644
--- a/src/backend/core/services/manager_bundle_upgrade.py
+++ b/src/backend/core/capabilities/plugins/management/bundle_upgrade.py
@@ -1,15 +1,15 @@
"""Upgrade installed official manager bundles; retain a rollback snapshot per owner."""
from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin
-from core.services.management_contract import MANAGERS
+from core.capabilities.plugins.management.contract import MANAGERS
from extensions.paths import PLUGINS_DIR
def refresh(db):
- from core.capabilities.paths import capabilities_enabled
- if capabilities_enabled():
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ if has_run_capabilities():
return 0 # Device legacy packages are handled by local_legacy_plugin_migration.
- from core.plugins import management as ps
+ from core.capabilities.plugins import management as ps
root = PLUGINS_DIR / "marketplace"
count = 0
for row in db.query(InstalledPlugin).filter(InstalledPlugin.slug.in_(list(MANAGERS))).all():
diff --git a/src/backend/core/services/cloud_management.py b/src/backend/core/capabilities/plugins/management/cloud.py
similarity index 78%
rename from src/backend/core/services/cloud_management.py
rename to src/backend/core/capabilities/plugins/management/cloud.py
index 443fd23b..fa464472 100644
--- a/src/backend/core/services/cloud_management.py
+++ b/src/backend/core/capabilities/plugins/management/cloud.py
@@ -1,6 +1,6 @@
"""Cloud adapters for the plugin-owned management contract."""
-from core.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging import sources as plugin_sources
from contextlib import contextmanager
import hashlib
import json
@@ -9,11 +9,11 @@
from sqlalchemy import text
from pathlib import Path
from core.capabilities import archive
-from core.agent_skills.binary_files import encode_upload
-from core.services.management_package import snapshot
+from core.capabilities.skills.binary_files import encode_upload
+from core.capabilities.plugins.management.package import snapshot
from core.db.engine import SessionLocal
from core.db.models import AdminSkill, AdminMcpServer, InstalledPlugin, Artifact, ContentBlock
-from core.services import capability_workcopies as copies
+from core.harness.registry.capability_revisions import capability_revisions
@contextmanager
@@ -27,7 +27,7 @@ def artifact_package(user_id, source):
if artifact.size_bytes > archive.MAX_TOTAL_BYTES:
raise ValueError("artifact package too large")
key = artifact.storage_key
- from core.storage import get_storage
+ from core.files.storage import get_storage
data = get_storage().download_bytes(key)
if len(data) > archive.MAX_TOTAL_BYTES:
@@ -67,7 +67,7 @@ def _revision(db, kind, row, snap):
def _owned(db, user, kind, iid):
if not user:
raise PermissionError("authentication required")
- row = copies._model(db, kind, iid)
+ row = capability_revisions().model(db, kind, iid)
if row is None or row.owner_user_id != user:
raise PermissionError("installation does not exist or is not owned by current user")
if kind == "skill" and row.source_plugin:
@@ -77,7 +77,7 @@ def _owned(db, user, kind, iid):
def _view(db, user, kind, row):
iid = row.skill_id if kind == "skill" else row.install_id
- snap = copies._snapshot(db, user, kind, iid)[0]
+ snap = capability_revisions().snapshot_in_session(db, user, kind, iid)[0]
revision = _revision(db, kind, row, snap)
return {
"ok": True,
@@ -109,13 +109,13 @@ def list_installed(user_id, kind):
def _skill_files(root, user_id, key=None):
- from core.agent_skills.registry import _split_frontmatter, _load_skill_metadata_from_str
- from core.services.marketplace_service import compute_install_id, _rewrite_frontmatter_name
+ from core.capabilities.skills.registry import split_frontmatter, load_skill_metadata_from_str
+ from core.capabilities.skills.installation import compute_install_id as compute_install_id, rewrite_frontmatter_name as _rewrite_frontmatter_name
files = {n: encode_upload(n, p.read_bytes()) for n, p in archive.iter_files(root)}
md = files.get("SKILL.md", "")
- fm, _ = _split_frontmatter(md)
- _load_skill_metadata_from_str(md, str(fm.get("name") or ""))
+ fm, _ = split_frontmatter(md)
+ load_skill_metadata_from_str(md, str(fm.get("name") or ""))
key = key or compute_install_id(str(fm["name"]), user_id)
files["SKILL.md"] = _rewrite_frontmatter_name(md, key)
return key, files
@@ -125,12 +125,12 @@ def install(user_id, kind, source):
with artifact_package(user_id, source) as root:
if kind == "skill":
key, files = _skill_files(root, user_id)
- before = copies.snapshot(user_id, kind, key)
+ before = capability_revisions().snapshot(user_id, kind, key)
if before["exists"]:
if before["files"] != files:
raise ValueError("name_conflict: use update_skill")
return get(user_id, kind, key)
- result = copies.commit(
+ result = capability_revisions().commit(
user_id,
kind,
key,
@@ -139,14 +139,14 @@ def install(user_id, kind, source):
if not result["applied"]:
raise ValueError("package could not be activated")
else:
- from core.plugins import management as ps
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins import management as ps
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
np = normalize_plugin_dir(root)
- key = plugin_sources._make_plugin_install_id(np.slug, user_id)
+ key = plugin_sources.make_plugin_install_id(np.slug, user_id)
with SessionLocal() as db:
_permission(db, user_id, "can_import_plugin")
- if copies._model(db, kind, key) is not None:
+ if capability_revisions().model(db, kind, key) is not None:
raise ValueError("name_conflict: use update_plugin")
ps.import_plugin(db, root, owner_user_id=user_id, created_by=user_id)
return get(user_id, kind, key)
@@ -164,12 +164,12 @@ def update(user_id, kind, install_id, source, expected_revision):
with SessionLocal() as db:
_write_lock(db)
row = _owned(db, user_id, kind, install_id)
- before = copies._snapshot(db, user_id, kind, install_id)[0]
+ before = capability_revisions().snapshot_in_session(db, user_id, kind, install_id)[0]
if not expected_revision or _revision(db, kind, row, before) != expected_revision:
raise ValueError("revision_conflict")
if kind == "plugin":
- from core.plugins import management as ps
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins import management as ps
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
_permission(db, user_id, "can_import_plugin")
if normalize_plugin_dir(root).slug != row.slug:
@@ -178,7 +178,7 @@ def update(user_id, kind, install_id, source, expected_revision):
else:
key, files = _skill_files(root, user_id, install_id)
if kind == "skill":
- copies.commit(
+ capability_revisions().commit(
user_id,
kind,
key,
@@ -187,7 +187,7 @@ def update(user_id, kind, install_id, source, expected_revision):
"expected_revision": expected_revision,
"expected_model_version": before["model_version"],
"request_id": hashlib.sha256(
- (user_id + key + expected_revision + copies.revision(files)).encode()
+ (user_id + key + expected_revision + capability_revisions().revision(files)).encode()
).hexdigest(),
},
)
@@ -198,21 +198,21 @@ def uninstall(user_id, kind, install_id, expected_revision):
with SessionLocal() as db:
_write_lock(db)
row = _owned(db, user_id, kind, install_id)
- before = copies._snapshot(db, user_id, kind, install_id)[0]
+ before = capability_revisions().snapshot_in_session(db, user_id, kind, install_id)[0]
if not expected_revision or _revision(db, kind, row, before) != expected_revision:
raise ValueError("revision_conflict")
_permission(db, user_id, "can_add_skill" if kind == "skill" else "can_import_plugin")
- record = db.get(ContentBlock, copies._id("cap-cloud-copy:", user_id, kind, install_id))
+ record = db.get(ContentBlock, capability_revisions().record_id("cap-cloud-copy:", user_id, kind, install_id))
if record:
db.delete(record)
if kind == "plugin":
- from core.plugins.management import uninstall_plugin
+ from core.capabilities.plugins.management import uninstall_plugin
uninstall_plugin(db, install_id, owner_user_id=user_id)
else:
db.delete(row)
db.commit()
- from core.agent_skills.cache_refresh import refresh_skill_caches
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
refresh_skill_caches()
return {"ok": True, "install_id": install_id, "action": "uninstalled", "source": "cloud"}
diff --git a/src/backend/core/plugins/management/components.py b/src/backend/core/capabilities/plugins/management/components.py
similarity index 88%
rename from src/backend/core/plugins/management/components.py
rename to src/backend/core/capabilities/plugins/management/components.py
index 2ec15c08..a0a66e32 100644
--- a/src/backend/core/plugins/management/components.py
+++ b/src/backend/core/capabilities/plugins/management/components.py
@@ -1,4 +1,4 @@
-"""Plugin components responsibilities."""
+"""Upsert a plugin's skills and MCP servers as admin catalog rows."""
from __future__ import annotations
@@ -6,24 +6,24 @@
import re
from typing import Any, Dict, List, Optional
-from core.agent_skills.binary_files import is_binary_value
-from core.agent_skills.deps_detector import detect_dependencies
-from core.agent_skills.registry import _load_skill_metadata_from_str
+from core.capabilities.skills.binary_files import is_binary_value
+from core.capabilities.skills.deps.detector import detect_dependencies
+from core.capabilities.skills.registry import load_skill_metadata_from_str
from core.db.models import AdminMcpServer, AdminSkill
from core.infra.exceptions import BadRequestError
from core.infra.time import utc_now
-from core.ontology.build_validator import ensure_ontology_build_valid
-from core.plugins.packaging import sources as plugin_sources
-from core.plugins.packaging.importer import _rewrite_path_vars
-from core.plugins.packaging.models import NormalizedSkill
-from core.services.marketplace_service import _inject_secrets, _rewrite_frontmatter_name
+from core.harness.registry.build_governance import ensure_ontology_build_valid
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging.importer import _rewrite_path_vars
+from core.capabilities.plugins.packaging.models import NormalizedSkill
+from core.capabilities.skills.installation import inject_secrets as _inject_secrets, rewrite_frontmatter_name as _rewrite_frontmatter_name
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
-def _merge_tool_metadata(stored: Any, manifest: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def merge_tool_metadata(stored: Any, manifest: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Keep captured tool schemas while refreshing manifest-owned display text.
``tools_json`` is the schema of record: probing the running MCP server fills in
@@ -91,7 +91,7 @@ def _rewrite_sibling_refs(text: str, sibling_ids: Dict[str, str]) -> str:
return text
-def _apply_skill(
+def apply_skill(
db: Session,
sk: NormalizedSkill,
*,
@@ -104,7 +104,7 @@ def _apply_skill(
sibling_ids: Optional[Dict[str, str]] = None,
) -> str:
"""Upsert one normalized skill as an AdminSkill (tagged with source_plugin). Returns the skill_id."""
- skill_id = plugin_sources._make_skill_id(slug, sk.name, owner_user_id)
+ skill_id = plugin_sources.make_skill_id(slug, sk.name, owner_user_id)
sandbox_dir = f"/workspace/skills/{skill_id}"
# Path-variable rewrite: SKILL.md body + text attachments (binaries untouched)
@@ -124,7 +124,7 @@ def _apply_skill(
# Display name comes from the original frontmatter name (before rewriting into the namespaced id), falling back to the skill directory name.
try:
- _display_name = _load_skill_metadata_from_str(content, skill_id).name
+ _display_name = load_skill_metadata_from_str(content, skill_id).name
except Exception: # noqa: BLE001
_display_name = None
@@ -135,7 +135,7 @@ def _apply_skill(
content = _inject_secrets(content, extra_files, required_secrets, secrets)
try:
- meta = _load_skill_metadata_from_str(content, skill_id)
+ meta = load_skill_metadata_from_str(content, skill_id)
except Exception as exc: # noqa: BLE001
raise BadRequestError(message=f"技能 {sk.name!r} 的 SKILL.md 不合法:{exc}")
@@ -190,7 +190,7 @@ def _apply_mcp(
stdio (needs_runtime) is force-disabled even when enabled is requested
(it can only be enabled once the runtime is fully in place).
"""
- server_id = plugin_sources._make_server_id(slug, mc.name, owner_user_id)
+ server_id = plugin_sources.make_server_id(slug, mc.name, owner_user_id)
effective_enabled = bool(enabled) and not mc.needs_runtime
tools_meta = [item for item in list(getattr(mc, "tools", None) or []) if isinstance(item, dict)]
if validate_ontology_build:
@@ -235,7 +235,7 @@ def _apply_mcp(
# the desktop's only source of tool parameters. Overwriting them with the
# manifest list would strip every parameter, so merge by name instead:
# manifest text wins, captured schemas survive.
- fields["tools_json"] = _merge_tool_metadata(existing.tools_json, tools_meta)
+ fields["tools_json"] = merge_tool_metadata(existing.tools_json, tools_meta)
for key, val in fields.items():
setattr(existing, key, val)
for col in ("args", "env_vars", "headers", "tools_json"):
diff --git a/src/backend/core/services/management_contract.py b/src/backend/core/capabilities/plugins/management/contract.py
similarity index 100%
rename from src/backend/core/services/management_contract.py
rename to src/backend/core/capabilities/plugins/management/contract.py
diff --git a/src/backend/core/plugins/management/defaults.py b/src/backend/core/capabilities/plugins/management/defaults.py
similarity index 93%
rename from src/backend/core/plugins/management/defaults.py
rename to src/backend/core/capabilities/plugins/management/defaults.py
index de1e397e..518fe639 100644
--- a/src/backend/core/plugins/management/defaults.py
+++ b/src/backend/core/capabilities/plugins/management/defaults.py
@@ -1,7 +1,7 @@
"""Add shared defaults without replaying an edition's earlier onboarding."""
from core.db.models import ContentBlock, InstalledPlugin
-from core.plugins.management.installation import install_plugin
+from core.capabilities.plugins.management.installation import install_plugin
from sqlalchemy.orm import Session
VISUALIZE_BOOTSTRAP_MARKER_ID = "visualize_plugin_bootstrap_v1"
diff --git a/src/backend/core/plugins/management/details.py b/src/backend/core/capabilities/plugins/management/details.py
similarity index 58%
rename from src/backend/core/plugins/management/details.py
rename to src/backend/core/capabilities/plugins/management/details.py
index 09ef4d71..15288a9c 100644
--- a/src/backend/core/plugins/management/details.py
+++ b/src/backend/core/capabilities/plugins/management/details.py
@@ -1,58 +1,29 @@
-"""Plugin details responsibilities."""
+"""Marketplace plugin details and per-skill exclusions."""
from __future__ import annotations
-import base64
import logging
from typing import Any, Dict, List, Optional
-from core.db.models import PluginMarketSkillExclusion
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.plugins.management import admin_config as plugin_admin_config
-from core.plugins.management import market as plugin_market
-from core.plugins.management import packages as plugin_packages
-from core.plugins.packaging import sources as plugin_sources
-from core.plugins.packaging.archive import _extract_plugin_zip
-from core.plugins.packaging.importer import normalize_plugin_dir
-from core.plugins.packaging.models import NormalizedPlugin, NormalizedSkill
-from core.services.marketplace_service import _strip_frontmatter
+from core.harness.registry.marketplace_catalog import marketplace_catalog
+from core.infra.exceptions import BadRequestError
+from core.capabilities.plugins.management import admin_config as plugin_admin_config
+from core.capabilities.plugins.packaging.presentation import overlay_market_meta
+from core.capabilities.plugins.packaging.models import NormalizedPlugin, NormalizedSkill
+from core.capabilities.skills.installation import strip_frontmatter as _strip_frontmatter
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def _normalize_market_plugin(slug: str, db: Optional[Session]) -> NormalizedPlugin:
- """Fetch and normalize a plugin by slug: filesystem preset bundle first, DB-published package as fallback."""
- from core.services.marketplace_version_store import active, decode
- from core.services.marketplace_version_sources import normalized_plugin
- if plugin_sources._resolve_plugin_dir(slug) is None and (
- db is None or plugin_packages._market_row(db, slug) is None
- ):
- raise ResourceNotFoundError("plugin", slug)
- snapshot = active(db, "plugin", slug)
- if snapshot:
- return normalized_plugin(decode(snapshot))
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
- if plugin_dir is not None:
- return normalize_plugin_dir(plugin_dir)
- if db is not None:
- row = plugin_packages._market_row(db, slug)
- if row is not None:
- with _extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root:
- return normalize_plugin_dir(plugin_root)
- raise ResourceNotFoundError("plugin", slug)
+ from core.harness.registry.package_sources import package_source
+ return package_source("marketplace").normalize(db, slug)
def get_market_skill_exclusions(db: Optional[Session], slug: str) -> set:
"""The set of skill names the admin has "removed" from the marketplace for this plugin."""
- if db is None:
- return set()
- return {
- row[0]
- for row in db.query(PluginMarketSkillExclusion.skill_name)
- .filter(PluginMarketSkillExclusion.slug == slug)
- .all()
- }
+ return marketplace_catalog().excluded_skills(db, slug)
def get_plugin_detail(slug: str, db: Optional[Session] = None) -> Dict[str, Any]:
@@ -64,8 +35,8 @@ def get_plugin_detail(slug: str, db: Optional[Session] = None) -> Dict[str, Any]
detail.
"""
np = _normalize_market_plugin(slug, db)
- detail = _normalized_to_detail(np, excluded=get_market_skill_exclusions(db, slug))
- plugin_market._overlay_market_meta(detail, plugin_market.resolve_market_meta(db, slug))
+ detail = normalized_to_detail(np, excluded=get_market_skill_exclusions(db, slug))
+ overlay_market_meta(detail, marketplace_catalog().plugin_metadata(db, slug))
return detail
@@ -81,35 +52,26 @@ def exclude_market_skill(
raw_names = {s.name for s in _normalize_market_plugin(slug, db).skills}
if skill_name not in raw_names:
raise BadRequestError(message=f"技能 {skill_name!r} 不属于插件 {slug!r}")
- existing = (
- db.query(PluginMarketSkillExclusion)
- .filter(
- PluginMarketSkillExclusion.slug == slug,
- PluginMarketSkillExclusion.skill_name == skill_name,
- )
- .first()
- )
- if existing is None:
- db.add(PluginMarketSkillExclusion(slug=slug, skill_name=skill_name, created_by=created_by))
- db.commit()
+ if marketplace_catalog().exclude_skill(db, slug, skill_name, created_by=created_by):
logger.info("plugin_market_skill_excluded: slug=%s skill=%s", slug, skill_name)
return {"slug": slug, "skill_name": skill_name, "excluded": True}
def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]:
"""Preview component for a not-yet-installed builtin/external skill: includes body instructions + file list for pre-install inspection."""
- from core.agent_skills.registry import _split_frontmatter
+ from core.capabilities.skills.registry import split_frontmatter
desc = ""
tags: List[str] = []
try:
- fm, _ = _split_frontmatter(sk.skill_content or "")
+ fm, _ = split_frontmatter(sk.skill_content or "")
desc = (fm.get("description") or "").strip()
raw_tags = fm.get("tags") or ""
if isinstance(raw_tags, str) and raw_tags:
tags = [x.strip() for x in raw_tags.replace(",", ",").split(",") if x.strip()]
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.plugins.management.details:_skill_component_preview', fallback_error)
files = sorted(k for k in (sk.extra_files or {}).keys() if k != "secrets.json")
return {
"skill_id": sk.name,
@@ -124,7 +86,7 @@ def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]:
}
-def _normalized_to_detail(
+def normalized_to_detail(
np: NormalizedPlugin, *, excluded: Optional[set] = None
) -> Dict[str, Any]:
excluded = excluded or set()
diff --git a/src/backend/core/services/management_errors.py b/src/backend/core/capabilities/plugins/management/errors.py
similarity index 100%
rename from src/backend/core/services/management_errors.py
rename to src/backend/core/capabilities/plugins/management/errors.py
diff --git a/src/backend/core/plugins/management/installation.py b/src/backend/core/capabilities/plugins/management/installation.py
similarity index 81%
rename from src/backend/core/plugins/management/installation.py
rename to src/backend/core/capabilities/plugins/management/installation.py
index bb8ba224..74993792 100644
--- a/src/backend/core/plugins/management/installation.py
+++ b/src/backend/core/capabilities/plugins/management/installation.py
@@ -1,4 +1,4 @@
-"""Plugin installation responsibilities."""
+"""Install plugins from the marketplace, a directory or a zip, and bootstrap the defaults."""
from __future__ import annotations
@@ -10,24 +10,24 @@
from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin
from core.infra.exceptions import ResourceNotFoundError
from core.infra.time import utc_now
-from core.plugins.management import components as plugin_components
-from core.plugins.management import details as plugin_details
-from core.plugins.management import market as plugin_market
-from core.plugins.management import packages as plugin_packages
-from core.plugins.management import projection as plugin_projection
-from core.plugins.packaging import definitions as plugin_definitions
-from core.plugins.packaging import sources as plugin_sources
-from core.plugins.packaging.archive import _extract_plugin_zip
-from core.plugins.packaging.importer import normalize_plugin_dir
-from core.plugins.packaging.models import NormalizedPlugin
-from core.services.ontology_policy import resolve_plugin_import_ontology_validation
+from core.capabilities.plugins.management import components as plugin_components
+from core.capabilities.plugins.management import details as plugin_details
+from core.harness.registry.marketplace_catalog import marketplace_catalog
+from core.capabilities.plugins.management import packages as plugin_packages
+from core.capabilities.plugins.management import projection as plugin_projection
+from core.capabilities.plugins.packaging import definitions as plugin_definitions
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging.archive import extract_plugin_zip
+from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
+from core.capabilities.plugins.packaging.models import NormalizedPlugin
+from core.harness.registry.build_governance import resolve_plugin_import_ontology_validation
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
-def _apply_normalized(
+def apply_normalized(
db: Session,
np: NormalizedPlugin,
*,
@@ -38,15 +38,15 @@ def _apply_normalized(
approved_revision: Optional[str] = None,
) -> Dict[str, Any]:
"""Persist a NormalizedPlugin and return the installation result (including the import_report)."""
- from core.applications.coordination import configuration
- from core.applications.plugin_upgrades import prepare
+ from core.harness.registry.package_sources import package_upgrade_policy
- configuration(db)
+ policy = package_upgrade_policy()
+ policy.lock_configuration(db)
db.expire_all()
- install_id = plugin_sources._make_plugin_install_id(np.slug, owner_user_id)
+ install_id = plugin_sources.make_plugin_install_id(np.slug, owner_user_id)
existing = db.query(InstalledPlugin).filter(InstalledPlugin.install_id == install_id).first()
try:
- staged = prepare(db, existing, np, source, approved_revision)
+ staged = policy.prepare(db, existing, np, source, approved_revision)
except BaseException:
db.rollback()
raise
@@ -69,12 +69,12 @@ def _apply_normalized(
# Sibling skill name → final skill_id mapping (used to rewrite inter-skill ../ relative references)
sibling_ids = {
- sk.name: plugin_sources._make_skill_id(np.slug, sk.name, owner_user_id)
+ sk.name: plugin_sources.make_skill_id(np.slug, sk.name, owner_user_id)
for sk in install_skills
}
for sk in install_skills:
- sid = plugin_components._apply_skill(
+ sid = plugin_components.apply_skill(
db,
sk,
slug=np.slug,
@@ -112,7 +112,7 @@ def _apply_normalized(
import_report = {"imported": imported, "adapted": adapted, "dropped": np.dropped}
if np.package_dir and np.ui:
- from core.plugins.packaging.runtime_assets import retain
+ from core.capabilities.plugins.packaging.runtime_assets import retain
import_report["package_revision"] = retain(db, Path(np.package_dir))
component_ids = {"skills": skill_ids, "mcp": server_ids, "prompts": []}
@@ -120,10 +120,10 @@ def _apply_normalized(
# A plugin update is a replacement of its declared component set. Remove
# components that belonged to the previous version but no longer appear in
# the new manifest; otherwise retired tools/skills remain visible forever.
- stale_skill_ids = set(plugin_sources._component_keys(previous_component_ids, "skills")) - set(
+ stale_skill_ids = set(plugin_sources.component_keys(previous_component_ids, "skills")) - set(
skill_ids
)
- stale_server_ids = set(plugin_sources._component_keys(previous_component_ids, "mcp")) - set(
+ stale_server_ids = set(plugin_sources.component_keys(previous_component_ids, "mcp")) - set(
server_ids
)
owner_skill_filter = (
@@ -161,7 +161,7 @@ def _apply_normalized(
now = utc_now()
# Display metadata for the installed record: UI-configured market metadata
# (DB override → builtin seed) wins over whatever the manifest carried.
- market_meta = plugin_market.resolve_market_meta(db, np.slug)
+ market_meta = marketplace_catalog().plugin_metadata(db, np.slug)
fields = dict(
slug=np.slug,
name=market_meta.get("display_name") or np.name,
@@ -191,7 +191,7 @@ def _apply_normalized(
action = "installed"
db.commit()
- plugin_projection._refresh_after_change(owner_user_id)
+ plugin_projection.refresh_after_change(owner_user_id)
plugin_projection._project_plugin_to_store(
{
"install_id": install_id,
@@ -210,7 +210,7 @@ def _apply_normalized(
)
# 插件在本机需要的资产(如站点插件的建站模板)跟着插件走:装到哪台机器上,就在
# 哪台机器上铺。云端同步装的那条走 desktop_cloud_bundles 的发布钩子,同一张表。
- from core.plugins.local.assets import provision_for
+ from core.harness.registry.local_publication import provision_plugin_assets as provision_for
provision_for(np.slug)
logger.info(
@@ -245,19 +245,19 @@ def install_plugin(
created_by: Optional[str] = None,
) -> Dict[str, Any]:
"""Install a marketplace plugin package (filesystem default / marketplace, or a DB-published marketplace package)."""
- from core.services.marketplace_version_store import active, archive, decode
+ from core.harness.registry.package_sources import package_source
plugin_details.get_plugin_detail(slug, db)
- snapshot = active(db, "plugin", slug)
- if snapshot:
- with _extract_plugin_zip(archive(decode(snapshot))) as root:
+ source_archive = package_source("marketplace").archive(db, slug)
+ if source_archive is not None:
+ with extract_plugin_zip(source_archive) as root:
return import_plugin(
db, root, owner_user_id=owner_user_id, secrets=secrets or {}, created_by=created_by
)
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
if plugin_dir is not None:
np = normalize_plugin_dir(plugin_dir)
- return _apply_normalized(
+ return apply_normalized(
db,
np,
owner_user_id=owner_user_id,
@@ -266,9 +266,9 @@ def install_plugin(
created_by=created_by,
)
# DB-published marketplace package: extract the original zip and go through the same path as import
- row = plugin_packages._market_row(db, slug)
+ row = plugin_packages.market_row(db, slug)
if row is not None:
- with _extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root:
+ with extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root:
return import_plugin(
db,
plugin_root,
@@ -333,7 +333,7 @@ def import_plugin(
"""Import an external plugin directory (native / Claude Code / Codex), persist it, and return the import_report."""
np = normalize_plugin_dir(plugin_dir)
source = {"claude": "imported_claude", "codex": "imported_codex"}.get(np.kind, "builtin")
- return _apply_normalized(
+ return apply_normalized(
db,
np,
owner_user_id=owner_user_id,
@@ -352,7 +352,7 @@ def import_plugin_from_zip(
created_by: Optional[str] = None,
) -> Dict[str, Any]:
"""Import a plugin from uploaded zip bytes (extract → locate root → import_plugin). Shared by user and admin uploads."""
- with _extract_plugin_zip(raw) as plugin_root:
+ with extract_plugin_zip(raw) as plugin_root:
return import_plugin(
db,
plugin_root,
diff --git a/src/backend/core/plugins/management/lifecycle.py b/src/backend/core/capabilities/plugins/management/lifecycle.py
similarity index 87%
rename from src/backend/core/plugins/management/lifecycle.py
rename to src/backend/core/capabilities/plugins/management/lifecycle.py
index 0fc91fae..08ac1a2f 100644
--- a/src/backend/core/plugins/management/lifecycle.py
+++ b/src/backend/core/capabilities/plugins/management/lifecycle.py
@@ -1,4 +1,4 @@
-"""Plugin lifecycle responsibilities."""
+"""Uninstall plugins and toggle them or their components, globally or per user."""
from __future__ import annotations
@@ -8,9 +8,9 @@
from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from core.infra.time import utc_now
-from core.plugins.management import market as plugin_market
-from core.plugins.management import projection as plugin_projection
-from core.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging.presentation import validate_icon
+from core.capabilities.plugins.management import projection as plugin_projection
+from core.capabilities.plugins.packaging import sources as plugin_sources
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -56,7 +56,7 @@ def _owner_filter(model):
db.delete(row)
db.commit()
plugin_projection._purge_sandbox_skill_files(removed_skill_ids)
- plugin_projection._refresh_after_change(owner_user_id)
+ plugin_projection.refresh_after_change(owner_user_id)
plugin_projection._remove_plugin_from_store(slug)
logger.info("plugin_uninstalled: id=%s slug=%s skills=%d mcp=%d", install_id, slug, n_sk, n_mcp)
return {"install_id": install_id, "slug": slug, "removed_skills": n_sk, "removed_mcp": n_mcp}
@@ -76,12 +76,12 @@ def set_plugin_enabled(
if row.owner_user_id != owner_user_id:
raise BadRequestError(message="无权操作该插件")
- from core.plugins.ui.workspace_enabled import set_enabled
+ from core.harness.registry.package_views import set_enabled
set_enabled(db, row, enabled)
cids = row.component_ids or {}
- skill_ids = plugin_sources._component_keys(cids, "skills")
- server_ids = plugin_sources._component_keys(cids, "mcp")
+ skill_ids = plugin_sources.component_keys(cids, "skills")
+ server_ids = plugin_sources.component_keys(cids, "mcp")
if skill_ids:
db.query(AdminSkill).filter(AdminSkill.skill_id.in_(skill_ids)).update(
{AdminSkill.is_enabled: enabled}, synchronize_session=False
@@ -91,7 +91,7 @@ def set_plugin_enabled(
for srv in db.query(AdminMcpServer).filter(AdminMcpServer.server_id.in_(server_ids)).all():
srv.is_enabled = bool(enabled) and srv.transport != "stdio"
db.commit()
- plugin_projection._refresh_after_change(owner_user_id)
+ plugin_projection.refresh_after_change(owner_user_id)
return {"install_id": install_id, "enabled": enabled}
@@ -123,7 +123,7 @@ def set_plugin_component_enabled(
cids = row.component_ids or {}
if kind == "skill":
- if component_id not in (plugin_sources._component_keys(cids, "skills")):
+ if component_id not in (plugin_sources.component_keys(cids, "skills")):
raise BadRequestError(message="该技能不属于此插件")
sk = db.query(AdminSkill).filter(AdminSkill.skill_id == component_id).first()
if sk is None:
@@ -131,7 +131,7 @@ def set_plugin_component_enabled(
sk.is_enabled = bool(enabled)
effective = sk.is_enabled
else:
- if component_id not in (plugin_sources._component_keys(cids, "mcp")):
+ if component_id not in (plugin_sources.component_keys(cids, "mcp")):
raise BadRequestError(message="该 MCP 不属于此插件")
srv = db.query(AdminMcpServer).filter(AdminMcpServer.server_id == component_id).first()
if srv is None:
@@ -140,7 +140,7 @@ def set_plugin_component_enabled(
srv.is_enabled = bool(enabled) and srv.transport != "stdio"
effective = srv.is_enabled
db.commit()
- plugin_projection._refresh_after_change(owner_user_id)
+ plugin_projection.refresh_after_change(owner_user_id)
return {
"install_id": install_id,
"kind": kind,
@@ -168,19 +168,19 @@ def set_plugin_enabled_for_user(
if row.owner_user_id is not None and row.owner_user_id != user_id:
raise BadRequestError(message="无权操作该插件")
- from core.plugins.ui.workspace_enabled import set_enabled
+ from core.harness.registry.package_views import set_enabled
set_enabled(db, row, enabled, user_id)
- from core.services.catalog_service import CatalogService
+ from core.capabilities.catalog.overrides import CatalogService
svc = CatalogService(db)
cids = row.component_ids or {}
- for sid in plugin_sources._component_keys(cids, "skills"):
+ for sid in plugin_sources.component_keys(cids, "skills"):
svc.update_override(user_id, "skill", sid, enabled)
- for sid in plugin_sources._component_keys(cids, "mcp"):
+ for sid in plugin_sources.component_keys(cids, "mcp"):
svc.update_override(user_id, "mcp", sid, enabled)
- plugin_projection._refresh_after_change(user_id)
+ plugin_projection.refresh_after_change(user_id)
return {"install_id": install_id, "enabled": enabled}
@@ -211,14 +211,14 @@ def set_plugin_component_enabled_for_user(
raise BadRequestError(message="无权操作该插件")
cids = row.component_ids or {}
- pool = plugin_sources._component_keys(cids, "skills" if kind == "skill" else "mcp")
+ pool = plugin_sources.component_keys(cids, "skills" if kind == "skill" else "mcp")
if component_id not in (pool or []):
raise BadRequestError(message="该组件不属于此插件")
- from core.services.catalog_service import CatalogService
+ from core.capabilities.catalog.overrides import CatalogService
CatalogService(db).update_override(user_id, kind, component_id, enabled)
- plugin_projection._refresh_after_change(user_id)
+ plugin_projection.refresh_after_change(user_id)
return {
"install_id": install_id,
"kind": kind,
@@ -253,7 +253,7 @@ def set_installed_plugin_meta(
if category is not None:
row.category = category.strip()
if icon is not None:
- row.icon = plugin_market._validate_icon(icon) or None
+ row.icon = validate_icon(icon) or None
row.updated_at = utc_now()
db.commit()
return {
diff --git a/src/backend/core/services/management_package.py b/src/backend/core/capabilities/plugins/management/package.py
similarity index 91%
rename from src/backend/core/services/management_package.py
rename to src/backend/core/capabilities/plugins/management/package.py
index dba71369..ebd2e4bc 100644
--- a/src/backend/core/services/management_package.py
+++ b/src/backend/core/capabilities/plugins/management/package.py
@@ -6,7 +6,7 @@
import tempfile
import zipfile
from core.capabilities import archive
-from core.capabilities.errors import IntegrityFailed
+from core.harness.capability_errors import IntegrityFailed
def _tar_files(data):
@@ -25,7 +25,7 @@ def _tar_files(data):
if (len(files) >= archive.MAX_MEMBERS or entry.size > archive.MAX_MEMBER_BYTES
or total > archive.MAX_TOTAL_BYTES):
raise IntegrityFailed("package exceeds size limits")
- archive._normalize_member(name)
+ archive.normalize_member(name)
with package.extractfile(entry) as stream:
files[name] = stream.read(entry.size + 1)
if len(files[name]) != entry.size:
@@ -76,8 +76,8 @@ def serialized(function):
from functools import wraps
@wraps(function)
def run(*args, **kwargs):
- from core.capabilities.lockfile import locked
- from core.capabilities.paths import meta_root
- with locked(meta_root() / "management.lock", timeout=35):
+ from core.infra.file_lock import locked
+ from core.harness.registry.local_publication import local_publication
+ with locked(local_publication().management_lock_path(), timeout=35):
return function(*args, **kwargs)
return run
diff --git a/src/backend/core/plugins/management/packages.py b/src/backend/core/capabilities/plugins/management/packages.py
similarity index 70%
rename from src/backend/core/plugins/management/packages.py
rename to src/backend/core/capabilities/plugins/management/packages.py
index e92759d9..7289ad0b 100644
--- a/src/backend/core/plugins/management/packages.py
+++ b/src/backend/core/capabilities/plugins/management/packages.py
@@ -6,24 +6,22 @@
import logging
from typing import Any, Dict, Optional
-from core.db.models import PluginMarketPackage
-from core.infra.exceptions import ResourceNotFoundError
+from core.harness.registry.marketplace_catalog import marketplace_catalog
from core.infra.time import utc_now
-from core.plugins.packaging.archive import _extract_plugin_zip
-from core.plugins.packaging.importer import normalize_plugin_dir
+from core.capabilities.plugins.packaging.archive import extract_plugin_zip
+from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
from sqlalchemy.orm import Session
-from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
-def _market_row(db: Session, slug: str) -> Optional[PluginMarketPackage]:
+def market_row(db: Session, slug: str) -> Any:
if not slug:
return None
- return db.query(PluginMarketPackage).filter(PluginMarketPackage.slug == slug).first()
+ return marketplace_catalog().market_package(db, slug)
-def _market_meta_dict(row: PluginMarketPackage) -> Dict[str, Any]:
+def market_meta_dict(row: Any) -> Dict[str, Any]:
"""DB marketplace package → marketplace list metadata isomorphic to filesystem bundles."""
return {
"slug": row.slug,
@@ -47,14 +45,13 @@ def publish_plugin_zip_to_market(
Once published it appears in the plugin marketplace list and can be
explicitly installed by admins/users; re-uploading the same slug updates it.
"""
- with _extract_plugin_zip(raw) as plugin_root:
+ with extract_plugin_zip(raw) as plugin_root:
np = normalize_plugin_dir(
plugin_root
) # parse/validate; invalid input raises immediately, nothing persisted
package_b64 = base64.b64encode(raw).decode("ascii")
has_admin_config = bool(np.admin_config and (np.admin_config.get("fields")))
now = utc_now()
- existing = _market_row(db, np.slug)
fields = dict(
name=np.name,
version=np.version,
@@ -68,15 +65,7 @@ def publish_plugin_zip_to_market(
package_b64=package_b64,
updated_at=now,
)
- if existing is not None:
- for key, val in fields.items():
- setattr(existing, key, val)
- flag_modified(existing, "required_secrets")
- action = "updated"
- else:
- db.add(PluginMarketPackage(slug=np.slug, created_at=now, created_by=created_by, **fields))
- action = "published"
- db.commit()
+ action = marketplace_catalog().publish_package(db, np.slug, fields, created_by=created_by, now=now)
logger.info(
"plugin_market_%s: slug=%s kind=%s skills=%d", action, np.slug, np.kind, len(np.skills)
)
@@ -92,10 +81,6 @@ def publish_plugin_zip_to_market(
def delete_market_package(db: Session, slug: str) -> Dict[str, Any]:
"""Remove an admin-uploaded DB marketplace package from the plugin marketplace (does not affect installed instances)."""
- row = _market_row(db, slug)
- if row is None:
- raise ResourceNotFoundError("plugin_market_package", slug)
- db.delete(row)
- db.commit()
+ marketplace_catalog().delete_package(db, slug)
logger.info("plugin_market_deleted: slug=%s", slug)
return {"slug": slug, "deleted": True}
diff --git a/src/backend/core/plugins/management/projection.py b/src/backend/core/capabilities/plugins/management/projection.py
similarity index 63%
rename from src/backend/core/plugins/management/projection.py
rename to src/backend/core/capabilities/plugins/management/projection.py
index 9ff26e66..d613c001 100644
--- a/src/backend/core/plugins/management/projection.py
+++ b/src/backend/core/capabilities/plugins/management/projection.py
@@ -1,40 +1,40 @@
-"""Plugin projection responsibilities."""
+"""Project plugin changes into the desktop store and refresh dependent caches."""
from __future__ import annotations
import logging
from typing import Any, Dict, Optional
-from core.agent_skills.cache_refresh import refresh_skill_caches
-from core.config.catalog_resolver import invalidate_capability_cache
+from core.capabilities.skills.cache_refresh import refresh_skill_caches
+from core.capabilities.catalog.resolver import invalidate_capability_cache
logger = logging.getLogger(__name__)
def _project_plugin_to_store(definition: Dict[str, Any], *, owner_user_id: Optional[str]) -> None:
"""Desktop store: keep ``R/plugins/local//`` and the component edges in step."""
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
return
- from core.capabilities import plugins as caps_plugins
+ from core.harness.registry.local_publication import local_publication
- caps_plugins.publish_local_plugin(definition, owner_user_id=owner_user_id)
+ local_publication().publish_local_plugin(definition, owner_user_id=owner_user_id)
def _remove_plugin_from_store(slug: str) -> None:
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
return
- from core.capabilities import plugins as caps_plugins
+ from core.harness.registry.local_publication import local_publication
- caps_plugins.remove_local_plugin(slug)
+ local_publication().remove_local_plugin(slug)
def _purge_sandbox_skill_files(skill_ids) -> None:
"""Drop the materialized files of skills a plugin no longer owns, so they stop showing up under /workspace/skills/."""
- from core.agent_skills.config import purge_skill_sandbox_files
+ from core.capabilities.skills.config import purge_skill_sandbox_files
for sid in skill_ids or []:
try:
@@ -43,7 +43,7 @@ def _purge_sandbox_skill_files(skill_ids) -> None:
logger.debug("purge sandbox files for skill %s failed: %s", sid, exc)
-def _refresh_after_change(owner_user_id: Optional[str]) -> None:
+def refresh_after_change(owner_user_id: Optional[str]) -> None:
"""Invalidate related caches after install/uninstall: skill cache + capability-resolution cache + MCP config cache."""
try:
refresh_skill_caches()
@@ -57,7 +57,7 @@ def _refresh_after_change(owner_user_id: Optional[str]) -> None:
except Exception as exc: # noqa: BLE001
logger.debug("invalidate_capability_cache failed: %s", exc)
try:
- from core.services.mcp_service import McpServerConfigService
+ from core.capabilities.mcp.service import McpServerConfigService
McpServerConfigService.get_instance().invalidate_cache()
except Exception as exc: # noqa: BLE001
diff --git a/src/backend/core/plugins/management/queries.py b/src/backend/core/capabilities/plugins/management/queries.py
similarity index 86%
rename from src/backend/core/plugins/management/queries.py
rename to src/backend/core/capabilities/plugins/management/queries.py
index 108a0ff7..c029e998 100644
--- a/src/backend/core/plugins/management/queries.py
+++ b/src/backend/core/capabilities/plugins/management/queries.py
@@ -1,4 +1,4 @@
-"""Plugin queries responsibilities."""
+"""List installed plugins and read their full details."""
from __future__ import annotations
@@ -7,9 +7,9 @@
from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.plugins.management import admin_config as plugin_admin_config
-from core.plugins.packaging import sources as plugin_sources
-from core.services.marketplace_service import _strip_frontmatter
+from core.capabilities.plugins.management import admin_config as plugin_admin_config
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.skills.installation import strip_frontmatter as _strip_frontmatter
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -30,7 +30,7 @@ def list_installed(
plugins (frontend user view; global items are read-only — consistent
with the skill library showing global skills).
"""
- from core.capabilities import device_catalog
+ from core.harness.registry import device_catalog
from sqlalchemy import or_
if device_catalog.active():
@@ -66,8 +66,8 @@ def list_installed(
all_mcp_ids: set = set()
for r in rows:
cids = r.component_ids or {}
- all_skill_ids.update(plugin_sources._component_keys(cids, "skills"))
- all_mcp_ids.update(plugin_sources._component_keys(cids, "mcp"))
+ all_skill_ids.update(plugin_sources.component_keys(cids, "skills"))
+ all_mcp_ids.update(plugin_sources.component_keys(cids, "mcp"))
callable_skills = (
{
@@ -96,17 +96,9 @@ def list_installed(
)
if all_mcp_ids:
- from core.db.models import McpMarketInstallation
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- suspended = {
- sid
- for (sid,) in db.query(McpMarketInstallation.server_id)
- .filter(
- McpMarketInstallation.server_id.in_(all_mcp_ids),
- McpMarketInstallation.status == "suspended",
- )
- .all()
- }
+ suspended = marketplace_catalog().suspended_mcp_ids(db, all_mcp_ids)
callable_mcps.difference_update(suspended)
# Enabled state:
@@ -115,7 +107,7 @@ def list_installed(
# personal toggles on global plugins are reflected correctly.
# - admin view (owner_user_id empty): determined by components' global is_enabled.
if owner_user_id is not None:
- from core.config.catalog_resolver import resolve_all_runtime_enabled
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
eff_skills, _eff_agents, eff_mcps = resolve_all_runtime_enabled(db, owner_user_id)
enabled_skills = set(eff_skills or [])
@@ -157,12 +149,12 @@ def list_installed(
for r in rows:
cids = r.component_ids or {}
enabled = any(
- s in enabled_skills for s in (plugin_sources._component_keys(cids, "skills"))
- ) or any(m in enabled_mcps for m in (plugin_sources._component_keys(cids, "mcp")))
+ s in enabled_skills for s in (plugin_sources.component_keys(cids, "skills"))
+ ) or any(m in enabled_mcps for m in (plugin_sources.component_keys(cids, "mcp")))
callable_now = any(
- s in callable_skills for s in (plugin_sources._component_keys(cids, "skills"))
- ) or any(m in callable_mcps for m in (plugin_sources._component_keys(cids, "mcp")))
- from core.plugins.ui.workspace_enabled import enabled as workspace_enabled, workspace_only
+ s in callable_skills for s in (plugin_sources.component_keys(cids, "skills"))
+ ) or any(m in callable_mcps for m in (plugin_sources.component_keys(cids, "mcp")))
+ from core.harness.registry.package_views import enabled as workspace_enabled, workspace_only
enabled = enabled or workspace_enabled(db, r, owner_user_id)
callable_now = callable_now or workspace_only(r)
@@ -195,8 +187,8 @@ def _installed_to_dict(
# Hard runtime availability, independent of the current user's
# personal catalog switch. Explicit pickers hide false entries.
"callable": callable_now,
- "skills": plugin_sources._component_keys(cids, "skills"),
- "mcp": plugin_sources._component_keys(cids, "mcp"),
+ "skills": plugin_sources.component_keys(cids, "skills"),
+ "mcp": plugin_sources.component_keys(cids, "mcp"),
"import_report": r.import_report or {},
"created_at": r.created_at.isoformat() if r.created_at else None,
"has_admin_config": plugin_admin_config._has_admin_config_for_slug(r.slug),
@@ -219,14 +211,14 @@ def get_installed_detail(
raise BadRequestError(message="无权查看该插件")
cids = row.component_ids or {}
- skill_ids = plugin_sources._component_keys(cids, "skills")
- server_ids = plugin_sources._component_keys(cids, "mcp")
+ skill_ids = plugin_sources.component_keys(cids, "skills")
+ server_ids = plugin_sources.component_keys(cids, "mcp")
# Component enabled state: user view uses their "effectively enabled" set (including per-user overrides); admin view uses is_enabled.
eff_skills: Optional[set] = None
eff_mcps: Optional[set] = None
if owner_user_id is not None:
- from core.config.catalog_resolver import resolve_all_runtime_enabled
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
es, _ea, em = resolve_all_runtime_enabled(db, owner_user_id)
eff_skills, eff_mcps = set(es or []), set(em or [])
@@ -298,5 +290,5 @@ def get_installed_detail(
plugin_admin_config._admin_config_for_slug(row.slug), with_values=False
),
# Account connection type (dingtalk / lark / None): the frontend uses this to render the account-connection panel on the detail page
- "connection": plugin_admin_config._connection_for_slug(row.slug),
+ "connection": plugin_admin_config.connection_for_slug(row.slug),
}
diff --git a/src/backend/core/plugins/packaging/__init__.py b/src/backend/core/capabilities/plugins/packaging/__init__.py
similarity index 100%
rename from src/backend/core/plugins/packaging/__init__.py
rename to src/backend/core/capabilities/plugins/packaging/__init__.py
diff --git a/src/backend/core/plugins/packaging/archive.py b/src/backend/core/capabilities/plugins/packaging/archive.py
similarity index 96%
rename from src/backend/core/plugins/packaging/archive.py
rename to src/backend/core/capabilities/plugins/packaging/archive.py
index 81453832..1432e4df 100644
--- a/src/backend/core/plugins/packaging/archive.py
+++ b/src/backend/core/capabilities/plugins/packaging/archive.py
@@ -12,7 +12,7 @@
from core.infra.exceptions import BadRequestError
-from . import definitions as plugin_definitions
+from core.capabilities.plugins.packaging import definitions as plugin_definitions
def _locate_plugin_root(extract_dir: Path) -> Path:
@@ -39,7 +39,7 @@ def _has_manifest(d: Path) -> bool:
@contextmanager
-def _extract_plugin_zip(raw: bytes) -> Iterator[Path]:
+def extract_plugin_zip(raw: bytes) -> Iterator[Path]:
"""Extract the plugin zip to a temp dir and locate the plugin root; yields that root dir and cleans up the temp dir on exit."""
if len(raw) > plugin_definitions.MAX_ZIP_BYTES:
raise BadRequestError(
diff --git a/src/backend/core/plugins/packaging/definitions.py b/src/backend/core/capabilities/plugins/packaging/definitions.py
similarity index 94%
rename from src/backend/core/plugins/packaging/definitions.py
rename to src/backend/core/capabilities/plugins/packaging/definitions.py
index 31cae0b5..8bf6da21 100644
--- a/src/backend/core/plugins/packaging/definitions.py
+++ b/src/backend/core/capabilities/plugins/packaging/definitions.py
@@ -1,4 +1,4 @@
-"""Plugin definitions responsibilities."""
+"""Paths, size limits and metadata keys for plugin bundles."""
from __future__ import annotations
@@ -53,7 +53,7 @@
}
-_META_KEYS = ("display_name", "category", "icon")
+META_KEYS = ("display_name", "category", "icon")
MAX_ICON_LEN = 200_000
diff --git a/src/backend/core/plugins/packaging/importer.py b/src/backend/core/capabilities/plugins/packaging/importer.py
similarity index 96%
rename from src/backend/core/plugins/packaging/importer.py
rename to src/backend/core/capabilities/plugins/packaging/importer.py
index 9c22f784..e4ab2550 100644
--- a/src/backend/core/plugins/packaging/importer.py
+++ b/src/backend/core/capabilities/plugins/packaging/importer.py
@@ -21,11 +21,11 @@
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
-from core.agent_skills.binary_files import pack_directory
+from core.capabilities.skills.binary_files import pack_directory
from core.infra.exceptions import BadRequestError
-from core.plugins.ui.contract import normalize_ui
+from core.harness.registry.package_views import normalize_ui
-from .models import NormalizedMcp, NormalizedPlugin, NormalizedSkill
+from core.capabilities.plugins.packaging.models import NormalizedMcp, NormalizedPlugin, NormalizedSkill
logger = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ def manifest_extensions(manifest: Dict[str, Any]) -> Dict[str, Any]:
return {}
-def _ext_or_top(manifest: Dict[str, Any], ext: Dict[str, Any], key: str) -> Any:
+def ext_or_top(manifest: Dict[str, Any], ext: Dict[str, Any], key: str) -> Any:
"""Read a platform field: extensions namespace first, legacy top-level as fallback."""
if key in ext:
return ext.get(key)
@@ -214,7 +214,7 @@ def _normalize_mcp_entry(name: str, raw: Dict[str, Any]) -> NormalizedMcp:
"""
url = raw.get("url")
if isinstance(url, str) and url.startswith("builtin://"):
- from core.config.mcp_config import _mcp_http_url
+ from core.config.mcp_config import mcp_http_url as _mcp_http_url
from extensions.mcp_servers._ports import PORTS
server_id = url.removeprefix("builtin://")
@@ -285,7 +285,7 @@ def _normalize_required_secrets(
"""
out: List[Dict[str, Any]] = []
# native: ["api_key", ...] or [{key,label,required}]
- rs = _ext_or_top(manifest, ext or {}, "required_secrets")
+ rs = ext_or_top(manifest, ext or {}, "required_secrets")
if isinstance(rs, list):
for item in rs:
if isinstance(item, str):
@@ -376,7 +376,7 @@ def _slugify(value: str) -> str:
return s or "plugin"
-def _normalize_admin_config(
+def normalize_admin_config(
manifest: Dict[str, Any], ext: Optional[Dict[str, Any]] = None
) -> Optional[Dict[str, Any]]:
"""Normalize the ``admin_config`` in the plugin manifest (admin-level provider credential declaration).
@@ -385,7 +385,7 @@ def _normalize_admin_config(
Missing or empty fields → None (this plugin needs no admin config). Read from
the extension namespace first, legacy top-level as fallback.
"""
- ac = _ext_or_top(manifest, ext or {}, "admin_config")
+ ac = ext_or_top(manifest, ext or {}, "admin_config")
if not isinstance(ac, dict):
return None
raw_fields = ac.get("fields")
@@ -453,15 +453,15 @@ def normalize_plugin_dir(plugin_dir: Path) -> NormalizedPlugin:
mcp = [_normalize_mcp_entry(n, raw) for n, raw in sorted(mcp_map.items())]
required_secrets = _normalize_required_secrets(manifest, ext)
- admin_config = _normalize_admin_config(manifest, ext)
- connection = _ext_or_top(manifest, ext, "connection")
+ admin_config = normalize_admin_config(manifest, ext)
+ connection = ext_or_top(manifest, ext, "connection")
connection = str(connection).strip() if connection else None
dropped = _collect_dropped(plugin_dir, manifest, kind)
- ui, ui_dropped = normalize_ui(_ext_or_top(manifest, ext, "ui"))
+ ui, ui_dropped = normalize_ui(ext_or_top(manifest, ext, "ui"))
dropped.extend(ui_dropped)
- de = _ext_or_top(manifest, ext, "default_enabled")
+ de = ext_or_top(manifest, ext, "default_enabled")
if isinstance(de, dict):
default_enabled = {
"skills": [str(x) for x in (de.get("skills") or [])],
diff --git a/src/backend/core/plugins/packaging/models.py b/src/backend/core/capabilities/plugins/packaging/models.py
similarity index 100%
rename from src/backend/core/plugins/packaging/models.py
rename to src/backend/core/capabilities/plugins/packaging/models.py
diff --git a/src/backend/core/capabilities/plugins/packaging/presentation.py b/src/backend/core/capabilities/plugins/packaging/presentation.py
new file mode 100644
index 00000000..182c024b
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/packaging/presentation.py
@@ -0,0 +1,31 @@
+"""Validate plugin icons and apply display metadata without marketplace storage."""
+from typing import Any, Dict
+from core.infra.exceptions import BadRequestError
+from core.capabilities.plugins.packaging import definitions as plugin_definitions
+
+def validate_icon(icon: str) -> str:
+ """Validate an icon value from the UI picker; returns the stripped value ('' = clear)."""
+ icon = (icon or "").strip()
+ if not icon:
+ return ""
+ if icon.startswith("data:"):
+ if not icon.startswith("data:image/"):
+ raise BadRequestError(message="图标 data URI 必须是 image 类型")
+ if len(icon) > plugin_definitions.MAX_ICON_LEN:
+ raise BadRequestError(message="图标过大(上传原图请控制在 80KB 以内)")
+ return icon
+ if not (icon.startswith("/") or icon.startswith("http://") or icon.startswith("https://")):
+ raise BadRequestError(message="图标须从图标库选择或上传(不支持任意文本)")
+ if len(icon) > plugin_definitions.MAX_ICON_URL_LEN:
+ raise BadRequestError(message="图标地址过长")
+ return icon
+
+
+def overlay_market_meta(item: Dict[str, Any], meta: Dict[str, Any]) -> None:
+ """Apply effective display metadata onto a market list/detail dict (in place)."""
+ if meta.get("display_name"):
+ item["name"] = meta["display_name"]
+ if meta.get("category"):
+ item["category"] = meta["category"]
+ if meta.get("icon"):
+ item["icon"] = meta["icon"]
diff --git a/src/backend/core/plugins/packaging/runtime_assets.py b/src/backend/core/capabilities/plugins/packaging/runtime_assets.py
similarity index 100%
rename from src/backend/core/plugins/packaging/runtime_assets.py
rename to src/backend/core/capabilities/plugins/packaging/runtime_assets.py
diff --git a/src/backend/core/plugins/packaging/skill_transform.py b/src/backend/core/capabilities/plugins/packaging/skill_transform.py
similarity index 96%
rename from src/backend/core/plugins/packaging/skill_transform.py
rename to src/backend/core/capabilities/plugins/packaging/skill_transform.py
index cc79c63b..f4d7ed87 100644
--- a/src/backend/core/plugins/packaging/skill_transform.py
+++ b/src/backend/core/capabilities/plugins/packaging/skill_transform.py
@@ -3,7 +3,7 @@
import re
import yaml
-from core.agent_skills.binary_files import decode_binary, is_binary_value
+from core.capabilities.skills.binary_files import decode_binary, is_binary_value
def files(child, skill_ids, server_ids, directory, plugin_directory):
diff --git a/src/backend/core/plugins/packaging/sources.py b/src/backend/core/capabilities/plugins/packaging/sources.py
similarity index 83%
rename from src/backend/core/plugins/packaging/sources.py
rename to src/backend/core/capabilities/plugins/packaging/sources.py
index 0f930a6b..8a96c814 100644
--- a/src/backend/core/plugins/packaging/sources.py
+++ b/src/backend/core/capabilities/plugins/packaging/sources.py
@@ -1,4 +1,4 @@
-"""Plugin sources responsibilities."""
+"""Locate plugin bundles on disk and derive namespaced component ids."""
from __future__ import annotations
@@ -8,14 +8,14 @@
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
-from core.plugins.packaging import definitions as plugin_definitions
-from core.plugins.packaging.importer import _ext_or_top, manifest_extensions
-from core.services.marketplace_service import compute_install_id
+from core.capabilities.plugins.packaging import definitions as plugin_definitions
+from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions
+from core.capabilities.skills.installation import compute_install_id
logger = logging.getLogger(__name__)
-def _iter_plugin_dirs():
+def iter_plugin_dirs():
"""Iterate over all plugin bundle directories containing plugin.json under default + marketplace."""
for root in plugin_definitions.PLUGIN_SOURCE_DIRS:
if not root.is_dir():
@@ -25,7 +25,7 @@ def _iter_plugin_dirs():
yield child
-def _resolve_plugin_dir(slug: str) -> Optional[Path]:
+def resolve_plugin_dir(slug: str) -> Optional[Path]:
"""Locate a plugin bundle directory by slug in default / marketplace (default takes precedence)."""
if not slug or "/" in slug or ".." in slug:
return None
@@ -36,7 +36,7 @@ def _resolve_plugin_dir(slug: str) -> Optional[Path]:
return None
-def _make_plugin_install_id(slug: str, owner_user_id: Optional[str]) -> str:
+def make_plugin_install_id(slug: str, owner_user_id: Optional[str]) -> str:
return f"{slug}@{owner_user_id or 'global'}"
@@ -55,7 +55,7 @@ def _sanitize_component_id(value: str, maxlen: int) -> str:
return f"{prefix}-{digest}"
-def _make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> str:
+def make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> str:
"""Namespaced skill id: {slug}-{skill} (+ user fingerprint). Constrained by _ID_RE (<=63)."""
base = _sanitize_component_id(f"{slug}-{skill_name}", 50)
return compute_install_id(
@@ -63,7 +63,7 @@ def _make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) ->
) # appends -<6-char fingerprint> when owner is non-empty
-def _make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) -> str:
+def make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) -> str:
"""Namespaced MCP server id: {slug}-{server} (+ user fingerprint), same rules as skill ids."""
return compute_install_id(
_sanitize_component_id(f"{slug}-{server_name}", 80),
@@ -87,11 +87,11 @@ def builtin_plugin_component_ids() -> Tuple[set, set]:
"""
import json
- from core.plugins.packaging.importer import _find_mcp_map
+ from core.capabilities.plugins.packaging.importer import _find_mcp_map
skill_ids: set = set()
mcp_ids: set = set()
- for child in _iter_plugin_dirs():
+ for child in iter_plugin_dirs():
skills_root = child / "skills"
if skills_root.is_dir():
for c in skills_root.iterdir():
@@ -103,12 +103,13 @@ def builtin_plugin_component_ids() -> Tuple[set, set]:
m = {}
try:
mcp_ids.update(_find_mcp_map(child, m if isinstance(m, dict) else {}).keys())
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.plugins.packaging.sources:builtin_plugin_component_ids', fallback_error)
return skill_ids, mcp_ids
-def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]:
+def scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]:
"""Lightweight read of a builtin plugin bundle's metadata (no full normalize).
Standard manifests carry no display fields — display metadata is overlaid
@@ -148,7 +149,7 @@ def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]:
}
-def _component_keys(components: Dict[str, Any], kind: str) -> List[str]:
+def component_keys(components: Dict[str, Any], kind: str) -> List[str]:
"""Project declarative component entries to IDs for legacy catalog operations.
Keep InstalledPlugin.component_ids unchanged: version/required/platform
diff --git a/src/backend/core/capabilities/plugins/resources/__init__.py b/src/backend/core/capabilities/plugins/resources/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/plugins/resources/artifacts.py b/src/backend/core/capabilities/plugins/resources/artifacts.py
similarity index 93%
rename from src/backend/core/plugins/resources/artifacts.py
rename to src/backend/core/capabilities/plugins/resources/artifacts.py
index 6fd9217b..0eeb7706 100644
--- a/src/backend/core/plugins/resources/artifacts.py
+++ b/src/backend/core/capabilities/plugins/resources/artifacts.py
@@ -9,7 +9,7 @@
import httpx
from fastapi import HTTPException
-from . import store
+from core.capabilities.plugins.resources import store
async def retain(row, download_id):
@@ -35,7 +35,7 @@ async def retain(row, download_id):
if total > limit:
raise HTTPException(413, "download_limit")
output.write(chunk)
- from core.llm.tools._tool_helpers import _store_generated_file_path
+ from core.files.artifacts.generated import store_generated_file_path as _store_generated_file_path
ref = await asyncio.to_thread(
_store_generated_file_path,
diff --git a/src/backend/core/plugins/resources/authentication.py b/src/backend/core/capabilities/plugins/resources/authentication.py
similarity index 93%
rename from src/backend/core/plugins/resources/authentication.py
rename to src/backend/core/capabilities/plugins/resources/authentication.py
index 8722ded7..a7af7469 100644
--- a/src/backend/core/plugins/resources/authentication.py
+++ b/src/backend/core/capabilities/plugins/resources/authentication.py
@@ -14,7 +14,7 @@ def capture(request):
headers["cookie"] = settings.session.cookie_name + "=" + token
return {"headers": headers, "path": request.url.path, "method": request.method}
-async def validate(context, user_id, db):
+async def validate(context, user_id):
from core.auth.backend import get_current_user
from core.config.local_mode import local_mode_enabled
headers = context.get("headers", {})
@@ -29,6 +29,6 @@ async def validate(context, user_id, db):
authorization = headers.get("authorization", "")
scheme, _, value = authorization.partition(" ")
credentials = HTTPAuthorizationCredentials(scheme=scheme, credentials=value) if value else None
- current = await get_current_user(request, credentials=credentials, db=db)
+ current = await get_current_user(request, credentials)
if str(current.user_id) != user_id:
raise HTTPException(403, "viewer_identity_changed")
diff --git a/src/backend/core/plugins/resources/configuration.py b/src/backend/core/capabilities/plugins/resources/configuration.py
similarity index 100%
rename from src/backend/core/plugins/resources/configuration.py
rename to src/backend/core/capabilities/plugins/resources/configuration.py
diff --git a/src/backend/core/plugins/resources/confinement.py b/src/backend/core/capabilities/plugins/resources/confinement.py
similarity index 83%
rename from src/backend/core/plugins/resources/confinement.py
rename to src/backend/core/capabilities/plugins/resources/confinement.py
index f2f2a93d..9f830b60 100644
--- a/src/backend/core/plugins/resources/confinement.py
+++ b/src/backend/core/capabilities/plugins/resources/confinement.py
@@ -7,12 +7,13 @@ def launch_policy(provider, user_id, chat_id):
from core.config.local_mode import local_mode_enabled
if not local_mode_enabled() or not provider.runs_on_host:
return None
- from core.llm.tool_permissions import resolve_approval_mode, LocalCommandAuthorization
- from core.services.local_grant_service import policy_for_gate
+ from core.sandbox.permissions.approval import resolve_approval_mode
+ from core.sandbox.permissions.dispatch_models import LocalCommandAuthorization
+ from core.harness.registry.local_files import policy_for_gate
from core.sandbox.permissions.os_gate import LocalAccessDecision
from core.sandbox.permissions.os_gate import protected_read_paths
from core.sandbox.permissions.models import NETWORK
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
mode = resolve_approval_mode(None, user_id=user_id)
policy = policy_for_gate(mode)
if policy.disposition_for(NETWORK) == "block":
diff --git a/src/backend/core/plugins/resources/crypto.py b/src/backend/core/capabilities/plugins/resources/crypto.py
similarity index 100%
rename from src/backend/core/plugins/resources/crypto.py
rename to src/backend/core/capabilities/plugins/resources/crypto.py
diff --git a/src/backend/core/capabilities/plugins/resources/deadlines.py b/src/backend/core/capabilities/plugins/resources/deadlines.py
new file mode 100644
index 00000000..fa529f5f
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/deadlines.py
@@ -0,0 +1,53 @@
+"""Generic runner deadline support; installed owners supply lease policy."""
+from fastapi import HTTPException
+from core.capabilities.plugins.resources import store
+
+def provider_for(row):
+ from core.sandbox.factory import get_sandbox_provider
+
+ provider = get_sandbox_provider()
+ descriptor = store.descriptor(row)
+ method = getattr(provider, "set_process_deadline", None)
+ if descriptor.get("provider") != provider.name or not callable(method):
+ raise HTTPException(403, detail={"code": "unsupported_capability"})
+ if not descriptor.get("process_id"):
+ raise HTTPException(410, detail={"code": "instance_closed"})
+ return method, descriptor["process_id"]
+
+
+
+def invalidate(db, row):
+ from core.harness.registry.resource_owners import resource_owner
+ return resource_owner().invalidate(db, row)
+
+
+async def arm(resource_id, *, require_background=False):
+ """Arm the runner deadline of an application resource (no-op for chat resources)."""
+ from core.harness.registry.resource_owners import resource_owner
+ return await resource_owner().arm(resource_id, require_background=require_background)
+
+
+def _active_application_ids(db, after, limit):
+ return [
+ resource_id
+ for (resource_id,) in db.query(store.PluginResource.resource_id)
+ .filter(
+ store.PluginResource.application_key.is_not(None),
+ store.PluginResource.status == "active",
+ store.PluginResource.resource_id > after,
+ )
+ .order_by(store.PluginResource.resource_id)
+ .limit(limit)
+ ]
+
+
+async def restore(*, after="", limit=100):
+ from core.db.threads import run_in_session
+
+ resource_ids = await run_in_session(_active_application_ids, after, limit)
+ for resource_id in resource_ids:
+ try:
+ await arm(resource_id)
+ except HTTPException:
+ pass # Existing janitor owns persisted closing cleanup.
+ return resource_ids[-1] if len(resource_ids) == limit else ""
diff --git a/src/backend/core/plugins/resources/installation.py b/src/backend/core/capabilities/plugins/resources/installation.py
similarity index 56%
rename from src/backend/core/plugins/resources/installation.py
rename to src/backend/core/capabilities/plugins/resources/installation.py
index 401f6a58..bbb34bc5 100644
--- a/src/backend/core/plugins/resources/installation.py
+++ b/src/backend/core/capabilities/plugins/resources/installation.py
@@ -16,31 +16,10 @@ class Installation:
def resolve(db, slug, user_id, install_id=None):
- from core.capabilities import device_catalog, store
- from core.capabilities.device_plugin_catalog import _projected_plugin
- from core.capabilities.local_plugin_runtime import enabled_for
- from core.capabilities.plugins import load_manifest
-
- candidate = _projected_plugin(install_id or slug, user_id=user_id)
- if candidate is not None:
- if candidate.key != slug or not candidate.ready or not enabled_for(candidate, user_id):
- raise HTTPException(403, "plugin_not_ready")
- component = store.get(
- "plugin", candidate.profile_id, candidate.key, candidate.resolved_revision
- )
- if component is None:
- raise HTTPException(409, "plugin_package_missing")
- manifest = load_manifest(component)
- package = component.path / "package"
- if not package.is_dir():
- raise HTTPException(409, "plugin_package_missing")
- return Installation(
- candidate.install_id,
- candidate.resolved_revision,
- slug,
- manifest.get("ui_contributions") or {},
- package,
- )
+ from core.harness.registry import device_catalog
+ selected = device_catalog.resolve_plugin(slug, user_id, install_id=install_id)
+ if selected is not None:
+ return selected
if device_catalog.active():
raise HTTPException(404, "plugin_source_unavailable")
from core.db.models import InstalledPlugin
@@ -59,12 +38,12 @@ def resolve(db, slug, user_id, install_id=None):
if not rows:
raise HTTPException(404, "plugin_source_unavailable")
row = rows[0]
- from core.config.catalog_resolver import resolve_all_runtime_enabled
- from core.plugins.packaging.sources import _component_keys
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+ from core.capabilities.plugins.packaging.sources import component_keys as _component_keys
skills, _, mcps = resolve_all_runtime_enabled(db, user_id)
cids = row.component_ids or {}
- from core.plugins.ui.workspace_enabled import enabled as workspace_enabled
+ from core.harness.registry.package_views import enabled as workspace_enabled
if not (
workspace_enabled(db, row, user_id)
@@ -77,13 +56,13 @@ def resolve(db, slug, user_id, install_id=None):
raise HTTPException(404, "plugin_ui_unavailable")
revision = (row.import_report or {}).get("package_revision")
if revision:
- from core.plugins.packaging.runtime_assets import materialize
+ from core.capabilities.plugins.packaging.runtime_assets import materialize
package = materialize(db, revision)
else:
- from core.plugins.packaging.sources import _resolve_plugin_dir
+ from core.capabilities.plugins.packaging.sources import resolve_plugin_dir
- package = _resolve_plugin_dir(slug)
+ package = resolve_plugin_dir(slug)
if package is None:
raise HTTPException(409, "plugin_package_missing")
return Installation(row.install_id, revision or row.version or "", slug, ui, Path(package))
diff --git a/src/backend/core/plugins/resources/lifecycle.py b/src/backend/core/capabilities/plugins/resources/lifecycle.py
similarity index 59%
rename from src/backend/core/plugins/resources/lifecycle.py
rename to src/backend/core/capabilities/plugins/resources/lifecycle.py
index de1a6779..77b2ec84 100644
--- a/src/backend/core/plugins/resources/lifecycle.py
+++ b/src/backend/core/capabilities/plugins/resources/lifecycle.py
@@ -7,14 +7,12 @@
from contextlib import asynccontextmanager
import httpx
-from core.db.engine import SessionLocal
+from core.db.threads import run_in_session
-from . import store
+from core.capabilities.plugins.resources import store
-async def reconcile(db, user_id=None, after="", limit=100):
- from . import service
-
+def _candidates(db, user_id, after, limit):
query = db.query(store.PluginResource).filter(
store.PluginResource.status.in_(["active", "starting", "warming", "warm", "closing"])
)
@@ -26,6 +24,26 @@ async def reconcile(db, user_id=None, after="", limit=100):
.limit(limit)
.all()
)
+ for row in rows:
+ db.expunge(row)
+ return rows
+
+
+def _validate_binding(db, resource_id, user_id):
+ from core.capabilities.plugins.resources import service
+
+ service.validate_binding(db, db.get(store.PluginResource, resource_id), user_id)
+
+
+async def reconcile(user_id=None, after="", limit=100):
+ """Close persisted workers that are expired, unbound or no longer answering.
+
+ Rows are read, and each binding is checked, on short sessions of their own;
+ the liveness probes and shutdowns in between hold no connection.
+ """
+ from core.capabilities.plugins.resources import service
+
+ rows = await run_in_session(_candidates, user_id, after, limit)
async def valid(row):
if row.status == "closing" or row.expires_at <= time.time():
@@ -33,7 +51,7 @@ async def valid(row):
if row.status in {"starting", "warming"}:
return True
try:
- service.validate_binding(db, row, row.user_id)
+ await run_in_session(_validate_binding, row.resource_id, row.user_id)
target = store.descriptor(row)
async with httpx.AsyncClient(
timeout=3, trust_env=False, headers=target["headers"]
@@ -47,25 +65,30 @@ async def valid(row):
for row, alive in zip(rows, results):
if not alive:
try:
- await service.close(db, row)
+ await service.close(row)
except Exception as exc:
- db.rollback()
logging.getLogger(__name__).warning(
"Resource cleanup failed for %s: %s", row.resource_id, type(exc).__name__
)
return rows[-1].resource_id if len(rows) == limit else ""
+def _collect(db):
+ from core.harness.registry.resource_owners import resource_owner
+
+ resource_owner().collect(db)
+ db.commit()
+
+
@asynccontextmanager
async def lifespan(app):
- from . import prewarm
+ from core.capabilities.plugins.resources import prewarm
- from .deadlines import restore
+ from core.capabilities.plugins.resources.deadlines import restore
cursor = ""
while True:
- with SessionLocal() as db:
- cursor = await restore(db, after=cursor)
+ cursor = await restore(after=cursor)
if not cursor:
break
prewarm.resume()
@@ -76,18 +99,11 @@ async def sweep():
while True:
await asyncio.sleep(int(os.getenv("PLUGIN_RESOURCE_RECONCILE_SECONDS", "30")))
try:
- from core.applications.revocation_sweep import sweep as revoke_stale
-
- owner_cursor = await revoke_stale(owner_cursor)
- with SessionLocal() as db:
- cursor = await reconcile(db, after=cursor)
- from core.applications.plugin_revisions import collect
-
- collect(db)
- from core.applications.plugin_candidate_storage import janitor
+ from core.harness.registry.resource_owners import resource_owner
- janitor(db)
- db.commit()
+ owner_cursor = await resource_owner().sweep(owner_cursor)
+ cursor = await reconcile(after=cursor)
+ await run_in_session(_collect)
except Exception as exc:
logging.getLogger(__name__).warning(
"Resource reconciliation failed: %s", type(exc).__name__
@@ -99,6 +115,6 @@ async def sweep():
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
- from .prewarm import stop
+ from core.capabilities.plugins.resources.prewarm import stop
await stop()
diff --git a/src/backend/core/capabilities/plugins/resources/owners.py b/src/backend/core/capabilities/plugins/resources/owners.py
new file mode 100644
index 00000000..5439e77e
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/owners.py
@@ -0,0 +1,66 @@
+"""Resource owner validation and provider session identity."""
+
+import hashlib
+import json
+
+from core.config.local_mode import local_mode_enabled
+from fastapi import HTTPException
+
+
+def execution_scope():
+ return "local" if local_mode_enabled() else "cloud"
+
+
+def runtime_session(row):
+ if row.chat_id is not None:
+ return row.chat_id
+ # Provider session IDs are execution containers, never conversation records.
+ binding = [
+ row.user_id,
+ row.application_key,
+ row.install_id,
+ row.revision,
+ row.scope,
+ row.instance_id,
+ ]
+ return "workbench-" + hashlib.sha256(json.dumps(binding).encode()).hexdigest()
+
+
+def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None):
+ from core.harness.registry.resource_owners import resource_owner
+ return resource_owner().binding(db, user_id, key, install_id, revision, scope, module_id, selected)
+
+
+def resolve_installation(db, row, user_id):
+ if row.application_key is not None:
+ from core.harness.registry.resource_owners import resource_owner
+ return resource_owner().resolve_installation(db, row, user_id)
+ from core.capabilities.plugins.resources.installation import resolve
+ selected = resolve(db, row.slug, user_id, row.install_id)
+ if selected.revision != row.revision:
+ raise HTTPException(409, "plugin_revision_changed")
+ return selected
+
+
+def validate(db, row, user_id, selected=None):
+ if row.chat_id is not None:
+ if row.application_key is not None or row.instance_id is not None:
+ raise HTTPException(403, "resource_owner_invalid")
+ from core.db.models import ChatSession
+
+ chat = db.get(ChatSession, row.chat_id)
+ if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
+ raise HTTPException(403, "conversation_unavailable")
+ else:
+ if not row.application_key or not row.instance_id:
+ raise HTTPException(403, "resource_owner_invalid")
+ application_binding(
+ db,
+ user_id,
+ row.application_key,
+ row.install_id,
+ row.revision,
+ row.scope,
+ row.module_id,
+ selected,
+ )
diff --git a/src/backend/core/capabilities/plugins/resources/prewarm.py b/src/backend/core/capabilities/plugins/resources/prewarm.py
new file mode 100644
index 00000000..162895da
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/prewarm.py
@@ -0,0 +1,177 @@
+"""Background opt-in resource startup, bound to a conversation sandbox lifetime."""
+from __future__ import annotations
+import asyncio
+import logging
+import threading
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+import httpx
+
+from core.db.threads import run_in_session
+from fastapi import HTTPException
+from sqlalchemy import or_
+from core.capabilities.plugins.resources import installation, service, store
+
+logger = logging.getLogger(__name__)
+
+
+def candidates(db, user_id):
+ from core.harness.registry import device_catalog
+ if device_catalog.active():
+ from core.harness.registry.device_catalog import plugin_entries
+ sources = [(entry["slug"], entry["install_id"]) for entry in plugin_entries(user_id)
+ if entry["enabled"] and entry["callable"]]
+ else:
+ from core.db.models import InstalledPlugin
+ sources = [(row.slug, row.install_id) for row in db.query(InstalledPlugin).filter(
+ or_(InstalledPlugin.owner_user_id == user_id, InstalledPlugin.owner_user_id.is_(None))).all()]
+ for slug, install_id in sources:
+ try:
+ selected = installation.resolve(db, slug, user_id, install_id)
+ except HTTPException:
+ continue
+ for module in (selected.ui.get("contributes") or {}).get("modules") or []:
+ if (module.get("resource") or {}).get("prewarm") is True:
+ yield selected, module
+
+def schedule(provider, chat_id, user_id, sandbox_id):
+ if not user_id or not chat_id or not sandbox_id or str(chat_id).startswith("eval_"):
+ return
+ key = (chat_id, user_id, str(sandbox_id))
+ loop = asyncio.get_running_loop()
+ with process_state().guard:
+ if process_state().shutdown or chat_id in process_state().closing or key in process_state().jobs:
+ return
+ # Bound completed admission markers; router providers notify repeatedly.
+ if len(process_state().jobs) >= 1024:
+ for old in [old for old, job in process_state().jobs.items() if job.done()][:256]:
+ process_state().jobs.pop(old, None)
+ process_state().jobs[key] = loop.create_task(_run(provider, chat_id, user_id, str(sandbox_id)))
+
+def _prewarm_targets(db, chat_id, user_id):
+ """``(slug, module_id, install_id)`` of every prewarmed module the chat's owner has."""
+ from core.db.models import ChatSession
+
+ chat = db.get(ChatSession, chat_id)
+ if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
+ return []
+ return [(selected.slug, module["id"], selected.install_id) for selected, module in candidates(db, user_id)]
+
+
+async def _run(provider, chat_id, user_id, sandbox_id):
+ from core.sandbox.sessions.binding import bind
+ with bind(sandbox_id):
+ try:
+ for slug, module_id, install_id in await run_in_session(_prewarm_targets, chat_id, user_id):
+ await service.create(slug, module_id, user_id, chat_id,
+ install_id=install_id, _prewarm=True, _provider=provider)
+ except Exception as exc:
+ logger.info("Resource prewarm skipped for chat=%s: %s", chat_id, type(exc).__name__)
+ return False
+
+async def _cancel(task):
+ if not task.done():
+ task.cancel()
+ result, = await asyncio.gather(task, return_exceptions=True)
+ return result is not False and (not isinstance(result, BaseException) or isinstance(result, asyncio.CancelledError))
+
+@asynccontextmanager
+async def closing(chat_id):
+ with process_state().guard:
+ process_state().closing[chat_id] = process_state().closing.get(chat_id, 0) + 1
+ try:
+ yield
+ finally:
+ with process_state().guard:
+ process_state().closing[chat_id] -= 1
+ if process_state().closing[chat_id] == 0:
+ process_state().closing.pop(chat_id)
+
+def resume():
+ pass
+ with process_state().guard:
+ process_state().shutdown = False
+
+async def stop(chat_id=None):
+ pass
+ if str(chat_id or "").startswith("eval_"):
+ return True
+ if chat_id is None:
+ with process_state().guard:
+ process_state().shutdown = True
+ async with closing(chat_id):
+ return await _stop(chat_id)
+
+def _warm_targets(db, chat_id):
+ """``(resource_id, target)`` of every blank worker; target None when unreadable."""
+ query = db.query(store.PluginResource).filter(store.PluginResource.status.in_(("warm", "warming")))
+ if chat_id is not None:
+ query = query.filter_by(chat_id=chat_id)
+ targets = []
+ for row in query.all():
+ try:
+ targets.append((row.resource_id, store.descriptor(row)))
+ except HTTPException:
+ targets.append((row.resource_id, None))
+ return targets
+
+
+def _close_rows(db, resource_ids):
+ if resource_ids:
+ db.query(store.PluginResource).filter(
+ store.PluginResource.resource_id.in_(resource_ids)
+ ).update({"status": "closed"}, synchronize_session=False)
+ db.commit()
+
+
+async def _stop(chat_id):
+ """Stop starts and blank workers. False forbids returning a dirty sandbox to its pool."""
+ with process_state().guard:
+ keys = [key for key in process_state().jobs if chat_id is None or key[0] == chat_id]
+ jobs = [process_state().jobs.pop(key) for key in keys]
+ loop = asyncio.get_running_loop()
+ clean = True
+ for task in jobs:
+ owner = task.get_loop()
+ if owner is loop:
+ clean = await _cancel(task) and clean
+ elif owner.is_running():
+ clean = await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(_cancel(task), owner)) and clean
+ elif not task.done():
+ clean = False
+ else:
+ try:
+ clean = task.result() is not False and clean
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ clean = False
+ targets = await run_in_session(_warm_targets, chat_id)
+ for resource_id, target in targets:
+ if target is None:
+ clean = False
+ continue
+ if target.get("url"):
+ try:
+ async with httpx.AsyncClient(timeout=3, trust_env=False, headers=target["headers"]) as client:
+ response = await client.post(target["url"] + "/shutdown", json={})
+ response.raise_for_status()
+ except httpx.HTTPError:
+ clean = False
+ await run_in_session(_close_rows, [resource_id for resource_id, _ in targets])
+ return clean
+
+
+@dataclass
+class PrewarmState:
+ guard: object = field(default_factory=lambda: threading.Lock())
+ jobs: object = field(default_factory=lambda: {})
+ closing: object = field(default_factory=lambda: {})
+ shutdown: object = False
+
+
+_STATE = PrewarmState()
+
+
+def process_state() -> PrewarmState:
+ return _STATE
diff --git a/src/backend/core/capabilities/plugins/resources/reservation.py b/src/backend/core/capabilities/plugins/resources/reservation.py
new file mode 100644
index 00000000..cbc6b020
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/reservation.py
@@ -0,0 +1,84 @@
+"""Serialize resource admission and one-time warm worker adoption per owner."""
+
+import os
+import time
+
+from fastapi import HTTPException
+
+from core.capabilities.plugins.resources import store
+
+LIVE_STATUSES = ("active", "starting", "warming", "warm")
+
+
+def lock_owner(db, user_id):
+ from core.db.models import UserShadow
+
+ if db.bind.dialect.name == "sqlite":
+ from sqlalchemy import text
+
+ db.execute(text("BEGIN IMMEDIATE"))
+ else:
+ db.query(UserShadow).filter_by(user_id=user_id).with_for_update().one()
+
+
+def adopt_once(db, selected, module_id, user_id, chat_id, *, prewarm, checkpoint_id):
+ """Take the owner lock and look for a warm worker to adopt.
+
+ Returns ``("adopted", row)``, ``("exists", None)`` (prewarm: one is already
+ live), ``("wait", None)`` (one is still warming; the lock is released), or
+ ``("reserve", None)`` — permission to reserve under the lock still held.
+ """
+ lock_owner(db, user_id)
+ query = (
+ db.query(store.PluginResource)
+ .filter_by(
+ user_id=user_id,
+ chat_id=chat_id,
+ install_id=selected.install_id,
+ revision=selected.revision,
+ slug=selected.slug,
+ module_id=module_id,
+ )
+ .filter(store.PluginResource.expires_at > time.time())
+ )
+ if prewarm:
+ if query.filter(store.PluginResource.status.in_(LIVE_STATUSES)).first():
+ db.commit()
+ return "exists", None
+ return "reserve", None
+ row = (
+ None
+ if checkpoint_id
+ else query.filter(store.PluginResource.status.in_(("warm", "warming")))
+ .order_by(store.PluginResource.created_at)
+ .first()
+ )
+ if row is None:
+ return "reserve", None
+ if row.status == "warm":
+ row.status = "active"
+ row.expires_at = time.time() + int(os.getenv("PLUGIN_RESOURCE_TTL_SECONDS", "86400"))
+ db.commit()
+ return "adopted", row
+ db.commit()
+ return "wait", None
+
+
+def check_quota(db, user_id):
+ active = (
+ db.query(store.PluginResource)
+ .filter(
+ store.PluginResource.user_id == user_id,
+ (
+ (
+ store.PluginResource.status.in_(LIVE_STATUSES)
+ & (store.PluginResource.expires_at > time.time())
+ )
+ | (store.PluginResource.status == "closing")
+ ),
+ )
+ .count()
+ )
+ if active >= int(os.getenv("PLUGIN_RESOURCE_MAX_PER_USER", "8")):
+ db.rollback()
+ raise HTTPException(429, "resource_limit")
diff --git a/src/backend/core/capabilities/plugins/resources/service.py b/src/backend/core/capabilities/plugins/resources/service.py
new file mode 100644
index 00000000..8e717b24
--- /dev/null
+++ b/src/backend/core/capabilities/plugins/resources/service.py
@@ -0,0 +1,460 @@
+"""Authorized resource lifecycle. Runtime behavior remains package-owned."""
+
+import asyncio
+import json
+import os
+import secrets
+import time
+import uuid
+
+from core.harness.registry.package_views import find_module
+from fastapi import HTTPException
+
+from core.capabilities.plugins.resources import installation, store
+from core.capabilities.plugins.resources.crypto import encrypt_secret
+from core.capabilities.plugins.resources.transport import RuntimeTransport
+from core.db.threads import run_in_session
+
+
+def authorized(db, resource_id, user_id):
+ row = store.get(db, resource_id, user_id)
+ if row.status != "active" or row.expires_at < time.time():
+ raise HTTPException(410, "resource_expired")
+ validate_binding(db, row, user_id)
+ return row
+
+
+def validate_binding(db, row, user_id):
+ if row.application_key is not None:
+ from core.harness.registry.resource_owners import resource_owner
+
+ resource_owner().authorize(db, row)
+ from core.capabilities.plugins.resources.owners import resolve_installation, validate
+
+ selected = resolve_installation(db, row, user_id)
+ validate(db, row, user_id, selected)
+ return row
+
+
+def detached(db, row):
+ """``row`` with its columns loaded, detached so it can be read after the session closes."""
+ db.refresh(row)
+ db.expunge(row)
+ return row
+
+
+def authorized_row(db, resource_id, user_id):
+ """``authorized`` for async callers: the row comes back detached."""
+ return detached(db, authorized(db, resource_id, user_id))
+
+
+def _authorized_public(db, resource_id, user_id):
+ return store.public(authorized(db, resource_id, user_id))
+
+
+def _existing(db, resource_id, user_id, chat_id, slug, module_id):
+ row = authorized(db, resource_id, user_id)
+ if row.chat_id != chat_id or row.slug != slug or row.module_id != module_id:
+ raise HTTPException(403, "resource_binding_mismatch")
+ return detached(db, row)
+
+
+def _admission_inputs(db, slug, module_id, user_id, chat_id, install_id, checkpoint_id, prewarm, application):
+ """Check the owner, resolve the installation and module, and build the runner config."""
+ if application is None:
+ from core.db.models import ChatSession
+
+ chat = db.get(ChatSession, chat_id)
+ if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
+ raise HTTPException(403, "conversation_unavailable")
+ else:
+ if chat_id is not None or checkpoint_id or prewarm:
+ raise HTTPException(403, "resource_owner_invalid")
+ from core.capabilities.plugins.resources.owners import application_binding
+
+ application_binding(
+ db,
+ user_id,
+ application["application_key"],
+ install_id,
+ application["revision"],
+ application["scope"],
+ module_id,
+ )
+ resolver = installation.resolve
+ if application:
+ from core.harness.registry.resource_owners import resource_owner
+ resolver = resource_owner().exact_installation
+
+ selected = resolver(db, slug, user_id, install_id)
+ if application and selected.install_id != install_id:
+ raise HTTPException(403, "resource_installation_mismatch")
+ if application and selected.revision != application["revision"]:
+ raise HTTPException(409, "plugin_revision_changed")
+ module = find_module(selected.ui, module_id)
+ if module is None or not module.get("resource"):
+ raise HTTPException(404, "resource_module_unavailable")
+ if application and "resource.attach" not in module.get("grants", []):
+ raise HTTPException(403, "resource_attach_not_granted")
+ from core.capabilities.plugins.resources.configuration import runtime_configuration
+
+ config = {
+ **runtime_configuration(module["resource"]),
+ "token": secrets.token_urlsafe(32),
+ "idle_seconds": int(os.getenv("PLUGIN_RESOURCE_IDLE_SECONDS", "1800")),
+ }
+ if checkpoint_id:
+ checkpoint = db.get(store.PluginResourceCheckpoint, checkpoint_id)
+ if (
+ checkpoint is None
+ or checkpoint.user_id != user_id
+ or checkpoint.install_id != selected.install_id
+ ):
+ raise HTTPException(404, "checkpoint_unavailable")
+ from core.capabilities.plugins.resources.crypto import decrypt_secret
+
+ state = decrypt_secret(checkpoint.state_enc)
+ if state is None:
+ raise HTTPException(410, "checkpoint_expired")
+ config["checkpoint"] = json.loads(state)
+ return selected, module, config
+
+
+def _reserve(db, selected, module_id, user_id, chat_id, prewarm, application):
+ """Insert a starting row under the owner lock the caller already holds."""
+ from core.capabilities.plugins.resources.reservation import check_quota
+ from core.capabilities.plugins.resources.owners import runtime_session
+ from core.config.settings import settings
+
+ check_quota(db, user_id)
+ row = store.PluginResource(
+ resource_id=application["resource_id"] if application else uuid.uuid4().hex,
+ user_id=user_id,
+ chat_id=chat_id,
+ application_key=application["application_key"] if application else None,
+ instance_id=application["instance_id"] if application else None,
+ request_digest=application["request_digest"] if application else None,
+ install_id=selected.install_id,
+ revision=selected.revision,
+ slug=selected.slug,
+ module_id=module_id,
+ scope=(
+ application["scope"]
+ if application
+ else ("local" if settings.deploy.is_local else "cloud")
+ ),
+ descriptor=encrypt_secret("{}"),
+ status="warming" if prewarm else "starting",
+ created_at=time.time(),
+ expires_at=time.time() + 90,
+ )
+ session_id = runtime_session(row)
+ db.add(row)
+ db.commit()
+ return detached(db, row), session_id
+
+
+def _admit(db, selected, module_id, user_id, chat_id, prewarm, checkpoint_id, application, attempt, fresh):
+ """One admission attempt under the owner lock.
+
+ Returns ``("adopted", row)`` for a reusable worker, ``("exists", None)`` when
+ a prewarm finds one already live, ``("wait", None)`` while another start is
+ in flight, or ``("reserved", (row, session_id))`` for a new starting row.
+ """
+ from core.capabilities.plugins.resources.reservation import adopt_once, lock_owner
+
+ if attempt:
+ # Each wait releases the transaction: installation switches and revocation
+ # must be noticed before the worker can be adopted.
+ if application is None:
+ current = installation.resolve(db, selected.slug, user_id, selected.install_id)
+ if current.revision != selected.revision:
+ raise HTTPException(409, "plugin_revision_changed")
+ else:
+ from core.capabilities.plugins.resources.owners import application_binding
+
+ application_binding(
+ db,
+ user_id,
+ application["application_key"],
+ selected.install_id,
+ selected.revision,
+ application["scope"],
+ module_id,
+ )
+ if fresh:
+ lock_owner(db, user_id)
+ outcome, row = "reserve", None
+ elif application is None:
+ outcome, row = adopt_once(
+ db, selected, module_id, user_id, chat_id, prewarm=prewarm, checkpoint_id=checkpoint_id
+ )
+ else:
+ from core.harness.registry.resource_owners import resource_owner
+
+ outcome, row = resource_owner().reserve(db, selected, module_id, user_id, application)
+ if outcome == "reserve":
+ return "reserved", _reserve(db, selected, module_id, user_id, chat_id, prewarm, application)
+ if outcome == "adopted":
+ return outcome, detached(db, row)
+ return outcome, None
+
+
+def _mark(db, resource_id, status, descriptor=None):
+ row = db.get(store.PluginResource, resource_id)
+ row.status = status
+ if descriptor is not None:
+ # Retain the exact handle before cleanup so a failed stop is janitor-retryable.
+ row.descriptor = encrypt_secret(json.dumps(descriptor))
+ db.commit()
+
+
+def _activate(db, resource_id, user_id, descriptor, prewarm, application):
+ row = db.get(store.PluginResource, resource_id)
+ validate_binding(db, row, user_id)
+ updated = (
+ db.query(store.PluginResource)
+ .filter_by(resource_id=resource_id, status="warming" if prewarm else "starting")
+ .update(
+ {
+ "descriptor": encrypt_secret(json.dumps(descriptor)),
+ "status": "warm" if prewarm else "active",
+ "expires_at": time.time()
+ + (
+ 90
+ if application
+ else (
+ 300
+ if prewarm
+ else int(os.getenv("PLUGIN_RESOURCE_TTL_SECONDS", "86400"))
+ )
+ ),
+ },
+ synchronize_session=False,
+ )
+ )
+ db.commit()
+ if updated != 1:
+ raise HTTPException(410, "resource_start_cancelled")
+
+
+def _observe_started(db, resource_id):
+ from core.harness.registry.resource_owners import resource_owner
+
+ row = db.get(store.PluginResource, resource_id)
+ resource_owner().observe(db, row, "resource", "ok")
+ return store.public(row)
+
+
+async def _worker_alive(row, provider, chat_id):
+ target = store.descriptor(row)
+ if (
+ target.get("sandbox_id") != await provider.current_sandbox_id(chat_id)
+ or target.get("provider") != provider.name
+ ):
+ return False
+ try:
+ state = await request(row, "/state")
+ except HTTPException:
+ return False
+ return not state.get("closed")
+
+
+async def create(
+ slug,
+ module_id,
+ user_id,
+ chat_id,
+ *,
+ resource_id=None,
+ install_id=None,
+ checkpoint_id=None,
+ _prewarm=False,
+ _provider=None,
+ _application=None,
+):
+ """Start (or adopt) a resource worker for a conversation or an application.
+
+ Every database step runs on a session of its own; the owner lock is only
+ held inside one step, so no connection waits on a runner or the sandbox.
+ """
+ if resource_id:
+ row = await run_in_session(_existing, resource_id, user_id, chat_id, slug, module_id)
+ await request(row, "/state")
+ return store.public(row)
+ selected, module, config = await run_in_session(
+ _admission_inputs, slug, module_id, user_id, chat_id, install_id, checkpoint_id,
+ _prewarm, _application,
+ )
+ from core.capabilities.plugins.resources.lifecycle import reconcile
+ from core.sandbox.factory import get_sandbox_provider
+
+ await reconcile(user_id)
+ provider = _provider or get_sandbox_provider()
+ deadline = time.monotonic() + (95 if _application else 90)
+ attempt, fresh = 0, False
+ while True:
+ outcome, value = await run_in_session(
+ _admit, selected, module_id, user_id, chat_id, _prewarm, checkpoint_id,
+ _application, attempt, fresh,
+ )
+ attempt += 1
+ if outcome == "exists":
+ return None
+ if outcome == "reserved":
+ break
+ if outcome == "adopted":
+ if _application is not None:
+ return store.public(value)
+ if await _worker_alive(value, provider, chat_id):
+ return await run_in_session(_authorized_public, value.resource_id, user_id)
+ await close(value)
+ # A dead warm worker has performed no user actions; reserve a fresh one.
+ fresh = True
+ continue
+ if time.monotonic() >= deadline:
+ raise HTTPException(
+ 503, "resource_start_pending" if _application else "resource_prewarm_timeout"
+ )
+ await asyncio.sleep(0.05)
+ row, session_id = value
+ from core.sandbox.processes.interactive import launch
+
+ try:
+ descriptor = await launch(provider, selected, module, user_id, session_id, config)
+ except BaseException as exc:
+ await run_in_session(_mark, row.resource_id, "closed")
+ if not isinstance(exc, Exception):
+ raise
+ code = (
+ str(exc)
+ if isinstance(exc, ValueError)
+ and str(exc)
+ in {
+ "chromium_sandbox_unavailable",
+ "chromium_runtime_missing",
+ "chromium_launch_failed",
+ "runtime_process_failed",
+ "runtime_start_timeout",
+ "runtime_start_failed",
+ }
+ else type(exc).__name__
+ )
+ raise HTTPException(503, "resource_runtime_unavailable: " + code) from exc
+ try:
+ await run_in_session(_activate, row.resource_id, user_id, descriptor, _prewarm, _application)
+ if _application:
+ from core.capabilities.plugins.resources.deadlines import arm
+
+ await arm(row.resource_id)
+ except BaseException:
+ if _application:
+ await run_in_session(_mark, row.resource_id, "closing", descriptor)
+ try:
+ await close(row)
+ except Exception:
+ pass
+ else:
+ await provider.write_stdin(
+ descriptor["process_id"],
+ sandbox_session_id=session_id,
+ user_id=user_id,
+ chars="\x03",
+ yield_time_ms=1000,
+ )
+ await run_in_session(_mark, row.resource_id, "closed")
+ raise
+ return await run_in_session(_observe_started, row.resource_id)
+
+
+async def request(row, path, body=None, *, transport=None):
+ if transport is not None:
+ return await transport.request(path, body)
+ async with RuntimeTransport(store.descriptor(row)) as connection:
+ return await connection.request(path, body)
+
+
+async def command(row, payload, *, actor, connection_id="", transport=None):
+ body = {**payload, "actor": actor, "connection_id": connection_id}
+ if body.get("action") in {"storage_state", "checkpoint"}:
+ raise HTTPException(403, "checkpoint_requires_explicit_save")
+ return await request(row, "/command", body, transport=transport)
+
+
+def _save_checkpoint(db, user_id, install_id, name, state):
+ record = store.PluginResourceCheckpoint(
+ checkpoint_id=uuid.uuid4().hex,
+ user_id=user_id,
+ install_id=install_id,
+ name=name[:80],
+ state_enc=encrypt_secret(json.dumps(state)),
+ )
+ db.add(record)
+ db.commit()
+ return {"checkpoint_id": record.checkpoint_id, "name": record.name}
+
+
+async def checkpoint(row, name, connection_id, *, transport=None):
+ if row.application_key is not None:
+ raise HTTPException(403, "application_checkpoint_not_granted")
+ state = await request(
+ row,
+ "/command",
+ {
+ "id": uuid.uuid4().hex,
+ "actor": "user",
+ "connection_id": connection_id,
+ "action": "checkpoint",
+ "params": {},
+ },
+ transport=transport,
+ )
+ return await run_in_session(_save_checkpoint, row.user_id, row.install_id, name, state["state"])
+
+
+def _mark_closed(db, resource_id):
+ db.query(store.PluginResource).filter_by(resource_id=resource_id).update(
+ {"status": "closed"}, synchronize_session=False
+ )
+ db.query(store.PluginResourceTicket).filter_by(resource_id=resource_id).delete()
+ db.query(store.PluginResourceAssetTicket).filter_by(resource_id=resource_id).delete()
+ db.commit()
+
+
+async def close(row):
+ """Stop a resource worker, then retire its row and tickets.
+
+ ``row`` only supplies the worker's coordinates; the database work runs on a
+ session of its own after the network calls, so callers never hold a
+ connection across the shutdown.
+ """
+ if row.application_key is not None:
+ from core.harness.registry.resource_owners import resource_owner
+
+ return await resource_owner().close(row)
+ from core.capabilities.plugins.resources.owners import runtime_session
+
+ try:
+ target = store.descriptor(row)
+ except HTTPException:
+ target = {}
+ if target.get("url") and target.get("headers"):
+ try:
+ await request(row, "/shutdown", {})
+ except HTTPException:
+ from core.sandbox.factory import get_sandbox_provider
+
+ provider = get_sandbox_provider()
+ if target.get("provider") == provider.name and target.get("process_id"):
+ try:
+ await provider.write_stdin(
+ target["process_id"],
+ sandbox_session_id=runtime_session(row),
+ user_id=row.user_id,
+ chars="\x03",
+ yield_time_ms=1000,
+ )
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.plugins.resources.service:close', fallback_error)
+ await run_in_session(_mark_closed, row.resource_id)
diff --git a/src/backend/core/plugins/resources/store.py b/src/backend/core/capabilities/plugins/resources/store.py
similarity index 97%
rename from src/backend/core/plugins/resources/store.py
rename to src/backend/core/capabilities/plugins/resources/store.py
index e9c97eaf..a247ba6b 100644
--- a/src/backend/core/plugins/resources/store.py
+++ b/src/backend/core/capabilities/plugins/resources/store.py
@@ -12,7 +12,7 @@
PluginResourceTicket,
)
-from .crypto import decrypt_secret, encrypt_secret
+from core.capabilities.plugins.resources.crypto import decrypt_secret, encrypt_secret
def public(row):
diff --git a/src/backend/core/plugins/resources/stream.py b/src/backend/core/capabilities/plugins/resources/stream.py
similarity index 72%
rename from src/backend/core/plugins/resources/stream.py
rename to src/backend/core/capabilities/plugins/resources/stream.py
index 7ca2af07..8313867f 100644
--- a/src/backend/core/plugins/resources/stream.py
+++ b/src/backend/core/capabilities/plugins/resources/stream.py
@@ -6,9 +6,10 @@
import uuid
import httpx
from fastapi import HTTPException, WebSocket
-from core.db.engine import SessionLocal
-from . import service, store
-from .transport import RuntimeTransport
+from core.db.threads import run_in_session
+from core.capabilities.plugins.resources import service, store
+from core.capabilities.plugins.resources.authentication import validate as validate_authentication
+from core.capabilities.plugins.resources.transport import RuntimeTransport
MAX_PACKET = 8 * 1024 * 1024
@@ -34,7 +35,22 @@ async def packets(stream):
if len(buffer) > MAX_PACKET + 65540:
raise ValueError("frame_buffer_overflow")
+def _consume_ticket(db, ticket, resource_id, origin):
+ return store.consume(db, ticket, resource_id, origin, with_authentication=True)
+
+
+def _authorized_target(db, resource_id, user_id):
+ row = service.authorized_row(db, resource_id, user_id)
+ return row, store.descriptor(row)
+
+
async def connect(websocket: WebSocket, resource_id: str):
+ """Relay one viewer to a resource worker.
+
+ Each database step (ticket, authorization, checkpoint) runs to completion on
+ a session of its own before the next network wait, so a long-lived viewer
+ never pins a pooled connection.
+ """
origin = websocket.headers.get("origin", "")
await websocket.accept()
connection_id = uuid.uuid4().hex
@@ -42,16 +58,18 @@ async def connect(websocket: WebSocket, resource_id: str):
row = None
try:
initial = await asyncio.wait_for(websocket.receive_json(), 5)
- with SessionLocal() as db:
- attachment = store.consume(db, str(initial.get("ticket", "")), resource_id, origin, with_authentication=True)
- if attachment is None:
- await websocket.close(4401)
- return
- user_id, authentication = attachment
- from .authentication import validate as validate_authentication
- await validate_authentication(authentication, user_id, db)
- row = service.authorized(db, resource_id, user_id)
- target = store.descriptor(row)
+ attachment = await run_in_session(_consume_ticket, str(initial.get("ticket", "")), resource_id, origin)
+ if attachment is None:
+ await websocket.close(4401)
+ return
+ user_id, authentication = attachment
+
+ async def authorize():
+ await validate_authentication(authentication, user_id)
+ return await run_in_session(service.authorized_row, resource_id, user_id)
+
+ await validate_authentication(authentication, user_id)
+ row, target = await run_in_session(_authorized_target, resource_id, user_id)
send_lock = asyncio.Lock()
async def send(data, binary=False):
async with send_lock:
@@ -71,19 +89,15 @@ async def receive():
payload = await websocket.receive_json()
if len(json.dumps(payload)) > 24 * 1024 * 1024:
raise ValueError("command_too_large")
- with SessionLocal() as db:
- await validate_authentication(authentication, user_id, db)
- current = service.authorized(db, resource_id, user_id)
+ current = await authorize()
try:
if payload.get("action") == "checkpoint":
- with SessionLocal() as db:
- data = await service.checkpoint(db, current, str(payload.get("params", {}).get("name", "账号")), connection_id, transport=transport)
+ data = await service.checkpoint(current, str(payload.get("params", {}).get("name", "账号")), connection_id, transport=transport)
else:
data = await service.command(current, payload, actor="user", connection_id=connection_id, transport=transport)
await send({"type": "result", "id": payload.get("id"), "ok": True, "data": data})
if payload.get("action") == "close":
- with SessionLocal() as db:
- await service.close(db, service.authorized(db, resource_id, user_id))
+ await service.close(await run_in_session(service.authorized_row, resource_id, user_id))
await websocket.close(4410)
return
except HTTPException as exc:
@@ -91,9 +105,7 @@ async def receive():
async def validate():
while True:
await asyncio.sleep(15)
- with SessionLocal() as db:
- await validate_authentication(authentication, user_id, db)
- service.authorized(db, resource_id, user_id)
+ await authorize()
async with RuntimeTransport(target) as transport:
tasks = [asyncio.create_task(fn()) for fn in (frames, receive, validate)]
try:
diff --git a/src/backend/core/plugins/resources/transport.py b/src/backend/core/capabilities/plugins/resources/transport.py
similarity index 100%
rename from src/backend/core/plugins/resources/transport.py
rename to src/backend/core/capabilities/plugins/resources/transport.py
diff --git a/src/backend/core/plugins/runtime/__init__.py b/src/backend/core/capabilities/plugins/runtime/__init__.py
similarity index 76%
rename from src/backend/core/plugins/runtime/__init__.py
rename to src/backend/core/capabilities/plugins/runtime/__init__.py
index f6c2ad25..f0d5b172 100644
--- a/src/backend/core/plugins/runtime/__init__.py
+++ b/src/backend/core/capabilities/plugins/runtime/__init__.py
@@ -30,27 +30,10 @@
transport MCP 的插件不延迟(激活期进程内起子进程的生命周期管理复杂度不值得)。
"""
-from core.plugins.runtime.activation import register_load_plugin
-from core.plugins.runtime.activation_history import (
- load_activated_plugin_slugs,
- record_plugin_activation,
-)
-from core.plugins.runtime.desktop_resolution import (
- prepare_desktop_plugin_skill_defaults,
- resolve_desktop_progressive_plugins,
-)
-from core.plugins.runtime.models import (
- DeferredPlugin,
- ProgressiveResolution,
- StickyPluginCapabilities,
- progressive_plugin_loading_enabled,
-)
-from core.plugins.runtime.resolution import (
- build_plugin_directory_section,
- resolve_bound_progressive_plugins,
- resolve_progressive_plugins,
-)
-from core.plugins.runtime.sticky_selection import resolve_sticky_plugin_capabilities
+from core.capabilities.plugins.runtime.activation_history import load_activated_plugin_slugs, record_plugin_activation
+from core.capabilities.plugins.runtime.models import DeferredPlugin, ProgressiveResolution, StickyPluginCapabilities, progressive_plugin_loading_enabled
+from core.capabilities.plugins.runtime.resolution import build_plugin_directory_section, resolve_bound_progressive_plugins, resolve_progressive_plugins
+from core.capabilities.plugins.runtime.sticky_selection import resolve_sticky_plugin_capabilities
__all__ = [
"progressive_plugin_loading_enabled",
@@ -63,7 +46,4 @@
"resolve_progressive_plugins",
"resolve_bound_progressive_plugins",
"build_plugin_directory_section",
- "prepare_desktop_plugin_skill_defaults",
- "resolve_desktop_progressive_plugins",
- "register_load_plugin",
]
diff --git a/src/backend/core/plugins/runtime/activation_history.py b/src/backend/core/capabilities/plugins/runtime/activation_history.py
similarity index 97%
rename from src/backend/core/plugins/runtime/activation_history.py
rename to src/backend/core/capabilities/plugins/runtime/activation_history.py
index 8215a99a..b00a8977 100644
--- a/src/backend/core/plugins/runtime/activation_history.py
+++ b/src/backend/core/capabilities/plugins/runtime/activation_history.py
@@ -10,7 +10,7 @@
_ACTIVATED_KEY = "activated_plugins"
-from core.plugins.runtime import sticky_selection as plugin_sticky_selection
+from core.capabilities.plugins.runtime import sticky_selection as plugin_sticky_selection
def load_activated_plugin_slugs(chat_id: Optional[str]) -> List[str]:
diff --git a/src/backend/core/plugins/runtime/models.py b/src/backend/core/capabilities/plugins/runtime/models.py
similarity index 100%
rename from src/backend/core/plugins/runtime/models.py
rename to src/backend/core/capabilities/plugins/runtime/models.py
diff --git a/src/backend/core/plugins/runtime/resolution.py b/src/backend/core/capabilities/plugins/runtime/resolution.py
similarity index 91%
rename from src/backend/core/plugins/runtime/resolution.py
rename to src/backend/core/capabilities/plugins/runtime/resolution.py
index d82300b7..9529a2f1 100644
--- a/src/backend/core/plugins/runtime/resolution.py
+++ b/src/backend/core/capabilities/plugins/runtime/resolution.py
@@ -5,11 +5,12 @@
import logging
from typing import Any, Dict, List, Optional, Sequence
-logger = logging.getLogger(__name__)
-from core.plugins.runtime import activation_history as plugin_activation_history
-from core.plugins.runtime import models as plugin_runtime_models
+from core.capabilities.plugins.runtime import activation_history as plugin_activation_history
+from core.capabilities.plugins.runtime import models as plugin_runtime_models
+
+logger = logging.getLogger(__name__)
def _http_transport(cfg: Any) -> bool:
@@ -21,15 +22,16 @@ def _http_transport(cfg: Any) -> bool:
def _server_configs(user_id: Optional[str] = None) -> Dict[str, Any]:
"""Transport lookup: global servers, plus the user's private ones when given."""
try:
- from core.services.mcp_service import McpServerConfigService
+ from core.capabilities.mcp.service import McpServerConfigService
svc = McpServerConfigService.get_instance()
cfgs = dict(svc.get_all_servers(enabled_only=True))
if user_id:
try:
cfgs.update(svc.get_owned_servers(str(user_id), enabled_only=False))
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.plugins.runtime.resolution:_server_configs', fallback_error)
return cfgs
except Exception as exc: # noqa: BLE001
logger.warning("[plugin-loader] server config lookup failed: %s", exc)
@@ -38,7 +40,7 @@ def _server_configs(user_id: Optional[str] = None) -> Dict[str, Any]:
def _skill_metadata() -> Dict[str, Any]:
try:
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
return get_skill_loader().load_all_metadata() or {}
except Exception: # noqa: BLE001
@@ -58,7 +60,7 @@ def _bound_mcp_ids(
return bound
-def _finalize_resolution(
+def finalize_resolution(
res: plugin_runtime_models.ProgressiveResolution,
eligible: Sequence[plugin_runtime_models.DeferredPlugin],
) -> plugin_runtime_models.ProgressiveResolution:
@@ -170,7 +172,7 @@ def resolve_progressive_plugins(
if newly_pinned:
plugin_activation_history.record_plugin_activation(chat_id, newly_pinned)
- return _finalize_resolution(res, eligible)
+ return finalize_resolution(res, eligible)
def resolve_bound_progressive_plugins(
@@ -215,8 +217,9 @@ def resolve_bound_progressive_plugins(
if skill_filter is not None:
try:
skills = list(skill_filter(skills))
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.plugins.runtime.resolution:resolve_bound_progressive_plugins', fallback_error)
mcps = [m for m in (cids.get("mcp") or []) if isinstance(m, str) and m.strip()]
if not skills and not mcps:
continue
@@ -234,7 +237,7 @@ def resolve_bound_progressive_plugins(
eligible.append(plugin)
res.deferred.append(plugin)
- return _finalize_resolution(res, eligible)
+ return finalize_resolution(res, eligible)
def build_plugin_directory_section(
diff --git a/src/backend/core/plugins/runtime/sticky_selection.py b/src/backend/core/capabilities/plugins/runtime/sticky_selection.py
similarity index 85%
rename from src/backend/core/plugins/runtime/sticky_selection.py
rename to src/backend/core/capabilities/plugins/runtime/sticky_selection.py
index 89dbc334..722807db 100644
--- a/src/backend/core/plugins/runtime/sticky_selection.py
+++ b/src/backend/core/capabilities/plugins/runtime/sticky_selection.py
@@ -1,32 +1,31 @@
"""Progressive plugin runtime: sticky selection."""
from __future__ import annotations
+from core.harness.registry import execution_account as skill_account
import logging
from typing import Any, Dict, List, Optional, Set
-logger = logging.getLogger(__name__)
-from core.plugins.runtime import activation_history as plugin_activation_history
-from core.plugins.runtime import models as plugin_runtime_models
+from core.capabilities.plugins.runtime import activation_history as plugin_activation_history
+from core.capabilities.plugins.runtime import models as plugin_runtime_models
+
+logger = logging.getLogger(__name__)
def _cloud_sticky_selection(tokens, *, user_id, allow_aliases=False, unavailable_out=None):
"""Keep saved cloud selections tied to their exact account and installation."""
- from core.capabilities import registry, skills
- from core.capabilities.errors import NameConflict, PermissionDenied
- from core.capabilities.paths import BUILTIN_PROFILE, LOCAL_PROFILE, capabilities_enabled
+ from core.harness.registry.device_catalog import account_plugin_installations
+ from core.harness.capability_errors import NameConflict, PermissionDenied
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE
- if not capabilities_enabled():
+ if not has_run_capabilities():
return []
- profile = skills.current_account_profile()
- authorized = skills.account_authorized_for(user_id)
- rows = (
- registry.list_installations(kind="plugin", profile_id=profile)
- if profile and authorized
- else []
- )
+ profile = skill_account.current_account_profile()
+ authorized = skill_account.account_authorized_for(user_id)
+ rows = account_plugin_installations(user_id) if profile and authorized else []
result = []
for token in tokens:
parts = str(token).split(":", 2)
@@ -93,8 +92,8 @@ def resolve_sticky_plugin_capabilities(
unavailable_out=result.unavailable_ids,
)
if cloud:
- from core.capabilities.errors import CapabilityError
- from core.capabilities.plugins import cloud_binding_ids
+ from core.harness.capability_errors import CapabilityError
+ from core.harness.registry.run_capabilities import cloud_binding_ids
for row in cloud:
try:
@@ -108,7 +107,7 @@ def resolve_sticky_plugin_capabilities(
result.mcp_ids.extend(mcp_ids)
try:
- from core.config.catalog_resolver import resolve_explicit_runtime_capabilities
+ from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities
from core.db.engine import SessionLocal
from core.db.models import InstalledPlugin
from sqlalchemy import or_
@@ -150,7 +149,7 @@ def resolve_sticky_plugin_capabilities(
requested_skills: List[str] = []
requested_mcps: List[str] = []
- from core.plugins.packaging.sources import _component_keys
+ from core.capabilities.plugins.packaging.sources import component_keys as _component_keys
for row in selected:
component_ids = row.component_ids or {}
diff --git a/src/backend/core/capabilities/ref.py b/src/backend/core/capabilities/reference.py
similarity index 98%
rename from src/backend/core/capabilities/ref.py
rename to src/backend/core/capabilities/reference.py
index 14fe95b0..1b4cfb0c 100644
--- a/src/backend/core/capabilities/ref.py
+++ b/src/backend/core/capabilities/reference.py
@@ -7,7 +7,7 @@
from typing import Any, Dict
from urllib.parse import urlsplit, urlunsplit
-from .paths import KINDS, LOCAL_PROFILE, safe_segment
+from core.capabilities.identity_rules import KINDS, LOCAL_PROFILE, safe_segment
LOCAL_ISSUER = "local"
LOCAL_NAMESPACE = "device"
diff --git a/src/backend/core/capabilities/runtime/__init__.py b/src/backend/core/capabilities/runtime/__init__.py
deleted file mode 100644
index 5773a355..00000000
--- a/src/backend/core/capabilities/runtime/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-"""Public capability runtime API; state and operations have one-way dependencies."""
-
-from core.capabilities.runtime.connectors import bind_mcp as bind_mcp
-from core.capabilities.runtime.connectors import pin_agent_definition as pin_agent_definition
-from core.capabilities.runtime.connectors import references as references
-from core.capabilities.runtime.dependencies import preflight as preflight
-from core.capabilities.runtime.dependencies import validate_activation as validate_activation
-from core.capabilities.runtime.preparation import prepare as prepare
-from core.capabilities.runtime.state import PreparedRun as PreparedRun
-from core.capabilities.runtime.state import child_scope as child_scope
-from core.capabilities.runtime.state import frozen_loader as frozen_loader
-from core.capabilities.runtime.state import get as get
-from core.capabilities.runtime.state import rebuild as rebuild
-from core.capabilities.runtime.state import record_tool_scope as record_tool_scope
-from core.capabilities.runtime.state import require_root_tool_scope as require_root_tool_scope
-from core.capabilities.runtime.state import save as save
-from core.capabilities.runtime.state import validate as validate
-from core.capabilities.runtime.state import view_for_execution as view_for_execution
-
-__all__ = [
- "child_scope",
- "PreparedRun",
- "get",
- "validate",
- "rebuild",
- "save",
- "frozen_loader",
- "view_for_execution",
- "record_tool_scope",
- "require_root_tool_scope",
- "prepare",
- "bind_mcp",
- "pin_agent_definition",
- "references",
- "preflight",
- "validate_activation",
-]
diff --git a/src/backend/core/capabilities/skills.py b/src/backend/core/capabilities/skills.py
deleted file mode 100644
index a7e75ee7..00000000
--- a/src/backend/core/capabilities/skills.py
+++ /dev/null
@@ -1,656 +0,0 @@
-"""Skill kind on the desktop store: candidates, resolution, runtime views, local publication.
-
-Two views are maintained, both link-only:
-
-- the **device view** (``get_sandbox_skills_dir()``, i.e. ``{workspace}/skills``):
- shipped built-ins plus device-local skills that are not private to one user.
- Process-level consumers (CLI shims on PATH) resolve through it.
-- the **user view** (``get_user_skills_dir(uid)``): everything the current user
- may use — device view entries plus the current cloud account's ready
- installations — after name resolution. Session workspaces link to it.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-import os
-import threading
-from collections import OrderedDict
-from concurrent.futures import ThreadPoolExecutor
-from contextvars import copy_context
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple
-
-from extensions.paths import SKILLS_DIR
-
-from . import registry, store
-from . import view as view_mod
-from .paths import (
- BUILTIN_PROFILE,
- KIND_SKILL,
- LOCAL_PROFILE,
- capability_root,
- revision_for_hash,
-)
-from .ref import builtin_ref, local_ref, profile_id
-from .resolver import (
- SOURCE_BUILTIN,
- SOURCE_CLOUD,
- SOURCE_LOCAL,
- SOURCE_PLUGIN,
- Candidate,
- Resolution,
- resolve,
-)
-
-logger = logging.getLogger(__name__)
-
-_SKIP_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"}
-_INVENTORY_NAME = ".inventory.json"
-
-_hash_lock = threading.Lock()
-_hash_cache: Dict[str, Tuple[Tuple, str]] = {}
-# Fresh reads still hash every byte. Reuse only the protocol JSON/base64 work
-# when the complete sequence of file names and byte digests is identical.
-_content_hash_cache: OrderedDict[tuple, str] = OrderedDict()
-
-_resolution_lock = threading.Lock()
-_last_resolution: Dict[str, Resolution] = {}
-_resolution_cache: Dict[str, Tuple[Tuple, Resolution]] = {}
-_view_generation = 0
-
-
-def view_generation() -> int:
- """Bumped whenever store contents change; view consumers refresh on a new value."""
- return _view_generation
-
-
-def bump_view_generation() -> int:
- global _view_generation
- with _resolution_lock:
- _view_generation += 1
- return _view_generation
-
-
-def builtin_dir() -> Path:
- return SKILLS_DIR / "default"
-
-
-def allowed_link_roots() -> List[Path]:
- roots = [builtin_dir()]
- root = capability_root()
- if root is not None:
- roots.insert(0, root)
- return roots
-
-
-def device_view_dir() -> Path:
- from core.agent_skills.config import get_sandbox_skills_dir
-
- return get_sandbox_skills_dir()
-
-
-def user_view_dir(user_id: Optional[str]) -> Optional[Path]:
- from core.agent_skills.config import get_user_skills_dir
-
- return get_user_skills_dir(user_id)
-
-
-# ── content hashing (same scheme the cloud uses for its manifest) ───────
-
-
-def _dir_signature(path: Path) -> Tuple:
- from .archive import iter_file_stats
-
- # A maximum mtime misses edits to any file older than the newest file.
- return tuple(
- (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino)
- for rel, _, metadata in iter_file_stats(path)
- if rel != _INVENTORY_NAME
- )
-
-
-def dir_digest(path: Path, compute: Callable[[], str], *, fresh: bool = False) -> str:
- """A package directory's content digest, cached on its file signature.
-
- Identity is *established* with ``fresh=True``: every byte is read, so the
- digest a binding freezes is always a true content hash. Later passes only
- *verify* that digest, and a run re-verifies its whole closure on every
- message — re-reading a hundred unchanged packages each time is pure cost.
- So verification recomputes exactly when the signature moved (a file was
- added, removed, or written), and reuses the frozen digest otherwise.
- """
- key = str(path.resolve())
- sig = _dir_signature(path)
- with _hash_lock:
- hit = _hash_cache.get(key)
- if not fresh and hit and hit[0] == sig:
- return hit[1]
- digest = compute()
- with _hash_lock:
- _hash_cache[key] = (sig, digest)
- return digest
-
-
-def skill_dir_hash(path: Path, *, fresh: bool = False) -> str:
- """Hash every package file; reject links and excess sizes rather than omit bytes."""
- from .archive import iter_file_stats
-
- key = str(path.resolve())
- entries = [item for item in iter_file_stats(path) if item[0] != _INVENTORY_NAME]
- sig = tuple(
- (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino)
- for rel, _, metadata in entries
- )
- with _hash_lock:
- hit = _hash_cache.get(key)
- if not fresh and hit and hit[0] == sig:
- return hit[1]
- # Reuse this traversal's metadata; fresh verification still reads every byte.
- digest = _hash_skill_package(path, entries=entries)
- with _hash_lock:
- _hash_cache[key] = (sig, digest)
- return digest
-
-
-def _hash_skill_package(path: Path, *, entries=None) -> str:
- from core.agent_skills.binary_files import encode_upload
- from core.services.desktop_capability_protocol import skill_content_hash
-
- from . import archive
- from .errors import IntegrityFailed
- from .junction import _native
-
- raw_files = {}
- total = 0
- for rel, file, metadata in archive.iter_file_stats(path) if entries is None else entries:
- if rel == _INVENTORY_NAME:
- continue
- size = metadata.st_size
- total += size
- if (
- size > archive.MAX_MEMBER_BYTES
- or total > archive.MAX_TOTAL_BYTES
- or len(raw_files) >= archive.MAX_MEMBERS
- ):
- raise IntegrityFailed("skill package exceeds content verification limits")
- # Scanning already supports extended Windows paths; byte reads must too.
- raw = _native(file).read_bytes()
- if len(raw) != size:
- raise IntegrityFailed("skill package changed during verification")
- raw_files[rel] = raw
- content_key = tuple((rel, hashlib.sha256(raw).digest()) for rel, raw in raw_files.items())
- with _hash_lock:
- cached = _content_hash_cache.get(content_key)
- if cached is not None:
- _content_hash_cache.move_to_end(content_key)
- return cached
- files = {rel: encode_upload(rel, raw) for rel, raw in raw_files.items()}
- digest = skill_content_hash(files.pop("SKILL.md", ""), files)
- with _hash_lock:
- _content_hash_cache[content_key] = digest
- _content_hash_cache.move_to_end(content_key)
- while len(_content_hash_cache) > 256:
- _content_hash_cache.popitem(last=False)
- return digest
-
-
-# ── account profile ───────────────────────────────────────────────────
-
-
-def current_account_profile() -> Optional[str]:
- """Profile id of the cloud account this device is currently bridged to."""
- from core.services.desktop_capability_protocol import token_subject
- from core.services.desktop_cloud_bridge import get_state
-
- st = get_state()
- if not st:
- return None
- subject = token_subject(str(st.get("token") or ""))
- if not subject:
- return None
- return profile_id(str(st["cloud_base"]), subject)
-
-
-_shadow_user_ids: Dict[str, str] = {}
-
-
-def current_local_user_id() -> Optional[str]:
- """Map the signed stable cloud subject to this device's shadow-user ID."""
- from core.services.desktop_cloud_bridge import get_identity_state
-
- identity = get_identity_state()
- # 本机影子用户按壳送来的命名空间标识建档(cloud:::),
- # 查找必须用同一标识,而不是凭据里的原始云端 id。
- center = str((identity or {}).get("shell_user_center_id") or "")
- if not center:
- return None
- # user_center_id → user_id is a primary-key mapping fixed at creation; only
- # hits are remembered so a shadow created later is still found.
- hit = _shadow_user_ids.get(center)
- if hit is not None:
- return hit
- from core.db.models import UserShadow
-
- with registry._session() as db:
- row = db.query(UserShadow).filter(UserShadow.user_center_id == center).first()
- if row is None:
- return None
- _shadow_user_ids[center] = str(row.user_id)
- return _shadow_user_ids[center]
-
-
-def account_authorized_for(user_id: Optional[str]) -> bool:
- from core.services.desktop_cloud_bridge import get_state
-
- st = get_state()
- if not st or not str(st.get("token") or "").startswith("dcap2."):
- return False
- current = current_local_user_id()
- return bool(user_id and current and str(user_id) == current)
-
-
-# ── candidates ────────────────────────────────────────────────────────
-
-
-def builtin_candidates() -> List[Candidate]:
- """安装包自带的技能。
-
- 混合模式下它们不进能力中心、也不会被启用(目录层在 ``get_runtime_catalog`` 里就清空
- 了),但**候选必须留着**:云端技能会声明对自带技能的依赖,自带的那份还是云端副本在
- 本平台跑不了时的兜底,候选一空这两条链路就断了。
- """
- root = builtin_dir()
- out: List[Candidate] = []
- if not root.is_dir():
- return out
- for d in sorted(p for p in root.iterdir() if p.is_dir()):
- if not (d / "SKILL.md").is_file():
- continue
- out.append(
- Candidate(
- install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, d.name),
- runtime_name=d.name,
- kind=KIND_SKILL,
- profile=BUILTIN_PROFILE,
- source=SOURCE_BUILTIN,
- path=d,
- content_hash=None, # computed lazily only on a name collision
- ref=builtin_ref(KIND_SKILL, d.name),
- usable=True,
- account_level=False,
- display_name=d.name,
- )
- )
- return out
-
-
-def _installation_candidate(inst: registry.Installation, *, account_level: bool) -> Candidate:
- path = None
- if inst.ready:
- from .errors import CapabilityError
-
- try:
- comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "")
- except (CapabilityError, OSError):
- comp = None
- path = comp.path if comp else None
- usable = inst.enabled and path is not None
- return Candidate(
- install_id=inst.install_id,
- runtime_name=str(inst.payload.get("runtime_name") or inst.key),
- kind=KIND_SKILL,
- profile=inst.profile_id,
- source=inst.source,
- path=path,
- content_hash=inst.payload.get("resolved_content_hash") or inst.content_hash,
- revision=inst.resolved_revision,
- ref=inst.ref,
- usable=usable,
- account_level=account_level and inst.enabled,
- state=inst.state if path is not None or not inst.ready else "files_missing",
- display_name=inst.display_name,
- description=inst.description,
- version=inst.version,
- )
-
-
-def _installation_candidates(rows):
- rows = list(rows)
- if os.name == "nt" and len(rows) >= 8:
- with ThreadPoolExecutor(max_workers=4, thread_name_prefix="cap-candidates") as pool:
- futures = [
- pool.submit(copy_context().run, _installation_candidate, row, account_level=True)
- for row in rows
- ]
- return [future.result() for future in futures]
- return [_installation_candidate(row, account_level=True) for row in rows]
-
-
-def local_candidates(user_id: Optional[str], *, shared_only: bool = False) -> List[Candidate]:
- rows = []
- for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE):
- owner = inst.payload.get("owner_user_id")
- if owner and (shared_only or owner != user_id):
- continue
- rows.append(inst)
- return _installation_candidates(rows)
-
-
-def account_candidates(profile: Optional[str] = None) -> List[Candidate]:
- profile = profile or current_account_profile()
- if not profile:
- return []
- return _installation_candidates(
- registry.list_installations(kind=KIND_SKILL, profile_id=profile)
- )
-
-
-def candidates(user_id: Optional[str]) -> List[Candidate]:
- return (
- builtin_candidates()
- + local_candidates(user_id)
- + (account_candidates() if account_authorized_for(user_id) else [])
- )
-
-
-def _fill_hashes(cands: List[Candidate]) -> List[Candidate]:
- """Hash only the members of multi-candidate names (identical-content detection)."""
- by_name: Dict[str, List[Candidate]] = {}
- for c in cands:
- by_name.setdefault(c.runtime_name, []).append(c)
- out: List[Candidate] = []
- for group in by_name.values():
- if len(group) < 2:
- out.extend(group)
- continue
- for c in group:
- if c.content_hash is None and c.path is not None:
- c = Candidate(**{**c.__dict__, "content_hash": skill_dir_hash(c.path)})
- out.append(c)
- return out
-
-
-def _builtin_signature() -> Tuple:
- """Shipped built-ins live on disk, outside every write signal — list them."""
- root = builtin_dir()
- try:
- return (str(root), tuple(sorted(p.name for p in root.iterdir() if p.is_dir())))
- except OSError:
- return (str(root), ())
-
-
-def resolve_for_user(user_id: Optional[str], *, requested: Optional[Set[str]] = None) -> Resolution:
- """Which candidate wins each runtime name, recomputed only when it can differ.
-
- The answer is a pure function of the installations, the store contents, the
- user's name preferences and the bound cloud account — every one of which
- moves ``registry.generation()``, ``view_generation()`` or the account
- profile. Agent assembly asks for this several times per message and the
- eligibility fixpoint walks every candidate's package each time, so without
- this the same answer was recomputed from disk two or three times a turn.
- """
- from .discovery import scan
- from .readiness import eligible_skill_candidates
- scan(user_id)
- key = user_id or ""
- signal = (
- registry.generation(),
- view_generation(),
- current_account_profile(),
- _builtin_signature(),
- frozenset(requested) if requested is not None else None,
- )
- with _resolution_lock:
- hit = _resolution_cache.get(key)
- if hit is not None and hit[0] == signal:
- _last_resolution[key] = hit[1]
- return hit[1]
-
- preferences = registry.preferences(KIND_SKILL, user_id=user_id)
- eligible = eligible_skill_candidates(
- _fill_hashes(candidates(user_id)), user_id, preferences=preferences, requested=requested
- )
- res = resolve(KIND_SKILL, eligible, preferences=preferences, requested=requested)
- with _resolution_lock:
- _last_resolution[key] = res
- # A cloud deployment shares this process across users; keep the memo
- # bounded rather than letting it grow with the user table.
- if len(_resolution_cache) >= 64 and key not in _resolution_cache:
- _resolution_cache.pop(next(iter(_resolution_cache)), None)
- _resolution_cache[key] = (signal, res)
- return res
-
-
-def filter_available_names(names, *, user_id=None):
- """Implicit defaults omit unusable names; explicit bindings still fail in preflight."""
- chosen = resolve_for_user(user_id).chosen
- return [name for name in dict.fromkeys(names) if name in chosen]
-
-
-def last_resolution(user_id: Optional[str]) -> Optional[Resolution]:
- with _resolution_lock:
- return _last_resolution.get(user_id or "")
-
-
-# ── views ─────────────────────────────────────────────────────────────
-
-
-def _targets(res: Resolution) -> Dict[str, Path]:
- return {name: c.path for name, c in res.chosen.items() if c.path is not None}
-
-
-def rebuild_device_view() -> view_mod.ViewReport:
- res = resolve(
- KIND_SKILL,
- _fill_hashes(builtin_candidates() + local_candidates(None, shared_only=True)),
- preferences=registry.preferences(KIND_SKILL),
- )
- report = view_mod.build_view(
- device_view_dir(), _targets(res), allowed_roots=allowed_link_roots()
- )
- _log_report("device", report)
- return report
-
-
-def rebuild_user_view(user_id: str) -> Optional[view_mod.ViewReport]:
- vdir = user_view_dir(user_id)
- if vdir is None:
- return None
- res = resolve_for_user(user_id)
- report = view_mod.build_view(vdir, _targets(res), allowed_roots=allowed_link_roots())
- _log_report(f"user:{user_id}", report)
- return report
-
-
-def rebuild_views(user_id: Optional[str]) -> Dict[str, view_mod.ViewReport]:
- reports = {"device": rebuild_device_view()}
- if user_id:
- user_report = rebuild_user_view(user_id)
- if user_report is not None:
- reports["user"] = user_report
- return reports
-
-
-def _log_report(label: str, report: view_mod.ViewReport) -> None:
- if report.changed:
- logger.info(
- "[caps-view] %s linked=%d relinked=%d removed=%d",
- label,
- len(report.linked),
- len(report.relinked),
- len(report.removed),
- )
- for name, why in report.blocked.items():
- logger.warning("[caps-view] %s name '%s' unavailable: %s", label, name, why)
-
-
-# ── local publication (device-created / DB-materialized skills) ───────
-
-
-def publish_local_skill(
- skill_id: str,
- *,
- files: Dict[str, bytes | str],
- content_hash: str,
- owner_user_id: Optional[str] = None,
- source: str = SOURCE_LOCAL,
- source_plugin: Optional[str] = None,
- display_name: str = "",
- description: str = "",
- version: str = "",
- enabled: bool = True,
- from_db: bool = False,
-) -> store.StoredComponent:
- """Write one revision of a device-local skill and make it the ready revision."""
- revision = revision_for_hash(content_hash)
- before_generation = registry.generation()
- published = False
- ref = local_ref(KIND_SKILL, skill_id)
- inst = registry.upsert(
- profile_id=LOCAL_PROFILE,
- ref=ref,
- display_name=display_name or skill_id,
- description=description,
- version=version,
- content_hash=content_hash,
- source=source,
- source_plugin=source_plugin,
- payload={"owner_user_id": owner_user_id, "from_db": from_db},
- enabled=enabled,
- )
- comp = store.get(KIND_SKILL, LOCAL_PROFILE, ref.key, revision)
- if comp is None:
- tx = registry.begin_transaction(inst.install_id, inst.generation + 1)
- try:
- comp = store.write_from_files(KIND_SKILL, LOCAL_PROFILE, ref.key, revision, files)
- published = True
- registry.advance_transaction(tx, "published", inventory=store.inventory(comp))
- except Exception as exc:
- registry.advance_transaction(tx, "failed", error=str(exc))
- registry.set_state(inst.install_id, "failed", last_error=str(exc))
- raise
- registry.set_state(inst.install_id, "ready", resolved_revision=revision)
- registry.advance_transaction(tx, "committed")
- elif inst.resolved_revision != revision or inst.state != "ready":
- registry.set_state(inst.install_id, "ready", resolved_revision=revision)
- # A DB-backed skill is materialized again after restart/cache refresh. An
- # identical publication must not invalidate every user's resolved closure.
- if published or registry.generation() != before_generation:
- bump_view_generation()
- return comp
-
-
-def remove_local_skill(skill_id: str) -> bool:
- from core.capabilities.runtime import references
-
- if references("skill", LOCAL_PROFILE, skill_id):
- # Source deletion revokes use; history still needs its exact bytes.
- return registry.mark_removed(registry.install_id("skill", LOCAL_PROFILE, skill_id))
- iid = registry.install_id(KIND_SKILL, LOCAL_PROFILE, skill_id)
- removed_files = store.remove_key(KIND_SKILL, LOCAL_PROFILE, skill_id) > 0
- removed_row = registry.delete(iid)
- registry.clear_preference(KIND_SKILL, skill_id)
- bump_view_generation()
- return removed_files or removed_row
-
-
-def prune_local(live_skill_owners: Dict[str, Optional[str]]) -> int:
- """Drop local-profile entries whose database row no longer exists."""
- removed = 0
- for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE):
- if inst.source in (SOURCE_LOCAL, SOURCE_PLUGIN) and inst.payload.get("from_db"):
- if inst.key not in live_skill_owners:
- registry.mark_removed(inst.install_id)
- removed += 1
- # Unindexed and superseded bytes may still be retained by history snapshots.
- return removed
-
-
-# ── loader integration: resolver-driven merge of skill sources ─────────
-
-
-def _info_candidate(info: Any) -> Candidate:
- """Map a backend ``SkillFileInfo`` onto a resolver candidate."""
- src = info.source_name
- origin = dict(getattr(info, "origin", None) or {})
- if src == "built-in":
- return Candidate(
- install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, info.skill_id),
- runtime_name=info.skill_id,
- kind=KIND_SKILL,
- profile=BUILTIN_PROFILE,
- source=SOURCE_BUILTIN,
- path=Path(info.file_path).parent,
- account_level=False,
- )
- if src in ("cloud", "device-store"):
- return Candidate(
- install_id=str(origin.get("install_id") or ""),
- runtime_name=info.skill_id,
- kind=KIND_SKILL,
- profile=str(origin.get("profile") or ""),
- source=SOURCE_CLOUD if src == "cloud" else SOURCE_LOCAL,
- path=Path(info.file_path).parent,
- content_hash=origin.get("content_hash"),
- revision=origin.get("revision"),
- account_level=True,
- )
- if src == "admin":
- # Device catalog row (created here, installed from a plugin or the market).
- return Candidate(
- install_id=registry.install_id(KIND_SKILL, LOCAL_PROFILE, info.skill_id),
- runtime_name=info.skill_id,
- kind=KIND_SKILL,
- profile=LOCAL_PROFILE,
- source=SOURCE_LOCAL,
- path=Path(""),
- account_level=True,
- )
- return Candidate(
- install_id=registry.install_id(KIND_SKILL, src, info.skill_id),
- runtime_name=info.skill_id,
- kind=KIND_SKILL,
- profile=src,
- source=SOURCE_LOCAL,
- path=Path(info.file_path).parent,
- account_level=False,
- )
-
-
-def merge_skill_infos(groups: Dict[str, List[Any]], *, backend_hash) -> Dict[str, Any]:
- """Composite-backend merge hook: one winner per id, conflicts load nothing.
-
- ``backend_hash(info) -> str`` computes a content hash for a backend entry; it
- is only called for ids with more than one candidate.
- """
- cands: List[Candidate] = []
- lookup: Dict[str, Any] = {}
- for skill_id, infos in groups.items():
- for info in infos:
- cand = _info_candidate(info)
- if len(infos) > 1 and cand.content_hash is None:
- cand = Candidate(**{**cand.__dict__, "content_hash": backend_hash(info)})
- cands.append(cand)
- lookup[cand.install_id] = info
- res = resolve(
- KIND_SKILL,
- cands,
- preferences=registry.preferences(KIND_SKILL, user_id=current_local_user_id()),
- )
- with _resolution_lock:
- _last_resolution["__loader__"] = res
- for name in res.conflicts:
- logger.warning("[caps] skill name conflict, nothing loaded for '%s'", name)
- return {name: lookup[c.install_id] for name, c in res.chosen.items()}
-
-
-def loader_resolution() -> Optional[Resolution]:
- with _resolution_lock:
- return _last_resolution.get("__loader__")
-
-
-def conflicted_names() -> Set[str]:
- res = loader_resolution()
- return set(res.conflicts) if res else set()
diff --git a/src/backend/core/agent_skills/__init__.py b/src/backend/core/capabilities/skills/__init__.py
similarity index 97%
rename from src/backend/core/agent_skills/__init__.py
rename to src/backend/core/capabilities/skills/__init__.py
index 4288a133..24812cf2 100644
--- a/src/backend/core/agent_skills/__init__.py
+++ b/src/backend/core/capabilities/skills/__init__.py
@@ -1,2 +1 @@
"""Agent skills registry and loaders."""
-
diff --git a/src/backend/core/capabilities/skills/backends/__init__.py b/src/backend/core/capabilities/skills/backends/__init__.py
new file mode 100644
index 00000000..9a8eb52c
--- /dev/null
+++ b/src/backend/core/capabilities/skills/backends/__init__.py
@@ -0,0 +1,14 @@
+"""Backend abstraction for multi-source skill loading."""
+
+from core.capabilities.skills.backends.protocol import SkillBackendProtocol, SkillFileInfo
+from core.capabilities.skills.backends.filesystem import FilesystemBackend
+from core.capabilities.skills.backends.composite import CompositeBackend
+from core.capabilities.skills.backends.database import DatabaseBackend
+
+__all__ = [
+ "SkillBackendProtocol",
+ "SkillFileInfo",
+ "FilesystemBackend",
+ "CompositeBackend",
+ "DatabaseBackend",
+]
diff --git a/src/backend/core/agent_skills/backends/composite.py b/src/backend/core/capabilities/skills/backends/composite.py
similarity index 97%
rename from src/backend/core/agent_skills/backends/composite.py
rename to src/backend/core/capabilities/skills/backends/composite.py
index 1524b48d..361c7daa 100644
--- a/src/backend/core/agent_skills/backends/composite.py
+++ b/src/backend/core/capabilities/skills/backends/composite.py
@@ -5,7 +5,7 @@
import threading
from typing import Any, Callable, Dict, List, Optional, Tuple
-from .protocol import SkillBackendProtocol, SkillFileInfo
+from core.capabilities.skills.backends.protocol import SkillBackendProtocol, SkillFileInfo
MergeHook = Callable[
[Dict[str, List[SkillFileInfo]], Callable[[SkillFileInfo], str]], Dict[str, SkillFileInfo]
@@ -115,7 +115,7 @@ def _select_visible(self, user_id: str | None) -> Dict[str, SkillFileInfo]:
def content_hash(self, info: SkillFileInfo) -> str:
"""Content hash of one backend entry (used only for colliding ids)."""
- from core.services.desktop_capability_protocol import skill_content_hash
+ from core.capabilities.manifest_protocol import skill_content_hash
if info.origin and info.origin.get("content_hash"):
return str(info.origin["content_hash"])
@@ -218,7 +218,7 @@ def get_extra_files(self, skill_id: str) -> dict:
# A single store backend can contain same-named installations. Read the
# selected concrete revision, not that backend's first runtime-name hit.
if info.source_name in ("cloud", "device-store") and not info.is_database:
- from .filesystem import FilesystemBackend
+ from core.capabilities.skills.backends.filesystem import FilesystemBackend
path = info.file_path.parent
return FilesystemBackend(path.parent, info.source_name).get_extra_files(path.name)
# Find the owning backend and delegate
diff --git a/src/backend/core/agent_skills/backends/database.py b/src/backend/core/capabilities/skills/backends/database.py
similarity index 99%
rename from src/backend/core/agent_skills/backends/database.py
rename to src/backend/core/capabilities/skills/backends/database.py
index 32014074..cb61205f 100644
--- a/src/backend/core/agent_skills/backends/database.py
+++ b/src/backend/core/capabilities/skills/backends/database.py
@@ -8,7 +8,7 @@
from sqlalchemy import func, or_
-from .protocol import SkillFileInfo
+from core.capabilities.skills.backends.protocol import SkillFileInfo
# Sentinel path used when skill content comes from DB (never actually read)
_DB_SENTINEL = Path("/db/admin_skills")
diff --git a/src/backend/core/agent_skills/backends/filesystem.py b/src/backend/core/capabilities/skills/backends/filesystem.py
similarity index 98%
rename from src/backend/core/agent_skills/backends/filesystem.py
rename to src/backend/core/capabilities/skills/backends/filesystem.py
index 702ce649..79838555 100644
--- a/src/backend/core/agent_skills/backends/filesystem.py
+++ b/src/backend/core/capabilities/skills/backends/filesystem.py
@@ -6,7 +6,7 @@
from pathlib import Path
from typing import List
-from .protocol import SkillBackendProtocol, SkillFileInfo
+from core.capabilities.skills.backends.protocol import SkillFileInfo
class FilesystemBackend:
diff --git a/src/backend/core/agent_skills/backends/protocol.py b/src/backend/core/capabilities/skills/backends/protocol.py
similarity index 100%
rename from src/backend/core/agent_skills/backends/protocol.py
rename to src/backend/core/capabilities/skills/backends/protocol.py
diff --git a/src/backend/core/agent_skills/binary_files.py b/src/backend/core/capabilities/skills/binary_files.py
similarity index 100%
rename from src/backend/core/agent_skills/binary_files.py
rename to src/backend/core/capabilities/skills/binary_files.py
diff --git a/src/backend/core/capabilities/skills/cache_refresh.py b/src/backend/core/capabilities/skills/cache_refresh.py
new file mode 100644
index 00000000..63dcdfba
--- /dev/null
+++ b/src/backend/core/capabilities/skills/cache_refresh.py
@@ -0,0 +1,50 @@
+"""Shared cache invalidation after admin skill mutations.
+
+Why centralize: skill changes ripple through 4 caches (loader metadata,
+catalog JSON, per-user capability cache, system prompt). Keeping them in
+one place prevents drift between admin_skills and admin_skill_drafts routes.
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def refresh_skill_caches() -> None:
+ from core.capabilities.skills.loader_factory import get_skill_loader
+ from core.capabilities.catalog.loader import invalidate_catalog_cache
+ from core.harness.registry.cache_invalidation import cache_invalidators
+ cache_invalidators().run("before_skill_refresh")
+ get_skill_loader(reset=True)
+ invalidate_catalog_cache()
+
+ try:
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
+
+ invalidate_capability_cache()
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error)
+
+ try:
+ from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache
+
+ invalidate_runtime_catalog_cache()
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error)
+
+ try:
+ cache_invalidators().run("after_skill_refresh")
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error)
+
+ try:
+ from core.capabilities.catalog.loader import load_catalog
+
+ load_catalog(include_runtime_details=False)
+ except Exception as exc:
+ logger.warning("Failed to eagerly refresh catalog cache after skill mutation: %s", exc)
diff --git a/src/backend/core/agent_skills/config.py b/src/backend/core/capabilities/skills/config.py
similarity index 89%
rename from src/backend/core/agent_skills/config.py
rename to src/backend/core/capabilities/skills/config.py
index 72216838..7faaf62b 100644
--- a/src/backend/core/agent_skills/config.py
+++ b/src/backend/core/capabilities/skills/config.py
@@ -76,9 +76,9 @@ def get_default_skill_sources() -> List[SkillSourceConfig]:
)
)
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- caps_mode = capabilities_enabled()
+ caps_mode = has_run_capabilities()
# 3. User skills (flat ``/SKILL.md`` folders). On a desktop with a
# capability store this directory *is* ``/skills`` and is owned by
@@ -127,12 +127,12 @@ def get_default_skill_sources() -> List[SkillSourceConfig]:
# sources is decided by the resolver (core.capabilities.skills) and shown
# to the user, never silently replaced.
if caps_mode:
- from core.capabilities.paths import kind_root
+ from core.harness.registry.local_publication import local_publication
sources.append(
SkillSourceConfig(
name="cloud",
- root_dir=kind_root("skill"),
+ root_dir=local_publication().skill_root(),
priority=150,
enabled=True,
)
@@ -202,18 +202,18 @@ def sync_builtin_skills_to_sandbox_dir() -> int:
Reconcile complete file trees rather than overlaying bundled files on DB
overrides. Returns the number of configured built-in source entries.
"""
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
+ if has_run_capabilities():
# Desktop store: the shared dir is a link view — built-ins are linked to
# the shipped bundle, nothing is copied.
- from core.capabilities.skills import rebuild_device_view
+ from core.harness.registry.local_publication import local_publication
- report = rebuild_device_view()
+ report = local_publication().rebuild_device_view()
logger.info("[skills-sync] device view rebuilt → %s", report.view_dir)
return len(report.linked) + len(report.relinked)
- from .publication import prepare_skill_view
+ from core.capabilities.skills.publication import prepare_skill_view
# Startup is a publisher too: resolve DB overrides and retired bundles
# under the same lock as every other writer.
@@ -241,9 +241,9 @@ def get_user_skills_root() -> Path:
shared = get_sandbox_skills_dir()
root = shared.parent / f"{shared.name}_u"
root.mkdir(parents=True, exist_ok=True)
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
+ if has_run_capabilities():
# Desktop store: user views link straight into the store, no shared hop.
return root
link = root / SHARED_LINK_NAME
@@ -284,7 +284,7 @@ def get_user_skills_dir(user_id: Optional[str]) -> Optional[Path]:
def skill_files_dir(skill_id: str, owner_user_id: Optional[str] = None) -> Path:
"""Where a skill's files are materialized: the owner's dir for private skills, the shared dir otherwise."""
- from .publication import safe_skill_id
+ from core.capabilities.skills.publication import safe_skill_id
safe_skill_id(skill_id)
owner_dir = get_user_skills_dir(owner_user_id)
@@ -305,12 +305,12 @@ def sync_user_skill_view(user_id: Optional[str]) -> Optional[Path]:
view = get_user_skills_dir(user_id)
if view is None:
return None
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities.skills import rebuild_user_view
+ if has_run_capabilities():
+ from core.harness.registry.local_publication import local_publication
- rebuild_user_view(user_id)
+ local_publication().rebuild_user_view(user_id)
return view
view.mkdir(parents=True, exist_ok=True)
shared = get_sandbox_skills_dir()
@@ -345,16 +345,16 @@ def purge_skill_sandbox_files(skill_id: str) -> bool:
return False
if (_builtin_skills_dir() / skill_id).is_dir():
return False
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities.skills import rebuild_views, remove_local_skill
+ if has_run_capabilities():
+ from core.harness.registry.local_publication import local_publication
- removed = remove_local_skill(skill_id)
- rebuild_views(None)
+ removed = local_publication().remove_local_skill(skill_id)
+ local_publication().rebuild_views(None)
return removed
- from .publication import publication_lock, remove_entry
+ from core.capabilities.skills.publication import publication_lock, remove_entry
with publication_lock():
removed = False
@@ -374,7 +374,7 @@ def _builtin_skills_dir() -> Path:
def prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int:
- from .publication import publication_lock
+ from core.capabilities.skills.publication import publication_lock
with publication_lock():
return _prune_orphan_sandbox_skill_dirs(live_skill_owners)
@@ -391,12 +391,12 @@ def _prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int:
"""
import shutil
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities.skills import prune_local
+ if has_run_capabilities():
+ from core.harness.registry.local_publication import local_publication
- return prune_local(live_skill_owners)
+ return local_publication().prune_local(live_skill_owners)
builtin = {d.name for d in _builtin_skills_dir().iterdir() if d.is_dir()}
removed = 0
@@ -429,6 +429,6 @@ def _prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int:
def model_facing_skill_dir(skill_id: str, real_dir: str) -> str:
"""Return the directory visible to this deployment's execution tools."""
- from core.llm.tools._paths import path_rules
+ from core.sandbox.workspace.resolution import path_rules
return path_rules().skill_directory(skill_id, real_dir)
diff --git a/src/backend/core/capabilities/skills/deps/__init__.py b/src/backend/core/capabilities/skills/deps/__init__.py
new file mode 100644
index 00000000..a4fd4881
--- /dev/null
+++ b/src/backend/core/capabilities/skills/deps/__init__.py
@@ -0,0 +1 @@
+"""Detect, probe and request the runtime dependencies that skills declare."""
diff --git a/src/backend/core/services/skill_deps_aggregator.py b/src/backend/core/capabilities/skills/deps/aggregator.py
similarity index 92%
rename from src/backend/core/services/skill_deps_aggregator.py
rename to src/backend/core/capabilities/skills/deps/aggregator.py
index d38dd5e7..c5a2496c 100644
--- a/src/backend/core/services/skill_deps_aggregator.py
+++ b/src/backend/core/capabilities/skills/deps/aggregator.py
@@ -3,12 +3,12 @@
Reads the `dependencies` JSON column from every enabled `admin_skills` row,
merges and deduplicates per ecosystem (pip / npm / apt), then writes:
- - `/docker/requirements-skills.txt` (pip)
- - `/docker/package-skills.json` (npm)
- - `/docker/apt-skills.txt` (apt)
+ - `/docker/sandbox/requirements-skills.txt` (pip)
+ - `/docker/sandbox/package-skills.json` (npm)
+ - `/docker/sandbox/apt-skills.txt` (apt)
-These files are picked up by `docker/Dockerfile.script-runner` and
-`docker/Dockerfile.opensandbox` during the rebuild flow.
+These files are picked up by `docker/sandbox/Dockerfile.script-runner` and
+`docker/sandbox/Dockerfile.opensandbox` during the rebuild flow.
"""
from __future__ import annotations
@@ -27,7 +27,7 @@
_PIP_HEADER = (
- "# Auto-generated by core.services.skill_deps_aggregator. DO NOT EDIT MANUALLY.\n"
+ "# Auto-generated by core.capabilities.skills.deps.aggregator. DO NOT EDIT MANUALLY.\n"
"# Managed via admin UI → 沙盒依赖 → 应用依赖。\n"
)
_NPM_DEFAULT: Dict[str, Any] = {
@@ -37,7 +37,7 @@
"dependencies": {},
}
_APT_HEADER = (
- "# Auto-generated by core.services.skill_deps_aggregator. DO NOT EDIT MANUALLY.\n"
+ "# Auto-generated by core.capabilities.skills.deps.aggregator. DO NOT EDIT MANUALLY.\n"
)
@@ -221,9 +221,9 @@ def write_manifests(
raise FileNotFoundError(f"repo_root not found: {repo_root}")
targets = {
- "pip": root / "docker" / "requirements-skills.txt",
- "npm": root / "docker" / "package-skills.json",
- "apt": root / "docker" / "apt-skills.txt",
+ "pip": root / "docker" / "sandbox" / "requirements-skills.txt",
+ "npm": root / "docker" / "sandbox" / "package-skills.json",
+ "apt": root / "docker" / "sandbox" / "apt-skills.txt",
}
changed: List[str] = []
for kind, path in targets.items():
diff --git a/src/backend/core/agent_skills/deps_detector.py b/src/backend/core/capabilities/skills/deps/detector.py
similarity index 100%
rename from src/backend/core/agent_skills/deps_detector.py
rename to src/backend/core/capabilities/skills/deps/detector.py
diff --git a/src/backend/core/services/skill_deps_prober.py b/src/backend/core/capabilities/skills/deps/prober.py
similarity index 98%
rename from src/backend/core/services/skill_deps_prober.py
rename to src/backend/core/capabilities/skills/deps/prober.py
index 1adceb33..58ab84d1 100644
--- a/src/backend/core/services/skill_deps_prober.py
+++ b/src/backend/core/capabilities/skills/deps/prober.py
@@ -7,7 +7,7 @@
- pip: ``importlib.metadata.distribution(name)`` queries the distribution in the sandbox Python interpreter (with PEP 503 normalization);
- apt: ``dpkg -s `` exit code;
-- npm: ``npm ls -g `` exit code (dependencies are baked in via ``npm install -g``, see docker/Dockerfile.opensandbox).
+- npm: ``npm ls -g `` exit code (dependencies are baked in via ``npm install -g``, see docker/sandbox/Dockerfile.opensandbox).
Returns the subset that is **still missing**. When the probe is uncertain (sandbox unreachable /
script error / output unparseable) it returns ``None``, and the caller handles it as
@@ -45,7 +45,7 @@ def _build_script(names: dict[str, list[str]]) -> str:
"""Generate the probe script to run inside the sandbox. Package names are embedded as JSON literals (the source is the detector, not free user input).
Key point: the sandbox's ``execute(language=python)`` runs the system ``/usr/bin/python3``, while the skill's pip
- dependencies are installed under ``$PY_BIN`` (see ``ENV PY_BIN=...`` in docker/Dockerfile.opensandbox +
+ dependencies are installed under ``$PY_BIN`` (see ``ENV PY_BIN=...`` in docker/sandbox/Dockerfile.opensandbox +
``$PY_BIN -m pip install -r requirements-skills.txt``). So pip must query distributions **against the $PY_BIN
interpreter**, not the currently running interpreter, otherwise everything is judged missing.
"""
diff --git a/src/backend/core/services/skill_deps_request_service.py b/src/backend/core/capabilities/skills/deps/requests.py
similarity index 81%
rename from src/backend/core/services/skill_deps_request_service.py
rename to src/backend/core/capabilities/skills/deps/requests.py
index 1e5e9942..b58400c5 100644
--- a/src/backend/core/services/skill_deps_request_service.py
+++ b/src/backend/core/capabilities/skills/deps/requests.py
@@ -30,7 +30,8 @@
from sqlalchemy.orm import Session
from core.db.models import AdminSkill, SkillDependencyRequest
-from core.services.skill_deps_prober import names_of, probe_still_missing
+from core.db.threads import run_in_session
+from core.capabilities.skills.deps.prober import names_of, probe_still_missing
logger = logging.getLogger(__name__)
@@ -74,8 +75,27 @@ def _upsert_pending(db: Session, skill_id: str, *, user_id: Optional[str], missi
req.reason = None
+def _skill_exists(db: Session, skill_id: str) -> bool:
+ return db.query(AdminSkill.skill_id).filter(AdminSkill.skill_id == skill_id).first() is not None
+
+
+def _record_verdict(
+ db: Session, skill_id: str, owner_user_id: Optional[str], missing: dict
+) -> bool:
+ skill = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
+ if skill is None:
+ return False
+ if not missing:
+ _mark_ready(db, skill)
+ db.commit()
+ return False
+ skill.dep_status = "installing"
+ _upsert_pending(db, skill_id, user_id=owner_user_id, missing=missing)
+ db.commit()
+ return True
+
+
async def gate_skill_deps(
- db: Session,
skill_id: str,
*,
owner_user_id: Optional[str],
@@ -87,14 +107,11 @@ async def gate_skill_deps(
- All already present in the sandbox -> ``ready``, return False (do not bother the admin);
- Packages truly missing (or probe inconclusive, conservatively treated as missing) -> ``installing`` + pending, return True.
"""
- skill = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
- if skill is None:
- return False
-
declared = _extract_declared(dependencies)
if not owner_user_id or not declared:
- _mark_ready(db, skill)
- db.commit()
+ await run_in_session(_record_verdict, skill_id, owner_user_id, {})
+ return False
+ if not await run_in_session(_skill_exists, skill_id):
return False
probed = await probe_still_missing(declared)
@@ -105,17 +122,12 @@ async def gate_skill_deps(
else:
missing = {k: v for k, v in probed.items() if v}
- if not missing:
- _mark_ready(db, skill)
- db.commit()
+ pending = await run_in_session(_record_verdict, skill_id, owner_user_id, missing)
+ if pending:
+ logger.info("[skill-deps] %s flagged installing; missing=%s", skill_id, missing)
+ else:
logger.info("[skill-deps] %s deps already satisfied in sandbox; ready", skill_id)
- return False
-
- skill.dep_status = "installing"
- _upsert_pending(db, skill_id, user_id=owner_user_id, missing=missing)
- db.commit()
- logger.info("[skill-deps] %s flagged installing; missing=%s", skill_id, missing)
- return True
+ return pending
def list_pending(db: Session) -> list[dict[str, Any]]:
@@ -187,40 +199,31 @@ def get_reject_reason(db: Session, skill_id: str) -> Optional[str]:
return (req.reason if req else None) or None
-async def verify_pending_after_rebuild(db: Session, run_id: str) -> dict[str, int]:
- """After a successful sandbox rebuild: **re-probe** the dependencies of each pending request's skill, one by one.
-
- Truly installed (no more missing packages in the sandbox) -> set satisfied + restore
- the skill to ready; still missing -> keep pending and refresh the missing list (this
- rebuild did not install it). Inconclusive probes also stay pending (conservative).
- Returns ``{"satisfied": n, "still_pending": m}``. Idempotent: returns all zeros when
- there are no pending requests.
- """
+def _pending_declarations(db: Session) -> list[tuple[str, dict]]:
rows = (
db.query(SkillDependencyRequest)
.filter(SkillDependencyRequest.status == "pending")
.all()
)
- if not rows:
- return {"satisfied": 0, "still_pending": 0}
+ out = []
+ for r in rows:
+ skill = db.query(AdminSkill).filter(AdminSkill.skill_id == r.skill_id).first()
+ out.append((r.request_id, _extract_declared(skill.dependencies if skill else None)))
+ return out
+
+def _record_probes(db: Session, run_id: str, probes: dict[str, Optional[dict]]) -> dict[str, int]:
now = utc_now()
satisfied = 0
still_pending = 0
- for r in rows:
- skill = db.query(AdminSkill).filter(AdminSkill.skill_id == r.skill_id).first()
- declared = _extract_declared(skill.dependencies if skill else None)
- if not declared:
- # Skill no longer declares dependencies (edited/emptied) -> treat as satisfied.
- probed: Optional[dict] = {k: [] for k in _DEP_KINDS}
- else:
- probed = await probe_still_missing(declared)
-
- if probed is None:
+ for request_id, probed in probes.items():
+ r = db.get(SkillDependencyRequest, request_id)
+ if probed is None or r is None or r.status != "pending":
still_pending += 1
continue
missing = {k: v for k, v in probed.items() if v}
if not missing:
+ skill = db.query(AdminSkill).filter(AdminSkill.skill_id == r.skill_id).first()
r.status = "satisfied"
r.satisfied_at = now
r.satisfied_by_run_id = run_id
@@ -231,8 +234,31 @@ async def verify_pending_after_rebuild(db: Session, run_id: str) -> dict[str, in
r.missing = missing
still_pending += 1
db.commit()
+ return {"satisfied": satisfied, "still_pending": still_pending}
+
+
+async def verify_pending_after_rebuild(run_id: str) -> dict[str, int]:
+ """After a successful sandbox rebuild: **re-probe** the dependencies of each pending request's skill, one by one.
+
+ Truly installed (no more missing packages in the sandbox) -> set satisfied + restore
+ the skill to ready; still missing -> keep pending and refresh the missing list (this
+ rebuild did not install it). Inconclusive probes also stay pending (conservative).
+ Returns ``{"satisfied": n, "still_pending": m}``. Idempotent: returns all zeros when
+ there are no pending requests.
+ """
+ pending = await run_in_session(_pending_declarations)
+ if not pending:
+ return {"satisfied": 0, "still_pending": 0}
+
+ probes: dict[str, Optional[dict]] = {}
+ for request_id, declared in pending:
+ # Skill no longer declares dependencies (edited/emptied) -> treat as satisfied.
+ probes[request_id] = (
+ await probe_still_missing(declared) if declared else {k: [] for k in _DEP_KINDS}
+ )
+ result = await run_in_session(_record_probes, run_id, probes)
logger.info(
"[skill-deps] rebuild %s verified: satisfied=%d still_pending=%d",
- run_id, satisfied, still_pending,
+ run_id, result["satisfied"], result["still_pending"],
)
- return {"satisfied": satisfied, "still_pending": still_pending}
+ return result
diff --git a/src/backend/core/services/skill_icon_service.py b/src/backend/core/capabilities/skills/icons.py
similarity index 100%
rename from src/backend/core/services/skill_icon_service.py
rename to src/backend/core/capabilities/skills/icons.py
diff --git a/src/backend/core/capabilities/skills/installation.py b/src/backend/core/capabilities/skills/installation.py
new file mode 100644
index 00000000..ccd9c2ef
--- /dev/null
+++ b/src/backend/core/capabilities/skills/installation.py
@@ -0,0 +1,117 @@
+"""Shared skill installation identity, content and credentials transformations."""
+from __future__ import annotations
+import hashlib
+import json
+import re
+from typing import Any, Dict, List, Optional
+from core.capabilities.skills.registry import split_frontmatter
+from core.infra.exceptions import BadRequestError
+
+SECRETS_FILENAME = "secrets.json"
+
+def strip_frontmatter(raw: str) -> str:
+ try:
+ _, body = split_frontmatter(raw)
+ except Exception: # noqa: BLE001
+ body = raw
+ return (body or "").strip()
+
+
+def owner_suffix(owner_user_id: str) -> str:
+ """Stable 6-char fingerprint derived from user_id, used to deduplicate private-install skill ids."""
+ return hashlib.sha1(owner_user_id.encode("utf-8")).hexdigest()[:6]
+
+
+def compute_install_id(entry_name: str, owner_user_id: Optional[str]) -> str:
+ """The installed AdminSkill.skill_id: admin = original name (global); user = original name + fingerprint (private)."""
+ if owner_user_id is None:
+ return entry_name
+ return f"{entry_name}-{owner_suffix(owner_user_id)}"
+
+
+def base_entry_name(skill_id: str, owner_user_id: Optional[str]) -> str:
+ """Inverse of ``compute_install_id``: recover the underlying market entry_name from an install id.
+
+ Private install ids look like ``{entry_name}-{fingerprint}``; stripping that
+ user's 6-char fingerprint suffix yields the entry_name. Global ids (owner is
+ None) already are the entry_name and are returned as-is. Used to merge the
+ "privately installed version" and the "admin global version" onto the same
+ underlying skill for deduplication — preventing the same skill from
+ appearing twice in the skill library / at runtime.
+ """
+ if owner_user_id is None:
+ return skill_id
+ suffix = f"-{owner_suffix(owner_user_id)}"
+ if skill_id.endswith(suffix):
+ return skill_id[: -len(suffix)]
+ return skill_id
+
+
+def rewrite_frontmatter_name(content: str, new_name: str) -> str:
+ """Rewrite the ``name:`` in the SKILL.md frontmatter to ``new_name`` (touching only that line).
+
+ Replaces only the first ``name:`` line within the frontmatter block; all
+ other content (including multi-line descriptions) is preserved verbatim.
+ Returned unchanged when there is no frontmatter.
+ """
+ if not content.startswith("---"):
+ return content
+ end = content.find("\n---", 3)
+ if end == -1:
+ return content
+ head = content[: end + 1]
+ tail = content[end + 1 :]
+ head = re.sub(r"(?m)^name:[ \t]*.*$", f"name: {new_name}", head, count=1)
+ return head + tail
+
+
+def inject_secrets(
+ skill_content: str,
+ extra_files: Dict[str, str],
+ required_secrets: List[Dict[str, Any]],
+ secrets: Dict[str, str],
+) -> str:
+ """Write the user-provided credentials into secrets.json and append a "credentials configuration" section at the end of SKILL.md.
+
+ Returns the SKILL.md content with the section appended. Missing required
+ fields raise a 400.
+ """
+ cleaned: Dict[str, str] = {}
+ for field in required_secrets:
+ key = str(field.get("key") or "").strip()
+ if not key:
+ continue
+ val = (secrets.get(key) or "").strip()
+ if not val:
+ if field.get("required"):
+ label = field.get("label") or key
+ raise BadRequestError(message=f"请填写必填凭据:{label}")
+ continue
+ cleaned[key] = val
+ if not cleaned:
+ return skill_content
+
+ extra_files[SECRETS_FILENAME] = json.dumps(cleaned, ensure_ascii=False, indent=2)
+ lines = [
+ "",
+ "---",
+ "",
+ "## 🔑 凭据配置(由安装者提供)",
+ "",
+ f"运行本技能所需的凭据已写入技能目录下的 `{SECRETS_FILENAME}`,包含以下字段:",
+ "",
+ ]
+ for key in cleaned:
+ lines.append(f"- `{key}`")
+ lines += [
+ "",
+ f"执行技能脚本前,请先从 `{SECRETS_FILENAME}` 读取这些值,并以**同名环境变量**导出后再运行,例如:",
+ "",
+ "```bash",
+ f'export {next(iter(cleaned))}="$(python3 -c "import json;print(json.load(open(\'{SECRETS_FILENAME}\'))[\'{next(iter(cleaned))}\'])")"',
+ "```",
+ "",
+ "> 凭据仅存于本技能目录,请勿在对话中明文回显。",
+ "",
+ ]
+ return skill_content.rstrip() + "\n" + "\n".join(lines)
diff --git a/src/backend/core/agent_skills/loader.py b/src/backend/core/capabilities/skills/loader.py
similarity index 77%
rename from src/backend/core/agent_skills/loader.py
rename to src/backend/core/capabilities/skills/loader.py
index 2d7ead9e..500c17ad 100644
--- a/src/backend/core/agent_skills/loader.py
+++ b/src/backend/core/capabilities/skills/loader.py
@@ -1,88 +1,17 @@
-"""Multi-source skill loader with backend abstraction."""
-
+"""Load skills from several sources, resolving conflicts by source priority."""
from __future__ import annotations
-
-import logging
+from core.harness.registry.local_publication import local_publication
import time
-from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
+from core.capabilities.skills.backends import CompositeBackend, DatabaseBackend, FilesystemBackend
+from core.capabilities.skills.binary_files import decode_binary, is_binary_value
+from core.capabilities.skills.config import get_enabled_skill_sources, skill_files_dir
+from core.capabilities.skills.registry import AgentSkillMetadata, AgentSkillSpec, _load_skill_from_file, _load_skill_from_str, _load_skill_metadata_from_file, load_skill_metadata_from_str, parse_scripts_json
+import logging
+from core.capabilities.skills import script_discovery
logger = logging.getLogger(__name__)
-from .backends import CompositeBackend, DatabaseBackend, FilesystemBackend, SkillFileInfo
-from .binary_files import decode_binary, is_binary_value
-from .config import get_enabled_skill_sources, skill_files_dir
-from .registry import (
- AgentSkillMetadata,
- AgentSkillSpec,
- _load_skill_from_file,
- _load_skill_from_str,
- _load_skill_metadata_from_file,
- _load_skill_metadata_from_str,
- parse_scripts_json,
-)
-
-
-_SCRIPT_EXTENSIONS = {".py", ".js", ".sh", ".r", ".R"}
-
-
-def _bound_skill_id(skill_info: SkillFileInfo, declared_id: str) -> str:
- # Capability names come from the resolved view, independently of the file's
- # original frontmatter. Ordinary source files keep their existing ID rules.
- if skill_info.source_name == "prepared" or (skill_info.origin or {}).get("install_id"):
- return skill_info.skill_id
- return declared_id
-
-
-def _auto_detect_scripts(
- extra_files: Dict[str, str],
- skill_info: "SkillFileInfo",
-) -> List[Dict]:
- """Auto-detect executable scripts when _scripts.json is absent.
-
- Scans extra_files (DB skills) or the skill directory (filesystem skills)
- for files with known script extensions and returns a whitelist entry for
- each one, so users don't need to manually create _scripts.json.
- """
- scripts: List[Dict] = []
-
- # Collect candidate filenames
- candidates: List[str] = []
-
- # From extra_files (DB skills)
- for fname in extra_files:
- if fname.startswith("_"):
- continue
- ext = Path(fname).suffix.lower()
- if ext in _SCRIPT_EXTENSIONS:
- candidates.append(fname)
-
- # From filesystem (filesystem skills without extra_files)
- if not candidates and not skill_info.is_database and skill_info.content is None:
- skill_dir = skill_info.file_path.parent
- for p in skill_dir.iterdir():
- if (
- p.is_file()
- and not p.name.startswith("_")
- and p.suffix.lower() in _SCRIPT_EXTENSIONS
- ):
- candidates.append(p.name)
-
- # Build whitelist entries
- ext_to_lang = {".py": "python", ".js": "javascript", ".sh": "bash", ".r": "r", ".R": "r"}
- for name in sorted(candidates):
- ext = Path(name).suffix.lower()
- scripts.append(
- {
- "name": name,
- "language": ext_to_lang.get(ext, "python"),
- "timeout": 60,
- }
- )
-
- return scripts
-
-
class MultiSourceSkillLoader:
"""Loads skills from multiple sources with priority-based conflict resolution.
@@ -117,7 +46,7 @@ def _create_default_backend() -> CompositeBackend:
Returns:
CompositeBackend configured with enabled skill sources.
"""
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
sources = get_enabled_skill_sources()
backends = []
@@ -125,9 +54,8 @@ def _create_default_backend() -> CompositeBackend:
if src.name == "admin":
backends.append(DatabaseBackend(priority=src.priority))
elif src.name == "cloud":
- from .backends.capability_store import CapabilityStoreBackend
- backends.append(CapabilityStoreBackend(priority=src.priority))
+ backends.append(local_publication().skill_backend(priority=src.priority))
else:
backends.append(
FilesystemBackend(
@@ -136,18 +64,16 @@ def _create_default_backend() -> CompositeBackend:
priority=src.priority,
)
)
- if not capabilities_enabled():
+ if not has_run_capabilities():
return CompositeBackend(backends)
- from .backends.capability_store import CapabilityStoreBackend
- backends.append(CapabilityStoreBackend(local=True))
+ backends.append(local_publication().skill_backend(local=True))
# Desktop store: same-id clashes go to the resolver, never to priority.
- from core.capabilities.skills import merge_skill_infos
return CompositeBackend(
backends,
- merge=lambda groups, backend_hash: merge_skill_infos(groups, backend_hash=backend_hash),
+ merge=lambda groups, backend_hash: local_publication().merge_skill_infos(groups, backend_hash=backend_hash),
)
def _get_backend_change_token(self) -> Optional[Any]:
@@ -203,14 +129,14 @@ def load_all_metadata(self) -> Dict[str, AgentSkillMetadata]:
mcp_server_ids=list(item.get("mcp_server_ids") or []),
)
elif skill_info.content is not None:
- metadata = _load_skill_metadata_from_str(
+ metadata = load_skill_metadata_from_str(
skill_info.content, skill_info.skill_id
)
else:
metadata = _load_skill_metadata_from_file(skill_info.file_path)
# Add source information to skill_path for debugging
metadata_with_source = AgentSkillMetadata(
- id=_bound_skill_id(skill_info, metadata.id),
+ id=script_discovery.bound_skill_id(skill_info, metadata.id),
name=metadata.name,
description=metadata.description,
version=metadata.version,
@@ -263,8 +189,9 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]:
try:
ef = self._backend.get_extra_files(skill_id)
extra_file_names = sorted(ef.keys())
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.capabilities.skills.loader:load_skill_full', fallback_error)
# Resolve base_dir for {baseDir} substitution
base_dir = ""
@@ -296,7 +223,7 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]:
# Auto-generate whitelist from .py files when _scripts.json is absent
if not executable_scripts:
- auto_scripts = _auto_detect_scripts(ef, skill_info)
+ auto_scripts = script_discovery.auto_detect_scripts(ef, skill_info)
if auto_scripts:
executable_scripts = auto_scripts
logger.info(
@@ -308,7 +235,7 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]:
# Add source information to skill_path
spec_with_source = AgentSkillSpec(
- id=_bound_skill_id(skill_info, spec.id),
+ id=script_discovery.bound_skill_id(skill_info, spec.id),
name=spec.name,
description=spec.description,
version=spec.version,
@@ -402,10 +329,10 @@ def _materialize_skill_files(
Returns:
Absolute path to the materialized directory.
"""
- from core.capabilities.paths import capabilities_enabled
- from .publication import decode_files, publication_lock, publish_tree
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from core.capabilities.skills.publication import decode_files, publication_lock, publish_tree
- if capabilities_enabled():
+ if has_run_capabilities():
# The desktop capability store already publishes immutable revisions.
content, current_files, owner = self._backend.read_snapshot(skill_id)
return self._publish_to_store(skill_id, content, current_files, owner)
@@ -426,16 +353,15 @@ def _publish_to_store(
owner: Optional[str],
) -> str:
"""Desktop store path of a DB skill: publish its current content, link the views."""
- from core.capabilities import skills as caps_skills
- from core.services.desktop_capability_protocol import skill_content_hash
+ from core.capabilities.manifest_protocol import skill_content_hash
info = self._backend.get_skill_info(skill_id)
meta = dict(info.metadata or {}) if info else {}
files: Dict[str, bytes | str] = {"SKILL.md": content}
for filename, body in extra_files.items():
files[filename] = decode_binary(body) if is_binary_value(body) else str(body)
- before_views = caps_skills.view_generation()
- comp = caps_skills.publish_local_skill(
+ before_views = local_publication().view_generation()
+ comp = local_publication().publish_local_skill(
skill_id,
files=files,
content_hash=skill_content_hash(content, extra_files),
@@ -451,8 +377,8 @@ def _publish_to_store(
# view generation exactly when that set moved. Re-materializing unchanged
# content leaves every existing link pointing at the right revision, so
# relinking every name again would only re-derive the same answer.
- if caps_skills.view_generation() != before_views:
- caps_skills.rebuild_views(owner)
+ if local_publication().view_generation() != before_views:
+ local_publication().rebuild_views(owner)
logger.info("Published skill '%s' to store revision %s", skill_id, comp.revision)
return str(comp.path)
@@ -510,18 +436,10 @@ def register_skills_to_toolkit(
skill_dir = self.get_skill_dir(sid)
if skill_dir is None:
continue
- from core.capabilities.paths import capabilities_enabled
-
- prepared = getattr(self, "capability_run", None)
- if prepared is not None or capabilities_enabled():
- # Native Toolkit deduplicates by Skill.name. Preserve the
- # resolved alias even if frontmatter keeps a source name or
- # the physical directory ends in a content revision.
- toolkit.register_agent_skill(
- skill_dir, runtime_name=sid, capability_run=prepared
- )
- else:
- toolkit.register_agent_skill(skill_dir)
+ from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader
+ toolkit.skill_loaders.append(RuntimeNamedSkillLoader(
+ skill_dir, sid, getattr(self, "capability_run", None)
+ ))
count += 1
except Exception as e:
logger.warning("Failed to register skill '%s': %s", sid, e)
@@ -539,22 +457,3 @@ def get_skill_source(self, skill_id: str) -> Optional[str]:
self._sync_backend_cache()
skill_info = self._backend.get_skill_info(skill_id)
return skill_info.source_name if skill_info else None
-
-
-# Global singleton instance
-_global_loader: Optional[MultiSourceSkillLoader] = None
-
-
-def get_skill_loader(reset: bool = False) -> MultiSourceSkillLoader:
- """Get the global MultiSourceSkillLoader instance.
-
- Args:
- reset: If True, recreate the loader (useful for testing).
-
- Returns:
- The global MultiSourceSkillLoader instance.
- """
- global _global_loader
- if _global_loader is None or reset:
- _global_loader = MultiSourceSkillLoader()
- return _global_loader
diff --git a/src/backend/core/capabilities/skills/loader_factory.py b/src/backend/core/capabilities/skills/loader_factory.py
new file mode 100644
index 00000000..77a68da8
--- /dev/null
+++ b/src/backend/core/capabilities/skills/loader_factory.py
@@ -0,0 +1,21 @@
+"""The process-wide skill loader instance."""
+from __future__ import annotations
+from core.capabilities.skills import loader
+from typing import Optional
+
+global_loader: Optional[loader.MultiSourceSkillLoader] = None
+
+
+def get_skill_loader(reset: bool = False) -> loader.MultiSourceSkillLoader:
+ """Get the global MultiSourceSkillLoader instance.
+
+ Args:
+ reset: If True, recreate the loader (useful for testing).
+
+ Returns:
+ The global MultiSourceSkillLoader instance.
+ """
+ global global_loader
+ if global_loader is None or reset:
+ global_loader = loader.MultiSourceSkillLoader()
+ return global_loader
diff --git a/src/backend/core/services/skill_management_service.py b/src/backend/core/capabilities/skills/management.py
similarity index 93%
rename from src/backend/core/services/skill_management_service.py
rename to src/backend/core/capabilities/skills/management.py
index 14fa8d7c..d2df3c9b 100644
--- a/src/backend/core/services/skill_management_service.py
+++ b/src/backend/core/capabilities/skills/management.py
@@ -7,10 +7,10 @@
from datetime import datetime
from typing import Any, Dict, List, Optional
-from core.agent_skills.registry import _split_frontmatter
+from core.capabilities.skills.registry import split_frontmatter
from core.db.models import AdminMcpServer, AdminSkill
from core.infra.exceptions import BadRequestError
-from core.ontology.build_validator import ensure_ontology_build_valid
+from core.harness.registry.build_governance import ensure_ontology_build_valid
from fastapi import HTTPException
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
@@ -21,7 +21,7 @@
def refresh_skill_caches() -> None:
"""Refresh skill caches without turning a committed mutation into an HTTP 500."""
try:
- from core.agent_skills.cache_refresh import refresh_skill_caches as refresh
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches as refresh
refresh()
except Exception as exc: # noqa: BLE001
@@ -34,7 +34,7 @@ def _sanitize_frontmatter_value(value: str) -> str:
def _parse_frontmatter_list(skill_content: Optional[str], field_name: str) -> List[str]:
try:
- frontmatter, _ = _split_frontmatter(skill_content or "")
+ frontmatter, _ = split_frontmatter(skill_content or "")
except Exception:
return []
raw = str(frontmatter.get(field_name, "") or "")
@@ -97,10 +97,10 @@ def resolve_ontology_workflows(db: Session, tags: List[str]) -> List[str]:
selected = {tag for tag in tags if isinstance(tag, str) and tag.startswith("ontology:")}
if not selected:
return []
- from core.services.ontology_service import OntologyService
+ from core.harness.registry.build_governance import list_asset_tag_options
workflow_refs: List[str] = []
- for option in OntologyService(db).list_asset_tag_options("skill"):
+ for option in list_asset_tag_options(db, "skill"):
if option.get("value") not in selected:
continue
for workflow in option.get("workflows") or []:
@@ -145,7 +145,7 @@ def build_skill_content(
def validate_skill_file_path(filename: str) -> str:
"""Validate and normalize a relative skill-file path."""
- from core.agent_skills.publication import normalize_file_path
+ from core.capabilities.skills.publication import normalize_file_path
# Keep the established leading-slash UI convention; ZIP paths are strict.
name = (filename or "").strip().strip("/")
@@ -158,7 +158,7 @@ def validate_skill_file_path(filename: str) -> str:
def extract_instructions(skill_content: Optional[str]) -> str:
"""Return the Markdown body after the ``SKILL.md`` frontmatter."""
try:
- _, body = _split_frontmatter(skill_content or "")
+ _, body = split_frontmatter(skill_content or "")
except Exception:
return ""
return (body or "").strip()
@@ -171,7 +171,7 @@ def parse_and_upsert_skill_zip(
owner_user_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Parse a skill zip and upsert a global or owner-isolated DB skill."""
- from core.services.marketplace_service import parse_skill_zip
+ from core.capabilities.skills.zip_reader import parse_skill_zip
parsed = parse_skill_zip(data)
skill_id = parsed["skill_id"]
diff --git a/src/backend/core/agent_skills/publication.py b/src/backend/core/capabilities/skills/publication.py
similarity index 89%
rename from src/backend/core/agent_skills/publication.py
rename to src/backend/core/capabilities/skills/publication.py
index 25b33d98..41370b2a 100644
--- a/src/backend/core/agent_skills/publication.py
+++ b/src/backend/core/capabilities/skills/publication.py
@@ -13,21 +13,22 @@
import tempfile
import threading
from contextlib import contextmanager
+from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
-from core.capabilities.lockfile import locked as file_lock
+from core.infra.file_lock import locked as file_lock
logger = logging.getLogger(__name__)
-_thread_lock = threading.RLock()
+
_local = threading.local()
@contextmanager
def publication_lock():
- from .config import get_sandbox_skills_dir
+ from core.capabilities.skills.config import get_sandbox_skills_dir
- with _thread_lock:
+ with process_state().thread_lock:
if getattr(_local, "held", False):
yield
return
@@ -66,7 +67,7 @@ def safe_skill_id(skill_id: str) -> str:
def decode_files(content: str, extra: dict) -> dict[str, bytes]:
- from .binary_files import decode_binary, is_binary_value
+ from core.capabilities.skills.binary_files import decode_binary, is_binary_value
files = {"SKILL.md": content.encode("utf-8")}
for name, value in extra.items():
@@ -105,7 +106,7 @@ def publish_tree(
target: Path, files: dict[str, bytes], *, modes: dict[str, int] | None = None
) -> Path:
"""Publish an exact tree; failure before the swap leaves the old tree intact."""
- from .config import get_sandbox_skills_dir
+ from core.capabilities.skills.config import get_sandbox_skills_dir
safe_skill_id(target.name)
files = {safe_relative(name): data for name, data in files.items()}
@@ -161,15 +162,11 @@ def prepare_skill_view(user_id: str | None = None) -> Path | None:
The publication lock also fences old loader instances. Preparation errors
propagate: serving an unreconciled directory would violate deletion semantics.
"""
- from core.capabilities.paths import capabilities_enabled
- from .config import (
- get_sandbox_skills_dir,
- get_user_skills_dir,
- sync_user_skill_view,
- )
- from .loader import MultiSourceSkillLoader
-
- if capabilities_enabled():
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from core.capabilities.skills.config import get_sandbox_skills_dir, get_user_skills_dir, sync_user_skill_view
+ from core.capabilities.skills.loader import MultiSourceSkillLoader
+
+ if has_run_capabilities():
return sync_user_skill_view(user_id)
with publication_lock():
backend = MultiSourceSkillLoader._create_default_backend()
@@ -195,7 +192,7 @@ def prepare_skill_view(user_id: str | None = None) -> Path | None:
def _publish_selected(backend, user_id, private_only, wanted_shared, wanted_private):
- from .config import skill_files_dir
+ from core.capabilities.skills.config import skill_files_dir
for info in backend.list_skill_files():
owner = (info.metadata or {}).get("owner_user_id")
@@ -225,3 +222,15 @@ def _publish_selected(backend, user_id, private_only, wanted_shared, wanted_priv
modes = {name: (info.file_path.parent / name).stat().st_mode & 0o777 for name in files}
publish_tree(target, files, modes=modes)
(wanted_private if owner else wanted_shared).add(info.skill_id)
+
+
+@dataclass
+class PublicationState:
+ thread_lock: object = field(default_factory=lambda: threading.RLock())
+
+
+_STATE = PublicationState()
+
+
+def process_state() -> PublicationState:
+ return _STATE
diff --git a/src/backend/core/agent_skills/registry.py b/src/backend/core/capabilities/skills/registry.py
similarity index 97%
rename from src/backend/core/agent_skills/registry.py
rename to src/backend/core/capabilities/skills/registry.py
index 785d4fe0..6bb35389 100644
--- a/src/backend/core/agent_skills/registry.py
+++ b/src/backend/core/capabilities/skills/registry.py
@@ -66,7 +66,7 @@ def _require_id(value: str) -> str:
return v
-def _split_frontmatter(raw: str) -> Tuple[Dict[str, str], str]:
+def split_frontmatter(raw: str) -> Tuple[Dict[str, str], str]:
# ZIP uploads are commonly created on Windows, where Markdown files use
# CRLF line endings. Normalize all newline styles before matching the
# frontmatter delimiters so a valid ``---`` header is platform-agnostic.
@@ -215,9 +215,9 @@ def _parse_mcp_server_ids(fm: Dict[str, str]) -> List[str]:
)
-def _load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMetadata:
+def load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMetadata:
"""Load only metadata from SKILL.md content string (for DB-backed skills)."""
- fm, body = _split_frontmatter(content)
+ fm, body = split_frontmatter(content)
skill_id = _require_id(fm.get("name", skill_id))
name = fm.get("display_name", "") or skill_id
@@ -243,7 +243,7 @@ def _load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMeta
def _load_skill_from_str(content: str, skill_id: str) -> AgentSkillSpec:
"""Load full skill spec from SKILL.md content string (for DB-backed skills)."""
- fm, body = _split_frontmatter(content)
+ fm, body = split_frontmatter(content)
skill_id = _require_id(fm.get("name", skill_id))
name = fm.get("display_name", "") or skill_id
@@ -280,7 +280,7 @@ def _load_skill_from_str(content: str, skill_id: str) -> AgentSkillSpec:
def _load_skill_metadata_from_file(path: Path) -> AgentSkillMetadata:
"""Load only metadata from SKILL.md (fast, no body parsing)."""
raw = path.read_text(encoding="utf-8")
- fm, body = _split_frontmatter(raw)
+ fm, body = split_frontmatter(raw)
folder_id = _require_id(path.parent.name)
skill_id = _require_id(fm.get("name", folder_id))
@@ -308,7 +308,7 @@ def _load_skill_metadata_from_file(path: Path) -> AgentSkillMetadata:
def _load_skill_from_file(path: Path) -> AgentSkillSpec:
"""Load full skill spec from SKILL.md (includes instructions)."""
raw = path.read_text(encoding="utf-8")
- fm, body = _split_frontmatter(raw)
+ fm, body = split_frontmatter(raw)
folder_id = _require_id(path.parent.name)
skill_id = _require_id(fm.get("name", folder_id))
diff --git a/src/backend/core/capabilities/skills/runtime_loader.py b/src/backend/core/capabilities/skills/runtime_loader.py
new file mode 100644
index 00000000..6abc1b34
--- /dev/null
+++ b/src/backend/core/capabilities/skills/runtime_loader.py
@@ -0,0 +1,115 @@
+"""Run-scoped skill loaders with authorized names and bounded desktop I/O."""
+from __future__ import annotations
+import asyncio
+import logging
+import os
+from dataclasses import replace
+from functools import lru_cache
+from pathlib import Path
+import frontmatter
+from agentscope.skill import LocalSkillLoader, SkillLoaderBase, Skill
+from core.capabilities.skills.config import model_facing_skill_dir
+logger = logging.getLogger(__name__)
+
+def _parse_runtime_skill(text):
+ content = frontmatter.loads(text)
+ name, description = content.get("name"), content.get("description")
+ if not name or not description:
+ return None
+ return str(name), str(description), content.content
+
+
+_cached_runtime_skill = lru_cache(maxsize=256)(_parse_runtime_skill)
+
+
+class RuntimeNamedSkillLoader(SkillLoaderBase):
+ """Read frozen files and expose a deployment-appropriate directory."""
+
+ def __init__(self, directory: str, runtime_name: str, capability_run=None) -> None:
+ self.directory = directory
+ self.runtime_name = runtime_name
+ self.capability_run = capability_run
+ self._physical_loader = LocalSkillLoader(directory)
+
+ def _list_desktop_skill(self):
+ from core.harness.registry.capability_runtime import validate
+ from core.harness.capability_errors import CapabilityError
+
+ # Keep authorization outside the parse-error handling: it must fail the
+ # whole enumeration, never quietly omit a revoked or modified skill.
+ try:
+ validate(self.capability_run, only_skill=self.runtime_name)
+ except (OSError, CapabilityError):
+ # 被撤销或改动过的单个技能只从枚举里消失,不牵连这一轮其它工具。
+ return []
+ try:
+ entry = Path(self.directory) / "SKILL.md"
+ updated_at = entry.stat().st_mtime
+ text = entry.read_text(encoding="utf-8")
+ parsed = (
+ _cached_runtime_skill(text) if len(text) <= 262144 else _parse_runtime_skill(text)
+ )
+ if parsed is None:
+ return []
+ _, description, markdown = parsed
+ return [
+ Skill(
+ name=self.runtime_name,
+ description=description,
+ dir=model_facing_skill_dir(self.runtime_name, self.directory),
+ markdown=markdown,
+ updated_at=updated_at,
+ )
+ ]
+ except Exception as exc:
+ logger.warning("Failed to load skill metadata: %s", type(exc).__name__)
+ return []
+
+ async def list_skills(self):
+ # AgentScope's general filesystem scanner crosses the executor boundary
+ # eight times for one known file. Batch that I/O in one worker while
+ # preserving its parser, fields and sandbox alias.
+ if os.name == "nt" and self.capability_run is not None:
+ return await asyncio.to_thread(self._list_desktop_skill)
+ from core.harness.registry.capability_runtime import validate
+
+ if self.capability_run is not None:
+ # Re-check only this loader's own skill (authorization + fresh hash),
+ # not the whole run. N loaders each re-hashing all N skills — twice
+ # per assembly — was O(N²) filesystem work and the dominant cost of
+ # desktop agent setup; scoping to one skill makes it O(N).
+ from core.harness.capability_errors import CapabilityError
+
+ try:
+ await asyncio.to_thread(validate, self.capability_run, only_skill=self.runtime_name)
+ except (CapabilityError, OSError):
+ return []
+ physical = await self._physical_loader.list_skills()
+ return [
+ replace(skill, name=self.runtime_name, dir=model_facing_skill_dir(self.runtime_name, self.directory))
+ for skill in physical
+ ]
+
+
+class ParallelRuntimeSkillLoader(SkillLoaderBase):
+ """Enumerate independent desktop loaders with bounded I/O concurrency."""
+
+ def __init__(self, loaders):
+ self._loaders = tuple(loaders)
+
+ async def list_skills(self):
+ limit = asyncio.Semaphore(4)
+
+ async def load(loader):
+ async with limit:
+ return await loader.list_skills()
+
+ results = await asyncio.gather(
+ *(load(loader) for loader in self._loaders), return_exceptions=True
+ )
+ skills = []
+ for result in results:
+ if isinstance(result, BaseException):
+ raise result
+ skills.extend(result)
+ return skills
diff --git a/src/backend/core/capabilities/skills/script_discovery.py b/src/backend/core/capabilities/skills/script_discovery.py
new file mode 100644
index 00000000..3add1837
--- /dev/null
+++ b/src/backend/core/capabilities/skills/script_discovery.py
@@ -0,0 +1,65 @@
+"""Find the executable scripts a skill ships when it has no _scripts.json."""
+from __future__ import annotations
+from pathlib import Path
+from typing import Dict, List
+from core.capabilities.skills.backends import SkillFileInfo
+
+
+SCRIPT_EXTENSIONS = {".py", ".js", ".sh", ".r", ".R"}
+
+
+def bound_skill_id(skill_info: SkillFileInfo, declared_id: str) -> str:
+ # Capability names come from the resolved view, independently of the file's
+ # original frontmatter. Ordinary source files keep their existing ID rules.
+ if skill_info.source_name == "prepared" or (skill_info.origin or {}).get("install_id"):
+ return skill_info.skill_id
+ return declared_id
+
+
+def auto_detect_scripts(
+ extra_files: Dict[str, str],
+ skill_info: "SkillFileInfo",
+) -> List[Dict]:
+ """Auto-detect executable scripts when _scripts.json is absent.
+
+ Scans extra_files (DB skills) or the skill directory (filesystem skills)
+ for files with known script extensions and returns a whitelist entry for
+ each one, so users don't need to manually create _scripts.json.
+ """
+ scripts: List[Dict] = []
+
+ # Collect candidate filenames
+ candidates: List[str] = []
+
+ # From extra_files (DB skills)
+ for fname in extra_files:
+ if fname.startswith("_"):
+ continue
+ ext = Path(fname).suffix.lower()
+ if ext in SCRIPT_EXTENSIONS:
+ candidates.append(fname)
+
+ # From filesystem (filesystem skills without extra_files)
+ if not candidates and not skill_info.is_database and skill_info.content is None:
+ skill_dir = skill_info.file_path.parent
+ for p in skill_dir.iterdir():
+ if (
+ p.is_file()
+ and not p.name.startswith("_")
+ and p.suffix.lower() in SCRIPT_EXTENSIONS
+ ):
+ candidates.append(p.name)
+
+ # Build whitelist entries
+ ext_to_lang = {".py": "python", ".js": "javascript", ".sh": "bash", ".r": "r", ".R": "r"}
+ for name in sorted(candidates):
+ ext = Path(name).suffix.lower()
+ scripts.append(
+ {
+ "name": name,
+ "language": ext_to_lang.get(ext, "python"),
+ "timeout": 60,
+ }
+ )
+
+ return scripts
diff --git a/src/backend/core/agent_skills/selection_record.py b/src/backend/core/capabilities/skills/selection_record.py
similarity index 98%
rename from src/backend/core/agent_skills/selection_record.py
rename to src/backend/core/capabilities/skills/selection_record.py
index 2c83ac4d..d992d75e 100644
--- a/src/backend/core/agent_skills/selection_record.py
+++ b/src/backend/core/capabilities/skills/selection_record.py
@@ -149,9 +149,9 @@ def select_with_record(
)
try:
- from core.agent_skills.selector import select_skills_for_query
+ from core.harness.registry.model_services import rank_skills
- selected = select_skills_for_query(
+ selected = rank_skills(
user_query=user_query,
available_skill_ids=available_skill_ids,
enabled_skill_ids=enabled_skill_ids,
diff --git a/src/backend/core/agent_skills/skill_archive.py b/src/backend/core/capabilities/skills/skill_archive.py
similarity index 88%
rename from src/backend/core/agent_skills/skill_archive.py
rename to src/backend/core/capabilities/skills/skill_archive.py
index 93637419..24b4afe7 100644
--- a/src/backend/core/agent_skills/skill_archive.py
+++ b/src/backend/core/capabilities/skills/skill_archive.py
@@ -14,11 +14,11 @@
from pathlib import Path
from typing import Optional
-from .publication import publication_lock, read_tree, safe_skill_id
+from core.capabilities.skills.publication import publication_lock, read_tree, safe_skill_id
def _archives_dir() -> Path:
- from .config import get_sandbox_skills_dir
+ from core.capabilities.skills.config import get_sandbox_skills_dir
path = get_sandbox_skills_dir().parent / "skill_archives"
path.mkdir(parents=True, exist_ok=True)
@@ -60,8 +60,8 @@ def build_skill_tar(skill_id: str, src: Path) -> Optional[Path]:
def build_current_skill_tar(skill_id: str, user_id: str | None = None) -> Optional[Path]:
"""Resolve visibility and freeze current bytes under one publication lock."""
- from .publication import prepare_skill_view
- from .config import get_sandbox_skills_dir
+ from core.capabilities.skills.publication import prepare_skill_view
+ from core.capabilities.skills.config import get_sandbox_skills_dir
safe_skill_id(skill_id)
with publication_lock():
diff --git a/src/backend/core/capabilities/skills/zip_reader.py b/src/backend/core/capabilities/skills/zip_reader.py
new file mode 100644
index 00000000..82366201
--- /dev/null
+++ b/src/backend/core/capabilities/skills/zip_reader.py
@@ -0,0 +1,124 @@
+"""Validate skill upload archives without marketplace persistence."""
+from __future__ import annotations
+import io
+import os
+import zipfile
+from typing import Any, Dict, List
+from core.capabilities.skills.binary_files import BINARY_EXTENSIONS, JUNK_BASENAMES, MAX_SINGLE_FILE, MAX_TOTAL, encode_binary, is_binary_value
+from core.capabilities.skills.deps.detector import detect_dependencies
+from core.capabilities.skills.registry import SkillSpecError, load_skill_metadata_from_str, split_frontmatter
+from fastapi import HTTPException
+
+def parse_skill_zip(data: bytes) -> Dict[str, Any]:
+ """Parse a skill zip → returns {skill_id, skill_content (original text), meta, extra_files, dependencies, skipped}.
+
+ Parsing/validation only; nothing is persisted. The same unpacking logic is
+ shared by admin market-listing uploads and users' private uploads.
+ """
+ try:
+ zf = zipfile.ZipFile(io.BytesIO(data))
+ except zipfile.BadZipFile:
+ raise HTTPException(status_code=400, detail="Invalid zip file")
+
+ from core.capabilities.skills.publication import normalize_file_path, safe_skill_id
+
+ # Preserve ZipInfo for reads: normalized names may not exist in the archive.
+ entries = {}
+ for info in zf.infolist():
+ raw_name = info.filename.replace("\\", "/")
+ is_dir = raw_name.endswith("/")
+ try:
+ name = normalize_file_path(raw_name[:-1] if is_dir else raw_name)
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ if is_dir:
+ continue
+ if name in entries:
+ raise HTTPException(status_code=400, detail=f"Duplicate skill file path: {name}")
+ entries[name] = info
+ # A file cannot also be a parent directory of another file.
+ for name in entries:
+ parts = name.split("/")
+ if any("/".join(parts[:i]) in entries for i in range(1, len(parts))):
+ raise HTTPException(status_code=400, detail=f"Conflicting skill file path: {name}")
+
+ skill_md_paths = [n for n in entries if n.split("/")[-1] == "SKILL.md"]
+ if not skill_md_paths:
+ raise HTTPException(status_code=400, detail="No SKILL.md found in zip")
+
+ skill_md_path = skill_md_paths[0]
+ parts = skill_md_path.split("/")
+ if len(parts) == 1:
+ prefix = ""
+ elif len(parts) == 2:
+ prefix = parts[0] + "/"
+ else:
+ prefix = "/".join(parts[:-1]) + "/"
+
+ try:
+ raw = zf.read(entries[skill_md_path]).decode("utf-8")
+ fm, _ = split_frontmatter(raw)
+ skill_id = fm.get("name", "").strip()
+ if not skill_id:
+ if prefix:
+ skill_id = prefix.rstrip("/").split("/")[-1]
+ else:
+ raise HTTPException(status_code=400, detail="SKILL.md missing 'name' in frontmatter")
+ except SkillSpecError as e:
+ raise HTTPException(status_code=400, detail=f"Invalid SKILL.md: {e}")
+
+ try:
+ safe_skill_id(skill_id)
+ meta = load_skill_metadata_from_str(raw, skill_id)
+ except Exception as e: # noqa: BLE001
+ raise HTTPException(status_code=400, detail=f"Invalid skill: {e}")
+
+ JUNK_PATH_MARKERS = ("__MACOSX/", "/.git/", "/__pycache__/", "/.svn/", "/.hg/")
+ extra_files: Dict[str, str] = {}
+ skipped: List[Dict[str, str]] = []
+ total_size = 0
+ for entry, info in entries.items():
+ if entry == skill_md_path:
+ continue
+ if prefix and not entry.startswith(prefix):
+ continue
+ rel_name = entry[len(prefix):] if prefix else entry
+ if not rel_name:
+ continue
+ base = rel_name.rsplit("/", 1)[-1]
+ if base in JUNK_BASENAMES or any(m in f"/{entry}" for m in JUNK_PATH_MARKERS):
+ continue
+ if info.file_size > MAX_SINGLE_FILE:
+ skipped.append({"file": rel_name, "reason": f"exceeds {MAX_SINGLE_FILE // (1024 * 1024)}MB single-file ceiling"})
+ continue
+ try:
+ file_bytes = zf.read(info)
+ except (KeyError, zipfile.BadZipFile):
+ skipped.append({"file": rel_name, "reason": "unreadable zip entry"})
+ continue
+ _, ext = os.path.splitext(rel_name)
+ if ext.lower() in BINARY_EXTENSIONS:
+ stored = encode_binary(file_bytes)
+ else:
+ try:
+ stored = file_bytes.decode("utf-8")
+ except UnicodeDecodeError:
+ stored = encode_binary(file_bytes)
+ stored_size = len(stored.encode("utf-8"))
+ if total_size + stored_size > MAX_TOTAL:
+ skipped.append({"file": rel_name, "reason": f"would exceed {MAX_TOTAL // (1024 * 1024)}MB total ceiling"})
+ continue
+ total_size += stored_size
+ extra_files[rel_name] = stored
+
+ dependencies = detect_dependencies(
+ {fn: c for fn, c in extra_files.items() if not is_binary_value(c)}
+ )
+ return {
+ "skill_id": skill_id,
+ "skill_content": raw,
+ "meta": meta,
+ "extra_files": extra_files,
+ "dependencies": dependencies,
+ "skipped": skipped,
+ }
diff --git a/src/backend/core/channels/adapters/dingtalk.py b/src/backend/core/channels/adapters/dingtalk.py
deleted file mode 100644
index 1e96dbfb..00000000
--- a/src/backend/core/channels/adapters/dingtalk.py
+++ /dev/null
@@ -1,854 +0,0 @@
-"""DingTalk inbound channel adapter.
-
-- **Binding validation**: exchange AppKey/AppSecret for an access_token
- (DingTalk v1.0 OAuth) — obtaining one is treated as valid credentials.
-- **Inbound (Stream long connection)**: WebSocket via the official
- ``dingtalk-stream`` SDK, listening for bot message callbacks
- (``ChatbotMessage.TOPIC``). Zero public IP, no callback configuration. If the
- SDK is missing, this channel is unavailable but the process is unaffected.
-- **Outbound replies**: use the ``sessionWebhook`` carried by each inbound
- message (no access_token needed), unified for p2p / group.
- Agent replies go through ``send_markdown`` as ``msgtype=markdown`` (DingTalk
- natively renders headings/bold/links/lists, etc.; tables/code fences do not
- render on mobile, so content is downgraded via
- ``core.channels.markdown.downgrade_for_dingtalk`` before sending).
- Short system messages (placeholders/receipts) still use ``send_text`` (plain text).
- DingTalk bots **do not support editing messages** → ``edit_message`` returns
- failure; but they do support **silent recall** of messages sent via the robot
- API (group ``groupMessages/recall`` / one-to-one ``otoMessages/batchRecall``,
- using the processQueryKey returned at send time). So placeholder messages go
- through ``send_placeholder`` via the robot API (obtaining a recallable key),
- and before the final reply the upper layer calls ``recall_message`` to remove
- the placeholder — visually equivalent to "replace". When the robot API is
- unavailable (no permission, etc.), placeholders automatically fall back to
- sessionWebhook plain text (not recallable, same behavior as the old version).
-- **Outbound proactive delivery (automation scheduled tasks, etc., no inbound
- message)**: synthetic messages have no sessionWebhook → ``send_text`` /
- ``send_markdown`` automatically fall back to the robot API (``sampleText`` /
- ``sampleMarkdown``).
-- **Outbound files**: sessionWebhook only accepts text-type messages, cannot
- send files → ``push_file`` uses the robot API: first ``media/upload`` to get
- a mediaId, then send a robot message. The robot message picks its endpoint by
- conversation type: group ``groupMessages/send`` (openConversationId);
- one-to-one with a staffId uses ``oToMessages/batchSend``, without one
- (proactive delivery) uses the human-bot conversation's openConversationId via
- ``privateChatMessages/send``.
- The app must be granted the "internal enterprise robot send message"
- permission on the DingTalk open platform, otherwise the send endpoints return 403.
-
-Conversation keying (``external_conversation_id``) always uses DingTalk ``conversationId``:
- - one-to-one (conversationType==``1``) conversationId maps one-to-one to a user → naturally one conversation per person;
- - group (conversationType==``2``) conversationId is unique per group → the whole group shares one conversation.
-
-Isomorphic to the [[lark]] adapter. See internal design docs.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-import os
-import re
-import shutil
-import tempfile
-import time
-from typing import Any, Callable, Dict, List, Optional
-
-import httpx
-
-from datetime import datetime, timezone
-
-from core.channels.markdown import derive_title, downgrade_for_dingtalk
-from core.channels.protocol import (
- ChannelCaps,
- ChannelResourceError,
- HistoryItem,
- InboundMsg,
- SendResult,
- chunk_text,
-)
-from core.infra.crypto import decrypt_secret
-
-logger = logging.getLogger(__name__)
-
-DINGTALK_API_BASE = "https://api.dingtalk.com"
-
-
-class DingTalkAdapter:
- caps = ChannelCaps(
- channel_type="dingtalk",
- max_message_len=4000,
- supports_markdown=True,
- splits_long_messages=False,
- supports_long_conn=True,
- # DingTalk exposes ``isInAtList`` on every bot callback, so the gate is already
- # wired; whether non-@ group messages ever get delivered depends on the tenant
- # holding DingTalk's group-message read permission (applied for and reviewed on the
- # open platform). Without it, group listening stays inert rather than broken.
- supports_group_observe=True,
- bind_mode="credentials",
- credential_fields=("app_id", "app_secret"),
- )
-
- # access_token cache: {app_id: (token, expire_epoch)} (only for validate/testing; replies use sessionWebhook)
- _token_cache: Dict[str, tuple] = {}
-
- # ── Credentials ─────────────────────────────────────────────────────
- @staticmethod
- def _app_secret(conn: Any) -> str:
- cfg = conn.config if isinstance(conn.config, dict) else {}
- return decrypt_secret(cfg.get("app_secret_enc")) or ""
-
- async def _access_token(self, app_id: str, app_secret: str) -> str:
- cached = self._token_cache.get(app_id)
- if cached and cached[1] > time.time() + 60:
- return cached[0]
- url = f"{DINGTALK_API_BASE}/v1.0/oauth2/accessToken"
- async with httpx.AsyncClient(timeout=10) as client:
- resp = await client.post(url, json={"appKey": app_id, "appSecret": app_secret})
- data = resp.json()
- token = data.get("accessToken")
- if not token:
- raise RuntimeError(f"钉钉 access_token 获取失败: {data}")
- self._token_cache[app_id] = (token, time.time() + int(data.get("expireIn", 7000)))
- return token
-
- async def validate_credentials(self, conn: Any) -> Dict[str, Any]:
- secret = self._app_secret(conn)
- if not conn.app_id or not secret:
- raise RuntimeError("缺少 AppKey / AppSecret")
- await self._access_token(conn.app_id, secret)
- return {"app_id": conn.app_id}
-
- # ── Webhook verification (DingTalk uses the Stream long connection; no webhook entry point) ──
- def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool:
- return True
-
- # ── Event → InboundMsg (the dict from the Stream callback) ──────────
- def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]:
- """DingTalk bot callback data (dict) → InboundMsg. Returns None for non-text/unsupported messages."""
- msgtype = payload.get("msgtype")
- text = ""
- if msgtype == "text":
- text = ((payload.get("text") or {}).get("content") or "").strip()
- elif msgtype == "richText":
- # Rich text: concatenate the plain-text nodes inside
- parts = (payload.get("content") or {}).get("richText") or []
- text = " ".join(p.get("text", "") for p in parts if isinstance(p, dict) and p.get("text")).strip()
- attachments = self._extract_attachments(msgtype, payload)
- if not text and not attachments:
- return None
- conv_type = str(payload.get("conversationType") or "1")
- chat_type = "group" if conv_type == "2" else "p2p"
- conv_id = payload.get("conversationId") or ""
- return InboundMsg(
- channel_id=conn.channel_id,
- channel_type="dingtalk",
- text=text,
- chat_type=chat_type,
- external_conversation_id=conv_id,
- sender_id=payload.get("senderStaffId") or payload.get("senderId") or "",
- sender_name=payload.get("senderNick") or "",
- message_id=payload.get("msgId") or "",
- attachments=attachments,
- # ``isInAtList`` is DingTalk's own "was the bot in the @ list" flag. With only the
- # default permission DingTalk delivers @bot messages exclusively, so this is
- # always true and the gate is a no-op; once the tenant is granted group-message
- # read, non-@ messages start arriving with it false and silent observation kicks
- # in with no further code change. Absent field → treat as addressed.
- addressed_to_bot=(
- True if chat_type == "p2p" else bool(payload.get("isInAtList", True))
- ),
- raw={
- "dingtalk_session_webhook": payload.get("sessionWebhook") or "",
- "dingtalk_conversation_id": conv_id,
- # Needed by download_resource: messageFiles/download requires robotCode, and
- # per DingTalk's docs it is **not** the AppKey — the only reliable source is
- # this field on the callback payload (present in Stream mode, which is what
- # the long connection uses).
- "dingtalk_robot_code": payload.get("robotCode") or "",
- },
- )
-
- # ── Group history pull (via the official dws CLI) ────────────────────
- # DingTalk has **no** REST OpenAPI for reading a group's message history — the
- # server-side API list only covers sending, recalling and read receipts. The capability
- # is reachable two ways, and this takes the second:
- # 1. DingTalk's MCP gateway (`list_conversation_message_v2` on the group-chat MCP
- # server). Works, but needs credentials provisioned per app — extra setup.
- # 2. The official `dws` CLI, which the backend **already** runs as a subprocess for
- # this user's DingTalk connection and which is **already logged in**. No new
- # credentials, no extra configuration: it simply reuses the connection the owner
- # has set up. That is why this is the one implemented.
- #
- # Runs as the connection's owner (per-user $HOME is the credential isolation boundary,
- # same as every other backend dws call). The 8s cap keeps a slow CLI from eating into
- # the caller's own history budget; any failure returns [] and history is skipped.
- _DWS_HISTORY_TIMEOUT_S = 8
-
- async def fetch_history(
- self,
- conn: Any,
- conversation_id: str,
- *,
- since_ms: int,
- limit: int,
- newest_first: bool = False,
- ) -> List[HistoryItem]:
- """`dws chat message list --group --time ` → normalized history items."""
- conv = (conversation_id or "").strip()
- owner = getattr(conn, "owner_user_id", "") or ""
- if not conv or not owner:
- return []
- from core.services.dingtalk_service import _run_dws
-
- # dws takes a local-time string, not epoch ms. `--forward` picks the direction:
- # true = messages after --time (incremental from the cursor); false = messages
- # before --time, which anchored at *now* is how the cold start gets the most recent
- # ones instead of the oldest of a week-long window.
- anchor_ms = int(datetime.now(timezone.utc).timestamp() * 1000) if newest_first else since_ms
- anchor = datetime.fromtimestamp(max(anchor_ms, 0) / 1000, tz=timezone.utc).astimezone()
- args = [
- "chat", "message", "list",
- "--group", conv,
- "--time", anchor.strftime("%Y-%m-%d %H:%M:%S"),
- "--limit", str(max(1, min(int(limit or 50), 100))),
- "--forward=false" if newest_first else "--forward=true",
- "--format", "json",
- ]
- try:
- out, err, code = await _run_dws(owner, args, timeout=self._DWS_HISTORY_TIMEOUT_S)
- except Exception: # noqa: BLE001
- logger.warning("[dingtalk] dws 群历史拉取异常 conv=%s", conv[:24], exc_info=True)
- return []
- if code != 0 or not out.strip():
- # Not logged in / no permission / timeout all land here. Debug, not warning:
- # a tenant without the DingTalk connection set up would otherwise log on every
- # single group message.
- logger.debug(
- "[dingtalk] dws 群历史拉取未成功 code=%s err=%s", code, (err or "")[:160]
- )
- return []
- try:
- return self._parse_dws_history(json.loads(out))
- except Exception: # noqa: BLE001
- logger.debug("[dingtalk] dws 输出解析失败: %s", out[:200], exc_info=True)
- return []
-
- _DWS_DOWNLOAD_TIMEOUT_S = 45
-
- @staticmethod
- def _dws_business_error(raw_out: str) -> Optional[str]:
- """Pull the human-readable reason out of a dws error envelope, if that's what this is.
-
- Worth surfacing rather than collapsing to "download failed": the messages are
- specific and actionable ("this data belongs to organization X while the tool is
- configured for organization Y"), and reading that as an expiry sends the user
- looking in entirely the wrong place.
- """
- try:
- err = (json.loads(raw_out or "{}") or {}).get("error")
- except Exception: # noqa: BLE001
- return None
- if not isinstance(err, dict):
- return None
- msg = str(err.get("message") or "").strip()
- return msg or None
-
- async def _read_doc_via_dws(self, conn: Any, node_id: str) -> Optional[bytes]:
- """`dws doc read --node --content-format markdown` → markdown bytes.
-
- A DingTalk online document has no file to download; it is read as content. Returning
- markdown keeps it in the same artifact pipeline as everything else.
- """
- owner = getattr(conn, "owner_user_id", "") or ""
- if not owner:
- return None
- from core.services.dingtalk_service import _run_dws
-
- out, err, code = await _run_dws(
- owner,
- ["doc", "read", "--node", node_id, "--content-format", "markdown", "--format", "json"],
- timeout=self._DWS_DOWNLOAD_TIMEOUT_S,
- )
- reason = self._dws_business_error(out) or self._dws_business_error(err)
- if reason:
- logger.warning("[dingtalk] 钉钉文档读取被拒 node=%s: %s", node_id[:16], reason[:200])
- raise ChannelResourceError(reason)
- if code != 0 or not out.strip():
- logger.debug("[dingtalk] 钉钉文档读取失败 code=%s err=%s", code, (err or "")[:160])
- return None
- try:
- payload = json.loads(out)
- except Exception: # noqa: BLE001
- return out.encode("utf-8")
- # Content sits under a few possible keys depending on CLI version; fall back to the
- # whole payload rather than silently returning nothing.
- for key in ("content", "markdown", "data", "result"):
- val = payload.get(key) if isinstance(payload, dict) else None
- if isinstance(val, str) and val.strip():
- return val.encode("utf-8")
- if isinstance(val, dict):
- for sub in ("content", "markdown", "text"):
- inner = val.get(sub)
- if isinstance(inner, str) and inner.strip():
- return inner.encode("utf-8")
- return json.dumps(payload, ensure_ascii=False).encode("utf-8")
-
- async def _download_via_dws(self, conn: Any, file_id: str) -> Optional[bytes]:
- """`dws drive download --node ` → bytes.
-
- The CLI only writes to a local path (it does the signed-URL dance internally), so the
- bytes are read back from a temp file that is always removed, including on failure —
- this runs on the backend host, not in a disposable sandbox.
- """
- owner = getattr(conn, "owner_user_id", "") or ""
- if not owner:
- return None
- from core.services.dingtalk_service import _run_dws
-
- tmpdir = tempfile.mkdtemp(prefix="dws_dl_")
- target = os.path.join(tmpdir, "download.bin")
- try:
- out, err, code = await _run_dws(
- owner,
- ["drive", "download", "--node", file_id, "--output", target, "--format", "json"],
- timeout=self._DWS_DOWNLOAD_TIMEOUT_S,
- )
- if code != 0 or not os.path.exists(target):
- logger.warning(
- "[dingtalk] dws 文件下载失败 file_id=%s code=%s err=%s",
- file_id[:16], code, (err or out or "")[:160],
- )
- return None
- with open(target, "rb") as fh:
- return fh.read()
- except Exception: # noqa: BLE001
- logger.warning("[dingtalk] dws 文件下载异常 file_id=%s", file_id[:16], exc_info=True)
- return None
- finally:
- shutil.rmtree(tmpdir, ignore_errors=True)
-
- @staticmethod
- def _dws_ts_ms(value: Any) -> int:
- """dws ``createTime`` → epoch ms.
-
- The CLI returns a **local-time string** (``"2026-08-08 21:57:33"``), not epoch
- milliseconds. Parsing it as an int silently yielded 0 for every message, which left
- the pull cursor pinned at the cold-start window — so every @ re-fetched the same 24
- hours forever. Numeric forms are still accepted in case a CLI version returns them.
- """
- if isinstance(value, (int, float)) and value > 0:
- return int(value)
- text = str(value or "").strip()
- if not text:
- return 0
- if text.isdigit():
- return int(text)
- try:
- naive = datetime.strptime(text, "%Y-%m-%d %H:%M:%S")
- except ValueError:
- return 0
- # Local time, as the CLI prints it; astimezone() attaches the process zone.
- return int(naive.astimezone().timestamp() * 1000)
-
- # dws renders a file message as **plain text**, not a structured attachment:
- # "[文件] 周报.docx fileId: X6GRez... 注意:如需下载使用dws drive download命令下载"
- # Parsed back out here, otherwise a pulled file is just an unusable sentence — the agent
- # can see a file was shared but has no handle to open it. Images use the same shape.
- _DWS_FILE_RE = re.compile(
- r"\[(?P文件|图片)\]\s*(?P.+?)\s+fileId:\s*(?P[A-Za-z0-9_\-]+)"
- r"(?:\s*注意[::][^\n]*)?"
- )
-
- # A DingTalk **online document** shared into a group arrives as a link card, not as a
- # file: a title line, an icon, and an alidocs URL. It is a different resource from a Drive
- # file — read with `dws doc read`, not downloaded by fileId — and without recognising it
- # the agent sees only an unusable link.
- _DWS_DOC_RE = re.compile(r"https?://alidocs\.dingtalk\.com/i/nodes/(?P[A-Za-z0-9_\-]+)")
-
- @classmethod
- def _extract_dws_docs(cls, content: str) -> List[Dict[str, Any]]:
- """DingTalk Docs link cards → attachment refs (deduped by node id)."""
- nodes = {m.group("node") for m in cls._DWS_DOC_RE.finditer(content or "")}
- if not nodes:
- return []
- # The card's first non-empty line is the document title; skip the markdown image line
- # and the "创建者:" byline, neither of which names the document.
- title = ""
- for line in (content or "").splitlines():
- line = line.strip()
- if line and not line.startswith("![") and not line.startswith("[http"):
- if line.startswith("创建者"):
- continue
- title = line
- break
- return [
- {"kind": "doc", "key": node, "name": title or "钉钉文档"}
- for node in sorted(nodes)
- ]
-
- @classmethod
- def _extract_dws_files(cls, content: str) -> tuple:
- """(cleaned_text, attachment_refs) for one history message body."""
- refs: List[Dict[str, Any]] = []
- for m in cls._DWS_FILE_RE.finditer(content or ""):
- refs.append({
- "kind": "image" if m.group("kind") == "图片" else "file",
- "key": m.group("fid"),
- "name": m.group("name").strip(),
- })
- doc_refs = cls._extract_dws_docs(content)
- if doc_refs:
- # The card body is title + icon + a long URL repeated twice; the attachment entry
- # carries title and handle, so keeping the raw card would just burn prompt space.
- return "", refs + doc_refs
- if not refs:
- return (content or "").strip(), []
- # Drop the matched sentences: the attachment ref already renders name + handle, and
- # the "use dws drive download" tail is instruction noise aimed at a CLI user.
- return cls._DWS_FILE_RE.sub("", content or "").strip(), refs
-
- @staticmethod
- def _parse_dws_history(body: Dict[str, Any]) -> List[HistoryItem]:
- """dws JSON output → HistoryItem list.
-
- Field names verified against live CLI output — they are **not** the ones the bot
- callback uses, which is the trap here:
- {"result": {"messages": [{"openMessageId", "sender", "content", "createTime",
- "openConversationId", "senderOpenDingTalkId"}], ...}}
- A bare ``{"messages": [...]}`` is accepted too, so a CLI version that drops the
- envelope does not silently yield nothing.
- """
- if not isinstance(body, dict):
- return []
- if body.get("success") is False or body.get("errorCode"):
- return []
- result = body.get("result") if isinstance(body.get("result"), dict) else body
- out: List[HistoryItem] = []
- for m in result.get("messages") or []:
- if not isinstance(m, dict):
- continue
- # Written out rather than chained: `a or b if cond else ""` binds the conditional
- # looser than the `or`, which would silently make every message look empty.
- text = m.get("content") or ""
- if not text and isinstance(m.get("text"), dict):
- text = m["text"].get("content") or ""
- if not isinstance(text, str):
- text = str(text)
- text, refs = DingTalkAdapter._extract_dws_files(text)
- if not text and not refs:
- continue
- out.append(HistoryItem(
- # openMessageId is the real id; msgId/messageId kept as fallbacks only.
- message_id=str(
- m.get("openMessageId") or m.get("msgId") or m.get("messageId") or ""
- ),
- # `sender` is already the display name — no directory lookup needed.
- sender_name=str(
- m.get("sender") or m.get("senderNick") or m.get("senderName") or "群成员"
- ),
- text=text,
- ts_ms=DingTalkAdapter._dws_ts_ms(m.get("createTime") or m.get("createAt")),
- attachments=refs,
- # Marks which download route this file needs: history files live on DingTalk
- # Drive and are fetched by fileId, not by the bot callback's downloadCode.
- raw={"dws_drive": True} if refs else {},
- ))
- return out
-
- # ── Inbound attachments ─────────────────────────────────────────────
- @staticmethod
- def _extract_attachments(msgtype: str, payload: Dict[str, Any]) -> list:
- """Message → attachment list. ``key`` carries DingTalk's ``downloadCode``.
-
- Content layout per DingTalk's "received message types" reference:
- picture → {pictureDownloadCode, downloadCode}
- file → {spaceId, fileName, downloadCode, fileId}
- audio → {downloadCode, recognition}
- video → {duration, videoType, downloadCode}
- richText → {richText: [ ...nodes, image nodes carry downloadCode... ]}
- Everything is read defensively (``downloadCode`` looked up in more than one place)
- because these payloads are only loosely specified and vary by client version — a
- missing field must degrade to "no attachment", never raise and kill the connection.
- """
- content = payload.get("content") if isinstance(payload.get("content"), dict) else {}
- out: list = []
-
- def _add(kind: str, code: Any, name: str) -> None:
- if code:
- out.append({"kind": kind, "key": str(code), "name": name})
-
- if msgtype == "picture":
- code = content.get("downloadCode") or content.get("pictureDownloadCode")
- _add("image", code, f"dingtalk_image_{payload.get('msgId', '')[:8] or 'x'}.png")
- elif msgtype == "file":
- _add("file", content.get("downloadCode"), content.get("fileName") or "file.bin")
- elif msgtype == "audio":
- _add("file", content.get("downloadCode"), "dingtalk_audio.amr")
- elif msgtype == "video":
- _add("file", content.get("downloadCode"), "dingtalk_video.mp4")
- elif msgtype == "richText":
- for i, node in enumerate(content.get("richText") or []):
- if not isinstance(node, dict):
- continue
- code = node.get("downloadCode") or node.get("pictureDownloadCode")
- _add("image", code, f"dingtalk_richtext_{i}.png")
- return out
-
- async def download_resource(
- self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any]
- ) -> Optional[bytes]:
- """Download an inbound attachment: downloadCode → temporary downloadUrl → bytes.
-
- Two hops, per DingTalk's design:
- POST /v1.0/robot/messageFiles/download {downloadCode, robotCode} → {downloadUrl}
- GET downloadUrl → bytes
- The temporary URL serves the file with a ``.file`` extension; we ignore that and keep
- the filename taken from the message, so the artifact keeps its real extension.
- Download codes expire, so a deferred fetch can legitimately fail — returning None
- (rather than raising) lets the caller degrade to "attachment unavailable".
- """
- code = attachment.get("key")
- if not code:
- return None
- # Files seen through *history* are DingTalk Drive entries addressed by fileId, not
- # bot-callback attachments addressed by a short-lived downloadCode — different
- # storage, different API. The marker is set when the history item is parsed.
- if attachment.get("kind") == "doc":
- return await self._read_doc_via_dws(conn, str(code))
- if (inbound.raw or {}).get("dws_drive"):
- return await self._download_via_dws(conn, str(code))
- robot_code = (inbound.raw or {}).get("dingtalk_robot_code") or ""
- if not robot_code:
- logger.warning("[dingtalk] 缺少 robotCode,无法下载附件 code=%s", str(code)[:12])
- return None
- try:
- token = await self._access_token(conn.app_id, self._app_secret(conn))
- async with httpx.AsyncClient(timeout=30) as client:
- resp = await client.post(
- f"{DINGTALK_API_BASE}/v1.0/robot/messageFiles/download",
- headers={"x-acs-dingtalk-access-token": token},
- json={"downloadCode": code, "robotCode": robot_code},
- )
- if resp.status_code != 200:
- logger.warning(
- "[dingtalk] 下载地址获取失败 status=%s body=%s",
- resp.status_code, resp.text[:200],
- )
- return None
- url = (resp.json() or {}).get("downloadUrl")
- if not url:
- return None
- fr = await client.get(url)
- if fr.status_code == 200:
- return fr.content
- logger.warning("[dingtalk] 附件下载失败 status=%s", fr.status_code)
- except Exception: # noqa: BLE001
- logger.exception("[dingtalk] 附件下载异常 code=%s", str(code)[:12])
- return None
-
- # ── Outbound push (via sessionWebhook, no access_token needed) ───────
- @staticmethod
- def _session_webhook(inbound: InboundMsg) -> str:
- return (inbound.raw or {}).get("dingtalk_session_webhook") or ""
-
- async def _post_webhook(self, webhook: str, payload: Dict[str, Any]) -> SendResult:
- if not webhook:
- return SendResult.fail("forbidden", "缺少 sessionWebhook(可能已过期)")
- try:
- async with httpx.AsyncClient(timeout=15) as client:
- resp = await client.post(webhook, json=payload)
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- if data.get("errcode", 0) == 0:
- return SendResult.ok()
- kind = "rate_limited" if data.get("errcode") in (130101, 88) else "unknown"
- return SendResult.fail(kind, f"errcode={data.get('errcode')} {data.get('errmsg')}")
-
- async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
- webhook = self._session_webhook(inbound)
- if webhook:
- return await self._post_webhook(
- webhook, {"msgtype": "text", "text": {"content": text}}
- )
- # Proactive delivery (automation scheduled tasks, etc.) has no inbound sessionWebhook → use the robot API
- try:
- token = await self._access_token(conn.app_id, self._app_secret(conn))
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- return await self._robot_send(token, conn, inbound, "sampleText", {"content": text})
-
- # Lets the upper layer do a whole-content downgrade **before** chunking
- # (avoids a table getting bisected by a chunk boundary and becoming unrecognizable for conversion).
- prepare_markdown = staticmethod(downgrade_for_dingtalk)
-
- async def send_markdown(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
- """Send a DingTalk markdown message (natively rendered). The downgrade is idempotent; calling directly or via prepare_markdown is equally safe.
-
- title is required for DingTalk's markdown type; it shows in the
- conversation-list summary / push notification and is derived from the
- first line of the body.
- """
- md = downgrade_for_dingtalk(text)
- title = derive_title(md)
- webhook = self._session_webhook(inbound)
- if webhook:
- return await self._post_webhook(
- webhook, {"msgtype": "markdown", "markdown": {"title": title, "text": md}}
- )
- # Proactive delivery → robot-API markdown message
- try:
- token = await self._access_token(conn.app_id, self._app_secret(conn))
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- return await self._robot_send(
- token, conn, inbound, "sampleMarkdown", {"title": title, "text": md}
- )
-
- async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult:
- md = downgrade_for_dingtalk(content)
- chunks = chunk_text(md, self.caps.max_message_len) or [md]
- first: Optional[SendResult] = None
- for i, c in enumerate(chunks):
- r = await self.send_markdown(conn, inbound, c)
- if i == 0:
- first = r
- if not r.success:
- return r
- return first or SendResult.fail("unknown", "空内容")
-
- async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult:
- # DingTalk bots cannot edit messages — the upper-layer placeholder logic
- # therefore switches to "recall + resend" (see recall_message).
- return SendResult.fail("bad_format", "钉钉不支持编辑消息")
-
- # ── Placeholder messages: sent via robot API (recallable) → recall as an equivalent of "replace" ──
- async def send_placeholder(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
- """Send a recallable placeholder message via the robot API; message_id is the processQueryKey (recall credential).
-
- If the robot API is unavailable (no permission / token failure) → fall
- back to sessionWebhook plain text; then there is no message_id and the
- upper layer will not attempt recall — behavior degrades to the old
- version (placeholder stays, reply is sent fresh).
- """
- try:
- token = await self._access_token(conn.app_id, self._app_secret(conn))
- r = await self._robot_send(token, conn, inbound, "sampleText", {"content": text})
- if r.success:
- return r
- except Exception: # noqa: BLE001
- logger.debug("[dingtalk] 机器人占位发送失败,回退 webhook", exc_info=True)
- return await self.send_text(conn, inbound, text)
-
- def _recall_url_body(
- self, conn: Any, inbound: InboundMsg, message_id: str
- ) -> Optional[tuple]:
- """Pick the recall endpoint by conversation type. Group chats need openConversationId; one-to-one batchRecall needs only the key."""
- body: Dict[str, Any] = {"robotCode": conn.app_id, "processQueryKeys": [message_id]}
- if inbound.chat_type == "group":
- conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
- if not conv_id:
- return None
- body["openConversationId"] = conv_id
- return f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/recall", body
- return f"{DINGTALK_API_BASE}/v1.0/robot/otoMessages/batchRecall", body
-
- async def recall_message(self, conn: Any, inbound: InboundMsg, message_id: str) -> SendResult:
- """Silently recall a message sent via the robot API (the client shows no recall notice)."""
- if not message_id:
- return SendResult.fail("bad_format", "缺少 processQueryKey")
- target = self._recall_url_body(conn, inbound, message_id)
- if target is None:
- return SendResult.fail("bad_format", "缺少 openConversationId")
- url, body = target
- try:
- token = await self._access_token(conn.app_id, self._app_secret(conn))
- async with httpx.AsyncClient(timeout=15) as client:
- resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body)
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- if resp.status_code == 200:
- return SendResult.ok(message_id)
- return SendResult.fail("unknown", f"撤回失败: {resp.status_code} {str(data)[:200]}")
-
- # ── Outbound file delivery (sessionWebhook cannot send files → use the robot API) ──
- async def push_file(
- self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str
- ) -> SendResult:
- secret = self._app_secret(conn)
- if not conn.app_id or not secret:
- return SendResult.fail("forbidden", "缺少 AppKey / AppSecret")
- try:
- token = await self._access_token(conn.app_id, secret)
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
-
- is_image = (mime_type or "").startswith("image/")
- media_id = await self._upload_media(token, content, filename, "image" if is_image else "file")
- if isinstance(media_id, SendResult):
- return media_id
-
- if is_image:
- # sampleImageMsg's photoURL accepts a mediaId directly
- msg_key, msg_param = "sampleImageMsg", {"photoURL": media_id}
- else:
- ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() or "file"
- msg_key = "sampleFile"
- msg_param = {"mediaId": media_id, "fileName": filename, "fileType": ext}
- return await self._robot_send(token, conn, inbound, msg_key, msg_param)
-
- async def _upload_media(self, token: str, content: bytes, filename: str, media_type: str):
- """Upload media in exchange for a mediaId; returns SendResult on failure (str on success)."""
- url = f"https://oapi.dingtalk.com/media/upload?access_token={token}&type={media_type}"
- try:
- async with httpx.AsyncClient(timeout=60) as client:
- resp = await client.post(url, files={"media": (filename, content)})
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", f"媒体上传失败: {exc}")
- media_id = data.get("media_id")
- if data.get("errcode", 0) != 0 or not media_id:
- return SendResult.fail("unknown", f"媒体上传失败: errcode={data.get('errcode')} {data.get('errmsg')}")
- return media_id
-
- async def _robot_send(
- self, token: str, conn: Any, inbound: InboundMsg, msg_key: str, msg_param: Dict[str, Any]
- ) -> SendResult:
- """Robot proactive message: group chats use openConversationId, one-to-one uses the recipient's staffId."""
- body: Dict[str, Any] = {
- "robotCode": conn.app_id, # for internal enterprise apps, robotCode == appKey
- "msgKey": msg_key,
- "msgParam": json.dumps(msg_param, ensure_ascii=False),
- }
- if inbound.chat_type == "group":
- conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
- if not conv_id:
- return SendResult.fail("bad_format", "缺少 openConversationId")
- url = f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send"
- body["openConversationId"] = conv_id
- else:
- staff_id = inbound.sender_id or ""
- if staff_id:
- url = f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend"
- body["userIds"] = [staff_id]
- else:
- # In proactive-delivery scenarios no staffId is available → send directly via the human-bot one-to-one conversation's openConversationId
- conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
- if not conv_id:
- return SendResult.fail("bad_format", "缺少接收人 staffId / openConversationId")
- url = f"{DINGTALK_API_BASE}/v1.0/robot/privateChatMessages/send"
- body["openConversationId"] = conv_id
- try:
- async with httpx.AsyncClient(timeout=30) as client:
- resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body)
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- key = data.get("processQueryKey") or data.get("processQueryKeys")
- if resp.status_code == 200 and key:
- # The processQueryKey is the recall credential, returned to the upper layer as message_id (used by recall_message)
- if isinstance(key, list):
- key = key[0] if key else None
- return SendResult.ok(key if isinstance(key, str) else None)
- code = data.get("code") or resp.status_code
- kind = "forbidden" if resp.status_code == 403 else "unknown"
- return SendResult.fail(kind, f"机器人消息发送失败: {code} {str(data)[:200]}")
-
- # ── Long connection (Stream, requires the dingtalk-stream SDK) ──────
- def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any:
- """Build the DingTalk Stream long-connection runner. ``on_message`` is invoked synchronously on the SDK thread (dispatched to the main loop).
-
- dingtalk_stream is lazily imported; if not installed a RuntimeError is
- raised, which the manager records as an error without affecting the
- process. The SDK's ``start()`` is async and ``start_forever()`` cannot
- be stopped → wrap it in a controllable runner so the manager can start
- and stop cleanly (actually disconnecting when a bot is disabled).
- """
- try:
- import dingtalk_stream
- except Exception as exc: # noqa: BLE001
- raise RuntimeError("dingtalk-stream SDK 未安装,长连接不可用") from exc
-
- adapter = self
- app_id = conn.app_id
- secret = self._app_secret(conn)
-
- class _Handler(dingtalk_stream.ChatbotHandler):
- async def process(self, callback): # type: ignore[override]
- try:
- inbound = adapter.parse_inbound(conn, callback.data or {})
- if inbound is not None:
- on_message(inbound)
- except Exception: # noqa: BLE001
- logger.exception("[dingtalk] Stream 事件处理失败 channel_id=%s", conn.channel_id)
- return dingtalk_stream.AckMessage.STATUS_OK, "OK"
-
- def _build_client():
- sdk_logger = logging.getLogger("dingtalk_stream")
- sdk_logger.setLevel(logging.WARNING)
- credential = dingtalk_stream.Credential(app_id, secret)
- client = dingtalk_stream.DingTalkStreamClient(credential, logger=sdk_logger)
- client.register_callback_handler(
- dingtalk_stream.chatbot.ChatbotMessage.TOPIC, _Handler()
- )
- return client
-
- return _DingTalkStreamRunner(_build_client)
-
-
-class _DingTalkStreamRunner:
- """Wraps dingtalk-stream's async ``client.start()`` into the "blocking start() + stoppable stop()" the manager expects.
-
- Creates a dedicated event loop in the worker thread to run ``client.start()``
- (the SDK reconnects on its own); ``stop()`` closes the websocket and cancels
- the task from the main thread via ``call_soon_threadsafe``, making
- ``start()`` return → the manager's thread body sees stop_flag set and exits
- cleanly.
- """
-
- def __init__(self, build_client: Callable[[], Any]):
- self._build_client = build_client
- self._loop: Optional[asyncio.AbstractEventLoop] = None
- self._client: Any = None
- self._task: Optional[asyncio.Task] = None
-
- def start(self) -> None:
- loop = asyncio.new_event_loop()
- self._loop = loop
- asyncio.set_event_loop(loop)
- try:
- self._client = self._build_client()
- self._task = loop.create_task(self._client.start())
- loop.run_until_complete(self._task)
- except asyncio.CancelledError:
- pass
- finally:
- try:
- loop.run_until_complete(loop.shutdown_asyncgens())
- except Exception: # noqa: BLE001
- pass
- loop.close()
-
- def stop(self) -> None:
- loop, task, client = self._loop, self._task, self._client
- if loop is None or loop.is_closed():
- return
-
- def _cancel() -> None:
- ws = getattr(client, "websocket", None)
- if ws is not None:
- try:
- asyncio.ensure_future(ws.close())
- except Exception: # noqa: BLE001
- pass
- if task is not None:
- task.cancel()
-
- try:
- loop.call_soon_threadsafe(_cancel)
- except Exception: # noqa: BLE001
- pass
diff --git a/src/backend/core/channels/adapters/lark.py b/src/backend/core/channels/adapters/lark.py
deleted file mode 100644
index 4b5240f3..00000000
--- a/src/backend/core/channels/adapters/lark.py
+++ /dev/null
@@ -1,683 +0,0 @@
-"""Feishu (Lark) inbound channel adapter.
-
-- **Binding validation / outbound push**: Feishu OpenAPI REST (httpx) with a
- tenant_access_token; no SDK needed.
-- **Webhook decryption/signature verification**: AES-256-CBC (Encrypt Key) +
- signature check, pure cryptography; no SDK needed.
-- **Long connection (WebSocket)**: uses the official ``lark_oapi`` SDK's
- ws.Client (lazy import; if the SDK is missing only the long connection is
- unavailable, webhook/push are unaffected). Long connection is recommended in
- production: the user only needs App ID/Secret, no public callback URL.
-
-Conversation keying (``external_conversation_id``) always uses Feishu
-``message.chat_id``:
- - a p2p chat_id maps one-to-one to a user → naturally one conversation per DM peer;
- - a group chat_id is unique per group → the whole group shares one conversation.
-Outbound push also sends by chat_id (``receive_id_type=chat_id``), unified for p2p/group.
-
-See internal design docs.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import base64
-import hashlib
-import json
-import logging
-import os
-import re
-import threading
-import time
-from typing import Any, Callable, Dict, List, Optional
-
-import httpx
-
-from core.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult
-from core.infra.crypto import decrypt_secret
-
-logger = logging.getLogger(__name__)
-
-LARK_API_BASE = os.getenv("LARK_API_BASE", "https://open.feishu.cn").rstrip("/")
-_MENTION_RE = re.compile(r"@_user_\d+")
-
-
-# ── lark_oapi ws event-loop isolation (supports multiple coexisting bots) ────
-# Root cause: ``lark_oapi/ws/client.py`` runs everything on a single
-# **module-level global** ``loop`` (``loop.run_until_complete`` in ``start()``,
-# ``loop.create_task`` in ``_connect`` ...). Each bot's long connection runs in
-# its own worker thread, but they all share that one global → when the second
-# bot starts it overwrites the global loop with its own thread's loop, and the
-# first bot's already-running coroutines then call ``loop.create_task`` on
-# someone else's loop → "This event loop is already running" / cross-thread
-# crashes. Single-bot setups hit the same cause sporadically (the global gets
-# rewritten on reconnect/retry).
-# Fix: replace that global ``loop`` with a **per-thread-resolving proxy** — each
-# worker thread always resolves to its own thread's loop (prepare_ws_thread
-# writes it into thread-local storage), fully eliminating cross-thread sharing
-# and overwriting.
-_ws_loop_tls = threading.local()
-_ws_proxy_installed = False
-_ws_proxy_lock = threading.Lock()
-
-
-class _PerThreadLoopProxy:
- """Masquerades as an event loop: every attribute/method access is forwarded to the real loop bound to the **current thread**."""
-
- def __getattr__(self, name: str) -> Any:
- loop = getattr(_ws_loop_tls, "loop", None)
- if loop is None:
- loop = asyncio.get_event_loop()
- return getattr(loop, name)
-
-
-def _install_ws_loop_proxy() -> None:
- """One-time replacement of lark_oapi.ws.client's module-level global loop with the per-thread proxy (idempotent)."""
- global _ws_proxy_installed
- if _ws_proxy_installed:
- return
- with _ws_proxy_lock:
- if _ws_proxy_installed:
- return
- try:
- import lark_oapi.ws.client as _wsc
- _wsc.loop = _PerThreadLoopProxy()
- _ws_proxy_installed = True
- except Exception: # noqa: BLE001 — if the SDK is missing, the long connection is unavailable anyway
- pass
-
-
-class LarkAdapter:
- caps = ChannelCaps(
- channel_type="lark",
- max_message_len=4000, # Feishu's per-message text cap is fairly high; leave headroom for chunking
- supports_markdown=False, # uses text messages; rich text (post) left for later
- splits_long_messages=False,
- supports_long_conn=True,
- # Feishu gates group delivery with three mutually distinct scopes:
- # im:message.group_at_msg → only messages @-ing this bot
- # im:message.group_at_msg.include_bot:readonly → messages @-ing any bot
- # im:message.group_msg (sensitive) → every group message
- # Group listening needs the last one, plus republishing the app version; with the
- # others Feishu never pushes a non-@ message and observe_all stays inert. Note the
- # @-gate is worth running under the middle scope too: it stops us answering a message
- # that @'d somebody else's bot.
- supports_group_observe=True,
- )
-
- # tenant_access_token cache: {app_id: (token, expire_epoch)}
- _token_cache: Dict[str, tuple] = {}
- # The bot's own open_id: {app_id: open_id}. Stable per app → cached without expiry.
- _bot_id_cache: Dict[str, str] = {}
-
- # ── Credentials ─────────────────────────────────────────────────────
- @staticmethod
- def _app_secret(conn: Any) -> str:
- cfg = conn.config if isinstance(conn.config, dict) else {}
- return decrypt_secret(cfg.get("app_secret_enc")) or ""
-
- @staticmethod
- def _encrypt_key(conn: Any) -> str:
- cfg = conn.config if isinstance(conn.config, dict) else {}
- return decrypt_secret(cfg.get("encrypt_key_enc")) or ""
-
- @staticmethod
- def _verification_token(conn: Any) -> str:
- cfg = conn.config if isinstance(conn.config, dict) else {}
- return decrypt_secret(cfg.get("verification_token_enc")) or ""
-
- async def _tenant_token(self, app_id: str, app_secret: str) -> str:
- cached = self._token_cache.get(app_id)
- if cached and cached[1] > time.time() + 60:
- return cached[0]
- url = f"{LARK_API_BASE}/open-apis/auth/v3/tenant_access_token/internal"
- async with httpx.AsyncClient(timeout=10) as client:
- resp = await client.post(url, json={"app_id": app_id, "app_secret": app_secret})
- data = resp.json()
- if data.get("code") != 0:
- raise RuntimeError(f"飞书 tenant_access_token 获取失败: {data.get('msg') or data}")
- token = data["tenant_access_token"]
- self._token_cache[app_id] = (token, time.time() + int(data.get("expire", 7000)))
- return token
-
- async def validate_credentials(self, conn: Any) -> Dict[str, Any]:
- """Successfully obtaining a token means the credentials are valid; returns an identity summary. Raises on failure."""
- secret = self._app_secret(conn)
- if not conn.app_id or not secret:
- raise RuntimeError("缺少 App ID / App Secret")
- await self._tenant_token(conn.app_id, secret)
- return {"app_id": conn.app_id}
-
- # ── Webhook decryption / signature verification ─────────────────────
- def decrypt_webhook(self, conn: Any, body: bytes) -> Dict[str, Any]:
- """If an Encrypt Key is configured, the body looks like {"encrypt": "..."} and is AES-256-CBC decrypted; otherwise plain JSON."""
- try:
- raw = json.loads(body.decode("utf-8"))
- except Exception:
- return {}
- if "encrypt" not in raw:
- return raw
- key = self._encrypt_key(conn)
- if not key:
- logger.warning("[lark] 收到加密事件但未配 Encrypt Key")
- return {}
- return self._aes_decrypt(raw["encrypt"], key)
-
- @staticmethod
- def _aes_decrypt(encrypt_b64: str, encrypt_key: str) -> Dict[str, Any]:
- from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-
- key = hashlib.sha256(encrypt_key.encode("utf-8")).digest()
- data = base64.b64decode(encrypt_b64)
- iv, ct = data[:16], data[16:]
- decryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor()
- plain = decryptor.update(ct) + decryptor.finalize()
- plain = plain[: -plain[-1]] # strip PKCS7 padding
- return json.loads(plain.decode("utf-8"))
-
- def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool:
- """Feishu signature: sha256(timestamp + nonce + encrypt_key + body). Without an Encrypt Key, falls back to token verification."""
- key = self._encrypt_key(conn)
- sig = headers.get("x-lark-signature") or headers.get("X-Lark-Signature")
- ts = headers.get("x-lark-request-timestamp") or headers.get("X-Lark-Request-Timestamp")
- nonce = headers.get("x-lark-request-nonce") or headers.get("X-Lark-Request-Nonce")
- if key and sig and ts is not None and nonce is not None:
- h = hashlib.sha256((ts + nonce + key).encode("utf-8") + body).hexdigest()
- return h == sig
- # No encryption configured → fall back to verification_token (re-checked at the parse stage after decryption)
- return True
-
- # ── Event → InboundMsg ──────────────────────────────────────────────
- def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]:
- """Decrypted event dict → InboundMsg. Returns None for non-message events."""
- # schema 2.0: header.event_type; legacy schema: payload["event"]["type"]
- header = payload.get("header") or {}
- event_type = header.get("event_type") or payload.get("type")
- if event_type != "im.message.receive_v1":
- return None
- event = payload.get("event") or {}
- message = event.get("message") or {}
- sender = event.get("sender") or {}
- mtype = message.get("message_type")
- text, attachments = self._extract_payload(mtype, message.get("content"))
- if not text and not attachments:
- return None # unsupported message type (sticker/location, etc.)
- sender_open_id = (sender.get("sender_id") or {}).get("open_id", "")
- return self._make_inbound(
- conn=conn,
- chat_id=message.get("chat_id", ""),
- chat_type=message.get("chat_type", "p2p"),
- message_id=message.get("message_id", ""),
- text=text,
- sender_open_id=sender_open_id,
- sender_name=(sender.get("sender_id") or {}).get("user_id", "") or "",
- attachments=attachments,
- mentioned_ids=self._mentioned_ids(message),
- raw=payload,
- )
-
- @staticmethod
- def _mentioned_ids(message: Dict[str, Any]) -> List[str]:
- """open_ids of everyone @'d in this message (``message.mentions``).
-
- Only relevant once the app holds ``im:message.group_msg`` and Feishu starts pushing
- non-@ group messages too — until then every delivered group message has the bot in
- here anyway. The bot's own open_id is not known synchronously, so matching happens
- later in ``resolve_addressed``.
- """
- out: List[str] = []
- for m in message.get("mentions") or []:
- if not isinstance(m, dict):
- continue
- oid = (m.get("id") or {}).get("open_id") if isinstance(m.get("id"), dict) else None
- if oid:
- out.append(oid)
- return out
-
- @staticmethod
- def _strip_mentions(text: str) -> str:
- return _MENTION_RE.sub("", text or "").strip()
-
- @classmethod
- def _extract_text(cls, content: Any) -> str:
- if not content:
- return ""
- try:
- obj = json.loads(content) if isinstance(content, str) else content
- text = obj.get("text", "") if isinstance(obj, dict) else ""
- except Exception:
- text = str(content)
- return cls._strip_mentions(text)
-
- @classmethod
- def _extract_payload(cls, mtype: str, content: Any):
- """Extract (text, attachments) by message type.
-
- - text: plain text body (@ mentions stripped)
- - image: image_key → attachment(kind=image)
- - file/media/audio: file_key + file_name → attachment(kind=file)
- - post (rich text): extract text + embedded image_key
- """
- try:
- obj = json.loads(content) if isinstance(content, str) else (content or {})
- except Exception:
- obj = {}
- if not isinstance(obj, dict):
- obj = {}
- attachments = []
- if mtype == "text":
- return cls._strip_mentions(obj.get("text", "")), attachments
- if mtype == "image":
- key = obj.get("image_key")
- if key:
- attachments.append({"kind": "image", "key": key, "name": f"{key}.png"})
- return "", attachments
- if mtype in ("file", "media", "audio"):
- key = obj.get("file_key")
- if key:
- name = obj.get("file_name") or f"{key}.bin"
- attachments.append({"kind": "file", "key": key, "name": name})
- return "", attachments
- if mtype == "post": # rich text: extract text + embedded images
- texts, img_keys = [], []
- for node in cls._post_nodes(obj):
- if node.get("tag") == "text":
- texts.append(node.get("text", ""))
- elif node.get("tag") == "img" and node.get("image_key"):
- img_keys.append(node["image_key"])
- title = obj.get("title", "")
- body = " ".join(texts)
- text = (title + "\n" + body) if title else body
- attachments = [{"kind": "image", "key": k, "name": f"{k}.png"} for k in img_keys]
- return text.strip(), attachments
- return "", attachments
-
- @staticmethod
- def _post_nodes(obj: Dict[str, Any]):
- """Flatten all nodes of a post rich-text message (structure may be {content:[[node...]]} or wrapped in a language layer)."""
- content = obj.get("content") or []
- if isinstance(content, dict): # language layer wrapper {"zh_cn": {"content": [...]}}
- content = next(iter(content.values()), {}).get("content", []) if content else []
- for line in content or []:
- for node in line or []:
- if isinstance(node, dict):
- yield node
-
- @staticmethod
- def _make_inbound(
- *, conn: Any, chat_id: str, chat_type: str, message_id: str,
- text: str, sender_open_id: str, sender_name: str,
- raw: Dict[str, Any], attachments=None, mentioned_ids=None,
- ) -> InboundMsg:
- return InboundMsg(
- channel_id=conn.channel_id,
- channel_type="lark",
- text=text,
- chat_type=chat_type,
- external_conversation_id=chat_id, # always keyed by chat_id
- sender_id=sender_open_id,
- sender_name=sender_name,
- message_id=message_id,
- attachments=attachments or [],
- # p2p is always aimed at the bot; in a group it depends on whether the bot's own
- # open_id is in the mention list — resolved asynchronously in resolve_addressed.
- addressed_to_bot=True if chat_type == "p2p" else None,
- mentioned_ids=list(mentioned_ids or []),
- raw={"lark_chat_id": chat_id, "lark_message_id": message_id},
- )
-
- # ── Group history pull (app credentials, independent of group listening) ──
- async def fetch_history(
- self,
- conn: Any,
- conversation_id: str,
- *,
- since_ms: int,
- limit: int,
- newest_first: bool = False,
- ) -> List[HistoryItem]:
- """GET /open-apis/im/v1/messages — this chat's own messages, oldest-first.
-
- Uses the tenant token (app credentials), so it is a property of the bound app rather
- than of any user's session. Note this sits behind a *different* permission from event
- push: an app that is not allowed to receive non-@ group messages may still be allowed
- to read the chat's history, which is precisely why pulling is worth having.
-
- Feishu takes ``start_time`` in **seconds**; the caller works in milliseconds.
- Any failure returns [] — the caller treats "unsupported" and "failed" alike.
- """
- chat_id = (conversation_id or "").strip()
- if not chat_id:
- return []
- try:
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- params = {
- "container_id_type": "chat",
- "container_id": chat_id,
- # Desc for the cold start so the first page holds the *newest* messages of
- # the window; Asc when walking forward from the cursor so nothing is skipped.
- "sort_type": "ByCreateTimeDesc" if newest_first else "ByCreateTimeAsc",
- "page_size": str(max(1, min(int(limit or 50), 50))), # Feishu caps page_size at 50
- }
- if since_ms > 0:
- params["start_time"] = str(int(since_ms // 1000))
- async with httpx.AsyncClient(timeout=15) as client:
- resp = await client.get(
- f"{LARK_API_BASE}/open-apis/im/v1/messages",
- params=params,
- headers={"Authorization": f"Bearer {token}"},
- )
- body = resp.json() or {}
- if resp.status_code != 200 or body.get("code") not in (0, None):
- logger.warning(
- "[lark] 群历史拉取失败 chat=%s code=%s msg=%s",
- chat_id, body.get("code"), str(body.get("msg"))[:120],
- )
- return []
- return [
- item
- for item in (
- self._history_item(raw) for raw in (body.get("data") or {}).get("items") or []
- )
- if item is not None
- ]
- except Exception: # noqa: BLE001
- logger.warning("[lark] 群历史拉取异常 chat=%s", chat_id, exc_info=True)
- return []
-
- @classmethod
- def _history_item(cls, raw: Dict[str, Any]) -> Optional[HistoryItem]:
- """One ``im/v1/messages`` entry → HistoryItem, reusing the inbound content parser."""
- if not isinstance(raw, dict) or raw.get("deleted"):
- return None
- body = raw.get("body") or {}
- text, attachments = cls._extract_payload(raw.get("msg_type"), body.get("content"))
- if not text and not attachments:
- return None
- sender = raw.get("sender") or {}
- message_id = raw.get("message_id") or ""
- try:
- ts_ms = int(raw.get("create_time") or 0)
- except (TypeError, ValueError):
- ts_ms = 0
- return HistoryItem(
- message_id=message_id,
- # Feishu returns only ids here; a display name would need a directory lookup per
- # sender, which is not worth an extra API call per message for background context.
- sender_name=(sender.get("id") or "")[-6:] or "群成员",
- text=text,
- ts_ms=ts_ms,
- attachments=attachments,
- # download_resource resolves resources by message_id — carry it so a file pulled
- # from history stays fetchable later via channel_read_attachment.
- raw={"lark_message_id": message_id},
- )
-
- # ── Group listening: is this group message @'ing the bot? ───────────
- async def resolve_addressed(self, conn: Any, inbound: InboundMsg) -> bool:
- """True when the bot's own open_id appears in the message's mention list.
-
- Fails open (True) whenever the bot identity cannot be fetched, so a transient
- ``bot/v3/info`` failure degrades to today's behavior (answer everything delivered)
- rather than making the bot go mute in the group.
- """
- try:
- bot_open_id = await self._bot_open_id(conn)
- except Exception: # noqa: BLE001
- logger.debug("[lark] 机器人 open_id 获取失败,按「已被 @」处理", exc_info=True)
- return True
- if not bot_open_id:
- return True
- return bot_open_id in (inbound.mentioned_ids or [])
-
- async def _bot_open_id(self, conn: Any) -> str:
- """The bot's own open_id (``/open-apis/bot/v3/info``), cached per app_id.
-
- Feishu never puts the bot identity in the event payload, so it has to be fetched once
- and kept. The value is stable for the lifetime of the app, hence a plain process cache
- with no expiry (a restart just re-fetches).
- """
- cached = self._bot_id_cache.get(conn.app_id)
- if cached:
- return cached
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- async with httpx.AsyncClient(timeout=10) as client:
- resp = await client.get(
- f"{LARK_API_BASE}/open-apis/bot/v3/info",
- headers={"Authorization": f"Bearer {token}"},
- )
- open_id = ((resp.json() or {}).get("bot") or {}).get("open_id") or ""
- if open_id:
- self._bot_id_cache[conn.app_id] = open_id
- return open_id
-
- # ── Outbound push ───────────────────────────────────────────────────
- def _chat_id(self, inbound: InboundMsg) -> str:
- return (inbound.raw or {}).get("lark_chat_id") or inbound.external_conversation_id
-
- async def _post_message(self, conn: Any, chat_id: str, msg_type: str, content: Dict[str, Any]) -> SendResult:
- if not chat_id:
- return SendResult.fail("bad_format", "缺少 chat_id")
- try:
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- url = f"{LARK_API_BASE}/open-apis/im/v1/messages?receive_id_type=chat_id"
- payload = {"receive_id": chat_id, "msg_type": msg_type,
- "content": json.dumps(content, ensure_ascii=False)}
- try:
- async with httpx.AsyncClient(timeout=15) as client:
- resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- if data.get("code") == 0:
- return SendResult.ok((data.get("data") or {}).get("message_id"))
- code = data.get("code")
- kind = "rate_limited" if code in (99991400, 99991401) else "forbidden" if code in (230002, 230013) else "unknown"
- return SendResult.fail(kind, f"code={code} msg={data.get('msg')}")
-
- async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
- return await self._post_message(conn, self._chat_id(inbound), "text", {"text": text})
-
- async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult:
- """Over-length content is auto-chunked by caps.max_message_len and sent piece by piece."""
- from core.channels.protocol import chunk_text
-
- chunks = chunk_text(content, self.caps.max_message_len) or [content]
- first: Optional[SendResult] = None
- cont: list = []
- for i, c in enumerate(chunks):
- r = await self.send_text(conn, inbound, c)
- if i == 0:
- first = r
- if not r.success:
- return r
- elif r.message_id:
- cont.append(r.message_id)
- if first is not None:
- first.continuation_ids = cont
- return first
- return SendResult.fail("unknown", "空内容")
-
- async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult:
- if not message_id:
- return SendResult.fail("bad_format", "缺少 message_id")
- try:
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{message_id}"
- payload = {"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)}
- try:
- async with httpx.AsyncClient(timeout=15) as client:
- resp = await client.put(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
- data = resp.json()
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
- if data.get("code") == 0:
- return SendResult.ok(message_id)
- return SendResult.fail("unknown", f"code={data.get('code')} msg={data.get('msg')}")
-
- # ── Inbound file download ───────────────────────────────────────────
- async def download_resource(
- self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any]
- ) -> Optional[bytes]:
- """Download a resource file from a message (requires the im:resource permission).
-
- GET /open-apis/im/v1/messages/{message_id}/resources/{file_key}?type=file|image
- """
- msg_id = (inbound.raw or {}).get("lark_message_id") or inbound.message_id
- key = attachment.get("key")
- res_type = "image" if attachment.get("kind") == "image" else "file"
- if not msg_id or not key:
- return None
- try:
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{msg_id}/resources/{key}?type={res_type}"
- async with httpx.AsyncClient(timeout=30) as client:
- resp = await client.get(url, headers={"Authorization": f"Bearer {token}"})
- if resp.status_code == 200 and not resp.headers.get("content-type", "").startswith("application/json"):
- return resp.content
- logger.warning("[lark] 资源下载失败 msg=%s key=%s body=%s", msg_id, key, resp.text[:200])
- except Exception: # noqa: BLE001
- logger.exception("[lark] 资源下载异常 msg=%s key=%s", msg_id, key)
- return None
-
- # ── Outbound file delivery ──────────────────────────────────────────
- async def push_file(
- self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str
- ) -> SendResult:
- chat_id = self._chat_id(inbound)
- if not chat_id:
- return SendResult.fail("bad_format", "缺少 chat_id")
- try:
- token = await self._tenant_token(conn.app_id, self._app_secret(conn))
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", str(exc))
-
- is_image = (mime_type or "").startswith("image/")
- try:
- if is_image:
- key = await self._upload_image(token, content)
- msg_type, msg_content = "image", {"image_key": key}
- else:
- key = await self._upload_file(token, content, filename)
- msg_type, msg_content = "file", {"file_key": key}
- except Exception as exc: # noqa: BLE001
- return SendResult.fail("transient", f"上传失败: {exc}")
- if not key:
- return SendResult.fail("unknown", "未拿到资源 key")
- return await self._post_message(conn, chat_id, msg_type, msg_content)
-
- async def _upload_image(self, token: str, content: bytes) -> Optional[str]:
- url = f"{LARK_API_BASE}/open-apis/im/v1/images"
- async with httpx.AsyncClient(timeout=30) as client:
- resp = await client.post(
- url, headers={"Authorization": f"Bearer {token}"},
- data={"image_type": "message"},
- files={"image": ("img.png", content)},
- )
- data = resp.json()
- return (data.get("data") or {}).get("image_key") if data.get("code") == 0 else None
-
- async def _upload_file(self, token: str, content: bytes, filename: str) -> Optional[str]:
- # file_type: Feishu distinguishes by extension; anything not in the known set is sent as stream
- ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower()
- known = {"opus", "mp4", "pdf", "doc", "xls", "ppt"}
- file_type = ext if ext in known else "stream"
- url = f"{LARK_API_BASE}/open-apis/im/v1/files"
- async with httpx.AsyncClient(timeout=60) as client:
- resp = await client.post(
- url, headers={"Authorization": f"Bearer {token}"},
- data={"file_type": file_type, "file_name": filename},
- files={"file": (filename, content)},
- )
- data = resp.json()
- return (data.get("data") or {}).get("file_key") if data.get("code") == 0 else None
-
- # ── Long connection (WebSocket, requires the lark_oapi SDK) ─────────
- def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any:
- """Build the Feishu long-connection client. ``on_message`` is invoked synchronously on the SDK thread; the caller dispatches it to the main loop.
-
- lark_oapi is lazily imported; if not installed a RuntimeError is raised,
- which the manager records as an error state without affecting the process.
- """
- try:
- import lark_oapi as lark
- from lark_oapi.api.im.v1 import P2ImMessageReceiveV1
- except Exception as exc: # noqa: BLE001
- raise RuntimeError("lark_oapi SDK 未安装,长连接不可用") from exc
-
- secret = self._app_secret(conn)
-
- def _do_message(data: "P2ImMessageReceiveV1") -> None:
- try:
- inbound = self._inbound_from_sdk_event(conn, data)
- if inbound is not None:
- on_message(inbound)
- except Exception: # noqa: BLE001
- logger.exception("[lark] 长连接事件处理失败 channel_id=%s", conn.channel_id)
-
- handler = (
- lark.EventDispatcherHandler.builder("", "")
- .register_p2_im_message_receive_v1(_do_message)
- .build()
- )
- return lark.ws.Client(
- conn.app_id, secret, event_handler=handler, log_level=lark.LogLevel.WARNING
- )
-
- def prepare_ws_thread(self, loop: Any) -> None:
- """Called inside the worker thread: install the per-thread loop proxy + write this thread's loop into thread-local storage.
-
- From then on, every access to the global ``loop`` inside
- lark_oapi.ws.client is forwarded by the proxy to **this thread's** loop —
- multiple Feishu bots each get their own thread and their own loop with
- no mutual overwriting (fixes "This event loop is already running"). The
- proxy is installed only once (process-level idempotent); each thread
- writes its own thread-local value.
- """
- _install_ws_loop_proxy()
- _ws_loop_tls.loop = loop
-
- def _inbound_from_sdk_event(self, conn: Any, data: Any) -> Optional[InboundMsg]:
- """lark_oapi's typed P2ImMessageReceiveV1 event → InboundMsg."""
- event = getattr(data, "event", None)
- message = getattr(event, "message", None)
- sender = getattr(event, "sender", None)
- if message is None:
- return None
- mtype = getattr(message, "message_type", None)
- text, attachments = self._extract_payload(mtype, getattr(message, "content", None))
- if not text and not attachments:
- return None
- sender_id_obj = getattr(sender, "sender_id", None)
- open_id = getattr(sender_id_obj, "open_id", "") if sender_id_obj else ""
- return self._make_inbound(
- conn=conn,
- chat_id=getattr(message, "chat_id", "") or "",
- chat_type=getattr(message, "chat_type", "p2p") or "p2p",
- message_id=getattr(message, "message_id", "") or "",
- text=text,
- sender_open_id=open_id or "",
- sender_name="",
- attachments=attachments,
- mentioned_ids=self._sdk_mentioned_ids(message),
- raw={},
- )
-
- @staticmethod
- def _sdk_mentioned_ids(message: Any) -> List[str]:
- """SDK-typed counterpart of ``_mentioned_ids`` (``MentionEvent.id.open_id``).
-
- The long connection is the production path, so mention extraction must exist on both
- sides; missing it here would make every group message look un-@'d.
- """
- out: List[str] = []
- for m in getattr(message, "mentions", None) or []:
- ident = getattr(m, "id", None)
- oid = getattr(ident, "open_id", "") if ident is not None else ""
- if oid:
- out.append(oid)
- return out
diff --git a/src/backend/core/channels/inbound.py b/src/backend/core/channels/inbound.py
deleted file mode 100644
index d8915037..00000000
--- a/src/backend/core/channels/inbound.py
+++ /dev/null
@@ -1,1005 +0,0 @@
-"""Inbound message orchestration: InboundMsg → reuse the chat pipeline as the owner → send the reply back to the channel.
-
-Owner service-account model:
- - Always runs as the bot's ``owner_user_id`` (no group-member resolution, no team hookup).
- - p2p / group share one path; the only difference is session keying:
- p2p → (channel_id, sender open_id) one session per private-chat peer
- group → (channel_id, group chat_id) the whole group shares one session
- - The resource_scope whitelist (if any) narrows the owner's full capabilities to the specified KBs / skills.
- - Speaker open_id / nickname are recorded for audit only, never mapped to a platform account.
-
-Reuses ``chat_run_executor.start_run`` + ``follow_run`` with zero changes to the orchestration layer.
-See internal design docs.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import mimetypes
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional
-
-from core.channels.protocol import InboundMsg
-from core.channels.registry import get_adapter
-from core.chat.context import resolve_enabled_capabilities
-from core.db.engine import SessionLocal
-from core.db.models import Artifact, ChannelConnection, ChatSession
-from core.db.repository.channel import ChannelConnectionRepository
-from core.services.chat_service import ChatService
-
-logger = logging.getLogger(__name__)
-
-# Inbound file size cap (same as /v1/file/upload)
-_MAX_INBOUND_FILE_BYTES = 50 * 1024 * 1024
-
-# #1 Per-conversation serialization: one asyncio lock per (channel_id, conversation); messages
-# arriving while a run is in progress queue up. handle_inbound always runs on the main event loop,
-# so asyncio.Lock suffices. Different conversations have their own locks → still concurrent.
-_conv_locks: "defaultdict[str, asyncio.Lock]" = defaultdict(asyncio.Lock)
-
-
-class ChannelRunError(RuntimeError):
- """A user-facing terminal error emitted by the shared chat-run stream."""
-
-
-def _conv_key(msg: "InboundMsg") -> str:
- return f"{msg.channel_id}:{msg.external_conversation_id}"
-
-# In-process idempotent dedup: channels redeliver events (webhook retries / long-connection
-# reconnect replays). Deduplicate by message_id with a bounded LRU (enough to cover short-window
-# redelivery; strict cross-process exactly-once is not a goal).
-_SEEN_MAX = 4096
-_seen_message_ids: "OrderedDict[str, None]" = OrderedDict()
-
-
-def _already_handled(message_id: str) -> bool:
- if not message_id:
- return False
- if message_id in _seen_message_ids:
- return True
- _seen_message_ids[message_id] = None
- if len(_seen_message_ids) > _SEEN_MAX:
- _seen_message_ids.popitem(last=False)
- return False
-
-
-# Channel-side "new conversation / clear context" commands. Only triggers when the whole
-# message (after trim) exactly equals one of them, so normal sentences like "帮我清空购物车"
-# ("clear my shopping cart") are never falsely matched.
-_RESET_COMMANDS = frozenset({
- "/new", "/clear", "/reset", "/restart",
- "新对话", "新会话", "清空", "清空上下文", "清除上下文",
- "重置", "重置对话", "重新开始",
-})
-
-
-def _is_reset_command(text: Optional[str]) -> bool:
- return (text or "").strip().lower() in _RESET_COMMANDS
-
-
-def _reset_session(db, conn: ChannelConnection, msg: InboundMsg) -> None:
- """Soft-delete the current channel session. The next non-command message creates a fresh empty session via _find_or_create_session."""
- existing = (
- db.query(ChatSession)
- .filter(
- ChatSession.channel_id == conn.channel_id,
- ChatSession.external_conversation_id == msg.external_conversation_id,
- ChatSession.deleted_at.is_(None),
- )
- .first()
- )
- if existing is not None:
- ChatService(db).delete_session_force(
- existing.chat_id, actor_user_id=conn.owner_user_id
- )
-
-
-def _find_or_create_session(
- db, conn: ChannelConnection, msg: InboundMsg
-) -> ChatSession:
- """Reuse or create a session keyed by (channel_id, external_conversation_id), owned by the owner."""
- existing = (
- db.query(ChatSession)
- .filter(
- ChatSession.channel_id == conn.channel_id,
- ChatSession.external_conversation_id == msg.external_conversation_id,
- # Exclude soft-deleted sessions: the channel-side /new clear works by soft-deleting
- # the current session — without this filter the next message would hit the old
- # session again, making the clear a no-op.
- ChatSession.deleted_at.is_(None),
- )
- .first()
- )
- if existing is not None:
- # Backfill the p2p peer id: proactive delivery (automation etc.) needs it to locate
- # the recipient when sending back in a private chat; old sessions predate this field,
- # so we backfill on every inbound message.
- if msg.chat_type == "p2p" and msg.sender_id:
- meta = dict(existing.extra_data or {})
- if meta.get("channel_peer_id") != msg.sender_id:
- meta["channel_peer_id"] = msg.sender_id
- existing.extra_data = meta
- db.commit()
- return existing
-
- chat_service = ChatService(db)
- title = (msg.text or "渠道会话").strip()[:40] or "渠道会话"
- extra: Dict[str, Any] = {
- "source": f"channel:{conn.channel_type}",
- "channel_chat_type": msg.chat_type,
- }
- if msg.chat_type == "p2p" and msg.sender_id:
- extra["channel_peer_id"] = msg.sender_id
- session = chat_service.create_session(
- user_id=conn.owner_user_id,
- title=title,
- extra_data=extra,
- )
- session.channel_id = conn.channel_id
- session.external_conversation_id = msg.external_conversation_id
- db.commit()
- db.refresh(session)
- return session
-
-
-def _load_history(db, chat_id: str, owner_id: str) -> List[Dict[str, Any]]:
- """Load history as session_messages — same source as the web path (checkpoint-aware + keeps tool calls/results).
-
- Goes through ``compaction_service.load_session_history`` (the same entry point as the web
- pipeline), not the old "``list_all_messages`` + keep only user/assistant plain text". The
- old approach **dropped wholesale** the assistant turns' ``tool_calls`` and tool results, as
- well as empty-text pure-tool turns — so in multi-turn channel sessions the model couldn't
- see which tools it called last turn or what results it got. With thinking disabled it would
- very easily redo the same work from scratch, degenerating into "I'll get right on generating…"
- style idling until hitting max_iters. With the same-source loader, the model gets real tool
- context across turns and reuses compaction checkpoints (large sessions no longer replay in full).
-
- When there is no access permission (theoretically impossible — the session was just created
- by _find_or_create_session and belongs to the owner), load_session_history returns None;
- we fall back to empty history here.
- """
- from core.services.compaction_service import load_session_history
-
- chat_service = ChatService(db)
- return load_session_history(chat_service, chat_id, owner_id) or []
-
-
-def _resolve_enabled(
- db, conn: ChannelConnection, owner_id: str
-) -> Dict[str, Optional[List[str]]]:
- """Resolve the owner's capabilities, then narrow them by the resource_scope whitelist."""
- skills, agents, mcps = resolve_enabled_capabilities(db, owner_id)
- scope = conn.resource_scope if isinstance(conn.resource_scope, dict) else {}
- scoped_skills = scope.get("skill_ids")
- scoped_kbs = scope.get("kb_ids")
- return {
- # Whitelist present → narrow to it (the owner must still own these; catalog gating applies as usual)
- "enabled_skills": scoped_skills if isinstance(scoped_skills, list) else skills,
- "enabled_agents": agents,
- "enabled_mcps": mcps,
- "enabled_kbs": scoped_kbs if isinstance(scoped_kbs, list) else None,
- }
-
-
-async def _ingest_attachments(
- db, adapter, conn: ChannelConnection, owner_id: str, chat_id: str, msg: InboundMsg
-) -> List[Dict[str, Any]]:
- """Download inbound attachments → store as Artifacts (mirrors /v1/file/upload) → return uploaded_files items.
-
- Each item contains metadata only. Parsing is deferred to the shared
- file-context/read_artifact path and cached on first use.
- """
- download = getattr(adapter, "download_resource", None)
- if download is None or not msg.attachments:
- return []
- from core.services.artifact_service import store_bytes_as_artifact
-
- out: List[Dict[str, Any]] = []
- for att in msg.attachments:
- try:
- content = await download(conn, msg, att)
- if not content or len(content) > _MAX_INBOUND_FILE_BYTES:
- continue
- name = att.get("name") or "file.bin"
- mime = mimetypes.guess_type(name)[0] or "application/octet-stream"
- art = store_bytes_as_artifact(
- db, user_id=owner_id, content=content, filename=name, mime_type=mime,
- chat_id=chat_id, source="channel_upload",
- extra={"channel_id": conn.channel_id},
- )
- out.append({
- "file_id": art.artifact_id,
- "download_url": f"/files/{art.artifact_id}",
- "name": name,
- "mime_type": mime,
- })
- except Exception: # noqa: BLE001
- logger.exception("[channels] 入站附件处理失败 key=%s", att.get("key"))
- return out
-
-
-async def _collect_reply(run_id: str):
- """Follow the run's event stream, accumulating assistant text + capturing artifact files generated this turn (meta.artifacts).
-
- Structured-reasoning models emit thinking as separate ``thinking`` events (ignored
- here), but inline-thinking models embed ``… `` in the content deltas
- themselves — strip those before handing the text to the channel, or the raw chain
- of thought goes out as message text (the web UI parses the tags; IM cannot).
- """
- from core.channels.markdown import strip_inline_thinking
- from orchestration import chat_run_executor
-
- full = ""
- artifacts: List[Dict[str, Any]] = []
- from core.channels.registry import current_relay_adapter
-
- relay = current_relay_adapter()
- async for event in chat_run_executor.follow_run(run_id):
- et = event.get("type")
- if relay and et in {"file_confirm", "ask_user_question"}:
- relay.error = "此操作需要确认,请在桌面端处理后重试"
- await relay.cancel()
- raise ChannelRunError(relay.error)
- if et == "content":
- full += event.get("delta", "") or ""
- elif et == "meta":
- arts = event.get("artifacts")
- if isinstance(arts, list):
- artifacts = arts
- elif et == "error":
- detail = str(event.get("error") or event.get("delta") or "").strip()
- raise ChannelRunError(detail or "请求处理失败,请稍后重试")
- return strip_inline_thinking(full).strip(), artifacts
-
-
-def _load_generated_files(artifacts: List[Dict[str, Any]]):
- """Read the bytes of artifact files generated this turn; returns a list of (content, name, mime) for pushing back (best-effort)."""
- if not artifacts:
- return []
- from core.storage import get_storage
-
- files = []
- with SessionLocal() as db:
- storage = get_storage()
- for art in artifacts:
- fid = art.get("file_id")
- if not fid:
- continue
- row = db.query(Artifact).filter(Artifact.artifact_id == fid).first()
- if row is None or not row.storage_key:
- continue
- try:
- content = storage.download_bytes(row.storage_key)
- except Exception: # noqa: BLE001
- logger.warning("[channels] 产物下载失败 %s", fid, exc_info=True)
- continue
- files.append((content, row.filename or art.get("name") or fid,
- row.mime_type or "application/octet-stream"))
- return files
-
-
-def _speaker_label(msg: InboundMsg) -> str:
- """Label the speaker in group scenarios (no directory lookup; falls back to existing fields)."""
- return (msg.sender_name or (msg.sender_id or "")[-6:] or "用户")
-
-
-# ── Group listening (observe_all): bystander messages as background context ──────────────
-# Cap on the buffered bystander messages carried into the next @bot turn. A busy group can
-# produce hundreds of messages between two @s; without a cap the buffer would blow up both the
-# JSON column and the prompt. Oldest entries are evicted first — recent context is what matters.
-_OBSERVE_MAX = 60
-# Per-message truncation. Long pasted blocks are background material, not the task.
-_OBSERVE_TEXT_MAX = 500
-# Cap on the lazy-fetch file index. Larger than _OBSERVE_MAX because a file stays worth
-# opening long after the chatter around it scrolled out of the message buffer.
-_OBSERVE_FILE_INDEX_MAX = 100
-
-# ── Deterministic history pull budget ────────────────────────────────────────────────────
-# Group history is pulled by code on every @, never by the model deciding to call a tool:
-# a capability that only works when the model remembers to use it is not a capability.
-# Every pull is incremental — bounded below by the stored cursor — so the same messages are
-# not fetched twice; the window below only applies to the very first pull in a conversation,
-# where there is no cursor yet.
-_HISTORY_COLDSTART_WINDOW_H = 168 # how far back the first pull reaches (7 days)
-_HISTORY_MAX_MESSAGES = 80 # hard cap per pull
-_HISTORY_MAX_TOTAL_CHARS = 12000 # total text budget per pull, so one chatty group can't flood the prompt
-_HISTORY_TIMEOUT_S = 10 # a slow platform must never hold up the reply
-# Consecutive failures after which we stop retrying for a conversation. Without this, a
-# permanently unavailable API would add its timeout to every single @ forever.
-_HISTORY_MAX_FAILURES = 3
-
-
-async def _resolve_addressed(adapter, conn: ChannelConnection, msg: InboundMsg) -> bool:
- """Is this message aimed at the bot? Adapters that cannot decide synchronously defer here.
-
- Fails open (True) on any resolver error or when the adapter has no resolver at all, so
- channels that never learned about group listening (WeCom / WeChat) keep behaving exactly
- as before.
- """
- if msg.addressed_to_bot is not None:
- return bool(msg.addressed_to_bot)
- resolver = getattr(adapter, "resolve_addressed", None)
- if resolver is None:
- return True
- try:
- return bool(await resolver(conn, msg))
- except Exception: # noqa: BLE001
- logger.debug("[channels] @ 判定失败,按「已被 @」处理 channel_id=%s", conn.channel_id, exc_info=True)
- return True
-
-
-def _observe_text(msg: InboundMsg) -> str:
- """The bystander message's own text, truncated. Attachments are handled separately."""
- text = (msg.text or "").strip()
- if len(text) > _OBSERVE_TEXT_MAX:
- text = text[:_OBSERVE_TEXT_MAX] + "…"
- return text
-
-
-def _observe_attachment_refs(msg: InboundMsg) -> List[Dict[str, Any]]:
- """Attachment handles for a bystander message — recorded, never downloaded.
-
- Eagerly downloading everything a group shares would pull the whole group's file traffic
- into our storage: large I/O and a far wider privacy footprint than reading text. But
- *discarding the key* (the earlier behavior) left the agent able to see that a file existed
- while being unable to ever open it. Keeping the key costs nothing and turns this into
- lazy loading: the model asks for a file by key via ``channel_read_attachment`` only when
- it decides the file matters.
- """
- return [
- {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""}
- for a in (msg.attachments or [])
- if a.get("key")
- ]
-
-
-def _observe_message(db, conn: ChannelConnection, msg: InboundMsg) -> None:
- """Record a bystander group message into the session's observation buffer. No agent run, no reply.
-
- The buffer lives in ``ChatSession.extra_data`` rather than being written as real session
- messages, for three reasons: it stays bounded (a quiet group can't inflate history), it
- avoids long runs of consecutive user-role turns in the model's message list, and draining
- it into the triggering turn keeps the group log in one coherent block instead of scattered
- fragments.
- """
- text = _observe_text(msg)
- refs = _observe_attachment_refs(msg)
- if not text and not refs:
- return
- session = _find_or_create_session(db, conn, msg)
- meta = dict(session.extra_data or {})
-
- entry: Dict[str, Any] = {"n": _speaker_label(msg), "t": text}
- if refs:
- entry["a"] = refs
- buf = list(meta.get("observed") or [])
- buf.append(entry)
- meta["observed"] = buf[-_OBSERVE_MAX:]
-
- if refs:
- # Separate, longer-lived index than the message buffer: the buffer is drained into a
- # prompt on the next @, but the model may only decide to open a file several turns
- # later. Keyed by attachment key, it stores what the adapter needs to fetch the bytes
- # (Lark resolves resources by message_id, DingTalk by the robotCode in raw) so none of
- # that plumbing has to appear in the prompt.
- index = dict(meta.get("observed_files") or {})
- for ref in refs:
- index.pop(ref["key"], None) # re-insert so recency ordering stays correct
- index[ref["key"]] = {
- "name": ref["name"], "kind": ref["kind"],
- "message_id": msg.message_id, "raw": dict(msg.raw or {}),
- }
- while len(index) > _OBSERVE_FILE_INDEX_MAX:
- index.pop(next(iter(index)))
- meta["observed_files"] = index
-
- session.extra_data = meta # reassign: in-place JSON mutation is not change-tracked
- db.commit()
-
-
-def _record_observed_batch(
- db, session: ChatSession, items: List[Dict[str, Any]], file_index: Dict[str, Any]
-) -> None:
- """Append a pulled batch to the observation buffer + file index, respecting the caps."""
- meta = dict(session.extra_data or {})
- buf = list(meta.get("observed") or [])
- buf.extend(items)
- meta["observed"] = buf[-_OBSERVE_MAX:]
- if file_index:
- index = dict(meta.get("observed_files") or {})
- for key, entry in file_index.items():
- index.pop(key, None)
- index[key] = entry
- while len(index) > _OBSERVE_FILE_INDEX_MAX:
- index.pop(next(iter(index)))
- meta["observed_files"] = index
- session.extra_data = meta
- db.commit()
-
-
-def _history_state(session: ChatSession) -> Dict[str, Any]:
- state = (session.extra_data or {}).get("history_pull")
- return dict(state) if isinstance(state, dict) else {}
-
-
-def _save_history_state(db, session: ChatSession, state: Dict[str, Any]) -> None:
- meta = dict(session.extra_data or {})
- meta["history_pull"] = state
- session.extra_data = meta
- db.commit()
-
-
-async def _pull_history(
- db, adapter, conn: ChannelConnection, msg: InboundMsg, session: ChatSession
-) -> int:
- """Deterministically pull this group's new messages and fold them into the buffer.
-
- Runs on every @ in a group, driven by code rather than by the model choosing to call a
- tool. Incremental by construction: the cursor stored on the session is the lower bound, so
- a pull only ever returns messages we have not already taken. The first pull in a
- conversation has no cursor and instead reaches back ``_HISTORY_COLDSTART_WINDOW_H`` hours
- — that is the cold start, and it is also what makes content from *before the bot joined*
- reachable at all.
-
- Deliberately independent of group listening: pushing and pulling sit behind different
- platform permissions, so a channel that will not push non-@ messages may still allow the
- pull. Returns the number of messages recorded.
- """
- fetch = getattr(adapter, "fetch_history", None)
- if fetch is None:
- return 0
- state = _history_state(session)
- if int(state.get("failures") or 0) >= _HISTORY_MAX_FAILURES:
- return 0
-
- cursor = int(state.get("cursor_ms") or 0)
- cold_start = not cursor
- if cold_start:
- # Timezone-aware on purpose: this value is compared against platform epoch-ms, and
- # the naive-utcnow habit elsewhere in this codebase is exactly what makes stored
- # timestamps read 8 hours off.
- cursor = int(
- (datetime.now(timezone.utc) - timedelta(hours=_HISTORY_COLDSTART_WINDOW_H)).timestamp() * 1000
- )
- seen_ids = set(state.get("seen_ids") or [])
-
- try:
- items = await asyncio.wait_for(
- fetch(
- conn, msg.external_conversation_id,
- since_ms=cursor, limit=_HISTORY_MAX_MESSAGES,
- # Cold start takes the newest end of the window: with a 7-day window and a
- # per-pull cap, taking the oldest would surface week-old chatter and leave
- # the recent messages — what the conversation is actually about — unread.
- newest_first=cold_start,
- ),
- timeout=_HISTORY_TIMEOUT_S,
- )
- # Adapters may return either order; the buffer must read chronologically. Items
- # without a timestamp keep their relative position rather than sinking to the front.
- items = sorted(items or [], key=lambda it: int(getattr(it, "ts_ms", 0) or 0))
- except Exception: # noqa: BLE001 — includes timeout; history is a bonus, never a blocker
- logger.warning(
- "[channels] 群历史拉取失败 channel_id=%s conv=%s",
- conn.channel_id, msg.external_conversation_id, exc_info=True,
- )
- state["failures"] = int(state.get("failures") or 0) + 1
- _save_history_state(db, session, state)
- return 0
-
- records: List[Dict[str, Any]] = []
- file_index: Dict[str, Any] = {}
- budget = _HISTORY_MAX_TOTAL_CHARS
- newest = cursor
- # Which end the budget cuts from differs by intent:
- # cold start — walk newest→oldest, so truncation drops the stale tail. There is no
- # continuity to preserve; this is a one-off seed.
- # incremental — walk oldest→newest and stop at the budget, advancing the cursor only
- # over what was actually recorded, so the next pull resumes exactly there
- # and no message is skipped.
- for it in (reversed(items) if cold_start else items):
- mid = getattr(it, "message_id", "") or ""
- # Dedup on two fronts: the cursor boundary is inclusive on some platforms, and a
- # message may also have already arrived via push. Either way it must not be recorded
- # twice — that is what made repeated pulls wasteful.
- if mid and (mid in seen_ids or _already_handled(f"hist:{mid}")):
- continue
- text = (getattr(it, "text", "") or "").strip()
- if len(text) > _OBSERVE_TEXT_MAX:
- text = text[:_OBSERVE_TEXT_MAX] + "…"
- refs = [
- {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""}
- for a in (getattr(it, "attachments", None) or [])
- if a.get("key")
- ]
- if not text and not refs:
- continue
- if len(text) > budget: # total budget exhausted → stop, keep the cursor honest
- break
- budget -= len(text)
- entry: Dict[str, Any] = {"n": getattr(it, "sender_name", "") or "用户", "t": text}
- if refs:
- entry["a"] = refs
- for ref in refs:
- file_index[ref["key"]] = {
- "name": ref["name"], "kind": ref["kind"],
- "message_id": mid, "raw": dict(getattr(it, "raw", None) or {}),
- }
- records.append(entry)
- if mid:
- seen_ids.add(mid)
- newest = max(newest, int(getattr(it, "ts_ms", 0) or 0))
-
- if cold_start:
- # Collected newest→oldest above; the buffer is read as a transcript, so flip it back.
- records.reverse()
- if records:
- _record_observed_batch(db, session, records, file_index)
- # Advance the cursor even when nothing was recorded: the window was still covered, and
- # not advancing would re-request it on every message forever.
- state.update({
- "cursor_ms": newest,
- "failures": 0,
- # Bounded id memory purely for boundary dedup; the cursor does the real work.
- "seen_ids": list(seen_ids)[-200:],
- })
- _save_history_state(db, session, state)
- if records:
- logger.info(
- "[channels] 群历史拉取 %d 条 channel_id=%s conv=%s",
- len(records), conn.channel_id, msg.external_conversation_id,
- )
- return len(records)
-
-
-def _drain_observed(db, session: ChatSession) -> List[Dict[str, Any]]:
- """Take and clear the observation buffer (it is folded into the turn being started)."""
- meta = dict(session.extra_data or {})
- buf = list(meta.get("observed") or [])
- if not buf:
- return []
- meta["observed"] = []
- session.extra_data = meta
- db.commit()
- return buf
-
-
-def _render_observed_line(item: Dict[str, Any]) -> str:
- """One group-log line: ``speaker:text [文件:name|key=…]``.
-
- Attachment keys are rendered inline so the model can pass one straight to
- ``channel_read_attachment``. The files themselves were never downloaded — the key is the
- only handle that exists, which is exactly why it has to reach the prompt.
- """
- parts = [item.get("t") or ""]
- for ref in item.get("a") or []:
- label = "图片" if ref.get("kind") == "image" else "文件"
- parts.append(f"[{label}:{ref.get('name') or '未命名'}|key={ref.get('key')}]")
- return f"{item.get('n', '用户')}:{' '.join(p for p in parts if p)}"
-
-
-def _render_observed_block(items: List[Dict[str, Any]]) -> str:
- """Render the buffer as a context preamble for the triggering message.
-
- Explicitly labelled as background and marked do-not-reply, otherwise the model treats the
- group log as a list of pending requests and tries to answer every line.
- """
- lines = "\n".join(_render_observed_line(it) for it in items)
- hint = (
- "\n(其中带 key= 的文件并未下载,只有句柄。确有需要时用 "
- "channel_read_attachment(file_key=\"…\") 取回再读,不要凭文件名臆测内容。)"
- if any(it.get("a") for it in items) else ""
- )
- return (
- "【群聊上下文】以下是你被 @ 之前群里的近期对话,仅供你理解背景,"
- "不要逐条回复,也不要在回复中复述:\n"
- f"{lines}{hint}\n"
- "【以下才是本次需要你处理的消息】\n"
- )
-
-
-async def _recall_placeholder(adapter, conn, msg: InboundMsg, placeholder_id: str) -> None:
- """Recall the placeholder message (best-effort): channels that can't edit but can recall (DingTalk) use "recall + resend" as an equivalent replacement."""
- recall = getattr(adapter, "recall_message", None)
- if recall is None:
- return
- try:
- r = await recall(conn, msg, placeholder_id)
- if not r.success:
- logger.debug("[channels] 占位撤回失败 kind=%s detail=%s", r.error_kind, r.error_detail)
- except Exception: # noqa: BLE001
- logger.debug("[channels] 占位撤回异常", exc_info=True)
-
-
-async def _replace_placeholder(
- adapter, conn, msg: InboundMsg, placeholder_id: Optional[str], text: str
-) -> None:
- """Finish a placeholder flow with text, always sending a terminal message.
-
- Edit in place when the channel returned a placeholder ID; otherwise recall
- (if supported) and resend. WeChat's send API does not return a message ID, so
- it must take the direct-send path instead of silently leaving the placeholder
- as the last visible message.
-
- Error notices / "no text reply" receipts go through here — the old logic only edited,
- and since DingTalk edits always fail, users were stuck on "processing" forever with
- no follow-up ever shown."""
- edit = getattr(adapter, "edit_message", None)
- if placeholder_id and edit is not None:
- r = await edit(conn, placeholder_id, text)
- if r.success:
- return
- if placeholder_id:
- await _recall_placeholder(adapter, conn, msg, placeholder_id)
- fr = await adapter.send_text(conn, msg, text)
- if not fr.success:
- logger.warning("[channels] 占位替换消息发送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail)
-
-
-async def _deliver_reply(adapter, conn, msg: InboundMsg, reply: str, placeholder_id: Optional[str]) -> None:
- """#2+#4: chunk long replies; if there is a placeholder message, edit the first chunk into it and send the rest as follow-up messages.
- Channel supports recall but not edit (DingTalk) → recall the placeholder then send, a visually equivalent replacement.
-
- When the channel supports markdown (caps.supports_markdown + send_markdown), send with native rendering;
- run prepare_markdown on the whole text before chunking (tables etc. can no longer be recognized and converted once cut by chunk boundaries).
- [ref:tool-N] citation markers only render on the web client, so they are always stripped before going out (all channels)."""
- from core.channels.markdown import strip_citation_markers
- from core.channels.protocol import chunk_text
-
- reply = strip_citation_markers(reply)
- send_md = (
- getattr(adapter, "send_markdown", None)
- if getattr(adapter.caps, "supports_markdown", False)
- else None
- )
- if send_md is not None:
- prepare = getattr(adapter, "prepare_markdown", None)
- if callable(prepare):
- reply = prepare(reply)
- chunks = chunk_text(reply, getattr(adapter.caps, "max_message_len", 0)) or [reply]
- edit = getattr(adapter, "edit_message", None)
- rest = chunks
- if placeholder_id and edit is not None:
- r = await edit(conn, placeholder_id, chunks[0])
- if r.success:
- rest = chunks[1:]
- else:
- # edit failed/unsupported → recall the placeholder (if supported), send all chunks as new messages
- await _recall_placeholder(adapter, conn, msg, placeholder_id)
- for c in rest:
- fr = await (send_md or adapter.send_text)(conn, msg, c)
- if not fr.success:
- logger.warning("[channels] 回复分段推送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail)
-
-
-def _module_route(db, conn: ChannelConnection):
- """The admit hook of the first enabled module that claims this bot, if any."""
- from core.extensions import channel_routes
-
- for handles, admit in channel_routes():
- if handles(db, conn):
- return admit
- return None
-
-
-async def handle_inbound(msg: InboundMsg) -> None:
- """Main entry point for inbound messages. Scheduled on the main event loop (long-connection threads submit via run_coroutine_threadsafe).
-
- All exceptions are swallowed and logged — one message's failure must not take down the long connection / process.
- """
- from core.services.channel_relay_dispatch import dispatch_to_desktop
- if await dispatch_to_desktop(msg, SessionLocal):
- return
- if _already_handled(f"{msg.channel_id}:{msg.message_id}" if msg.message_id else ""):
- return
- if not (msg.text or "").strip() and not msg.attachments:
- return
- # #1 Per-conversation serialization: until an in-progress run finishes, later messages in the same conversation queue up, avoiding history races/corruption.
- async with _conv_locks[_conv_key(msg)]:
- await _process_inbound(msg)
-
-
-async def _process_inbound(msg: InboundMsg) -> None:
- from core.chat.context import build_runtime_context, collect_historical_attachments
- from orchestration import chat_run_executor
-
- placeholder_id: Optional[str] = None
- accepted_run_id: Optional[str] = None
- db = SessionLocal()
- try:
- repo = ChannelConnectionRepository(db)
- conn = repo.get_by_id(msg.channel_id)
- if conn is None or not conn.enabled:
- logger.warning("[channels] inbound 无对应启用连接 channel_id=%s", msg.channel_id)
- db.close()
- return
-
- owner_id = conn.owner_user_id
- adapter = get_adapter(conn.channel_type)
- _ = conn.config # force-load the credential column so it remains usable after detaching
-
- # Group listening gate. Only reached at all when the platform actually delivers non-@
- # group messages (needs the channel app's "read all group messages" permission);
- # otherwise every inbound group message is an @ and this is a no-op.
- if msg.chat_type == "group" and not await _resolve_addressed(adapter, conn, msg):
- if conn.group_listen_mode == "observe_all":
- _observe_message(db, conn, msg)
- repo.touch_event(conn.channel_id) # bystander traffic still proves the connection is alive
- logger.info(
- "[channels] 旁听群消息 channel_id=%s type=%s conv=%s",
- conn.channel_id, conn.channel_type, msg.external_conversation_id,
- )
- else:
- # mention_only → drop entirely, exactly as before this feature existed.
- # Logged because reaching this line at all is the *only* evidence that the
- # platform is delivering non-@ group messages (i.e. that the "read all group
- # messages" permission is actually granted). Without it, "the bot ignores the
- # group" and "the platform never sends the group" look identical from outside.
- # DEBUG, not INFO: an active group would otherwise flood the log.
- logger.debug(
- "[channels] 丢弃未 @ 的群消息(旁听未开启)channel_id=%s type=%s",
- conn.channel_id, conn.channel_type,
- )
- # Either way: no reset-command handling, no run, no placeholder, no reply.
- db.close()
- return
-
- admit = _module_route(db, conn)
- if admit is not None:
- db.expunge(conn)
- db.close()
- await admit(conn, msg, adapter)
- return
-
- # Channel-side "new conversation / clear context": soft-deleting the current session is
- # enough (the next message automatically creates an empty session); no agent run. This is
- # where /new and clear-context actually take effect in Feishu and other clients —
- # otherwise the command would be fed to the agent as plain text and the history would keep
- # being reused, impossible to clear.
- if _is_reset_command(msg.text):
- _reset_session(db, conn, msg)
- db.refresh(conn)
- db.expunge(conn)
- db.close()
- try:
- await adapter.send_text(conn, msg, "✅ 已开启新对话,之前的上下文已清除。")
- except Exception: # noqa: BLE001
- logger.debug("[channels] 清空回执发送失败", exc_info=True)
- return
-
- session = _find_or_create_session(db, conn, msg)
- chat_id = session.chat_id
-
- # #4 Placeholder first. Everything below this line does real network work —
- # downloading attachments, pulling group history — and the placeholder exists
- # precisely so the user is not staring at nothing while that happens. It used to be
- # sent after the whole prep stage, which put ~1s of history pull ahead of any visible
- # feedback and read as the bot being slow to react.
- # Prefer the channel's send_placeholder (DingTalk: robot API, recallable) — only that
- # path yields a message_id supporting the later edit/recall-replace; otherwise plain
- # send_text.
- try:
- send_ph = getattr(adapter, "send_placeholder", None) or adapter.send_text
- ph = await send_ph(conn, msg, "🤔 正在处理,请稍候…")
- if ph.success:
- placeholder_id = ph.message_id
- except Exception: # noqa: BLE001
- logger.debug("[channels] 占位消息发送失败", exc_info=True)
-
- # Inbound attachments: download → store as Artifact → uploaded_files
- uploaded_files = await _ingest_attachments(db, adapter, conn, owner_id, chat_id, msg)
- # File-only message with no text → synthesize a one-line user prompt
- if (msg.text or "").strip():
- user_text = msg.text
- elif uploaded_files:
- user_text = "[收到文件] " + "、".join(f.get("name", "") for f in uploaded_files)
- else:
- db.close()
- return
- # #5 Label the speaker in group scenarios so the agent can tell multi-person conversations apart
- if msg.chat_type == "group":
- user_text = f"{_speaker_label(msg)}:{user_text}"
- # Deterministic incremental pull, before draining: whatever the platform pushed
- # (possibly nothing) plus whatever we can pull ends up in the same buffer, and the
- # drain below sees both. Failure is absorbed inside — it must not block the reply.
- await _pull_history(db, adapter, conn, msg, session)
- # observe_all: prepend everything the group said since the last @ as background.
- # Prepending into user_text (rather than a separate context field) means it is
- # persisted with the turn, so later turns still see the group log through normal
- # history loading and compaction — no second retention mechanism to maintain.
- observed = _drain_observed(db, session)
- if observed:
- user_text = _render_observed_block(observed) + user_text
-
- enabled = _resolve_enabled(db, conn, owner_id)
- # Enable thinking: same as the web client's default in non-fast mode. With thinking off,
- # models (Qwen family especially) tend to emit shallow filler like "I'll get right on X"
- # without actually landing on tool calls, idling repeatedly across turns. Thinking never
- # leaks into the channel reply: structured-reasoning models emit separate thinking events
- # (ignored by _collect_reply, which only accumulates content/meta), and inline-thinking
- # models' … spans inside content are stripped by _collect_reply.
- context = build_runtime_context(
- model_name=None, user_id=owner_id, chat_id=chat_id, enable_thinking=True,
- uploaded_files=uploaded_files,
- enabled_skills=enabled["enabled_skills"], enabled_agents=enabled["enabled_agents"],
- enabled_mcps=enabled["enabled_mcps"], enabled_kbs=enabled["enabled_kbs"],
- )
- from core.services.ontology_service import build_user_ontology_runtime
-
- ontology_enabled, ontology_runtime = build_user_ontology_runtime(
- user_id=owner_id,
- task=user_text,
- db=db,
- )
- context["ontology_enabled"] = ontology_enabled
- context["ontology_runtime"] = ontology_runtime
- # Cross-turn file readability: inject files uploaded/generated earlier in this session
- # (including last turn's Feishu attachments) as a summary block so the model gets real
- # file_ids to call read_artifact with. Exclude this turn's attachments (already injected
- # in full via uploaded_files) to avoid duplication. build_runtime_context doesn't include
- # this field, so we add it here (aligned with the web client's historical_files injection
- # in chats.py).
- current_file_ids = {f.get("file_id") for f in uploaded_files if f.get("file_id")}
- context["historical_files"] = collect_historical_attachments(
- chat_id, owner_id, exclude_file_ids=current_file_ids,
- )
- # Bind to a specific sub-agent: pin the whole run to that sub-agent (running with its own
- # prompt/tools/model/knowledge bases), via the workflow's direct sub-agent mode. NULL →
- # unset, run with the owner's default capabilities (main agent).
- # Note: the sub-agent carries its own capability bindings, which override the whitelist
- # narrowing from _resolve_enabled above — this is intended behavior.
- direct_agent_id = conn.agent_id or (conn.config or {}).get("desktop_relay_agent_id")
- if direct_agent_id:
- context["agent_id"] = direct_agent_id
- # #7 Trigger A: let the agent self-create scheduled delivery tasks within this conversation
- context["channel_origin"] = {
- "channel_id": conn.channel_id,
- "conversation_id": msg.external_conversation_id,
- "chat_type": msg.chat_type,
- # channel_type lets the agent be told *which* platform it is in, so it can reach
- # for that platform's CLI to pull group history the bot never received (anything
- # from before it joined). Delivery targets never needed it; the prompt does.
- "channel_type": conn.channel_type,
- }
- from core.services.chat_sequencer import ChatSequencer
-
- accepted = ChatSequencer(db).accept_main_run(
- chat_id=chat_id,
- user_id=owner_id,
- user_content=user_text,
- user_extra_data={
- "channel_sender_open_id": msg.sender_id,
- "channel_sender_name": msg.sender_name,
- "channel_message_id": msg.message_id,
- "channel_attachments": [f.get("file_id") for f in uploaded_files],
- "attachments": [
- {"file_id": f.get("file_id"), "name": f.get("name")} for f in uploaded_files
- ],
- },
- request_payload={
- "channel_id": msg.channel_id,
- "source": "desktop_channel" if (conn.config or {}).get("desktop_relay") else "channel",
- "kind": "chat",
- },
- )
- accepted_run_id = accepted.run.run_id
- session_messages = _load_history(db, chat_id, owner_id)
- repo.touch_event(conn.channel_id)
- # The commits in create_session / add_message / touch_event expire conn's column
- # attributes (expire_on_commit defaults to True). If we expunged as-is, reading app_id/
- # config during push after detaching would trigger a refresh and raise
- # DetachedInstanceError ("is not bound to a Session").
- # Refresh first to reload all columns, then expunge — afterwards attribute reads are
- # pure in-memory and no session is needed.
- db.refresh(accepted.run)
- db.refresh(conn)
- db.expunge(accepted.run)
- db.expunge(conn) # app_id/config remain readable after leaving the session, for push
- except Exception:
- logger.exception("[channels] inbound 准备阶段失败 channel_id=%s", msg.channel_id)
- from core.channels.registry import current_relay_adapter
- if current_relay_adapter():
- current_relay_adapter().error = "本机机器人准备失败"
- db.close()
- if accepted_run_id:
- with SessionLocal() as cleanup_db:
- ChatSequencer(cleanup_db).abandon_pending_run(
- accepted_run_id,
- reason="channel preparation failed",
- )
- return
- finally:
- db.close()
-
- # (The placeholder is sent inside the prep stage above, before attachment download and
- # history pull — those are the slow parts it needs to cover.)
-
- # §13 Channel adaptation: IM clients like Feishu have no approval UI for MySpace write
- # operations. Without pre-authorization, the gate would suspend on a MySpace write waiting
- # for out-of-band user confirmation (_collect_reply only recognizes content/meta;
- # file_confirm is dropped) → stuck until the 2h timeout with the placeholder never updated.
- # The bot runs as the owner, and in private chat the message sender is the owner themselves,
- # so pre-mark this session as allowed — equivalent to the owner approving their own write.
- # (Sub-agents remain non-interactive; their /myspace writes are still rejected.)
- try:
- from core.llm.tools import _tool_confirmation as _mc
- if not (conn.config or {}).get("desktop_relay"):
- _mc.allow_session(chat_id)
- except Exception: # noqa: BLE001 — a pre-authorization failure must not take down the whole run
- logger.debug("[channels] myspace 写预授权失败 chat_id=%s", chat_id, exc_info=True)
-
- try:
- run = await chat_run_executor.start_run(
- accepted_run=accepted.run,
- chat_id=chat_id,
- user_id=owner_id,
- session_messages=session_messages,
- effective_user_message=user_text,
- raw_user_message=user_text,
- context=context,
- model_name=None,
- )
- except Exception as exc:
- with SessionLocal() as cleanup_db:
- ChatSequencer(cleanup_db).abandon_pending_run(
- accepted_run_id,
- reason=str(exc),
- )
- logger.exception("[channels] inbound run 启动失败 channel_id=%s", msg.channel_id)
- from core.channels.registry import current_relay_adapter
- if current_relay_adapter():
- current_relay_adapter().error = "本机机器人启动失败"
- try:
- await _replace_placeholder(
- adapter, conn, msg, placeholder_id, "⚠️ 处理出错了,请稍后重试。"
- )
- except Exception: # noqa: BLE001
- logger.debug("[channels] 失败消息回推异常", exc_info=True)
- return
-
- try:
- from core.channels.registry import current_relay_adapter
- relay_adapter = current_relay_adapter()
- if relay_adapter:
- relay_adapter.record_run(run.run_id, owner_id)
- reply, gen_artifacts = await _collect_reply(run.run_id)
- except Exception as exc:
- logger.exception("[channels] inbound run 失败 channel_id=%s", msg.channel_id)
- if relay_adapter:
- relay_adapter.error = "本机机器人执行失败"
- notice = (
- str(exc).strip()
- if isinstance(exc, ChannelRunError) and str(exc).strip()
- else "处理出错了,请稍后重试。"
- )
- try:
- await _replace_placeholder(adapter, conn, msg, placeholder_id, f"⚠️ {notice}")
- except Exception: # noqa: BLE001
- logger.debug("[channels] 失败消息回推异常", exc_info=True)
- return
-
- # Note: the assistant reply is **not persisted here**. start_run's background executor
- # (chat_run_executor) already persists the assistant message together with
- # usage/model/tool_calls/artifacts when the stream ends. Saving another one here would create
- # a **duplicate assistant message**, and one without usage (tokens recorded as 0) — exactly
- # the root cause of "input/output tokens are both 0" in channel sessions. We only use the
- # reply text to push back to the channel, without persisting again.
-
- # Push back: text (placeholder edit + follow-up chunks) + generated files
- try:
- if reply:
- await _deliver_reply(adapter, conn, msg, reply, placeholder_id)
- else:
- note = "已为你生成文件。" if gen_artifacts else "(本次无文本回复)"
- await _replace_placeholder(adapter, conn, msg, placeholder_id, note)
- if getattr(adapter, "push_file", None):
- for content, name, mime in _load_generated_files(gen_artifacts):
- fr = await adapter.push_file(conn, msg, content, name, mime)
- if not fr.success:
- logger.warning("[channels] 文件回传失败 name=%s kind=%s", name, fr.error_kind)
- except Exception:
- logger.exception("[channels] inbound 回推阶段失败 channel_id=%s", msg.channel_id)
- if relay_adapter:
- relay_adapter.error = "本机机器人回复回传失败"
diff --git a/src/backend/core/services/audio_transcription.py b/src/backend/core/chat/audio_transcription.py
similarity index 98%
rename from src/backend/core/services/audio_transcription.py
rename to src/backend/core/chat/audio_transcription.py
index 690ca47e..ebd00ee1 100644
--- a/src/backend/core/services/audio_transcription.py
+++ b/src/backend/core/chat/audio_transcription.py
@@ -9,7 +9,7 @@
import httpx
from fastapi import HTTPException
-from core.db.model_repository import get_active_role_provider
+from core.models.repository import get_active_role_provider
MAX_SECONDS = 120
MAX_BYTES = 16000 * 2 * MAX_SECONDS + 4096
diff --git a/src/backend/core/chat/compaction/__init__.py b/src/backend/core/chat/compaction/__init__.py
new file mode 100644
index 00000000..75837be2
--- /dev/null
+++ b/src/backend/core/chat/compaction/__init__.py
@@ -0,0 +1 @@
+"""Context compaction for long chats: budgeting, summarising and checkpointing."""
diff --git a/src/backend/core/services/context_budget_checkpoint.py b/src/backend/core/chat/compaction/budget_checkpoint.py
similarity index 97%
rename from src/backend/core/services/context_budget_checkpoint.py
rename to src/backend/core/chat/compaction/budget_checkpoint.py
index 9537f0a6..525e9cfd 100644
--- a/src/backend/core/services/context_budget_checkpoint.py
+++ b/src/backend/core/chat/compaction/budget_checkpoint.py
@@ -5,8 +5,8 @@
from dataclasses import dataclass, field
from core.db.models import ChatMessage
-from core.llm.context_safety import ContextBudgetExceeded
-from core.services.run_journal import RunJournal
+from core.llm.context.safety import ContextBudgetExceeded
+from core.runs.journal import RunJournal
class ContextBudgetJournal(RunJournal):
diff --git a/src/backend/core/services/compaction_coordinator.py b/src/backend/core/chat/compaction/coordinator.py
similarity index 94%
rename from src/backend/core/services/compaction_coordinator.py
rename to src/backend/core/chat/compaction/coordinator.py
index 3c01374b..b3a2a02b 100644
--- a/src/backend/core/services/compaction_coordinator.py
+++ b/src/backend/core/chat/compaction/coordinator.py
@@ -8,11 +8,11 @@
from typing import Any, Dict, List, Optional
from core.config import settings as config_settings
-from core.llm import compaction as C
-from core.services import compaction_budget as budget
-from core.services import compaction_summary as summary_model
-from core.services.compaction_history import normalize_rows
-from core.services.compaction_progress import finish_progress, report_outcome, start_progress
+from core.llm.context import compaction as C
+from core.llm.context import budget
+from core.chat.compaction import summary as summary_model
+from core.chat.compaction.history import normalize_rows
+from core.chat.compaction.progress import finish_progress, report_outcome, start_progress
logger = logging.getLogger(__name__)
@@ -44,7 +44,7 @@ def __init__(self, chat_id: str, *, run_id: str = "", hook_bus: Any = None):
elif run_id:
from core.harness.events import EventSink
from core.harness.hooks import HookBus
- from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger
+ from core.harness.ledger import DurableEventStore, HarnessUsageLedger
self.hook_bus = HookBus(
event_sink=EventSink(DurableEventStore()),
@@ -55,7 +55,7 @@ def __init__(self, chat_id: str, *, run_id: str = "", hook_bus: Any = None):
def _acquire(self):
from core.db.engine import SessionLocal
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
with SessionLocal() as db:
return ChatService(db).acquire_compaction_snapshot(
@@ -68,7 +68,7 @@ def _acquire(self):
def _release(self) -> None:
from core.db.engine import SessionLocal
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
with SessionLocal() as db:
ChatService(db).release_compaction_lease(self.chat_id, owner=self.owner)
@@ -82,7 +82,7 @@ def _commit(
manifest: Dict[str, Any],
) -> None:
from core.db.engine import SessionLocal
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
with SessionLocal() as db:
ChatService(db).commit_compaction_checkpoint(
@@ -188,7 +188,7 @@ async def compact(
inputs = thaw_value(before.data["budget"])
inputs["messages"] = history
if "system_prompt" not in inputs:
- from core.services.compaction_summary import _load_base_system_prompt
+ from core.chat.compaction.summary import _load_base_system_prompt
inputs["system_prompt"] = _load_base_system_prompt()
estimate = budget.estimate_context_budget(**inputs)
@@ -215,7 +215,7 @@ async def compact(
)
return None
progress = await start_progress(self.chat_id, self.run_id, phase.value)
- from core.llm.model_usage import model_usage_scope
+ from core.models.usage import model_usage_scope
with model_usage_scope(
self.run_id,
@@ -253,7 +253,7 @@ async def compact(
}
replacement_inputs = {**inputs, "messages": replacement}
replacement_estimate = budget.estimate_context_budget(**replacement_inputs)
- from core.llm.context_usage import build_compaction_context_usage
+ from core.llm.context.usage import build_compaction_context_usage
manifest["replacement_budget_estimate"] = replacement_estimate
manifest["replacement_context_usage"] = build_compaction_context_usage(
diff --git a/src/backend/core/services/compaction_history.py b/src/backend/core/chat/compaction/history.py
similarity index 93%
rename from src/backend/core/services/compaction_history.py
rename to src/backend/core/chat/compaction/history.py
index b01671e8..6ab148eb 100644
--- a/src/backend/core/services/compaction_history.py
+++ b/src/backend/core/chat/compaction/history.py
@@ -6,10 +6,10 @@
from core.chat import inflight
from core.chat.context import build_effective_user_message
-from core.llm import compaction as C
-from core.llm.context_ir import SESSION_CONTEXT_META_KEY
-from core.llm.model_steps import assistant_row_replay, record_assistant_step
-from core.services.chat_reference_service import render_reference_block
+from core.llm.context import compaction as C
+from core.llm.context.ir_constants import SESSION_CONTEXT_META_KEY
+from core.llm.runtime.model_steps import assistant_row_replay, record_assistant_step
+from core.chat.references import render_reference_block
def _cancelled_steps(steps: Any, content: str) -> Any:
diff --git a/src/backend/core/services/compaction_progress.py b/src/backend/core/chat/compaction/progress.py
similarity index 100%
rename from src/backend/core/services/compaction_progress.py
rename to src/backend/core/chat/compaction/progress.py
diff --git a/src/backend/core/services/compaction_service.py b/src/backend/core/chat/compaction/service.py
similarity index 96%
rename from src/backend/core/services/compaction_service.py
rename to src/backend/core/chat/compaction/service.py
index 479a229d..c668eada 100644
--- a/src/backend/core/services/compaction_service.py
+++ b/src/backend/core/chat/compaction/service.py
@@ -9,15 +9,11 @@
from core.chat import inflight
from core.config import settings as config_settings
-from core.llm import compaction as C
-from core.llm.context_manager import (
- AUTO_COMPACT_MAX_RATIO,
- resolve_model_context_window,
- usable_context_window,
-)
-from core.services.compaction_budget import estimate_context_budget, should_compact
-from core.services.compaction_history import normalize_rows
-from core.services.compaction_summary import _load_base_system_prompt, summarize_without_checkpoint
+from core.llm.context import compaction as C
+from core.models.context_window import AUTO_COMPACT_MAX_RATIO, resolve_model_context_window
+from core.llm.context.budget import estimate_context_budget, should_compact
+from core.chat.compaction.history import normalize_rows
+from core.chat.compaction.summary import _load_base_system_prompt, summarize_without_checkpoint
logger = logging.getLogger(__name__)
@@ -86,7 +82,7 @@ def _load_history(chat_service: Any, chat_id: str, *, repair: bool = True) -> Li
``repair`` is forwarded to :func:`_rows_after_seq`; see the warning there
about taking a row lock on a session that outlives the read.
"""
- from core.services.commands.context import append_active_completions
+ from core.sandbox.commands.context import append_active_completions
ckpt = (
chat_service.get_latest_compaction_checkpoint(chat_id)
@@ -169,7 +165,7 @@ def resolve_trigger_ratio() -> float:
def _read_trigger_ratio() -> float:
fallback = config_settings.settings.compaction.trigger_ratio
try:
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
raw = (SystemConfigService.get_instance().get("chat.compress_in_turn_ratio") or "").strip()
except Exception as exc: # noqa: BLE001
@@ -214,7 +210,7 @@ def resolve_active_tokens(usage: Optional[Dict[str, Any]]) -> int:
return int(u.get("context_tokens") or u.get("total_tokens") or 0)
-from core.services.compaction_coordinator import CompactionCoordinator
+from core.chat.compaction.coordinator import CompactionCoordinator
async def run_compaction(
diff --git a/src/backend/core/services/compaction_summary.py b/src/backend/core/chat/compaction/summary.py
similarity index 93%
rename from src/backend/core/services/compaction_summary.py
rename to src/backend/core/chat/compaction/summary.py
index b4ea65c6..5bebd465 100644
--- a/src/backend/core/services/compaction_summary.py
+++ b/src/backend/core/chat/compaction/summary.py
@@ -8,18 +8,18 @@
from typing import Any, Dict, List, Optional
import httpx
-from core.llm import compaction as C
-from core.llm.context_manager import resolve_model_context_window, usable_context_window
+from core.llm.context import compaction as C
+from core.models.context_window import resolve_model_context_window, usable_context_window
logger = logging.getLogger(__name__)
def _resolve_summarizer_model():
"""Use the main chat model because compaction input may fill its window."""
- from core.llm.single_turn import Endpoint
+ from core.models.completions.single_turn import Endpoint
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
c = ModelConfigService.get_instance().resolve("main_agent")
if c and c.base_url and c.api_key and c.model_name:
@@ -148,8 +148,8 @@ async def _summarize(history: List[Dict[str, Any]], *, timeout: int) -> Optional
# (2) Call + reactive context-overflow self-rescue. The optional context is
# bound by CompactionCoordinator so each direct HTTP try becomes one
# append-only model attempt without changing this helper's public signature.
- from core.llm.model_usage import CURRENT_MODEL_USAGE
- from core.llm.single_turn import build_request, parse_response
+ from core.models.usage import CURRENT_MODEL_USAGE
+ from core.models.completions.single_turn import build_request, parse_response
usage_context = CURRENT_MODEL_USAGE.get()
previous_attempt_seq = None
@@ -192,8 +192,9 @@ async def record_attempt(resp=None, *, status: str, started: float) -> None:
),
)
previous_attempt_seq = recorded.attempt_seq
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.compaction.summary:record_attempt', fallback_error)
for attempt in range(_SUMMARIZE_MAX_ATTEMPTS):
started = time.monotonic()
@@ -262,7 +263,7 @@ async def summarize_without_checkpoint(
)
if not older:
return None
- from core.services.compaction_progress import finish_progress, start_progress
+ from core.chat.compaction.progress import finish_progress, start_progress
progress = await start_progress(chat_id, run_id, "mid_turn") if chat_id else None
summary = None
diff --git a/src/backend/core/services/context_usage_projection.py b/src/backend/core/chat/compaction/usage_projection.py
similarity index 97%
rename from src/backend/core/services/context_usage_projection.py
rename to src/backend/core/chat/compaction/usage_projection.py
index fa130844..127aa956 100644
--- a/src/backend/core/services/context_usage_projection.py
+++ b/src/backend/core/chat/compaction/usage_projection.py
@@ -5,9 +5,9 @@
from datetime import timezone
from typing import Any, Dict
-from core.llm.context_usage import SCHEMA_VERSION, build_compaction_context_usage
+from core.llm.context.usage import SCHEMA_VERSION, build_compaction_context_usage
from core.config import settings as config_settings
-from core.services import compaction_budget as compaction
+from core.llm.context import budget as compaction
def get_chat_context_state(chat_service: Any, chat_id: str) -> Dict[str, Any]:
diff --git a/src/backend/core/chat/compaction_checkpoints.py b/src/backend/core/chat/compaction_checkpoints.py
new file mode 100644
index 00000000..8f9a1513
--- /dev/null
+++ b/src/backend/core/chat/compaction_checkpoints.py
@@ -0,0 +1,259 @@
+"""Publish and read compaction checkpoints with lease and compare-and-swap fencing."""
+from __future__ import annotations
+import uuid
+from dataclasses import replace
+from typing import Any, Dict, List, Optional
+from core.db.models import ChatCompactionState, ChatMessage
+from core.chat import compaction_types
+from core.infra.time import utc_now
+
+def invalidate_compaction_lineage_for_rewrite(
+ self,
+ chat_id: str,
+ *,
+ rewritten_seq: int,
+) -> bool:
+ """Fence a snapshot that may contain a rewritten/deleted message."""
+ state = self._ensure_compaction_state(chat_id)
+ # Lock the lineage row where supported. SQLite serializes the following
+ # write transaction; PostgreSQL blocks a concurrent publisher here.
+ state = (
+ self.db.query(ChatCompactionState)
+ .filter(ChatCompactionState.chat_id == chat_id)
+ .with_for_update()
+ .populate_existing()
+ .one()
+ )
+ affects_checkpoint = bool(
+ state.active_checkpoint_id
+ and int(rewritten_seq) <= int(state.covered_seq or 0)
+ )
+ affects_snapshot = bool(state.lease_owner)
+ if not affects_checkpoint and not affects_snapshot:
+ return False
+
+ state.active_checkpoint_id = None
+ state.checkpoint_version = int(state.checkpoint_version or 0) + 1
+ state.covered_seq = 0
+ state.lease_owner = None
+ state.lease_expires_at = None
+ state.updated_at = utc_now()
+
+ from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND
+
+ checkpoints = (
+ self.db.query(ChatMessage)
+ .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system")
+ .all()
+ )
+ for checkpoint in checkpoints:
+ extra = dict(checkpoint.extra_data or {})
+ if extra.get("kind") != COMPACTION_CHECKPOINT_KIND:
+ continue
+ extra["active"] = False
+ extra["notice_pending"] = False
+ checkpoint.extra_data = extra
+ return True
+
+
+def add_compaction_checkpoint(
+ self,
+ chat_id: str,
+ *,
+ summary_text: str,
+ replacement_history: List[Dict],
+ covered_seq: Optional[int] = None,
+ source_hash: Optional[str] = None,
+ base_checkpoint_version: Optional[int] = None,
+ replacement_manifest: Optional[Dict[str, Any]] = None,
+) -> ChatMessage:
+ """Compatibility wrapper around the lease + CAS checkpoint protocol."""
+ owner = f"direct:{uuid.uuid4().hex}"
+ snapshot = self.acquire_compaction_snapshot(
+ chat_id, owner=owner, lease_seconds=60
+ )
+ if snapshot is None:
+ raise compaction_types.CompactionCASConflict(
+ "no uncompacted source or compaction lease is busy"
+ )
+ if (
+ base_checkpoint_version is not None
+ and snapshot.base_checkpoint_version != int(base_checkpoint_version)
+ ):
+ self.release_compaction_lease(chat_id, owner=owner)
+ raise compaction_types.CompactionCASConflict("checkpoint base changed")
+ if covered_seq is not None and int(covered_seq) != snapshot.covered_seq:
+ self.release_compaction_lease(chat_id, owner=owner)
+ raise ValueError("covered_seq must equal the acquired source watermark")
+ if source_hash:
+ snapshot = replace(snapshot, source_hash=source_hash)
+ return self.commit_compaction_checkpoint(
+ snapshot,
+ owner=owner,
+ summary_text=summary_text,
+ replacement_history=replacement_history,
+ replacement_manifest=replacement_manifest or {},
+ )
+
+
+def latest_compaction_checkpoint_fallback(
+ self, chat_id: str
+) -> Optional[ChatMessage]:
+ """Find the newest checkpoint without timestamps (legacy-state fallback)."""
+ from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND
+
+ from sqlalchemy.orm import defer
+
+ rows = (
+ self.db.query(ChatMessage).options(defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls))
+ .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system")
+ .order_by(ChatMessage.chat_seq.desc(), ChatMessage.message_id.desc())
+ .limit(20)
+ .all()
+ )
+ return next(
+ (
+ row
+ for row in rows
+ if (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND
+ ),
+ None,
+ )
+
+
+def checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int:
+ """Return only an explicitly captured source watermark.
+
+ A legacy checkpoint was inserted *after* summarization, so its own
+ sequence cannot prove which earlier rows were actually summarized.
+ Treating ``checkpoint.chat_seq - 1`` as covered can hide a message
+ that arrived while the summary was in flight. Legacy checkpoints are
+ therefore conservatively invalidated and rebuilt from raw history.
+ """
+ if checkpoint is None:
+ return 0
+ extra = checkpoint.extra_data or {}
+ explicit = extra.get("covered_seq")
+ if explicit is None:
+ explicit = extra.get("covers_up_to_chat_seq")
+ if explicit is not None:
+ return max(0, int(explicit))
+ return 0
+
+
+def commit_compaction_checkpoint(
+ self,
+ snapshot: compaction_types.CompactionSnapshot,
+ *,
+ owner: str,
+ summary_text: str,
+ replacement_history: List[Dict[str, Any]],
+ replacement_manifest: Dict[str, Any],
+) -> ChatMessage:
+ """Atomically publish one successor from ``snapshot`` using CAS."""
+ from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND
+
+ if owner != snapshot.owner:
+ raise compaction_types.CompactionCASConflict("snapshot owner mismatch")
+ message_id = f"cmpct_{uuid.uuid4().hex[:16]}"
+ next_version = snapshot.base_checkpoint_version + 1
+ manifest = dict(replacement_manifest)
+ manifest.update(
+ {
+ "source_message_ids": list(snapshot.source_message_ids),
+ "source_message_seqs": list(snapshot.source_message_seqs),
+ "source_count": len(snapshot.source_rows),
+ "replacement_count": len(replacement_history),
+ }
+ )
+ message = self.message_repo.create(
+ {
+ "message_id": message_id,
+ "chat_id": snapshot.chat_id,
+ "role": "system",
+ "content": summary_text,
+ "extra_data": {
+ "kind": COMPACTION_CHECKPOINT_KIND,
+ "replacement_history": replacement_history,
+ "covered_seq": snapshot.covered_seq,
+ "covers_up_to_chat_seq": snapshot.covered_seq,
+ "covers_up_to_message_id": (
+ snapshot.source_message_ids[-1]
+ if snapshot.source_message_ids
+ else None
+ ),
+ "source_hash": snapshot.source_hash,
+ "base_checkpoint_version": snapshot.base_checkpoint_version,
+ "checkpoint_version": next_version,
+ "replacement_manifest": manifest,
+ "active": True,
+ "notice_pending": True,
+ },
+ },
+ commit=False,
+ )
+ now = utc_now()
+ updated = (
+ self.db.query(ChatCompactionState)
+ .filter(
+ ChatCompactionState.chat_id == snapshot.chat_id,
+ ChatCompactionState.checkpoint_version
+ == snapshot.base_checkpoint_version,
+ ChatCompactionState.covered_seq == snapshot.base_covered_seq,
+ ChatCompactionState.lease_owner == owner,
+ )
+ .update(
+ {
+ ChatCompactionState.active_checkpoint_id: message_id,
+ ChatCompactionState.checkpoint_version: next_version,
+ ChatCompactionState.covered_seq: snapshot.covered_seq,
+ ChatCompactionState.lease_owner: None,
+ ChatCompactionState.lease_expires_at: None,
+ ChatCompactionState.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ )
+ if updated != 1:
+ self.db.rollback()
+ raise compaction_types.CompactionCASConflict("checkpoint base or lease changed")
+
+ if snapshot.base_checkpoint_id:
+ previous = self.db.get(ChatMessage, snapshot.base_checkpoint_id)
+ if previous is not None:
+ previous_extra = dict(previous.extra_data or {})
+ previous_extra["active"] = False
+ previous.extra_data = previous_extra
+ self.db.commit()
+ return message
+
+
+def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]:
+ """Return the chat's latest compaction checkpoint (None if there is none).
+
+ The kind filter is done on the Python side to avoid depending on Postgres-specific
+ ``->>`` (SQLite compatible). Each checkpoint's extra_data carries the entire
+ replacement_history (can reach ~100KB), and rolling compaction keeps accumulating —
+ scan only the latest few rows, never ``.all()`` load everything.
+ """
+ state = self.db.get(ChatCompactionState, chat_id)
+ if state is not None:
+ if not state.active_checkpoint_id:
+ return None
+ from sqlalchemy.orm import defer
+ checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id,
+ options=[defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)])
+ if (
+ checkpoint is not None
+ and (checkpoint.extra_data or {}).get("covered_seq") is not None
+ ):
+ return checkpoint
+ return None
+
+ checkpoint = self._latest_compaction_checkpoint_fallback(chat_id)
+ if (
+ checkpoint is None
+ or (checkpoint.extra_data or {}).get("covered_seq") is None
+ ):
+ return None
+ return checkpoint
diff --git a/src/backend/core/chat/compaction_snapshots.py b/src/backend/core/chat/compaction_snapshots.py
new file mode 100644
index 00000000..2b5eaa46
--- /dev/null
+++ b/src/backend/core/chat/compaction_snapshots.py
@@ -0,0 +1,265 @@
+"""Capture the message snapshot a compaction summarises and hold its writer lease."""
+from __future__ import annotations
+import hashlib
+import json
+from datetime import timedelta
+from typing import Optional
+from core.db.models import ChatCompactionState, ChatMessage, ChatSession
+from core.chat import compaction_types
+import logging
+from core.infra.time import as_utc, utc_now
+logger = logging.getLogger(__name__)
+
+def ensure_message_sequences(self, chat_id: str) -> None:
+ """Repair legacy/direct ORM rows before they enter a source snapshot."""
+ from sqlalchemy import func
+
+ # ``key_share=True`` renders ``FOR NO KEY UPDATE`` on PostgreSQL.
+ # A plain ``FOR UPDATE`` here is a full outage risk: inserting any row
+ # into ``chat_messages`` takes ``FOR KEY SHARE`` on this parent row for
+ # the foreign key, and ``FOR KEY SHARE`` conflicts with ``FOR UPDATE``.
+ # Holding that lock therefore blocks every message write for the chat —
+ # including the run journal's terminal commit, which runs on the event
+ # loop and freezes the whole process when it waits. We only bump the
+ # non-key column ``next_message_seq``, so the weaker lock is both
+ # sufficient (it still excludes a concurrent sequence repair) and
+ # compatible with the foreign-key lock.
+ chat = (
+ self.db.query(ChatSession)
+ .filter(ChatSession.chat_id == chat_id)
+ .with_for_update(key_share=True)
+ .one_or_none()
+ )
+ if chat is None:
+ return
+ max_seq = int(
+ self.db.query(func.max(ChatMessage.chat_seq))
+ .filter(ChatMessage.chat_id == chat_id)
+ .scalar()
+ or 0
+ )
+ next_seq = max(int(chat.next_message_seq or 1), max_seq + 1)
+ missing = (
+ self.db.query(ChatMessage)
+ .filter(ChatMessage.chat_id == chat_id, ChatMessage.chat_seq.is_(None))
+ .order_by(ChatMessage.created_at, ChatMessage.message_id)
+ .all()
+ )
+ for row in missing:
+ row.chat_seq = next_seq
+ next_seq += 1
+ chat.next_message_seq = next_seq
+ self.db.flush()
+
+
+def ensure_compaction_state(self, chat_id: str) -> ChatCompactionState:
+ state = self.db.get(ChatCompactionState, chat_id)
+ if state is not None:
+ return state
+ checkpoint = self._latest_compaction_checkpoint_fallback(chat_id)
+ extra = (checkpoint.extra_data or {}) if checkpoint is not None else {}
+ has_exact_watermark = (
+ checkpoint is not None
+ and (
+ extra.get("covered_seq") is not None
+ or extra.get("covers_up_to_chat_seq") is not None
+ )
+ )
+ state = ChatCompactionState(
+ chat_id=chat_id,
+ active_checkpoint_id=checkpoint.message_id if has_exact_watermark else None,
+ checkpoint_version=(
+ int(extra.get("checkpoint_version") or 1) if has_exact_watermark else 0
+ ),
+ covered_seq=self._checkpoint_covered_seq(checkpoint)
+ if has_exact_watermark
+ else 0,
+ updated_at=utc_now(),
+ )
+ self.db.add(state)
+ self.db.flush()
+ return state
+
+
+def acquire_compaction_snapshot(
+ self,
+ chat_id: str,
+ *,
+ owner: str,
+ lease_seconds: int,
+) -> Optional[compaction_types.CompactionSnapshot]:
+ """Capture an exact source watermark and acquire a short-lived lease.
+
+ Every exit path must end this transaction. The body takes a row lock on
+ the chat before it reads, so an exception escaping with the transaction
+ still open strands that lock on a pooled connection that no longer has
+ a Python owner — an ``idle in transaction`` backend that blocks the
+ chat's message writes until the process dies. The rollback below is the
+ guarantee that cannot be forgotten by a future early return.
+ """
+ try:
+ return self._acquire_compaction_snapshot(
+ chat_id, owner=owner, lease_seconds=lease_seconds
+ )
+ except BaseException:
+ try:
+ self.db.rollback()
+ except Exception: # noqa: BLE001 - never mask the original failure
+ logger.exception(
+ "[compaction] rollback failed after a snapshot error (chat=%s)",
+ chat_id,
+ )
+ raise
+
+
+def acquire_compaction_snapshot_impl(
+ self,
+ chat_id: str,
+ *,
+ owner: str,
+ lease_seconds: int,
+) -> Optional[compaction_types.CompactionSnapshot]:
+ from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND
+
+ self._ensure_message_sequences(chat_id)
+ state = self._ensure_compaction_state(chat_id)
+ now = utc_now()
+ if state.lease_owner and (as_utc(state.lease_expires_at) or now) > now:
+ self.db.rollback()
+ return None
+
+ checkpoint = (
+ self.db.get(ChatMessage, state.active_checkpoint_id)
+ if state.active_checkpoint_id
+ else None
+ )
+ replacement_history = (
+ tuple(
+ dict(item)
+ for item in (
+ (checkpoint.extra_data or {}).get("replacement_history") or []
+ )
+ )
+ if checkpoint is not None
+ else ()
+ )
+ candidates = (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ ChatMessage.chat_seq > int(state.covered_seq or 0),
+ )
+ .order_by(ChatMessage.chat_seq)
+ .all()
+ )
+ source_models = tuple(
+ row
+ for row in candidates
+ if not (
+ row.role == "system"
+ and (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND
+ )
+ )
+ if not source_models:
+ self.db.rollback()
+ return None
+
+ source_rows = tuple(
+ compaction_types.CompactionSourceRow(
+ message_id=row.message_id,
+ chat_seq=int(row.chat_seq or 0),
+ role=row.role,
+ content=row.content,
+ tool_calls=row.tool_calls,
+ extra_data=dict(row.extra_data or {}),
+ )
+ for row in source_models
+ )
+ covered_seq = max(row.chat_seq for row in source_rows)
+ canonical_source = [
+ {
+ "message_id": row.message_id,
+ "chat_seq": row.chat_seq,
+ "role": row.role,
+ "content": row.content,
+ "tool_calls": row.tool_calls,
+ "metadata": row.extra_data,
+ }
+ for row in source_rows
+ ]
+ source_hash = hashlib.sha256(
+ json.dumps(
+ {
+ "base_checkpoint_version": int(state.checkpoint_version or 0),
+ "base_covered_seq": int(state.covered_seq or 0),
+ "covered_seq": covered_seq,
+ "source": canonical_source,
+ },
+ ensure_ascii=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ default=str,
+ ).encode("utf-8")
+ ).hexdigest()
+
+ base_version = int(state.checkpoint_version or 0)
+ updated = (
+ self.db.query(ChatCompactionState)
+ .filter(
+ ChatCompactionState.chat_id == chat_id,
+ ChatCompactionState.checkpoint_version == base_version,
+ (
+ ChatCompactionState.lease_owner.is_(None)
+ | (ChatCompactionState.lease_expires_at <= now)
+ | (ChatCompactionState.lease_owner == owner)
+ ),
+ )
+ .update(
+ {
+ ChatCompactionState.lease_owner: owner,
+ ChatCompactionState.lease_expires_at: now
+ + timedelta(seconds=max(1, lease_seconds)),
+ ChatCompactionState.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ )
+ if updated != 1:
+ self.db.rollback()
+ return None
+ base_covered_seq = int(state.covered_seq or 0)
+ base_checkpoint_id = checkpoint.message_id if checkpoint is not None else None
+ self.db.commit()
+ return compaction_types.CompactionSnapshot(
+ chat_id=chat_id,
+ owner=owner,
+ base_checkpoint_id=base_checkpoint_id,
+ base_checkpoint_version=base_version,
+ base_covered_seq=base_covered_seq,
+ covered_seq=covered_seq,
+ source_hash=source_hash,
+ replacement_history=replacement_history,
+ source_rows=source_rows,
+ source_message_ids=tuple(row.message_id for row in source_rows),
+ source_message_seqs=tuple(row.chat_seq for row in source_rows),
+ )
+
+
+def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool:
+ updated = (
+ self.db.query(ChatCompactionState)
+ .filter(
+ ChatCompactionState.chat_id == chat_id,
+ ChatCompactionState.lease_owner == owner,
+ )
+ .update(
+ {
+ ChatCompactionState.lease_owner: None,
+ ChatCompactionState.lease_expires_at: None,
+ ChatCompactionState.updated_at: utc_now(),
+ },
+ synchronize_session=False,
+ )
+ )
+ self.db.commit()
+ return updated == 1
diff --git a/src/backend/core/chat/compaction_types.py b/src/backend/core/chat/compaction_types.py
new file mode 100644
index 00000000..f0219a50
--- /dev/null
+++ b/src/backend/core/chat/compaction_types.py
@@ -0,0 +1,36 @@
+"""Snapshot and conflict types shared by the compaction checkpoint protocol."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+class CompactionCASConflict(RuntimeError):
+ """The checkpoint base/lease is stale and another writer won."""
+
+
+@dataclass(frozen=True)
+class CompactionSourceRow:
+ """Detached, immutable copy of one message in the compacted snapshot."""
+
+ message_id: str
+ chat_seq: int
+ role: str
+ content: str
+ tool_calls: Any
+ extra_data: Dict[str, Any]
+
+
+@dataclass(frozen=True)
+class CompactionSnapshot:
+ """Immutable source boundary captured before the summary LLM call."""
+
+ chat_id: str
+ owner: str
+ base_checkpoint_id: Optional[str]
+ base_checkpoint_version: int
+ base_covered_seq: int
+ covered_seq: int
+ source_hash: str
+ replacement_history: tuple[Dict[str, Any], ...]
+ source_rows: tuple[CompactionSourceRow, ...]
+ source_message_ids: tuple[str, ...]
+ source_message_seqs: tuple[int, ...]
diff --git a/src/backend/core/chat/context.py b/src/backend/core/chat/context.py
index 21a9cbb4..ef08a5ea 100644
--- a/src/backend/core/chat/context.py
+++ b/src/backend/core/chat/context.py
@@ -5,16 +5,19 @@
"""
from __future__ import annotations
+from core.harness.registry import execution_catalog as bridge_gateway
from core.infra.time import utc_now
import logging
from datetime import datetime
from typing import Any, Dict, List, Optional
+from pydantic import ValidationError
from sqlalchemy.orm import Session
-from core.config.catalog_resolver import resolve_all_runtime_enabled
-from core.services import UserService
+from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+from core.auth.users import UserService
+from core.chat.schemas import QuoteFileSource, QuotedFollowUpItem
logger = logging.getLogger(__name__)
@@ -40,8 +43,9 @@ def resolve_db_user_id(
try:
db.rollback()
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.context:resolve_db_user_id', fallback_error)
external_user = normalize_external_user_id(request_user_id)
user_service = UserService(db)
@@ -55,7 +59,7 @@ def resolve_db_user_id(
def generate_smart_title(message: str) -> str:
# 命令原文当标题等于没标题:历史列表里躺着一条叫「/init」的会话,看不出是哪个项目、
# 干了什么。命令开头的会话按命令本身命名(与前端 seedChatTitle 一致)。
- from core.services.project_init import INIT_COMMANDS
+ from core.chat.project_init import INIT_COMMANDS
message = message.strip()
if not message:
@@ -71,48 +75,69 @@ def generate_smart_title(message: str) -> str:
return message if len(message) <= 20 else message[:20] + "..."
+def _parse_quote(quoted_follow_up: Optional[Any]) -> Optional[QuotedFollowUpItem]:
+ """Request model as-is; a stored ``extra_data`` dict is validated the same way, and an
+ unreadable one is dropped rather than failing the replayed turn."""
+ if quoted_follow_up is None or isinstance(quoted_follow_up, QuotedFollowUpItem):
+ return quoted_follow_up
+ try:
+ return QuotedFollowUpItem.model_validate(quoted_follow_up)
+ except ValidationError:
+ return None
+
+
+def _render_quote(quote: QuotedFollowUpItem, message: str) -> str:
+ source = quote.source
+ if isinstance(source, QuoteFileSource):
+ origin = f"文件《{source.file_name}》(文件 ID:{source.file_id}"
+ origin += f",位置:{source.locator})" if source.locator else ")"
+ return (
+ f"用户在右侧预览中打开了{origin},选取了其中一段原文并就它提问。\n"
+ "要求:\n"
+ "1. 将【引用原文】视为这次提问直接针对的内容;“这段/这里/它/上面”等指代优先指向它。\n"
+ "2. 回答时以引用原文为准;需要文件其余部分的上下文时,按上面的文件名或文件 ID 查阅原文件。\n"
+ "3. 直接回答用户的问题,不要重复无关背景,也不要提及本提示词。\n\n"
+ f"【引用原文】(摘自{origin})\n{quote.text}\n\n"
+ f"【用户提问】\n{message}"
+ )
+ return (
+ "你正在回答同一会话中的一条追问消息。请优先结合当前会话上下文,并重点参考下面的引用原文来理解代词、省略和上下文指向。\n"
+ "要求:\n"
+ "1. 将【引用原文】视为这次追问直接关联的内容。\n"
+ "2. 若用户问题中出现“这个/这个点/它/上述/刚才提到的”等指代,优先从【引用原文】和最近几轮会话补全语义。\n"
+ "3. 直接回答用户当前追问,不要重复无关背景,也不要提及本提示词或“根据引用原文”。\n\n"
+ f"【引用原文】\n{quote.text}\n\n"
+ f"【用户追问】\n{message}"
+ )
+
+
def build_effective_user_message(
message: str,
quoted_follow_up: Optional[Any],
reference_block: str = "",
) -> str:
- """Splice the "follow-up quote" into the user message (domain-level assembly shared by send and history replay, not HTTP-layer logic).
+ """Splice the user's quote into the user message (domain-level assembly shared by send and history replay, not HTTP-layer logic).
- ``quoted_follow_up`` accepts any object with a ``text`` attribute (pydantic model) or
- a ``{"text": ...}`` dict; returns unchanged when there is no quote.
+ ``quoted_follow_up`` is a ``QuotedFollowUpItem`` or its stored dict form; its
+ ``source`` decides the framing — a passage from an earlier message in this chat,
+ or a passage from a file the user has open in the canvas (named, with its ID and
+ locator, so the model knows which file and where). No quote → message unchanged.
``reference_block`` 是用户这一轮显式引用的历史会话名片(见
- ``core.services.chat_reference_service.render_reference_block``),拼在最前面:它交代
+ ``core.chat.references.render_reference_block``),拼在最前面:它交代
的是"回答这条消息要参考哪些会话",属于这一轮的背景,先于追问引用和用户原话。
"""
- if isinstance(quoted_follow_up, dict):
- quote_text = quoted_follow_up.get("text")
- else:
- quote_text = getattr(quoted_follow_up, "text", None) if quoted_follow_up is not None else None
prefix = (reference_block or "").strip()
prefix = f"{prefix}\n\n" if prefix else ""
-
- if not quote_text:
+ quote = _parse_quote(quoted_follow_up)
+ if quote is None:
return f"{prefix}{message}"
-
- quote = str(quote_text).strip()
- if not quote:
- return f"{prefix}{message}"
-
- return prefix + (
- "你正在回答同一会话中的一条追问消息。请优先结合当前会话上下文,并重点参考下面的引用原文来理解代词、省略和上下文指向。\n"
- "要求:\n"
- "1. 将【引用原文】视为这次追问直接关联的内容。\n"
- "2. 若用户问题中出现“这个/这个点/它/上述/刚才提到的”等指代,优先从【引用原文】和最近几轮会话补全语义。\n"
- "3. 直接回答用户当前追问,不要重复无关背景,也不要提及本提示词或“根据引用原文”。\n\n"
- f"【引用原文】\n{quote}\n\n"
- f"【用户追问】\n{message}"
- )
+ return prefix + _render_quote(quote, message)
def resolve_user_facing_error(exc: Exception) -> str:
"""Map exceptions to user-friendly Chinese error strings."""
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
from core.infra.exceptions import FirstResponseTimeoutError
if isinstance(exc, FirstResponseTimeoutError):
@@ -178,9 +203,8 @@ def _apply_desktop_cloud_bridge(ids, kind: str):
云端部署 / 纯本机模式下桥未激活,原样返回(零行为变化)。
"""
try:
- from core.services import desktop_cloud_bridge as bridge
- apply = bridge.apply_to_enabled_skill_ids if kind == "skill" else bridge.apply_to_enabled_mcp_ids
+ apply = bridge_gateway.apply_to_enabled_skill_ids if kind == "skill" else bridge_gateway.apply_to_enabled_mcp_ids
return apply(list(ids) if ids is not None else None)
except Exception: # noqa: BLE001 - 桥故障不能影响会话装配
return ids
@@ -280,11 +304,7 @@ def collect_historical_attachments(
try:
from core.db.engine import SessionLocal
from core.db.models import Artifact as ArtifactModel, ChatMessage
- from core.content.artifact_reader import (
- SOURCE_AI_GENERATED,
- SOURCE_USER_UPLOAD,
- infer_source,
- )
+ from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source
except Exception:
return []
diff --git a/src/backend/core/services/chat_edition.py b/src/backend/core/chat/edition.py
similarity index 100%
rename from src/backend/core/services/chat_edition.py
rename to src/backend/core/chat/edition.py
diff --git a/src/backend/orchestration/followups.py b/src/backend/core/chat/followups.py
similarity index 83%
rename from src/backend/orchestration/followups.py
rename to src/backend/core/chat/followups.py
index d3cc23ba..b0fbd1ce 100644
--- a/src/backend/orchestration/followups.py
+++ b/src/backend/core/chat/followups.py
@@ -6,7 +6,7 @@
structured data (not embedded in the response text), so the frontend
can render them as clickable buttons.
-走 core.llm.single_turn,端点用哪条协议由它按配置决定。
+走 core.models.completions.single_turn,端点用哪条协议由它按配置决定。
"""
from __future__ import annotations
@@ -16,10 +16,11 @@
import logging
import re
import time
+from dataclasses import dataclass
from typing import Any, Optional
from core.config.settings import settings
-from core.llm.single_turn import Endpoint, complete
+from core.models.completions.single_turn import Endpoint, complete
_LOGGER = logging.getLogger(__name__)
@@ -41,7 +42,7 @@
def _resolve_followup_endpoint() -> Optional[Endpoint]:
"""Resolve model config from DB: try 'followup' role, then 'summarizer', then 'main_agent'."""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
svc = ModelConfigService.get_instance()
for role in ("followup", "summarizer", "main_agent"):
@@ -123,13 +124,9 @@ async def generate(
status = "success"
response_usage = result.usage
raw = result.text
- _LOGGER.info(
- "[followup] raw LLM response (%d chars): %s", len(raw), raw[:200]
- )
+ _LOGGER.info("[followup] raw LLM response (%d chars): %s", len(raw), raw[:200])
questions = _parse_questions(raw)
- _LOGGER.info(
- "[followup] parsed %d questions: %s", len(questions), questions
- )
+ _LOGGER.info("[followup] parsed %d questions: %s", len(questions), questions)
return questions
except asyncio.CancelledError:
@@ -144,12 +141,8 @@ async def generate(
finally:
if run_id:
try:
- from core.harness.usage import (
- AttemptUsage,
- UsageAttempt,
- record_usage_safely,
- )
- from core.services.harness_ledger import HarnessUsageLedger
+ from core.harness.usage import AttemptUsage, UsageAttempt, record_usage_safely
+ from core.harness.ledger import HarnessUsageLedger
recorder = usage_recorder or HarnessUsageLedger()
await record_usage_safely(
@@ -163,15 +156,9 @@ async def generate(
status=status,
latency_ms=int((time.monotonic() - started) * 1_000),
usage=AttemptUsage(
- prompt_tokens=int(
- response_usage.get("prompt_tokens") or 0
- ),
- completion_tokens=int(
- response_usage.get("completion_tokens") or 0
- ),
- cache_read_tokens=int(
- response_usage.get("cache_read_tokens") or 0
- ),
+ prompt_tokens=int(response_usage.get("prompt_tokens") or 0),
+ completion_tokens=int(response_usage.get("completion_tokens") or 0),
+ cache_read_tokens=int(response_usage.get("cache_read_tokens") or 0),
cache_write_tokens=int(
response_usage.get("cache_write_tokens") or 0
),
@@ -227,11 +214,21 @@ def _clean_list(items: list) -> list[str]:
# ── Singleton ──────────────────────────────────────────────────────────
-_instance: FollowUpGenerator | None = None
-
def get_followup_generator() -> FollowUpGenerator:
- global _instance
- if _instance is None:
- _instance = FollowUpGenerator()
- return _instance
+ pass
+ if process_state().instance is None:
+ process_state().instance = FollowUpGenerator()
+ return process_state().instance
+
+
+@dataclass
+class FollowupsState:
+ instance: FollowUpGenerator | None = None
+
+
+_STATE = FollowupsState()
+
+
+def process_state() -> FollowupsState:
+ return _STATE
diff --git a/src/backend/core/services/chat_fork_service.py b/src/backend/core/chat/fork.py
similarity index 99%
rename from src/backend/core/services/chat_fork_service.py
rename to src/backend/core/chat/fork.py
index bce6bd87..31429432 100644
--- a/src/backend/core/services/chat_fork_service.py
+++ b/src/backend/core/chat/fork.py
@@ -7,7 +7,7 @@
from datetime import datetime, timezone
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
-from core.auth.permissions_iface import resolve_project_access
+from core.auth.edition import resolve_project_access
from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner
from core.db.models import ChatMessage, ChatRun, ChatSession, Project
from sqlalchemy import update
diff --git a/src/backend/core/chat/history_operations.py b/src/backend/core/chat/history_operations.py
new file mode 100644
index 00000000..84b0de6e
--- /dev/null
+++ b/src/backend/core/chat/history_operations.py
@@ -0,0 +1,49 @@
+"""User-scoped history operations for tool hosts."""
+
+def history_operations(*, user_id, chat_id=None, project_id=None):
+ def _list(query: str, limit: int):
+ from core.db.engine import SessionLocal
+ from core.chat.references import list_referencable_sessions, session_brief
+
+ with SessionLocal() as db:
+ sessions = list_referencable_sessions(
+ db,
+ user_id,
+ project_id=project_id,
+ query=query,
+ limit=limit,
+ exclude_chat_id=chat_id,
+ )
+ return {
+ "ok": True,
+ "scope": "project" if project_id else "user",
+ "count": len(sessions),
+ "chats": [session_brief(session) for session in sessions],
+ }
+
+
+ def _read(target_chat_id: str, mode: str, offset: int, limit: int):
+ from core.db.engine import SessionLocal
+ from core.chat.references import read_session_messages, session_card
+
+ with SessionLocal() as db:
+ card = session_card(db, target_chat_id, user_id)
+ if card is None:
+ return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
+ if mode == "card":
+ return {"ok": True, "mode": "card", **card}
+ page = read_session_messages(db, target_chat_id, user_id, offset=offset, limit=limit)
+ if page is None:
+ return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
+ return {"ok": True, "mode": "full", "title": card["title"], **page}
+
+
+ async def _digest(target_chat_id: str):
+ from core.chat.references import build_session_digest
+
+ result = await build_session_digest(target_chat_id, user_id)
+ if result is None:
+ return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
+ return {"ok": True, "mode": "digest", **result}
+
+ return _list, _read, _digest
diff --git a/src/backend/core/chat/limits.py b/src/backend/core/chat/limits.py
new file mode 100644
index 00000000..6dc3146f
--- /dev/null
+++ b/src/backend/core/chat/limits.py
@@ -0,0 +1,88 @@
+"""Length limits applied to message content and thinking before they are stored."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+import logging
+logger = logging.getLogger(__name__)
+
+CONTENT_MAX = 100_000
+
+
+TRUNC_NOTE = "\n\n…(本条回复过长,已截断保存;完整过程见工具调用记录)"
+
+
+THINKING_MAX = 200_000
+
+
+def clamp_message_content(content: str) -> str:
+ """Apply the database content limit independently of ChatService wiring."""
+
+ text = content or ""
+ if len(text) <= CONTENT_MAX:
+ return text
+ keep = CONTENT_MAX - len(TRUNC_NOTE)
+ logger.warning("[chat] message content truncated: %d -> %d chars", len(text), CONTENT_MAX)
+ kept = text[:keep]
+ # 老格式的思考是以 … 内联在正文里的。截断点可能落在块中间,
+ # 闭合标签一丢,历史重建就把整段思考当正文渲染出来(思考泄露到页面上)。
+ close = ""
+ if kept.count("") > kept.count(close):
+ kept = kept[: keep - len(close)] + close
+ return kept + TRUNC_NOTE
+
+
+def clamp_thinking(
+ blocks: Optional[List[Dict[str, Any]]],
+ segments: Optional[List[Dict[str, Any]]] = None,
+) -> tuple[Optional[List[Dict[str, Any]]], Optional[List[Dict[str, Any]]]]:
+ """思考单独存一列,有自己的额度,不再和正文抢 content 的 10 万字上限。
+
+ 超额时从最早的块开始丢——最后几轮的推理离最终答案最近,对用户回看最有用。
+
+ ``metadata.segments`` 按下标引用思考块,丢块会让下标错位,所以两者必须一起夹取:
+ 返回夹取后的块列表,以及下标已重映射、引用了被丢块的段落已剔除的段落表。
+ """
+ if not blocks:
+ return None, segments or None
+ if sum(len(str(b.get("content") or "")) for b in blocks) <= THINKING_MAX:
+ return blocks, segments or None
+ kept: List[Dict[str, Any]] = []
+ kept_indices: List[int] = []
+ budget = THINKING_MAX
+ for offset, block in enumerate(reversed(blocks)):
+ text = str(block.get("content") or "")
+ if not text:
+ continue
+ if len(text) > budget:
+ text = text[len(text) - budget :]
+ if not text:
+ break
+ budget -= len(text)
+ kept.append({**block, "content": text})
+ kept_indices.append(len(blocks) - 1 - offset)
+ if budget <= 0:
+ break
+ kept.reverse()
+ kept_indices.reverse()
+ if not kept:
+ return None, drop_thinking_segments(segments, {})
+ remap = {old: new for new, old in enumerate(kept_indices)}
+ return kept, drop_thinking_segments(segments, remap)
+
+
+def drop_thinking_segments(
+ segments: Optional[List[Dict[str, Any]]],
+ remap: Dict[int, int],
+) -> Optional[List[Dict[str, Any]]]:
+ """把段落表里的思考下标重映射到夹取后的位置,引用已丢块的段落直接剔除。"""
+ if not segments:
+ return segments or None
+ rebuilt: List[Dict[str, Any]] = []
+ for segment in segments:
+ if segment.get("type") != "thinking":
+ rebuilt.append(segment)
+ continue
+ new_index = remap.get(segment.get("index"))
+ if new_index is None:
+ continue
+ rebuilt.append({**segment, "index": new_index})
+ return rebuilt or None
diff --git a/src/backend/orchestration/message_parser.py b/src/backend/core/chat/message_parser.py
similarity index 97%
rename from src/backend/orchestration/message_parser.py
rename to src/backend/core/chat/message_parser.py
index a5ed5280..5d0217aa 100644
--- a/src/backend/orchestration/message_parser.py
+++ b/src/backend/core/chat/message_parser.py
@@ -99,7 +99,9 @@ def extract_text_from_messages_chunk(chunk: Any) -> Tuple[str, str, Any]:
message_like, metadata = chunk
node_type = "unknown"
if isinstance(metadata, dict):
- node_name = str(metadata.get("node_type", metadata.get("langgraph_node", ""))).strip().lower()
+ node_name = (
+ str(metadata.get("node_type", metadata.get("langgraph_node", ""))).strip().lower()
+ )
if node_name == "tools":
node_type = "tools"
elif node_name == "model":
@@ -112,6 +114,7 @@ def extract_text_from_messages_chunk(chunk: Any) -> Tuple[str, str, Any]:
# Source normalisation / dedup helpers
# ------------------------------------------------------------------
+
def source_rank(source_type: str) -> int:
mapping = {
"database": 0,
diff --git a/src/backend/core/chat/message_queries.py b/src/backend/core/chat/message_queries.py
new file mode 100644
index 00000000..dbdb7e31
--- /dev/null
+++ b/src/backend/core/chat/message_queries.py
@@ -0,0 +1,91 @@
+"""Read chat messages, with access checks."""
+from __future__ import annotations
+from typing import List, Optional, Tuple
+from core.db.models import ChatMessage
+
+def list_all_messages(
+ self, chat_id: str, user_id: str
+) -> Optional[List[ChatMessage]]:
+ """List all messages in chronological order with access check.
+
+ Edition-specific sharing policies may also grant read access.
+
+ **Excludes** compaction checkpoint rows (the only writer of role='system' in
+ chat_messages is add_compaction_checkpoint) — internal artifacts, invisible to all
+ downstream consumers (replay/export/sharing/title/memory), filtered at the SQL layer
+ (a checkpoint's extra_data carries the entire replacement_history; loading it just to
+ discard is pure waste). The replay layer (compaction_service) fetches them separately
+ via get_latest_compaction_checkpoint.
+ """
+ pair = self.get_session_with_access(chat_id, user_id)
+ if pair is None:
+ return None
+
+ return (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ ChatMessage.role != "system",
+ )
+ .order_by(ChatMessage.chat_seq)
+ .all()
+ )
+
+
+def list_messages(
+ self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50
+) -> Optional[Tuple[List[ChatMessage], int, int]]:
+ """List messages in a chat session."""
+ # Check ownership
+ session = self.get_session(chat_id, user_id)
+ if not session:
+ return None
+
+ messages, total = self.message_repo.list_by_chat(chat_id, page, page_size)
+ total_pages = (total + page_size - 1) // page_size
+
+ return messages, total, total_pages
+
+
+def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]:
+ """Get a single message by its ID."""
+ return (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.message_id == message_id,
+ )
+ .first()
+ )
+
+
+def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]:
+ """Get a message by its durable 0-based position in the chat."""
+ return (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ )
+ .order_by(ChatMessage.chat_seq)
+ .offset(index)
+ .limit(1)
+ .first()
+ )
+
+
+def get_user_message_before(
+ self, chat_id: str, message_id: str
+) -> Optional[ChatMessage]:
+ """Get the user message immediately before the given message."""
+ target = self.get_message_by_id(message_id)
+ if not target:
+ return None
+ return (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ ChatMessage.role == "user",
+ ChatMessage.chat_seq < target.chat_seq,
+ )
+ .order_by(ChatMessage.chat_seq.desc())
+ .first()
+ )
diff --git a/src/backend/core/chat/message_writes.py b/src/backend/core/chat/message_writes.py
new file mode 100644
index 00000000..c2e69cd8
--- /dev/null
+++ b/src/backend/core/chat/message_writes.py
@@ -0,0 +1,299 @@
+"""Add, upsert, stream-refresh and delete chat messages."""
+from __future__ import annotations
+from core.infra.time import as_utc, utc_now
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession
+from core.db.repository.chat import _strip_nul
+from sqlalchemy import update
+from core.ontology.revision import is_substantive_revision, normalize_revision_candidate
+from core.chat import limits
+
+def add_message(
+ self,
+ chat_id: str,
+ role: str,
+ content: str,
+ model: Optional[str] = None,
+ thinking: Optional[List[Dict[str, Any]]] = None,
+ tool_calls: Optional[List[Dict]] = None,
+ usage: Optional[Dict] = None,
+ error: Optional[Dict] = None,
+ extra_data: Dict = None,
+ message_id: Optional[str] = None,
+ commit: bool = True,
+ chat_seq: Optional[int] = None,
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+) -> ChatMessage:
+ """Add a message to a chat session."""
+ extra = dict(extra_data or {})
+ kept_thinking, kept_segments = limits.clamp_thinking(thinking, extra.get("segments"))
+ if "segments" in extra:
+ extra["segments"] = kept_segments
+ message_data = {
+ "message_id": message_id or f"msg_{uuid.uuid4().hex[:16]}",
+ "chat_id": chat_id,
+ "chat_seq": chat_seq,
+ "role": role,
+ "content": limits.clamp_message_content(content),
+ "model": model,
+ "thinking": kept_thinking,
+ "tool_calls": tool_calls,
+ "model_steps": model_steps,
+ "usage": usage,
+ "error": error,
+ "extra_data": extra,
+ }
+
+ message = self.message_repo.create(message_data, commit=commit)
+
+ # Keep session metadata in sync for list APIs.
+ session = self.session_repo.get_by_id(chat_id)
+ if session:
+ session.message_count = (session.message_count or 0) + 1
+ now = utc_now()
+ session.updated_at = now
+ session.last_message_at = now
+ if commit:
+ self.db.commit()
+
+ return message
+
+
+def refresh_streaming_message(
+ self,
+ *,
+ run_id: str,
+ owner: str,
+ message_id: str,
+ content: str,
+ thinking: Optional[List[Dict[str, Any]]],
+ tool_calls: Optional[List[Dict]],
+ extra_data: Dict[str, Any],
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+) -> bool:
+ """Overwrite the in-flight assistant row while its run streams.
+
+ The run row lock serializes this write with its final commit, so a
+ superseded or finishing worker cannot clobber the final message. Nothing else is touched — in particular no session
+ timestamps, so pollers do not mistake a checkpoint for a new message.
+ Returns False when the caller no longer owns the run.
+ """
+ # Lock the same run row as journal.complete before touching its message.
+ # An EXISTS predicate alone can see a live lease, wait for the message
+ # lock, and then overwrite a final commit using that stale MVCC view.
+ live_run = (self.db.query(ChatRun)
+ .filter(ChatRun.run_id == run_id, ChatRun.lease_owner == owner,
+ ChatRun.status.in_(("pending", "running")))
+ .with_for_update().first())
+ if live_run is None:
+ return False
+ extra = dict(extra_data)
+ kept_thinking, kept_segments = limits.clamp_thinking(thinking, extra.get("segments"))
+ if "segments" in extra:
+ extra["segments"] = kept_segments
+ result = self.db.execute(
+ update(ChatMessage)
+ .where(ChatMessage.message_id == message_id)
+ .values(
+ content=limits.clamp_message_content(content),
+ thinking=_strip_nul(kept_thinking),
+ tool_calls=_strip_nul(tool_calls),
+ model_steps=_strip_nul(model_steps),
+ extra_data=_strip_nul(extra),
+ )
+ )
+ return result.rowcount > 0
+
+
+def upsert_message(
+ self,
+ chat_id: str,
+ role: str,
+ content: str,
+ *,
+ message_id: str,
+ model: Optional[str] = None,
+ thinking: Optional[List[Dict[str, Any]]] = None,
+ tool_calls: Optional[List[Dict]] = None,
+ usage: Optional[Dict] = None,
+ error: Optional[Dict] = None,
+ extra_data: Dict = None,
+ commit: bool = True,
+ chat_seq: Optional[int] = None,
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+ invalidate_mid_turn_checkpoint: bool = False,
+) -> ChatMessage:
+ """Idempotently upsert a message by message_id: overwrite in place if it exists, otherwise create.
+
+ The autonomous loop uses this for "incrementally refreshing the same assistant message as
+ progress advances" — each requirement processed / each evaluation round flushes the
+ current accumulated body + tool cards into this message, so progress is visible in the DB
+ even after a mid-run crash/refresh, instead of only being written once at the terminal
+ state. The update path does not re-increment message_count (only +1 on creation).
+ """
+ existing = self.message_repo.get_by_id(message_id)
+ if existing is not None:
+ update: Dict[str, Any] = {"content": limits.clamp_message_content(content)}
+ extra = dict(extra_data) if extra_data is not None else None
+ kept_thinking, kept_segments = limits.clamp_thinking(
+ thinking, (extra or {}).get("segments")
+ )
+ if model is not None:
+ update["model"] = model
+ if thinking is not None:
+ update["thinking"] = kept_thinking
+ if tool_calls is not None:
+ update["tool_calls"] = tool_calls
+ if model_steps is not None:
+ update["model_steps"] = model_steps
+ if usage is not None:
+ update["usage"] = usage
+ if error is not None:
+ update["error"] = error
+ if extra is not None:
+ if "segments" in extra:
+ extra["segments"] = kept_segments
+ update["extra_data"] = extra
+ if invalidate_mid_turn_checkpoint:
+ # The completed turn has already crossed the post-turn trigger.
+ # Its new steps cannot be safely sliced from a row covered by a
+ # mid-turn checkpoint, so rebuild that successor from originals.
+ # Match checkpoint publisher lock order: session, then lineage.
+ self.db.query(ChatSession).filter(ChatSession.chat_id == chat_id).with_for_update().first()
+ state = (self.db.query(ChatCompactionState)
+ .filter(ChatCompactionState.chat_id == chat_id)
+ .with_for_update().populate_existing().first())
+ if (state is not None and state.active_checkpoint_id
+ and int(existing.chat_seq or 0) <= int(state.covered_seq or 0)):
+ checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id)
+ manifest = ((checkpoint.extra_data or {}).get("replacement_manifest") or {}) if checkpoint else {}
+ captured = (extra or {}).get("context_usage", {}).get("captured_at_ms", 0)
+ checkpoint_ms = int(as_utc(checkpoint.created_at).timestamp() * 1000) if checkpoint else 0
+ if (manifest.get("phase") == "mid_turn" and type(captured) is int
+ and captured > checkpoint_ms):
+ self._invalidate_compaction_lineage_for_rewrite(
+ chat_id, rewritten_seq=int(existing.chat_seq or 0)
+ )
+ msg = self.message_repo.update(message_id, update, commit=commit)
+ session = self.session_repo.get_by_id(chat_id)
+ if session:
+ now = utc_now()
+ session.updated_at = now
+ session.last_message_at = now
+ if commit:
+ self.db.commit()
+ return msg or existing
+ return self.add_message(
+ chat_id=chat_id,
+ role=role,
+ content=content,
+ model=model,
+ thinking=thinking,
+ tool_calls=tool_calls,
+ usage=usage,
+ error=error,
+ extra_data=extra_data,
+ message_id=message_id,
+ commit=commit,
+ chat_seq=chat_seq,
+ model_steps=model_steps,
+ )
+
+
+def delete_messages_from(self, chat_id: str, message_id: str) -> int:
+ """Delete a message and all subsequent messages in the chat.
+
+ History rewrites invalidate the entire compaction lineage in the same
+ transaction. This fences an in-flight summarizer through the lineage
+ version CAS and forces the next replay/compaction to rebuild from raw
+ retained messages instead of trusting a stale covered watermark.
+
+ Returns the number of messages deleted.
+ """
+ self._ensure_message_sequences(chat_id)
+ target = (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ ChatMessage.message_id == message_id,
+ )
+ .first()
+ )
+ if not target:
+ return 0
+
+ self._invalidate_compaction_lineage_for_rewrite(
+ chat_id, rewritten_seq=int(target.chat_seq)
+ )
+
+ deleted = (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ ChatMessage.chat_seq >= int(target.chat_seq),
+ )
+ .delete(synchronize_session="fetch")
+ )
+
+ # Update session message count
+ session = self.session_repo.get_by_id(chat_id)
+ if session:
+ remaining = (
+ self.db.query(ChatMessage)
+ .filter(
+ ChatMessage.chat_id == chat_id,
+ )
+ .count()
+ )
+ session.message_count = remaining
+ session.updated_at = utc_now()
+
+ self.db.commit()
+ return deleted
+
+
+def update_message_extra_data(
+ self,
+ message_id: str,
+ patch: Dict[str, Any],
+) -> bool:
+ """Merge *patch* into a message's extra_data. Returns True on success."""
+ return self.message_repo.update_extra_data(message_id, patch) is not None
+
+
+def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]:
+ """Replace an assistant message with its persisted ontology revision candidate."""
+ message = self.message_repo.get_by_id(message_id)
+ if not message or message.role != "assistant":
+ return None
+ extra_data = dict(message.extra_data or {})
+ governance = extra_data.get("ontology_governance")
+ if not isinstance(governance, dict):
+ return None
+ review = governance.get("review")
+ if not isinstance(review, dict):
+ return None
+ candidate = normalize_revision_candidate(review.get("candidate_answer"))
+ if not is_substantive_revision(candidate):
+ return None
+ updated_review = {**review, "candidate_answer": candidate, "accepted": True}
+ extra_data["ontology_governance"] = {**governance, "review": updated_review}
+ # Some pure unit adapters exercise this method without a database.
+ if not hasattr(self, "db"):
+ return self.message_repo.update(
+ message_id,
+ {"content": candidate, "extra_data": extra_data},
+ )
+
+ updated = self.message_repo.update(
+ message_id,
+ {"content": candidate, "extra_data": extra_data},
+ commit=False,
+ )
+ self._invalidate_compaction_lineage_for_rewrite(
+ message.chat_id,
+ rewritten_seq=int(message.chat_seq),
+ )
+ self.db.commit()
+ return updated
diff --git a/src/backend/core/chat/mode_seed.py b/src/backend/core/chat/mode_seed.py
new file mode 100644
index 00000000..6a90527f
--- /dev/null
+++ b/src/backend/core/chat/mode_seed.py
@@ -0,0 +1,51 @@
+"""Seed values for the built-in turbo chat mode."""
+from __future__ import annotations
+from typing import Any, Dict
+from core.db.models.chat_mode import TOOL_SCOPE_RESTRICTED
+import logging
+from core.chat import mode_types
+
+logger = logging.getLogger(__name__)
+
+def turbo_seed_fields() -> Dict[str, Any]:
+ """极速模式内置行的字段,取自历史 ``turbo.*`` SystemConfig 键。
+
+ 这是那批键的**最后一次**读取——播种之后本表才是真源,Config 台
+ 「系统配置 → 极速模式」那组入口随之下线。
+ """
+ from core.config.store.execution_options import turbo_manual_invoke_enabled, turbo_max_iters, turbo_mcp_server_ids, turbo_plugin_ids, turbo_skill_ids
+
+ try:
+ mcp_ids = sorted(turbo_mcp_server_ids())
+ skill_ids = list(turbo_skill_ids())
+ plugin_ids = list(turbo_plugin_ids())
+ manual = turbo_manual_invoke_enabled()
+ iters = turbo_max_iters()
+ except Exception: # noqa: BLE001 — 配置层不可用时用内置检索三件套兜底
+ logger.warning("[chat_mode] 读 turbo.* 配置失败,极速模式按内置默认播种", exc_info=True)
+ mcp_ids = ["internet_search", "retrieve_dataset_content", "web_fetch"]
+ skill_ids, plugin_ids, manual, iters = [], [], True, 4
+
+ return {
+ "id": "mode_builtin_turbo",
+ "slug": mode_types.SLUG_TURBO,
+ "name": "极速模式",
+ "description": "只留检索类工具、不思考,秒级直达结果",
+ "owner_user_id": None,
+ "is_builtin": True,
+ "enabled": True,
+ "sort_order": 20,
+ "tool_scope": TOOL_SCOPE_RESTRICTED,
+ "mcp_server_ids": mcp_ids,
+ "skill_ids": skill_ids,
+ "plugin_ids": plugin_ids,
+ "agent_ids": [],
+ "manual_invoke_enabled": manual,
+ # 极速的产品契约就是"检索直达、不执行代码",保持关。
+ "code_exec_enabled": False,
+ "max_iters": iters,
+ # 极速那条链路本就不思考,强度锁死——不是"默认不思考但可以改"。
+ "default_effort": "fast",
+ "effort_locked": True,
+ "prompt_kind": "turbo",
+ }
diff --git a/src/backend/core/chat/mode_types.py b/src/backend/core/chat/mode_types.py
new file mode 100644
index 00000000..9920e3cb
--- /dev/null
+++ b/src/backend/core/chat/mode_types.py
@@ -0,0 +1,48 @@
+"""The runtime contract a chat mode hands to agent assembly."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Optional, Tuple
+from core.db.models.chat_mode import TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED
+
+@dataclass(frozen=True)
+class ChatModeSpec:
+ """运行时装配契约——``agent_factory`` 只认这个,不认数据库行。"""
+
+ slug: str
+ name: str
+ tool_scope: str = TOOL_SCOPE_ALL
+ mcp_server_ids: Tuple[str, ...] = ()
+ skill_ids: Tuple[str, ...] = ()
+ plugin_ids: Tuple[str, ...] = ()
+ agent_ids: Tuple[str, ...] = ()
+ manual_invoke_enabled: bool = True
+ #: 收窄模式下是否保留沙箱代码执行与文件工具;``all`` 作用域不看这个位。
+ code_exec_enabled: bool = False
+ max_iters: Optional[int] = None
+ default_effort: str = "fast"
+ effort_locked: bool = False
+ prompt_kind: Optional[str] = None
+ prompt_text: Optional[str] = None
+
+ @property
+ def restricted(self) -> bool:
+ """是否收窄工具面。标准模式为 False(跟随 catalog 与用户授权)。"""
+ return self.tool_scope == TOOL_SCOPE_RESTRICTED
+
+
+SLUG_STANDARD = "standard"
+
+
+SLUG_TURBO = "turbo"
+
+
+BUILTIN_SLUGS = (SLUG_STANDARD, SLUG_TURBO)
+
+
+STANDARD_SPEC = ChatModeSpec(
+ slug=SLUG_STANDARD,
+ name="标准模式",
+ tool_scope=TOOL_SCOPE_ALL,
+ code_exec_enabled=True,
+ default_effort="fast",
+)
diff --git a/src/backend/core/chat/mode_validation.py b/src/backend/core/chat/mode_validation.py
new file mode 100644
index 00000000..3a56c337
--- /dev/null
+++ b/src/backend/core/chat/mode_validation.py
@@ -0,0 +1,107 @@
+"""Normalise and validate chat mode fields from request bodies."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models.chat_mode import EFFORT_VALUES, TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED
+from core.infra.exceptions import BadRequestError
+
+def as_id_list(value: Any) -> List[str]:
+ """JSON 列 → 干净的字符串 id 列表(去空白、去重、保序)。"""
+ if not isinstance(value, (list, tuple)):
+ return []
+ seen: Dict[str, None] = {}
+ for item in value:
+ if isinstance(item, str) and item.strip():
+ seen.setdefault(item.strip(), None)
+ return list(seen.keys())
+
+
+def writable_fields(
+ payload: Dict[str, Any], owner_user_id: Optional[str], creating: bool
+) -> Dict[str, Any]:
+ """把请求体收敛成可写字段。未出现的键在更新时保持原值。"""
+ out: Dict[str, Any] = {}
+
+ def take(key: str, value: Any) -> None:
+ if value is not None or creating:
+ out[key] = value
+
+ if creating or "name" in payload:
+ name = str(payload.get("name") or "").strip()
+ if not name:
+ raise BadRequestError("模式名称不能为空")
+ out["name"] = name[:100]
+ if creating or "description" in payload:
+ out["description"] = str(payload.get("description") or "").strip()[:500]
+ if creating or "enabled" in payload:
+ out["enabled"] = bool(payload.get("enabled", True))
+ if creating or "sort_order" in payload:
+ try:
+ out["sort_order"] = int(payload.get("sort_order", 100))
+ except (TypeError, ValueError):
+ out["sort_order"] = 100
+
+ if creating or "tool_scope" in payload:
+ scope = str(payload.get("tool_scope") or TOOL_SCOPE_RESTRICTED)
+ out["tool_scope"] = scope if scope in (TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED) else TOOL_SCOPE_RESTRICTED
+ for key in ("mcp_server_ids", "skill_ids", "plugin_ids", "agent_ids"):
+ if creating or key in payload:
+ out[key] = as_id_list(payload.get(key))
+ if creating or "manual_invoke_enabled" in payload:
+ out["manual_invoke_enabled"] = bool(payload.get("manual_invoke_enabled", True))
+ if creating or "code_exec_enabled" in payload:
+ out["code_exec_enabled"] = bool(payload.get("code_exec_enabled", False))
+ if creating or "max_iters" in payload:
+ raw = payload.get("max_iters")
+ try:
+ # 下限 2:配成 1 会把"回答那一轮"也掐掉,等于永远拿不到结果。
+ out["max_iters"] = max(2, int(raw)) if raw not in (None, "") else None
+ except (TypeError, ValueError):
+ out["max_iters"] = None
+
+ if creating or "default_effort" in payload:
+ effort = str(payload.get("default_effort") or "fast")
+ out["default_effort"] = effort if effort in EFFORT_VALUES else "fast"
+ if creating or "effort_locked" in payload:
+ out["effort_locked"] = bool(payload.get("effort_locked", False))
+
+ # 提示词两种来源二选一:绑一个提示词管理里的 tab(prompt_kind,正文在那边
+ # 编辑与版本化),或者就地手写一段正文(prompt_text)。互斥——同一行留两个
+ # 来源,运行时不知道该听谁的。装配侧优先级 prompt_text > prompt_kind。
+ #
+ # 版本池的 tab 现在可以新增(prompt_version_service.create_custom_kind),
+ # 所以"绑 tab"这条路对自建模式同样走得通,不再只有内置极速能用。
+ _wants_kind = "prompt_kind" in payload
+ _wants_text = "prompt_text" in payload
+ if creating or _wants_kind or _wants_text:
+ kind = str(payload.get("prompt_kind") or "").strip()
+ if owner_user_id is not None:
+ # 私有模式只许手写正文:分类是管理端的东西,绑上等于间接读管理员的
+ # 提示词内容,越过了"用户看不到管理端提示词"的边界。
+ kind = ""
+ text = str(payload.get("prompt_text") or "").strip()
+ if kind:
+ out["prompt_kind"] = kind
+ out["prompt_text"] = None
+ elif text:
+ out["prompt_text"] = text
+ out["prompt_kind"] = None
+ elif _wants_kind or _wants_text or creating:
+ # 两个都清空 = 显式取消专属提示词,退回默认装配
+ out["prompt_kind"] = None
+ out["prompt_text"] = None
+ take("updated_at", utc_now())
+ return out
+
+
+def normalize_slug(raw: Any, fallback_name: Any) -> str:
+ """slug 归一:小写、只留 [a-z0-9_-]。空则由名称派生,再空则随机。"""
+ candidate = str(raw or "").strip().lower()
+ if not candidate:
+ candidate = str(fallback_name or "").strip().lower()
+ cleaned = "".join(ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "-" for ch in candidate)
+ cleaned = cleaned.strip("-_")
+ if not cleaned:
+ cleaned = f"mode-{uuid.uuid4().hex[:8]}"
+ return cleaned[:64]
diff --git a/src/backend/core/services/chat_mode_service.py b/src/backend/core/chat/modes.py
similarity index 62%
rename from src/backend/core/services/chat_mode_service.py
rename to src/backend/core/chat/modes.py
index cbc95621..3d346d1b 100644
--- a/src/backend/core/services/chat_mode_service.py
+++ b/src/backend/core/chat/modes.py
@@ -1,95 +1,24 @@
-"""对话模式服务。
-
-职责三块:
-
-1. **解析**:给一段对话解析出它的装配契约(``resolve``)。运行时唯一真源——
- ``agent_factory`` 不再自己去读 ``turbo.*`` 配置键。
-2. **CRUD**:官方模式(Config 台)与私有模式(用户设置页)走同一套增删改,
- 只是可见范围和提示词落点不同。
-3. **播种/迁移**:首次启动时把内置的「标准 / 极速」两行写进去,并把历史
- ``turbo.*`` SystemConfig 键**一次性**搬到极速那一行上。
-
-可见范围规则(一处实现,别处只调):一个用户能用的模式 = 全部启用的官方模式
-+ 他自己的私有模式。私有模式永远不跨人可见,也永远不进 Config 台的列表。
-"""
-
+"""Create, edit and resolve chat modes, including marketplace skill bindings."""
from __future__ import annotations
+from core.llm import prompt_kinds
from core.infra.time import utc_now
-
-import logging
import uuid
-from dataclasses import dataclass, field
-from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-
-from core.db.models.chat_mode import (
- EFFORT_VALUES,
- TOOL_SCOPE_ALL,
- TOOL_SCOPE_RESTRICTED,
- ChatMode,
-)
+from core.db.models.chat_mode import TOOL_SCOPE_ALL, ChatMode
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from sqlalchemy import or_
from sqlalchemy.orm import Session
+import logging
+from core.chat import mode_seed
+from core.chat import mode_types
+from core.chat import mode_validation
logger = logging.getLogger(__name__)
-#: 内置模式的 slug。``standard`` 是"不收窄"的那条,``turbo`` 是历史极速模式。
-SLUG_STANDARD = "standard"
-SLUG_TURBO = "turbo"
-BUILTIN_SLUGS = (SLUG_STANDARD, SLUG_TURBO)
-#: 未安装的市场项在清单里用这个前缀标记;保存时换成安装后的真实 id。
MARKET_PREFIX = "market:"
-@dataclass(frozen=True)
-class ChatModeSpec:
- """运行时装配契约——``agent_factory`` 只认这个,不认数据库行。"""
-
- slug: str
- name: str
- tool_scope: str = TOOL_SCOPE_ALL
- mcp_server_ids: Tuple[str, ...] = ()
- skill_ids: Tuple[str, ...] = ()
- plugin_ids: Tuple[str, ...] = ()
- agent_ids: Tuple[str, ...] = ()
- manual_invoke_enabled: bool = True
- #: 收窄模式下是否保留沙箱代码执行与文件工具;``all`` 作用域不看这个位。
- code_exec_enabled: bool = False
- max_iters: Optional[int] = None
- default_effort: str = "fast"
- effort_locked: bool = False
- prompt_kind: Optional[str] = None
- prompt_text: Optional[str] = None
-
- @property
- def restricted(self) -> bool:
- """是否收窄工具面。标准模式为 False(跟随 catalog 与用户授权)。"""
- return self.tool_scope == TOOL_SCOPE_RESTRICTED
-
-
-#: 标准模式的兜底契约:库里那行被删/没播种时也不能让对话起不来。
-STANDARD_SPEC = ChatModeSpec(
- slug=SLUG_STANDARD,
- name="标准模式",
- tool_scope=TOOL_SCOPE_ALL,
- code_exec_enabled=True,
- default_effort="fast",
-)
-
-
-def _as_id_list(value: Any) -> List[str]:
- """JSON 列 → 干净的字符串 id 列表(去空白、去重、保序)。"""
- if not isinstance(value, (list, tuple)):
- return []
- seen: Dict[str, None] = {}
- for item in value:
- if isinstance(item, str) and item.strip():
- seen.setdefault(item.strip(), None)
- return list(seen.keys())
-
-
class ChatModeService:
def __init__(self, db: Session):
self.db = db
@@ -142,20 +71,20 @@ def get_visible(self, mode_id: str, user_id: Optional[str]) -> Optional[ChatMode
# ── 运行时解析 ──────────────────────────────────────────────────────
- def resolve(self, slug: Optional[str], user_id: Optional[str]) -> ChatModeSpec:
+ def resolve(self, slug: Optional[str], user_id: Optional[str]) -> mode_types.ChatModeSpec:
"""把线上传来的 ``mode_slug`` 解析成装配契约。
解析不到(空 slug / 已删 / 别人的私有模式)一律回落标准模式——模式是
产品增强而不是权限边界,解析失败该让对话照常跑,不该 500。
"""
wanted = (slug or "").strip()
- if not wanted or wanted == SLUG_STANDARD:
- return self._spec_of(self._find_by_slug(SLUG_STANDARD, user_id)) or STANDARD_SPEC
+ if not wanted or wanted == mode_types.SLUG_STANDARD:
+ return self._spec_of(self._find_by_slug(mode_types.SLUG_STANDARD, user_id)) or mode_types.STANDARD_SPEC
row = self._find_by_slug(wanted, user_id)
if row is None:
logger.info("[chat_mode] slug=%s 不可用,回落标准模式", wanted)
- return self._spec_of(self._find_by_slug(SLUG_STANDARD, user_id)) or STANDARD_SPEC
- return self._spec_of(row) or STANDARD_SPEC
+ return self._spec_of(self._find_by_slug(mode_types.SLUG_STANDARD, user_id)) or mode_types.STANDARD_SPEC
+ return self._spec_of(row) or mode_types.STANDARD_SPEC
def _find_by_slug(self, slug: str, user_id: Optional[str]) -> Optional[ChatMode]:
query = self.db.query(ChatMode).filter(
@@ -173,17 +102,17 @@ def _find_by_slug(self, slug: str, user_id: Optional[str]) -> Optional[ChatMode]
return query.order_by(ChatMode.owner_user_id.isnot(None).desc()).first()
@staticmethod
- def _spec_of(row: Optional[ChatMode]) -> Optional[ChatModeSpec]:
+ def _spec_of(row: Optional[ChatMode]) -> Optional[mode_types.ChatModeSpec]:
if row is None:
return None
- return ChatModeSpec(
+ return mode_types.ChatModeSpec(
slug=row.slug,
name=row.name,
tool_scope=row.tool_scope or TOOL_SCOPE_ALL,
- mcp_server_ids=tuple(_as_id_list(row.mcp_server_ids)),
- skill_ids=tuple(_as_id_list(row.skill_ids)),
- plugin_ids=tuple(_as_id_list(row.plugin_ids)),
- agent_ids=tuple(_as_id_list(row.agent_ids)),
+ mcp_server_ids=tuple(mode_validation.as_id_list(row.mcp_server_ids)),
+ skill_ids=tuple(mode_validation.as_id_list(row.skill_ids)),
+ plugin_ids=tuple(mode_validation.as_id_list(row.plugin_ids)),
+ agent_ids=tuple(mode_validation.as_id_list(row.agent_ids)),
manual_invoke_enabled=bool(row.manual_invoke_enabled),
code_exec_enabled=bool(getattr(row, "code_exec_enabled", False)),
max_iters=row.max_iters,
@@ -196,8 +125,8 @@ def _spec_of(row: Optional[ChatMode]) -> Optional[ChatModeSpec]:
# ── 写入 ────────────────────────────────────────────────────────────
def create(self, payload: Dict[str, Any], owner_user_id: Optional[str]) -> ChatMode:
- slug = self._normalize_slug(payload.get("slug"), payload.get("name"))
- if slug in BUILTIN_SLUGS:
+ slug = mode_validation.normalize_slug(payload.get("slug"), payload.get("name"))
+ if slug in mode_types.BUILTIN_SLUGS:
raise BadRequestError(f"slug「{slug}」是内置模式保留字,请换一个")
clash = (
self.db.query(ChatMode)
@@ -213,7 +142,7 @@ def create(self, payload: Dict[str, Any], owner_user_id: Optional[str]) -> ChatM
if clash is not None:
raise BadRequestError(f"已存在同名模式「{slug}」")
- fields = self._writable_fields(payload, owner_user_id, creating=True)
+ fields = mode_validation.writable_fields(payload, owner_user_id, creating=True)
self._install_market_refs(fields, owner_user_id)
row = ChatMode(
id=f"mode_{uuid.uuid4().hex[:16]}",
@@ -231,7 +160,7 @@ def update(
self, mode_id: str, payload: Dict[str, Any], owner_user_id: Optional[str]
) -> ChatMode:
row = self._owned_row(mode_id, owner_user_id)
- fields = self._writable_fields(payload, owner_user_id, creating=False)
+ fields = mode_validation.writable_fields(payload, owner_user_id, creating=False)
self._install_market_refs(fields, owner_user_id)
for key, value in fields.items():
setattr(row, key, value)
@@ -260,94 +189,6 @@ def _owned_row(self, mode_id: str, owner_user_id: Optional[str]) -> ChatMode:
raise ResourceNotFoundError("chat_mode", mode_id)
return row
- def _writable_fields(
- self, payload: Dict[str, Any], owner_user_id: Optional[str], creating: bool
- ) -> Dict[str, Any]:
- """把请求体收敛成可写字段。未出现的键在更新时保持原值。"""
- out: Dict[str, Any] = {}
-
- def take(key: str, value: Any) -> None:
- if value is not None or creating:
- out[key] = value
-
- if creating or "name" in payload:
- name = str(payload.get("name") or "").strip()
- if not name:
- raise BadRequestError("模式名称不能为空")
- out["name"] = name[:100]
- if creating or "description" in payload:
- out["description"] = str(payload.get("description") or "").strip()[:500]
- if creating or "enabled" in payload:
- out["enabled"] = bool(payload.get("enabled", True))
- if creating or "sort_order" in payload:
- try:
- out["sort_order"] = int(payload.get("sort_order", 100))
- except (TypeError, ValueError):
- out["sort_order"] = 100
-
- if creating or "tool_scope" in payload:
- scope = str(payload.get("tool_scope") or TOOL_SCOPE_RESTRICTED)
- out["tool_scope"] = scope if scope in (TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED) else TOOL_SCOPE_RESTRICTED
- for key in ("mcp_server_ids", "skill_ids", "plugin_ids", "agent_ids"):
- if creating or key in payload:
- out[key] = _as_id_list(payload.get(key))
- if creating or "manual_invoke_enabled" in payload:
- out["manual_invoke_enabled"] = bool(payload.get("manual_invoke_enabled", True))
- if creating or "code_exec_enabled" in payload:
- out["code_exec_enabled"] = bool(payload.get("code_exec_enabled", False))
- if creating or "max_iters" in payload:
- raw = payload.get("max_iters")
- try:
- # 下限 2:配成 1 会把"回答那一轮"也掐掉,等于永远拿不到结果。
- out["max_iters"] = max(2, int(raw)) if raw not in (None, "") else None
- except (TypeError, ValueError):
- out["max_iters"] = None
-
- if creating or "default_effort" in payload:
- effort = str(payload.get("default_effort") or "fast")
- out["default_effort"] = effort if effort in EFFORT_VALUES else "fast"
- if creating or "effort_locked" in payload:
- out["effort_locked"] = bool(payload.get("effort_locked", False))
-
- # 提示词两种来源二选一:绑一个提示词管理里的 tab(prompt_kind,正文在那边
- # 编辑与版本化),或者就地手写一段正文(prompt_text)。互斥——同一行留两个
- # 来源,运行时不知道该听谁的。装配侧优先级 prompt_text > prompt_kind。
- #
- # 版本池的 tab 现在可以新增(prompt_version_service.create_custom_kind),
- # 所以"绑 tab"这条路对自建模式同样走得通,不再只有内置极速能用。
- _wants_kind = "prompt_kind" in payload
- _wants_text = "prompt_text" in payload
- if creating or _wants_kind or _wants_text:
- kind = str(payload.get("prompt_kind") or "").strip()
- if owner_user_id is not None:
- # 私有模式只许手写正文:分类是管理端的东西,绑上等于间接读管理员的
- # 提示词内容,越过了"用户看不到管理端提示词"的边界。
- kind = ""
- text = str(payload.get("prompt_text") or "").strip()
- if kind:
- out["prompt_kind"] = kind
- out["prompt_text"] = None
- elif text:
- out["prompt_text"] = text
- out["prompt_kind"] = None
- elif _wants_kind or _wants_text or creating:
- # 两个都清空 = 显式取消专属提示词,退回默认装配
- out["prompt_kind"] = None
- out["prompt_text"] = None
- take("updated_at", utc_now())
- return out
-
- @staticmethod
- def _normalize_slug(raw: Any, fallback_name: Any) -> str:
- """slug 归一:小写、只留 [a-z0-9_-]。空则由名称派生,再空则随机。"""
- candidate = str(raw or "").strip().lower()
- if not candidate:
- candidate = str(fallback_name or "").strip().lower()
- cleaned = "".join(ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "-" for ch in candidate)
- cleaned = cleaned.strip("-_")
- if not cleaned:
- cleaned = f"mode-{uuid.uuid4().hex[:8]}"
- return cleaned[:64]
# ── 可选能力清单 ────────────────────────────────────────────────────
@@ -362,7 +203,7 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any]
保存模式时由 :meth:`_install_market_refs` 按需安装成真实条目。
"""
from core.db.models import AdminSkill
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
svc = UserAgentService(self.db)
resources = svc.list_available_resources(owner_user_id)
@@ -389,7 +230,7 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any]
for prow in pq.all():
cids = prow.component_ids or {}
plugin_component_skill_ids.update(str(x) for x in (cids.get("skills") or []))
- from core.plugins.management import builtin_plugin_component_ids
+ from core.capabilities.plugins.management import builtin_plugin_component_ids
_builtin_skill_ids, _ = builtin_plugin_component_ids()
plugin_component_skill_ids.update(str(x) for x in _builtin_skill_ids)
@@ -456,17 +297,17 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any]
# 提示词 tab 清单:只发给管理端(owner=None)。普通用户看不到「提示词管理」,
# 给他一个分类下拉等于让他绑一个自己既看不到内容、也改不了正文的黑盒——
# 私有模式的提示词只有手写正文一条路。
- prompt_kinds: List[Dict[str, Any]] = []
+ kinds: List[Dict[str, Any]] = []
if owner_user_id is None:
try:
- from core.services import prompt_version_service as _pvs
+ from core.llm import prompt_versions as _pvs
- prompt_kinds = _pvs.all_kinds(self.db)
+ kinds = prompt_kinds.all_kinds(self.db)
except Exception: # noqa: BLE001
logger.debug("[chat_mode] 提示词 kind 清单读取失败", exc_info=True)
return {
- "prompt_kinds": prompt_kinds,
+ "prompt_kinds": kinds,
"mcp": resources.get("mcp_servers") or [],
"skills": skills,
"plugins": resources.get("plugins") or [],
@@ -489,9 +330,9 @@ def _market_options(
agents: List[Dict[str, Any]] = []
plugins: List[Dict[str, Any]] = []
try:
- from core.services.marketplace_service import list_marketplace_skills
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- for item in list_marketplace_skills(self.db) or []:
+ for item in marketplace_catalog().list_marketplace_skills(self.db) or []:
slug = str(item.get("slug") or "")
# 内置技能全局常驻、装不了(install 会直接拒),而且已经在上面的
# 技能清单里了——列出来只会给出一个点了报错的选项。
@@ -510,9 +351,9 @@ def _market_options(
except Exception: # noqa: BLE001
logger.debug("[chat_mode] 技能市场清单读取失败", exc_info=True)
try:
- from core.services.agent_market_service import list_marketplace_agents
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- for item in list_marketplace_agents(self.db) or []:
+ for item in marketplace_catalog().list_marketplace_agents(self.db) or []:
slug = str(item.get("slug") or "")
if not slug or item.get("market_enabled") is False:
continue
@@ -527,9 +368,9 @@ def _market_options(
except Exception: # noqa: BLE001
logger.debug("[chat_mode] 子智能体市场清单读取失败", exc_info=True)
try:
- from core.plugins.management import list_plugins
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- for item in list_plugins(self.db, None) or []:
+ for item in marketplace_catalog().list_plugins(self.db, None) or []:
slug = str(item.get("slug") or "")
if not slug or item.get("installed") or item.get("market_enabled") is False:
continue
@@ -573,21 +414,21 @@ def _install_market_refs(
slug = sid[len(MARKET_PREFIX):]
try:
if installer == "skill":
- from core.services.marketplace_service import install_marketplace_skill
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- res = install_marketplace_skill(
+ res = marketplace_catalog().install_marketplace_skill(
self.db, slug, owner_user_id=owner_user_id
)
new_id = str(res.get("skill_id") or res.get("id") or "")
elif installer == "plugin":
- from core.plugins.management import install_plugin
+ from core.capabilities.plugins.management import install_plugin
res = install_plugin(self.db, slug, owner_user_id=owner_user_id)
new_id = str(res.get("install_id") or "")
else:
- from core.services.agent_market_service import install_marketplace_agent
+ from core.harness.registry.marketplace_catalog import marketplace_catalog
- res = install_marketplace_agent(
+ res = marketplace_catalog().install_marketplace_agent(
self.db, slug, owner_user_id=owner_user_id
)
new_id = str(res.get("agent_id") or res.get("id") or "")
@@ -615,11 +456,11 @@ def ensure_builtins(self) -> None:
.all()
}
created = False
- if SLUG_STANDARD not in existing:
+ if mode_types.SLUG_STANDARD not in existing:
self.db.add(
ChatMode(
id="mode_builtin_standard",
- slug=SLUG_STANDARD,
+ slug=mode_types.SLUG_STANDARD,
name="标准模式",
description="完整工具、技能与插件,思考强度可调,适合大部分任务",
owner_user_id=None,
@@ -641,58 +482,8 @@ def ensure_builtins(self) -> None:
)
)
created = True
- if SLUG_TURBO not in existing:
- self.db.add(ChatMode(**self._turbo_seed_fields()))
+ if mode_types.SLUG_TURBO not in existing:
+ self.db.add(ChatMode(**mode_seed.turbo_seed_fields()))
created = True
if created:
self.db.commit()
-
- @staticmethod
- def _turbo_seed_fields() -> Dict[str, Any]:
- """极速模式内置行的字段,取自历史 ``turbo.*`` SystemConfig 键。
-
- 这是那批键的**最后一次**读取——播种之后本表才是真源,Config 台
- 「系统配置 → 极速模式」那组入口随之下线。
- """
- from core.services.system_config import (
- turbo_manual_invoke_enabled,
- turbo_max_iters,
- turbo_mcp_server_ids,
- turbo_plugin_ids,
- turbo_skill_ids,
- )
-
- try:
- mcp_ids = sorted(turbo_mcp_server_ids())
- skill_ids = list(turbo_skill_ids())
- plugin_ids = list(turbo_plugin_ids())
- manual = turbo_manual_invoke_enabled()
- iters = turbo_max_iters()
- except Exception: # noqa: BLE001 — 配置层不可用时用内置检索三件套兜底
- logger.warning("[chat_mode] 读 turbo.* 配置失败,极速模式按内置默认播种", exc_info=True)
- mcp_ids = ["internet_search", "retrieve_dataset_content", "web_fetch"]
- skill_ids, plugin_ids, manual, iters = [], [], True, 4
-
- return {
- "id": "mode_builtin_turbo",
- "slug": SLUG_TURBO,
- "name": "极速模式",
- "description": "只留检索类工具、不思考,秒级直达结果",
- "owner_user_id": None,
- "is_builtin": True,
- "enabled": True,
- "sort_order": 20,
- "tool_scope": TOOL_SCOPE_RESTRICTED,
- "mcp_server_ids": mcp_ids,
- "skill_ids": skill_ids,
- "plugin_ids": plugin_ids,
- "agent_ids": [],
- "manual_invoke_enabled": manual,
- # 极速的产品契约就是"检索直达、不执行代码",保持关。
- "code_exec_enabled": False,
- "max_iters": iters,
- # 极速那条链路本就不思考,强度锁死——不是"默认不思考但可以改"。
- "default_effort": "fast",
- "effort_locked": True,
- "prompt_kind": "turbo",
- }
diff --git a/src/backend/core/services/session_operations/__init__.py b/src/backend/core/chat/operations/__init__.py
similarity index 100%
rename from src/backend/core/services/session_operations/__init__.py
rename to src/backend/core/chat/operations/__init__.py
diff --git a/src/backend/core/chat/operations/admission.py b/src/backend/core/chat/operations/admission.py
new file mode 100644
index 00000000..0e3e3e32
--- /dev/null
+++ b/src/backend/core/chat/operations/admission.py
@@ -0,0 +1,317 @@
+"""One durable admission path for streaming and complete agent responses."""
+
+import asyncio
+from dataclasses import dataclass
+
+import core.chat.context as chat_context
+import core.chat.runtime.admission as admission
+import core.db.engine as db_engine
+import core.chat.service as chat_services_ChatService_module
+import core.auth.users as chat_services_UserService_module
+import core.chat.operations.admission_errors as chat_admission
+import core.chat.operations.agent_targets as chat_agent_targets
+import core.chat.operations.invocation as chat_invocation
+import core.chat.operations.request_context as chat_request_context
+import core.chat.operations.session_context as chat_session_context
+from core.auth.backend import UserContext
+from core.chat.schemas import ChatRequest
+from core.infra.logging import get_logger, trace_id_var
+from core.chat.sequencer import ChatBusyError, ChatSequencer
+from core.db.threads import run_in_session
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+logger = get_logger(__name__)
+
+
+@dataclass(frozen=True)
+class StartedResponse:
+ run_id: str
+ message_id: str
+ chat_id: str
+ user_message_id: str
+
+
+async def _read_preferences(request: ChatRequest, user_id: str):
+ # Sessions cannot cross threads; each bounded read owns its connection.
+ def capabilities():
+ with db_engine.SessionLocal() as db:
+ return chat_context.resolve_enabled_capabilities(
+ db, user_id, request.enabled_skills, request.enabled_agents, request.enabled_mcps
+ )
+
+ def preferences():
+ with db_engine.SessionLocal() as db:
+ return chat_services_UserService_module.UserService(db).get_user_settings(user_id)
+
+ return await asyncio.gather(asyncio.to_thread(capabilities), asyncio.to_thread(preferences))
+
+
+def _check_project(db: Session, request: ChatRequest, user_id: str) -> None:
+ """A chat bound to a project always runs in that project, whichever client sends to it."""
+ from core.projects.access import require_visible_project
+
+ if request.chat_id:
+ pair = chat_services_ChatService_module.ChatService(db).get_session_with_access(request.chat_id, user_id)
+ if pair is not None and pair[0].project_id:
+ request.project_id = pair[0].project_id
+ if request.project_id:
+ require_visible_project(db, user_id, request.project_id)
+
+
+def _link_attachments(db: Session, request: ChatRequest, user_id: str) -> None:
+ ids = [item.file_id for item in request.attachments if item.file_id]
+ if not ids:
+ return
+ from core.db.models import Artifact
+
+ db.query(Artifact).filter(
+ Artifact.artifact_id.in_(ids),
+ Artifact.user_id == user_id,
+ Artifact.chat_id.is_(None),
+ ).update({"chat_id": request.chat_id}, synchronize_session="fetch")
+ db.commit()
+
+
+def _prepare(db: Session, request: ChatRequest, user: UserContext):
+ """Apply the credential's scope; open the agent-API call log; resolve the user."""
+ from core.auth.agent_api import begin_agent_api_call, prepare_agent_api_request
+
+ # Scope and API-session provenance come only from the authenticated key.
+ # prepare logs its own rejected scoped requests before it raises.
+ from core.projects.api import prepare_project_api_request
+
+ request = prepare_project_api_request(db, user, request)
+ request, scope = prepare_agent_api_request(db, user, request)
+ if user.public_chat_visitor:
+ from core.chat.public import validate_visitor_attachments
+
+ validate_visitor_attachments(db, user, request.attachments)
+ call_id = (
+ begin_agent_api_call(db, scope, request.stream, trace_id=trace_id_var.get())
+ if scope
+ else None
+ )
+ return request, scope, call_id
+
+
+def _resolve_user(db: Session, user: UserContext) -> str:
+ chat_session_context.ensure_main_model_configured()
+ return chat_context.resolve_db_user_id(db, chat_session_context.authenticated_user_id(user))
+
+
+def _resolve_targets(db: Session, request: ChatRequest, user_id: str, scope):
+ """Target agent, explicit capability invocation, effective message and model."""
+ if scope:
+ # Admission already verified this exact DB-backed agent. Never
+ # enumerate the owner's catalog or parse delegation commands here.
+ from core.db.models import UserAgent
+
+ agent = db.get(UserAgent, scope["agent_id"])
+ if agent is None:
+ raise HTTPException(status_code=403, detail={"code": "agent_unavailable"})
+ agent_name = agent.name
+ request._resolved_agent_profile = "local"
+ execution_message, command = request.message, None
+ else:
+ request, agent_name, execution_message, command = (
+ chat_agent_targets._resolve_chat_agent_targets(db, request, user_id)
+ )
+ request = chat_invocation.resolve_explicit_capability_invocation(db, request, user_id)
+ message = chat_context.build_effective_user_message(
+ execution_message,
+ request.quoted_follow_up,
+ chat_request_context._resolve_reference_block(db, request, user_id),
+ )
+ provider_id = chat_session_context.resolve_selected_model_provider_id(db, request, user_id)
+ model_name = chat_session_context.resolve_actual_chat_model_name(request, provider_id)
+ return request, agent_name, command, message, provider_id, model_name
+
+
+def _admit(
+ db: Session,
+ request: ChatRequest,
+ user: UserContext,
+ user_id: str,
+ scope,
+ call_id,
+ agent_name,
+ command,
+ provider_id,
+ model_name,
+ enabled,
+ settings,
+):
+ """Build the run context, accept the run and prepare its history in one transaction.
+
+ Returns the accepted run (detached), the user message id, the history and
+ the context. The pending writer is released again if anything here fails.
+ """
+ from core.auth.agent_api import bind_agent_api_run
+
+ skills, agents, mcps = enabled
+ if user.public_chat_visitor:
+ from core.chat.public import visitor_approval_mode
+
+ settings["tool_approval_mode"] = visitor_approval_mode(
+ db, user, settings.get("tool_approval_mode")
+ )
+ _check_project(db, request, user_id)
+ service = chat_services_ChatService_module.ChatService(db)
+ chat_session_context._ensure_chat_session(
+ service,
+ request.chat_id,
+ user_id,
+ request.message,
+ agent_id=request.agent_id,
+ agent_name=agent_name,
+ plan_chat=request.plan_chat,
+ workflow_chat=request.workflow_chat,
+ site_chat=request.site_chat,
+ project_id=request.project_id,
+ )
+ context = chat_request_context.build_ctx(
+ request,
+ user_id,
+ skills,
+ agents,
+ mcps,
+ memory_enabled=bool(settings.get("memory_enabled", False)) and not scope,
+ memory_write_enabled=bool(settings.get("memory_write_enabled", False)) and not scope,
+ reranker_enabled=bool(settings.get("reranker_enabled", False)),
+ model_provider_id=provider_id,
+ actual_model_name=model_name,
+ ontology_enabled=bool(settings.get("ontology_enabled", False)) and not scope,
+ ontology_pack_ids=settings.get("ontology_pack_ids") or None,
+ approval_mode=settings.get("tool_approval_mode"),
+ )
+ if user.api_key_project_id:
+ from core.projects.api import make_project_api_scope
+
+ context["project_api_scope"] = make_project_api_scope(user, request.chat_id)
+ payload = request.model_dump(exclude_none=True)
+ if user.api_key_project_id:
+ payload["project_api_scope"] = make_project_api_scope(user, request.chat_id)
+ # Authentication owns provenance; never accept a client-supplied origin.
+ payload["request_origin"] = "api" if user.api_key_id else "cloud"
+ if user.api_key_id:
+ payload["api_key_id"] = str(user.api_key_id)
+ if command:
+ explicit_command = {
+ "agent_id": command.agent_id,
+ "agent_name": command.agent_name,
+ "task": command.task,
+ }
+ context["explicit_subagent_command"] = explicit_command
+ payload["explicit_subagent_command"] = explicit_command
+ if provider_id:
+ payload["model_provider_id"] = provider_id
+ else:
+ payload.pop("model_provider_id", None)
+ if scope:
+ context["agent_api_scope"] = dict(scope)
+ payload["agent_api_scope"] = dict(scope)
+
+ sequencer = ChatSequencer(db)
+ try:
+ accepted = sequencer.accept_main_run(
+ chat_id=request.chat_id,
+ user_id=user_id,
+ user_content=request.message,
+ model=model_name,
+ user_extra_data=chat_request_context._build_user_extra_data(request, provider_id),
+ request_payload=payload,
+ )
+ except ChatBusyError as exc:
+ raise chat_admission.chat_busy_http_exception(exc) from exc
+ try:
+ if user.public_chat_visitor:
+ from core.db.models import ChatSession
+
+ session = db.get(ChatSession, request.chat_id)
+ if session and not (session.extra_data or {}).get("public_chat_title_set"):
+ session.title = request.message.strip().split("\n", 1)[0][:80] or session.title
+ session.extra_data = {
+ **(session.extra_data or {}),
+ "public_chat_title_set": True,
+ }
+ db.flush()
+ # No history or attachment mutation before writer admission.
+ messages = chat_session_context.load_session_messages(service, request.chat_id, user_id)
+ from core.chat.plan_progress import clear_plan_progress
+
+ clear_plan_progress(request.chat_id)
+ _link_attachments(db, request, user_id)
+ if call_id:
+ bind_agent_api_run(db, call_id, accepted.run.run_id)
+ db.commit()
+ except BaseException as exc:
+ db.rollback()
+ sequencer.abandon_pending_run(accepted.run.run_id, reason=str(exc) or type(exc).__name__)
+ raise
+ db.refresh(accepted.run)
+ db.expunge(accepted.run)
+ return accepted.run, accepted.user_message.message_id, messages, context
+
+
+def _abandon(db: Session, run_id: str, reason: str) -> None:
+ ChatSequencer(db).abandon_pending_run(run_id, reason=reason)
+
+
+async def start_response_run(request: ChatRequest, user: UserContext) -> StartedResponse:
+ """Admit one agent response and launch its run.
+
+ Each database phase runs on a session of its own; the evaluation check, the
+ preference reads and the launch wait between them hold no connection.
+ """
+ from core.auth.agent_api import fail_agent_api_call
+ from core.harness.registry.evaluation import authorize
+
+ request, scope, call_id = await run_in_session(_prepare, request, user)
+ try:
+ user_id = await run_in_session(_resolve_user, user)
+ await authorize(request, user_id, agent_scoped=bool(scope))
+ request, agent_name, command, message, provider_id, model_name = await run_in_session(
+ _resolve_targets, request, user_id, scope
+ )
+ enabled, settings = await _read_preferences(request, user_id)
+ accepted_run, user_message_id, messages, context = await run_in_session(
+ _admit, request, user, user_id, scope, call_id, agent_name, command,
+ provider_id, model_name, enabled, settings,
+ )
+ try:
+ run = await admission.start_run(
+ accepted_run=accepted_run,
+ chat_id=request.chat_id,
+ user_id=user_id,
+ session_messages=messages,
+ effective_user_message=message,
+ raw_user_message=request.message,
+ context=context,
+ model_name=model_name,
+ )
+ except BaseException as exc:
+ await run_in_session(_abandon, accepted_run.run_id, str(exc) or type(exc).__name__)
+ raise
+ # Title work is auxiliary: it starts after first-message admission and
+ # never waits for the agent answer or blocks a successfully launched run.
+ from core.chat.titles import schedule_first_title
+
+ try:
+ schedule_first_title(request.chat_id, user_id, run.run_id)
+ except Exception as title_error:
+ logger.warning("chat_title_schedule_failed: %s", type(title_error).__name__)
+ return StartedResponse(run.run_id, run.message_id, request.chat_id, user_message_id)
+ except Exception as exc:
+ if call_id:
+ status = exc.status_code if isinstance(exc, HTTPException) else 500
+ # Logs carry a bounded machine code, never raw model/tool errors or secrets.
+ detail = exc.detail if isinstance(exc, HTTPException) else None
+ code = detail.get("code") if isinstance(detail, dict) else None
+ await run_in_session(fail_agent_api_call, call_id, status, code or "request_rejected")
+ if isinstance(exc, HTTPException):
+ raise
+ logger.exception("agent_response_start_failed", chat_id=request.chat_id)
+ raise HTTPException(
+ status_code=500, detail=chat_context.resolve_user_facing_error(exc)
+ ) from exc
diff --git a/src/backend/core/services/session_operations/admission_errors.py b/src/backend/core/chat/operations/admission_errors.py
similarity index 92%
rename from src/backend/core/services/session_operations/admission_errors.py
rename to src/backend/core/chat/operations/admission_errors.py
index 83ebae1c..d65507f5 100644
--- a/src/backend/core/services/session_operations/admission_errors.py
+++ b/src/backend/core/chat/operations/admission_errors.py
@@ -2,7 +2,7 @@
from fastapi import HTTPException
-from core.services.chat_sequencer import ChatBusyError
+from core.chat.sequencer import ChatBusyError
def chat_busy_http_exception(exc: ChatBusyError) -> HTTPException:
diff --git a/src/backend/core/services/session_operations/agent_targets.py b/src/backend/core/chat/operations/agent_targets.py
similarity index 94%
rename from src/backend/core/services/session_operations/agent_targets.py
rename to src/backend/core/chat/operations/agent_targets.py
index 1c1b945b..ee0d27e1 100644
--- a/src/backend/core/services/session_operations/agent_targets.py
+++ b/src/backend/core/chat/operations/agent_targets.py
@@ -2,7 +2,7 @@
from typing import Any, Optional
-import core.services.user_service as user_service
+import core.auth.users as user_service
from core.chat.schemas import ChatRequest
from fastapi import HTTPException
from sqlalchemy.orm import Session
@@ -28,15 +28,15 @@ def _resolve_chat_agent_targets(
constrain its next real tool call to ``call_subagent``. Only a persistent
``agent_id`` conversation executes the selected sub-agent directly.
"""
- from core.llm.builtin_subagents import get_builtin_subagent, merge_builtin_subagents
- from core.services.project_init import resolve_project_init
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.builtin import get_builtin_subagent, merge_builtin_subagents
+ from core.chat.project_init import resolve_project_init
+ from core.capabilities.agents.edition import UserAgentService
initialized = resolve_project_init(db, request, user_id)
if initialized is not None:
return request, None, initialized, None
- from core.services.subagent_routing_service import may_be_explicit_subagent_command
+ from core.chat.subagent_routing import may_be_explicit_subagent_command
if not (
request.agent_id
@@ -74,7 +74,7 @@ def _resolve_chat_agent_targets(
mention_agent_id = request.mention_agent_id
mention_agent_name = request.mention_name
if not mention_agent_id and not mention_agent_name:
- from core.services.subagent_routing_service import parse_explicit_subagent_command
+ from core.chat.subagent_routing import parse_explicit_subagent_command
explicit_command = parse_explicit_subagent_command(
request.message,
@@ -109,7 +109,7 @@ def _resolve_chat_agent_targets(
selected_display_name,
)
else:
- from core.services.subagent_routing_service import parse_explicit_subagent_command
+ from core.chat.subagent_routing import parse_explicit_subagent_command
command = parse_explicit_subagent_command(request.message, [mentioned])
if command:
@@ -205,7 +205,7 @@ def _saved_agent_source_profile(
return None
-def _resolve_rerun_agent_targets(
+def resolve_rerun_agent_targets(
db, request, user_id, session, user_message, *, assistant_message_id=None
):
"""Restore legacy dedicated chats and recheck each saved source before admission."""
diff --git a/src/backend/core/services/session_operations/commands.py b/src/backend/core/chat/operations/commands.py
similarity index 82%
rename from src/backend/core/services/session_operations/commands.py
rename to src/backend/core/chat/operations/commands.py
index df681beb..5d624c44 100644
--- a/src/backend/core/services/session_operations/commands.py
+++ b/src/backend/core/chat/operations/commands.py
@@ -1,30 +1,26 @@
"""Shared commands operations; callers supply authenticated identity."""
+import core.runs.runtime as runtime
from core.infra.responses import success_response
-
-from core.services.chat_steer_service import put_pending_steer
-
-from core.services.steer_queue import SteerQueueConflict
-
+from core.chat.steer import put_pending_steer
+from core.chat.steer_queue import SteerQueueConflict
from fastapi import HTTPException
-from orchestration import chat_run_executor
-
async def cancel_chat_run(run_id, user=None):
"""取消正在执行的 chat run,真正杀掉后台任务;run 不存在返回 404,无权操作返回 403。"""
try:
- cancelled = await chat_run_executor.cancel_run(run_id, user_id=user.user_id)
- except chat_run_executor.ChatRunNotFound:
+ cancelled = await runtime.cancel_run(run_id, user_id=user.user_id)
+ except runtime.ChatRunNotFound:
raise HTTPException(status_code=404, detail="run not found")
- except chat_run_executor.ChatRunPermissionDenied:
+ except runtime.ChatRunPermissionDenied:
raise HTTPException(status_code=403, detail="无权取消该 run")
return success_response(data={"run_id": run_id, "cancelled": cancelled})
async def steer_chat_run(run_id, body, user=None):
"""Queue one instruction for the live ReAct loop's next safe boundary."""
- run = chat_run_executor.get_run(run_id)
+ run = runtime.get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail="run not found")
if run.user_id != user.user_id:
diff --git a/src/backend/core/services/session_operations/history.py b/src/backend/core/chat/operations/history.py
similarity index 92%
rename from src/backend/core/services/session_operations/history.py
rename to src/backend/core/chat/operations/history.py
index 260e2ea7..b99737f6 100644
--- a/src/backend/core/services/session_operations/history.py
+++ b/src/backend/core/chat/operations/history.py
@@ -1,6 +1,6 @@
"""Shared history operations; callers supply authenticated identity."""
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
from core.chat import inflight
@@ -10,7 +10,7 @@
from core.infra.responses import paginated_response
-from core.services.context_usage_projection import get_chat_context_state
+from core.chat.compaction.usage_projection import get_chat_context_state
_WAKE_MESSAGE_PREFIXES = (
"[系统] 进度播报:",
@@ -67,7 +67,7 @@ def list_messages(
chat_id, page=1, page_size=50, order="asc", cursor=False, before_seq=None, user=None, db=None
):
"""获取当前用户有权读取的会话消息列表;无权访问时返回 404。"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
user_id = str(user.user_id)
pair = chat_service.get_session_with_access(chat_id, user_id)
@@ -92,7 +92,7 @@ def list_messages(
newest_first=(order.lower() == "desc"),
display_only=True,
)
- from core.services.compaction_service import _live_run_ids
+ from core.chat.compaction.service import _live_run_ids
live = _live_run_ids(chat_service, messages)
items = [_message_to_dict(m, live) for m in messages if not _is_internal_message(m)]
@@ -110,7 +110,7 @@ def list_messages(
# checkpoint baseline separately so context-usage estimates do not keep
# counting messages that the model now sees only through the summary.
response["data"].update(get_chat_context_state(chat_service, chat_id))
- from core.services.compaction_progress import read_progress
+ from core.chat.compaction.progress import read_progress
response["data"]["compaction_progress"] = read_progress(chat_service, chat_id)
return response
diff --git a/src/backend/core/services/session_operations/invocation.py b/src/backend/core/chat/operations/invocation.py
similarity index 85%
rename from src/backend/core/services/session_operations/invocation.py
rename to src/backend/core/chat/operations/invocation.py
index ca5a4632..8dccd1fd 100644
--- a/src/backend/core/services/session_operations/invocation.py
+++ b/src/backend/core/chat/operations/invocation.py
@@ -10,7 +10,7 @@
logger = get_logger(__name__)
-def _resolve_explicit_capability_invocation(
+def resolve_explicit_capability_invocation(
db: Session,
request: ChatRequest,
user_id: str,
@@ -22,7 +22,7 @@ def _resolve_explicit_capability_invocation(
its enabled switch is off, but cannot cross ownership, missing-installation,
or dependency-readiness boundaries.
"""
- from core.config.catalog_resolver import resolve_explicit_runtime_capabilities
+ from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities
def _ids(raw) -> List[str]:
return list(
@@ -45,14 +45,14 @@ def _ids(raw) -> List[str]:
.first()
)
if installed is None:
- from core.capabilities.errors import CapabilityError
- from core.capabilities.invocation import cloud_plugin_selection
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.capability_errors import CapabilityError
+ from core.harness.registry.run_capabilities import cloud_plugin_selection
+ from core.harness.registry.run_capabilities import has_run_capabilities
try:
cloud_plugin = (
cloud_plugin_selection(request.plugin_id, user_id=user_id)
- if capabilities_enabled()
+ if has_run_capabilities()
else None
)
except CapabilityError as exc:
@@ -68,7 +68,7 @@ def _ids(raw) -> List[str]:
else:
if installed.owner_user_id is not None and installed.owner_user_id != user_id:
raise HTTPException(status_code=403, detail="无法访问该插件,可能已卸载")
- from core.plugins.packaging.sources import _component_keys
+ from core.capabilities.plugins.packaging.sources import component_keys as _component_keys
component_ids = installed.component_ids or {}
plugin_skill_ids = _component_keys(component_ids, "skills")
@@ -81,7 +81,7 @@ def _ids(raw) -> List[str]:
# authorization both refer to the same package.
if request.skill_id:
from core.db.models import AdminSkill
- from core.services.marketplace_service import compute_install_id
+ from core.capabilities.skills.installation import compute_install_id
private_id = compute_install_id(request.skill_id, user_id)
private_row = (
@@ -114,22 +114,15 @@ def _ids(raw) -> List[str]:
unavailable_skills,
unavailable_mcps,
)
- from core.capabilities import registry, skills
- from core.capabilities.paths import LOCAL_PROFILE, capabilities_enabled
+ from core.harness.registry.device_catalog import owned_identities
+ from core.harness.registry.run_capabilities import has_run_capabilities
# An owned desktop installation may be missing files or temporarily
# disabled. Retain its selection for an explanatory answer, without
# granting anything the resolver rejected. Foreign identities still fail.
owned = set()
- if capabilities_enabled():
- profiles = [LOCAL_PROFILE]
- if skills.account_authorized_for(user_id):
- profiles.append(skills.current_account_profile())
- owned = {
- (row.kind, row.key)
- for row in registry.list_installations(profiles=profiles)
- if row.payload.get("owner_user_id") in (None, "", user_id)
- }
+ if has_run_capabilities():
+ owned = owned_identities(user_id)
rejected = [("skill", name) for name in unavailable_skills] + [
("mcp", name) for name in unavailable_mcps
]
diff --git a/src/backend/core/services/session_operations/projects.py b/src/backend/core/chat/operations/projects.py
similarity index 90%
rename from src/backend/core/services/session_operations/projects.py
rename to src/backend/core/chat/operations/projects.py
index 7a0b3b97..8dc08faa 100644
--- a/src/backend/core/services/session_operations/projects.py
+++ b/src/backend/core/chat/operations/projects.py
@@ -1,12 +1,12 @@
"""Shared projects operations; callers supply authenticated identity."""
-from core.auth.permissions_iface import ProjectAccessInfo
+from core.auth.edition import ProjectAccessInfo
from core.infra.responses import paginated_response, success_response
-from core.services.project_file_service import ProjectFileService
+from core.projects.files import ProjectFileService
-from core.services.project_service import ProjectService
+from core.projects.edition import ProjectService
from fastapi import HTTPException
diff --git a/src/backend/core/services/session_operations/request_context.py b/src/backend/core/chat/operations/request_context.py
similarity index 91%
rename from src/backend/core/services/session_operations/request_context.py
rename to src/backend/core/chat/operations/request_context.py
index 44372b26..bf2e1704 100644
--- a/src/backend/core/services/session_operations/request_context.py
+++ b/src/backend/core/chat/operations/request_context.py
@@ -2,20 +2,20 @@
from typing import Any, Dict, List, Optional
-import core.services.session_operations.session_context as chat_session_context
+import core.chat.operations.session_context as chat_session_context
import core.chat.context as chat_context
import core.db.engine as db_engine
from core.chat.schemas import ChatRequest
from core.infra.exceptions import ServiceUnavailableError
from core.infra.logging import get_logger
-from core.llm.tool_permissions import normalize_approval_mode
+from core.sandbox.permissions.approval import normalize_approval_mode
from fastapi import HTTPException
from sqlalchemy.orm import Session
logger = get_logger(__name__)
-def _build_ctx(
+def build_ctx(
request: ChatRequest,
db_user_id: str,
enabled_skills,
@@ -69,7 +69,7 @@ def _build_ctx(
if project_id:
try:
from core.db.engine import SessionLocal as _Sess
- from core.services.project_scope import build_project_ctx
+ from core.projects.edition import build_project_ctx
with _Sess() as _db:
resolved_project_ctx = build_project_ctx(_db, project_id)
@@ -85,16 +85,13 @@ def _build_ctx(
workspace_id_value = f"project:{project_id}" if project_id else "default"
memory_scope_user_id_value = project_ctx.pop("memory_scope_user_id", None)
- from core.services.ontology_service import (
- build_ontology_runtime_for_preference,
- disabled_ontology_runtime,
- )
+ from core.ontology.runtime_policy import build_ontology_runtime_for_preference, disabled_ontology_runtime
ontology_runtime: Dict[str, Any] = disabled_ontology_runtime()
if ontology_enabled:
try:
with db_engine.SessionLocal() as _ontology_db:
- from core.services.ontology_policy import user_can_use_ontology_validation
+ from core.ontology.policy import user_can_use_ontology_validation
if user_can_use_ontology_validation(_ontology_db, db_user_id):
ontology_enabled, ontology_runtime = build_ontology_runtime_for_preference(
@@ -132,20 +129,20 @@ def _build_ctx(
# Preserve None so downstream (SkillsMiddleware) falls back to catalog defaults.
# Only call _clean_id_list when there's an actual list to normalize.
"enabled_skills": (
- chat_session_context._clean_id_list(enabled_skills)
+ chat_session_context.clean_id_list(enabled_skills)
if enabled_skills is not None
else None
),
"enabled_agents": (
- chat_session_context._clean_id_list(enabled_agents)
+ chat_session_context.clean_id_list(enabled_agents)
if enabled_agents is not None
else None
),
"enabled_mcps": (
- chat_session_context._clean_id_list(enabled_mcps) if enabled_mcps is not None else None
+ chat_session_context.clean_id_list(enabled_mcps) if enabled_mcps is not None else None
),
"enabled_kbs": (
- chat_session_context._clean_id_list(request.enabled_kbs)
+ chat_session_context.clean_id_list(request.enabled_kbs)
if request.enabled_kbs is not None
else None
),
@@ -236,7 +233,7 @@ def _resolve_reference_block(db: Session, request: ChatRequest, user_id: str) ->
名片按当前用户的权限现查,并挂回 ``request`` 上,随后与用户消息一起落库——历史重放
时照这份快照渲染,模型看到的和当时看到的是同一份。
"""
- from core.services.chat_reference_service import render_reference_block, resolve_reference_cards
+ from core.chat.references import render_reference_block, resolve_reference_cards
cards = resolve_reference_cards(db, user_id, getattr(request, "referenced_chats", None))
request._resolved_reference_cards = cards
diff --git a/src/backend/core/services/session_operations/runs.py b/src/backend/core/chat/operations/runs.py
similarity index 90%
rename from src/backend/core/services/session_operations/runs.py
rename to src/backend/core/chat/operations/runs.py
index 1f507bcb..c28b7971 100644
--- a/src/backend/core/services/session_operations/runs.py
+++ b/src/backend/core/chat/operations/runs.py
@@ -1,8 +1,8 @@
"""Shared runs operations; callers supply authenticated identity."""
-from core.auth.backend import UserContext
import core.chat.context as chat_context
-
+import core.runs.runtime as runtime
+from core.auth.backend import UserContext
from core.infra.responses import success_response
@@ -13,13 +13,13 @@ def list_active_chat_runs(user=None, db=None, include_status=False, recent_secon
侧边栏对没点开过的会话一无所知。这里按用户一次查完:查询只命中 pending /
running 这一小撮行,与历史 run 的数量无关。
"""
- from orchestration import chat_run_executor
+ pass
db_user_id = chat_context.resolve_db_user_id(
db, user.user_id if isinstance(user, UserContext) else None
)
- runs, recent_chat_ids = chat_run_executor.list_active_runs_for_user(db_user_id, recent_seconds)
- from core.services.sidebar_status import sidebar_run_statuses
+ runs, recent_chat_ids = runtime.list_active_runs_for_user(db_user_id, recent_seconds)
+ from core.chat.sidebar_status import sidebar_run_statuses
return success_response(
data={
@@ -44,12 +44,12 @@ def chat_active_run(chat_id, user=None, db=None):
有则返回 run_id / message_id / status / 刷新重放 offset / thinking 模式等元信息,
无则返回 data=null。需认证(有效 cookie 或 API-Key)。
"""
- from orchestration import chat_run_executor
+ pass
db_user_id = chat_context.resolve_db_user_id(
db, user.user_id if isinstance(user, UserContext) else None
)
- run = chat_run_executor.get_active_run_for_chat(chat_id, db_user_id)
+ run = runtime.get_active_run_for_chat(chat_id, db_user_id)
if run is None:
return success_response(data=None)
payload = run.request_payload if isinstance(run.request_payload, dict) else {}
diff --git a/src/backend/core/services/session_operations/schemas.py b/src/backend/core/chat/operations/schemas.py
similarity index 100%
rename from src/backend/core/services/session_operations/schemas.py
rename to src/backend/core/chat/operations/schemas.py
diff --git a/src/backend/core/services/session_operations/session_context.py b/src/backend/core/chat/operations/session_context.py
similarity index 69%
rename from src/backend/core/services/session_operations/session_context.py
rename to src/backend/core/chat/operations/session_context.py
index 3eeb7564..19d5da8b 100644
--- a/src/backend/core/services/session_operations/session_context.py
+++ b/src/backend/core/chat/operations/session_context.py
@@ -2,38 +2,35 @@
from typing import Any, Dict, List, Optional
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
from core.chat.schemas import ChatRequest
from core.auth.backend import UserContext
from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner
from core.chat.context import generate_smart_title
from core.infra.logging import get_logger
-from core.services.model_config import ModelConfigService
-from core.services.user_model_selection import (
- UserModelSelectionError,
- resolve_effective_chat_model_name,
- resolve_user_model_provider_id,
-)
+from core.models.config import ModelConfigService
+from core.auth.model_selection import UserModelSelectionError, resolve_user_model_provider_id
+from core.models.selection import resolve_effective_chat_model_name
from fastapi import HTTPException
from sqlalchemy.orm import Session
logger = get_logger(__name__)
-def _clean_id_list(raw: Optional[list]) -> List[str]:
+def clean_id_list(raw: Optional[list]) -> List[str]:
"""Normalize a list of capability IDs: strip whitespace, remove empties."""
if not isinstance(raw, list):
return []
return [str(s).strip() for s in raw if str(s).strip()]
-def _authenticated_user_id(user: Optional[UserContext]) -> Optional[str]:
+def authenticated_user_id(user: Optional[UserContext]) -> Optional[str]:
if isinstance(user, UserContext):
return user.user_id
return None
-def _ensure_main_model_configured() -> None:
+def ensure_main_model_configured() -> None:
"""Fail fast with a user-facing error when the main chat model is missing."""
resolved = ModelConfigService.get_instance().resolve("main_agent")
if resolved is not None:
@@ -44,7 +41,7 @@ def _ensure_main_model_configured() -> None:
)
-def _resolve_selected_model_provider_id(
+def resolve_selected_model_provider_id(
db: Session, request: ChatRequest, user_id: str
) -> Optional[str]:
try:
@@ -53,7 +50,7 @@ def _resolve_selected_model_provider_id(
raise HTTPException(status_code=400, detail=str(exc)) from exc
-def _resolve_actual_chat_model_name(
+def resolve_actual_chat_model_name(
request: ChatRequest,
selected_model_provider_id: Optional[str],
) -> Optional[str]:
@@ -64,7 +61,7 @@ def _resolve_actual_chat_model_name(
def _ensure_chat_session(
- chat_service: chat_services.ChatService,
+ chat_service: chat_services_ChatService_module.ChatService,
chat_id: str,
user_id: str,
first_message: str,
@@ -132,41 +129,14 @@ def _ensure_chat_session(
return session
-def _load_session_messages(
- chat_service: chat_services.ChatService, chat_id: str, user_id: str
+def load_session_messages(
+ chat_service: chat_services_ChatService_module.ChatService, chat_id: str, user_id: str
) -> List[Dict[str, Any]]:
# Checkpoint-aware history loading: when a checkpoint exists, fetch only the tail messages from the DB (no more load-everything-then-discard).
- # Cross-turn tool results are not truncated. See core/services/compaction_service.py.
- from core.services.compaction_service import load_session_history
+ # Cross-turn tool results are not truncated. See core/chat/compaction/service.py.
+ from core.chat.compaction.service import load_session_history
messages = load_session_history(chat_service, chat_id, user_id)
if messages is None:
raise HTTPException(status_code=404, detail=f"Session {chat_id} not found")
return messages
-
-
-def _release_request_session(db: Session) -> None:
- """End the request transaction before an SSE response is handed over.
-
- A streaming response outlives its handler: FastAPI keeps the request-scoped
- session (``Depends(get_db)``) open until the stream completes. Anything left
- open here is therefore a transaction — and any row lock in it — held for the
- entire turn. The background run then blocks on its own chat row when it
- commits the reply, so the run cannot finish, so the stream cannot finish, so
- this session is never released: a circular wait that only a client
- disconnect breaks.
-
- Committing rather than rolling back matches what the handler already did on
- every other exit: everything it meant to persist is committed by this point,
- so this only ends the read transaction the history load left behind.
- """
- try:
- db.commit()
- except Exception: # noqa: BLE001 - never turn cleanup into a failed request
- logger.warning("release_request_session_commit_failed", exc_info=True)
- try:
- db.rollback()
- except Exception: # noqa: BLE001
- logger.warning("release_request_session_rollback_failed", exc_info=True)
- finally:
- db.close()
diff --git a/src/backend/core/services/session_operations/sessions.py b/src/backend/core/chat/operations/sessions.py
similarity index 86%
rename from src/backend/core/services/session_operations/sessions.py
rename to src/backend/core/chat/operations/sessions.py
index b93f4e33..16002c18 100644
--- a/src/backend/core/services/session_operations/sessions.py
+++ b/src/backend/core/chat/operations/sessions.py
@@ -2,9 +2,9 @@
from typing import Any, Dict, Optional
-import core.services as chat_services
+import core.chat.service as chat_services_ChatService_module
-from core.auth.permissions_iface import can_delete_session
+from core.auth.edition import can_delete_session
from core.infra.exceptions import ResourceNotFoundError
@@ -17,7 +17,7 @@
from sqlalchemy.orm import Session
-def _session_to_dict(s) -> dict:
+def session_to_dict(s) -> dict:
"""Convert a ChatSession ORM object to the edition-neutral API response."""
return {
"chat_id": s.chat_id,
@@ -34,11 +34,11 @@ def _session_to_dict(s) -> dict:
}
-def _session_view_for_user(db: Session, s, user_id: str, level: str) -> dict:
+def session_view_for_user(db: Session, s, user_id: str, level: str) -> dict:
"""Render a session for the current user, then let the edition extend it."""
- from core.services.chat_edition import extend_session_view
+ from core.chat.edition import extend_session_view
- return extend_session_view(db, s, user_id, level, _session_to_dict(s))
+ return extend_session_view(db, s, user_id, level, session_to_dict(s))
def list_chats(
@@ -64,7 +64,7 @@ def list_chats(
- -created_at - Most recently created first
- created_at - Oldest created first
"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
# Parse filters
pinned_only = filter == "pinned=true" if filter else False
@@ -80,7 +80,7 @@ def list_chats(
exclude_automation=exclude_automation,
)
- items = [_session_to_dict(s) for s in sessions]
+ items = [session_to_dict(s) for s in sessions]
return paginated_response(
items=items,
@@ -97,9 +97,9 @@ def create_chat(request, user=None, db=None):
The session is automatically associated with the current authenticated user.
"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
if request.project_id:
- from core.services.project_access import require_visible_project
+ from core.projects.access import require_visible_project
require_visible_project(db, user.user_id, request.project_id)
@@ -111,7 +111,7 @@ def create_chat(request, user=None, db=None):
)
return created_response(
- data=_session_to_dict(session), message="Chat session created successfully"
+ data=session_to_dict(session), message="Chat session created successfully"
)
@@ -124,7 +124,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None)
Returns sessions with match_type ("title" or "content") and matched_snippet for content matches.
"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
results, total = chat_service.search_sessions(
user_id=user.user_id,
@@ -136,7 +136,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None)
items = []
for r in results:
- item = _session_to_dict(r["session"])
+ item = session_to_dict(r["session"])
item["match_type"] = r["match_type"]
item["matched_snippet"] = r["matched_snippet"]
items.append(item)
@@ -148,7 +148,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None)
def get_chat(chat_id, user=None, db=None):
"""获取当前用户有权读取的会话详情。"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
pair = chat_service.get_session_with_access(chat_id, str(user.user_id))
if pair is None:
@@ -156,14 +156,14 @@ def get_chat(chat_id, user=None, db=None):
session, level = pair
return success_response(
- data=_session_view_for_user(db, session, str(user.user_id), level),
+ data=session_view_for_user(db, session, str(user.user_id), level),
message="Chat session retrieved successfully",
)
def update_chat(chat_id, request, user=None, db=None):
"""更新当前用户有权修改的会话元信息。"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
user_id = str(user.user_id)
pair = chat_service.get_session_with_access(chat_id, user_id)
@@ -193,7 +193,7 @@ def update_chat(chat_id, request, user=None, db=None):
raise HTTPException(status_code=403, detail="会话元数据仅创建者可改")
metadata_change = request.metadata
- from core.services.chat_edition import update_member_state
+ from core.chat.edition import update_member_state
member_state_updated = False
if request.pinned is not None or request.favorite is not None:
@@ -237,14 +237,14 @@ def update_chat(chat_id, request, user=None, db=None):
raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id)
s2, level2 = pair2
return success_response(
- data=_session_view_for_user(db, s2, user_id, level2),
+ data=session_view_for_user(db, s2, user_id, level2),
message="Chat session updated successfully",
)
def delete_chat(chat_id, user=None, db=None):
"""软删当前用户有权管理的会话。"""
- chat_service = chat_services.ChatService(db)
+ chat_service = chat_services_ChatService_module.ChatService(db)
user_id = str(user.user_id)
# First check whether the session is visible to the current user — return 404 if not, to avoid leaking its existence
@@ -256,5 +256,7 @@ def delete_chat(chat_id, user=None, db=None):
if not can_delete_session(db, user_id, session):
raise HTTPException(status_code=403, detail="共享会话仅创建者或项目管理员可删")
+ from core.llm.mcp.chat_sessions import invalidate_chat
+ invalidate_chat(chat_id)
chat_service.delete_session_force(chat_id, actor_user_id=user_id)
return Response(status_code=status.HTTP_204_NO_CONTENT)
diff --git a/src/backend/core/services/plan_service.py b/src/backend/core/chat/plans.py
similarity index 100%
rename from src/backend/core/services/plan_service.py
rename to src/backend/core/chat/plans.py
diff --git a/src/backend/core/services/project_init.py b/src/backend/core/chat/project_init.py
similarity index 91%
rename from src/backend/core/services/project_init.py
rename to src/backend/core/chat/project_init.py
index 6dd8ed34..c2353cf0 100644
--- a/src/backend/core/services/project_init.py
+++ b/src/backend/core/chat/project_init.py
@@ -7,8 +7,8 @@
from fastapi import HTTPException
from sqlalchemy.orm import Session
-from core.auth.permissions_iface import resolve_project_permission
-from core.services.project_instructions import ProjectInstructionsService
+from core.auth.edition import resolve_project_permission
+from core.projects.instructions import ProjectInstructionsService
INIT_COMMANDS = frozenset({"/init", "/初始化指令"})
@@ -17,7 +17,7 @@ def resolve_project_init(db: Session, request, user_id: str) -> str | None:
"""Authorize against the saved chat binding, not a client-provided file path."""
if request.message.strip() not in INIT_COMMANDS:
return None
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
project_id = request.project_id
pair = ChatService(db).get_session_with_access(request.chat_id, user_id)
@@ -31,7 +31,7 @@ def resolve_project_init(db: Session, request, user_id: str) -> str | None:
project_id = chat.project_id
if not project_id:
raise HTTPException(400, "请先选择具体项目,再使用 /init 或 /初始化指令")
- from core.services.project_access import require_visible_project
+ from core.projects.access import require_visible_project
project = require_visible_project(db, user_id, project_id)
if resolve_project_permission(db, user_id, project) not in ("admin", "edit"):
diff --git a/src/backend/core/chat/projection/__init__.py b/src/backend/core/chat/projection/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/backend/core/chat/projection/codec.py b/src/backend/core/chat/projection/codec.py
new file mode 100644
index 00000000..20dee0ae
--- /dev/null
+++ b/src/backend/core/chat/projection/codec.py
@@ -0,0 +1,8 @@
+"""Compose chat projection reducers and checkpoint codecs for the run host."""
+from core.runs.events.projection import ProjectionCodec
+from core.chat.projection.reducer import fold_projection, new_projection
+from core.chat.projection.storage import affected_tools, decode_checkpoint, encode_checkpoint
+
+
+def chat_projection_codec() -> ProjectionCodec:
+ return ProjectionCodec(new_projection, fold_projection, affected_tools, encode_checkpoint, decode_checkpoint)
diff --git a/src/backend/orchestration/run_projection.py b/src/backend/core/chat/projection/reducer.py
similarity index 62%
rename from src/backend/orchestration/run_projection.py
rename to src/backend/core/chat/projection/reducer.py
index b1990bb2..f5cfe367 100644
--- a/src/backend/orchestration/run_projection.py
+++ b/src/backend/core/chat/projection/reducer.py
@@ -6,16 +6,29 @@
def new_projection(run_id):
- return {"run_id": run_id, "message_id": "", "started_at": None,
- "event_offset": 0, "blocks": [], "tools": [], "signals": {},
- "terminal": False, "history": [], "citations": []}
+ return {
+ "run_id": run_id,
+ "message_id": "",
+ "started_at": None,
+ "event_offset": 0,
+ "blocks": [],
+ "tools": [],
+ "signals": {},
+ "terminal": False,
+ "history": [],
+ "citations": [],
+ }
def _text(state, kind, content, **fields):
if not content:
return
blocks = state["blocks"]
- if blocks and blocks[-1]["kind"] == kind and all(blocks[-1].get(k) == v for k, v in fields.items()):
+ if (
+ blocks
+ and blocks[-1]["kind"] == kind
+ and all(blocks[-1].get(k) == v for k, v in fields.items())
+ ):
blocks[-1]["content"] += content
else:
blocks.append({"kind": kind, "content": content, **fields})
@@ -27,9 +40,16 @@ def _tool(state, event):
index = next((i for i, t in enumerate(tools) if identity and t.get("id") == identity), -1)
if index < 0:
index = len(tools)
- tools.append({"id": identity or None, "name": event.get("tool_name") or "",
- "displayName": event.get("tool_display_name"), "input": {}, "status": "running",
- "timestamp": event.get("started_at") or event.get("server_ts")})
+ tools.append(
+ {
+ "id": identity or None,
+ "name": event.get("tool_name") or "",
+ "displayName": event.get("tool_display_name"),
+ "input": {},
+ "status": "running",
+ "timestamp": event.get("started_at") or event.get("server_ts"),
+ }
+ )
state["blocks"].append({"kind": "tool", "index": index})
return tools[index]
@@ -44,18 +64,32 @@ def _child(tool, event):
else:
steps.append({"kind": kind, "text": str(event.get("delta") or "")})
elif kind in {"tool_call", "tool_call_start", "tool_call_delta", "tool_result"}:
- step = next((s for s in steps if s["kind"] == "tool" and tid and s.get("toolId") == tid), None)
+ step = next(
+ (s for s in steps if s["kind"] == "tool" and tid and s.get("toolId") == tid), None
+ )
if step is None:
- step = {"kind": "tool", "toolId": tid, "name": event.get("tool_name") or "", "status": "running"}
+ step = {
+ "kind": "tool",
+ "toolId": tid,
+ "name": event.get("tool_name") or "",
+ "status": "running",
+ }
steps.append(step)
if kind == "tool_call":
step["input"] = event.get("input", event.get("tool_args", {}))
if kind == "tool_call_delta":
step["inputText"] = step.get("inputText", "") + str(event.get("arguments_delta") or "")
if kind == "tool_result":
- step.update(output=event.get("output", event.get("result")), status=event.get("status") or "success")
+ step.update(
+ output=event.get("output", event.get("result")),
+ status=event.get("status") or "success",
+ )
elif kind == "end":
- tool["status"] = "interrupted" if event.get("status") == "cancelled" else "success" if event.get("ok") else "error"
+ tool["status"] = (
+ "interrupted"
+ if event.get("status") == "cancelled"
+ else "success" if event.get("ok") else "error"
+ )
for step in steps:
if step["kind"] == "tool" and step.get("status") == "running":
step["status"] = "interrupted"
@@ -63,7 +97,6 @@ def _child(tool, event):
tool["subagentName"] = event["agent_name"]
-
def _signal(state, event):
kind = event.get("type", "")
identity = event.get("request_id") or event.get("confirm_id") or event.get("tool_id") or ""
@@ -82,18 +115,26 @@ def _signal(state, event):
if kind in {"file_confirm", "design_pick"} and event.get("expired"):
state["signals"].pop(key, None)
return
- value = {**state["signals"].get(key, {}), **{k: v for k, v in event.items() if not k.startswith("_")}}
+ value = {
+ **state["signals"].get(key, {}),
+ **{k: v for k, v in event.items() if not k.startswith("_")},
+ }
for field in ("delta", "arguments_delta"):
if field in event and key in state["signals"]:
value[field] = str(state["signals"][key].get(field) or "") + str(event[field] or "")
state["signals"][key] = value
+
def fold_projection(previous, event):
"""Copy-on-write: concurrent appenders never mutate a published base."""
- state = {**previous, "blocks": [dict(b) for b in previous["blocks"]],
- "tools": [dict(t) for t in previous["tools"]],
- "signals": dict(previous["signals"]), "history": list(previous["history"]),
- "citations": list(previous.get("citations", []))}
+ state = {
+ **previous,
+ "blocks": [dict(b) for b in previous["blocks"]],
+ "tools": [dict(t) for t in previous["tools"]],
+ "signals": dict(previous["signals"]),
+ "history": list(previous["history"]),
+ "citations": list(previous.get("citations", [])),
+ }
kind = event.get("type")
if kind == "run_started":
state = new_projection(state["run_id"])
@@ -105,18 +146,36 @@ def fold_projection(previous, event):
previous_message["terminal"] = True
history.append({"role": "assistant", "state": previous_message})
if event.get("message"):
- history.append({"role": "user", "message_id": event.get("message_id"),
- "content": event["message"], "timestamp": event.get("server_ts")})
+ history.append(
+ {
+ "role": "user",
+ "message_id": event.get("message_id"),
+ "content": event["message"],
+ "timestamp": event.get("server_ts"),
+ }
+ )
state = new_projection(state["run_id"])
state["history"] = history
- state.update(message_id=event.get("next_assistant_message_id") or "",
- started_at=event.get("server_ts"))
+ state.update(
+ message_id=event.get("next_assistant_message_id") or "",
+ started_at=event.get("server_ts"),
+ )
elif kind in {"content", "ai_message", "text", "delta"}:
- _text(state, "text", str(event.get("delta") or event.get("content") or event.get("text") or ""))
+ _text(
+ state,
+ "text",
+ str(event.get("delta") or event.get("content") or event.get("text") or ""),
+ )
elif kind == "compaction_progress":
block = {"kind": "compaction", "progress": dict(event)}
- index = next((i for i, b in enumerate(state["blocks"])
- if b["kind"] == "compaction" and b.get("progress", {}).get("id") == event.get("id")), -1)
+ index = next(
+ (
+ i
+ for i, b in enumerate(state["blocks"])
+ if b["kind"] == "compaction" and b.get("progress", {}).get("id") == event.get("id")
+ ),
+ -1,
+ )
if index >= 0:
state["blocks"][index] = block
else:
@@ -125,10 +184,16 @@ def fold_projection(previous, event):
if event.get("structured_reasoning") is True and not state.get("structured_reasoning"):
state["blocks"].append({"kind": "protocol", "structured": True})
state["structured_reasoning"] = True
- _text(state, "thinking", str(event.get("delta") or event.get("message") or ""),
- structured=event.get("structured_reasoning") is True)
+ _text(
+ state,
+ "thinking",
+ str(event.get("delta") or event.get("message") or ""),
+ structured=event.get("structured_reasoning") is True,
+ )
elif kind == "content_replace":
- state["blocks"].append({"kind": "answer", "content": str(event.get("content") or event.get("text") or "")})
+ state["blocks"].append(
+ {"kind": "answer", "content": str(event.get("content") or event.get("text") or "")}
+ )
state["structured_reasoning"] = True
elif event.get("scope") == "ontology_revision":
_signal(state, event)
@@ -140,15 +205,22 @@ def fold_projection(previous, event):
elif kind == "tool_call_delta":
tool["inputText"] = tool.get("inputText", "") + str(event.get("arguments_delta") or "")
elif kind == "tool_result":
- tool.update(output=event.get("result"), status=event.get("status") or "success",
- durationMs=event.get("duration_ms"))
+ tool.update(
+ output=event.get("result"),
+ status=event.get("status") or "success",
+ durationMs=event.get("duration_ms"),
+ )
state["blocks"].append({"kind": "phase"})
if tool["name"] == "choose_design":
- state["signals"] = {k: v for k, v in state["signals"].items() if v.get("type") != "design_pick"}
+ state["signals"] = {
+ k: v for k, v in state["signals"].items() if v.get("type") != "design_pick"
+ }
elif kind == "subagent_event" and event.get("sub_type") == "job_progress":
_signal(state, event)
elif kind == "subagent_event":
- parent = next((t for t in state["tools"] if t.get("id") == event.get("parent_tool_id")), None)
+ parent = next(
+ (t for t in state["tools"] if t.get("id") == event.get("parent_tool_id")), None
+ )
if parent is not None:
parent["subSteps"] = [dict(step) for step in parent.get("subSteps", [])]
_child(parent, event)
diff --git a/src/backend/orchestration/run_projection_storage.py b/src/backend/core/chat/projection/storage.py
similarity index 89%
rename from src/backend/orchestration/run_projection_storage.py
rename to src/backend/core/chat/projection/storage.py
index 7fceb460..98b3e0d6 100644
--- a/src/backend/orchestration/run_projection_storage.py
+++ b/src/backend/core/chat/projection/storage.py
@@ -1,4 +1,5 @@
"""Checkpoint fields: changing one child card must not rewrite every card."""
+
import json
@@ -28,5 +29,8 @@ def affected_tools(state, event):
if kind in {"run_started", "steer_applied"}:
return set(range(len(state["tools"])))
identity = event.get("parent_tool_id") if kind == "subagent_event" else event.get("tool_id")
- return {index for index, tool in enumerate(state["tools"])
- if identity and tool.get("id") == str(identity)}
+ return {
+ index
+ for index, tool in enumerate(state["tools"])
+ if identity and tool.get("id") == str(identity)
+ }
diff --git a/src/backend/orchestration/run_subscription.py b/src/backend/core/chat/projection/subscription.py
similarity index 80%
rename from src/backend/orchestration/run_subscription.py
rename to src/backend/core/chat/projection/subscription.py
index 485dfb26..7d977ceb 100644
--- a/src/backend/orchestration/run_subscription.py
+++ b/src/backend/core/chat/projection/subscription.py
@@ -1,12 +1,15 @@
"""Native snapshot + tail subscription. Transport EOF carries no task verdict."""
+
import asyncio
import contextlib
import json
import time
-from core.llm import interaction_store
-from orchestration import chat_run_executor
-from orchestration.run_event_stream import START, _sort_key, get_run_event_stream
+import core.runs.clock as clock
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+from core.llm.runtime import interaction_store
+from core.runs.events.store import START, sort_key, get_run_event_stream
async def subscribe_run(run_id):
@@ -20,27 +23,32 @@ async def snapshot_frame(state, run):
if signal.get("type") not in {"file_confirm", "design_pick"}:
continue
record = await interaction_store.read("confirm", run.chat_id, signal.get("confirm_id"))
- if (record is None or record.get("decision")
- or record["expires_at"] <= time.time() or record["lease_until"] <= time.time()):
+ if (
+ record is None
+ or record.get("decision")
+ or record["expires_at"] <= time.time()
+ or record["lease_until"] <= time.time()
+ ):
signals.pop(key)
state = {**state, "signals": signals}
return {"type": "run_snapshot", "state": state, "event_offset": state["event_offset"]}
async def frames():
state, cursor = await stream.capture(run_id)
- run = await asyncio.to_thread(chat_run_executor.get_run, run_id)
+ run = await asyncio.to_thread(runtime.get_run, run_id)
if run is None:
raise RuntimeError("run no longer exists")
if state is None and cursor == START and not run.last_event_offset:
# Admission already owns an empty assistant row, before event 1.
# This is the empty event prefix, not a reconstructed history row.
- from orchestration.run_projection import new_projection
+ from core.chat.projection.reducer import new_projection
+
state = new_projection(run_id)
state["message_id"] = run.message_id
- state["started_at"] = chat_run_executor._epoch_ms(run.started_at)
+ state["started_at"] = clock._epoch_ms(run.started_at)
if state is None or not state.get("message_id"):
raise RuntimeError("run projection unavailable")
- if run.status in chat_run_executor._TERMINAL_STATUSES:
+ if run.status in settings._TERMINAL_STATUSES:
state, cursor = await stream.capture(run_id)
if state is None:
raise RuntimeError("run projection expired")
@@ -51,15 +59,15 @@ async def frames():
while True:
batch = await stream.wait(run_id, after=cursor, limit=100, timeout_ms=5000)
if not batch:
- run = await asyncio.to_thread(chat_run_executor.get_run, run_id)
- if run and run.status in chat_run_executor._TERMINAL_STATUSES:
+ run = await asyncio.to_thread(runtime.get_run, run_id)
+ if run and run.status in settings._TERMINAL_STATUSES:
state, _ = await stream.capture(run_id)
if state is not None:
state["terminal"] = True
yield await snapshot_frame(state, run)
return
first = await stream.read(run_id, limit=1)
- if cursor != START and first and _sort_key(first[0][0]) > _sort_key(cursor):
+ if cursor != START and first and sort_key(first[0][0]) > sort_key(cursor):
state, cursor = await stream.capture(run_id)
if state is None:
raise RuntimeError("run projection expired")
diff --git a/src/backend/core/services/public_chat_service.py b/src/backend/core/chat/public.py
similarity index 78%
rename from src/backend/core/services/public_chat_service.py
rename to src/backend/core/chat/public.py
index ed25cb94..f429f328 100644
--- a/src/backend/core/services/public_chat_service.py
+++ b/src/backend/core/chat/public.py
@@ -1,26 +1,23 @@
"""Public entry links reuse scoped API credentials; visitor scope never grants account access."""
-import hashlib
-import re
import secrets
+from core.auth.public_identity import digest
import uuid
from datetime import datetime, timezone
from core.db.models import ContentBlock, UserApiKey, UserShadow
from core.infra.crypto import decrypt_secret, encrypt_secret
-from core.services.api_key_service import ApiKeyService, resolve_api_key_identity
+from core.auth.api_keys import ApiKeyService
from fastapi import HTTPException
from sqlalchemy.exc import IntegrityError
-def digest(value):
- return hashlib.sha256(value.encode()).hexdigest()
def require_manager(db, user_id, kind, resource_id):
from core.auth.capabilities import resolve_user_capabilities
- from core.services.agent_api_service import require_agent_manager
- from core.services.project_api_service import require_project_key_access
+ from core.auth.agent_api import require_agent_manager
+ from core.projects.api import require_project_key_access
if kind not in ("agents", "projects"):
raise HTTPException(404, detail="Unknown resource")
@@ -113,30 +110,6 @@ def set_enabled(db, user_id, kind, resource_id, enabled):
return get_link(db, user_id, kind, resource_id)
-def resolve_public_user(db, token, visitor):
- from core.auth.backend import UserContext
-
- if not re.fullmatch(r"[A-Za-z0-9_-]{43}", token) or not re.fullmatch(
- r"[A-Za-z0-9_-]{32,64}", visitor
- ):
- raise HTTPException(401, detail="Invalid public chat link")
- row = db.get(ContentBlock, "publicchat:token:" + digest(token))
- key = db.get(UserApiKey, row.payload["key_id"]) if row else None
- identity = (
- resolve_api_key_identity(db, decrypt_secret(key.key_enc)) if key and key.key_enc else None
- )
- if not identity:
- raise HTTPException(401, detail="Public chat link is unavailable")
- owner, key = identity
- return UserContext(
- user_id=owner.user_id,
- user_center_id=owner.user_center_id or "",
- username=owner.username,
- api_key_id=key.id,
- api_key_agent_id=key.agent_id,
- api_key_project_id=key.project_id,
- public_chat_visitor=digest(visitor),
- )
def visitor_preference_id(user):
@@ -144,14 +117,14 @@ def visitor_preference_id(user):
def visitor_approval_mode(db, user, default=None):
- from core.llm.tool_permissions import normalize_approval_mode
+ from core.sandbox.permissions.approval import normalize_approval_mode
row = db.get(ContentBlock, visitor_preference_id(user))
return normalize_approval_mode((row.payload if row else {}).get("tool_approval_mode", default))
def set_visitor_approval_mode(db, user, mode):
- from core.llm.tool_permissions import normalize_approval_mode
+ from core.sandbox.permissions.approval import normalize_approval_mode
key = visitor_preference_id(user)
row = db.get(ContentBlock, key)
@@ -165,8 +138,8 @@ def set_visitor_approval_mode(db, user, mode):
def prepare_visitor_upload(db, user, chat_id, folder_id=None):
from core.chat.schemas import ChatRequest
- from core.services.agent_api_service import prepare_agent_api_request
- from core.services.project_api_service import prepare_project_api_request
+ from core.auth.agent_api import prepare_agent_api_request
+ from core.projects.api import prepare_project_api_request
if not chat_id or folder_id:
raise HTTPException(422, detail="A public upload requires its conversation")
diff --git a/src/backend/core/services/chat_reference_service.py b/src/backend/core/chat/references.py
similarity index 98%
rename from src/backend/core/services/chat_reference_service.py
rename to src/backend/core/chat/references.py
index 4be774e7..f6a6637a 100644
--- a/src/backend/core/services/chat_reference_service.py
+++ b/src/backend/core/chat/references.py
@@ -17,7 +17,7 @@
from core.db.models import ChatMessage, ChatSession
from core.db.reserved_chats import ordinary_chat_filter
-from core.services.chat_service import ChatService
+from core.chat.service import ChatService
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
@@ -286,8 +286,8 @@ async def build_session_digest(
import asyncio
from core.db.engine import SessionLocal
- from core.services.compaction_service import _load_history
- from core.services.compaction_summary import _summarize
+ from core.chat.compaction.service import _load_history
+ from core.chat.compaction.summary import _summarize
def _prepare():
with SessionLocal() as db:
diff --git a/src/backend/core/chat/runtime/__init__.py b/src/backend/core/chat/runtime/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/backend/core/chat/runtime/admission.py b/src/backend/core/chat/runtime/admission.py
new file mode 100644
index 00000000..63a1434c
--- /dev/null
+++ b/src/backend/core/chat/runtime/admission.py
@@ -0,0 +1,98 @@
+"""Admit a normal chat run and start its lease-owning worker."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+import core.chat.runtime.worker as worker
+import core.db.engine as engine
+import core.db.models as models
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.runtime as runtime
+from core.db.models import ChatRun
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def start_run(
+ *,
+ accepted_run: Optional[ChatRun] = None,
+ chat_id: str,
+ user_id: str,
+ session_messages: List[Dict[str, Any]],
+ effective_user_message: str,
+ raw_user_message: str,
+ context: Dict[str, Any],
+ request_payload: Optional[Dict[str, Any]] = None,
+ model_name: Optional[str] = None,
+) -> ChatRun:
+ """Launch a ChatRun that ChatSequencer has already durably accepted.
+
+ Acceptance (user message + pending run) must commit before this function is
+ called. Keeping launch separate makes a process crash between the two
+ recoverable by the Run Journal in #64 instead of losing the request.
+ """
+
+ def persist_launch():
+ recovery_snapshot = writer._json_safe(
+ {
+ "kind": "chat",
+ "worker_args": {
+ "session_messages": session_messages,
+ "effective_user_message": effective_user_message,
+ "raw_user_message": raw_user_message,
+ "context": context,
+ "model_name": model_name,
+ },
+ }
+ )
+ if accepted_run is None:
+ run = ownership.create_run_record(
+ chat_id=chat_id,
+ user_id=user_id,
+ request_payload=dict(request_payload or {}),
+ recovery_snapshot=recovery_snapshot,
+ )
+ else:
+ run = accepted_run
+ if run.chat_id != chat_id or run.user_id != user_id or run.status != "pending":
+ raise ValueError("accepted_run does not match the launch request")
+ with engine.SessionLocal() as db:
+ durable_run = db.get(models.ChatRun, run.run_id)
+ if durable_run is None or durable_run.status != "pending":
+ raise ValueError("accepted_run is no longer pending")
+ durable_run.recovery_snapshot = recovery_snapshot
+ durable_run.snapshot_version = max(int(durable_run.snapshot_version or 0), 1)
+ durable_run.run_phase = durable_run.run_phase or "accepted"
+ durable_run.last_operation_safety = (
+ durable_run.last_operation_safety or "replayable"
+ )
+ durable_run.updated_at = clock._utcnow()
+ db.commit()
+ return run
+
+ run = await asyncio.to_thread(persist_launch)
+ journal_owner = ownership.new_worker_owner(run.run_id)
+ runtime.register_run_task(
+ run.run_id,
+ worker._run_workflow(
+ run_id=run.run_id,
+ chat_id=chat_id,
+ user_id=user_id,
+ message_id=run.message_id,
+ assistant_chat_seq=run.assistant_chat_seq,
+ session_messages=session_messages,
+ effective_user_message=effective_user_message,
+ raw_user_message=raw_user_message,
+ context=context,
+ model_name=model_name,
+ journal_owner=journal_owner,
+ ),
+ name=f"chat_run:{run.run_id}",
+ )
+ logger.info("chat_run_started", run_id=run.run_id, chat_id=chat_id, user_id=user_id)
+ return run
diff --git a/src/backend/core/chat/runtime/chunks.py b/src/backend/core/chat/runtime/chunks.py
new file mode 100644
index 00000000..544b62a5
--- /dev/null
+++ b/src/backend/core/chat/runtime/chunks.py
@@ -0,0 +1,183 @@
+"""Apply workflow events to chat output and durable projections."""
+
+from __future__ import annotations
+
+import core.llm.runtime.model_steps as model_steps
+from core.infra.text_bounds import bound_result_for_display
+from core.chat.tool_log import (
+ attach_subagent_step,
+ build_thinking_event,
+ build_tool_call_delta_event,
+ build_tool_call_event,
+ build_tool_call_start_event,
+ build_tool_result_event,
+ build_user_question_event,
+ build_user_question_resolved_event,
+)
+from core.infra.logging import get_logger
+
+
+from core.chat.runtime import completion, snapshots, steering
+
+logger = get_logger(__name__)
+
+
+async def consume_chunk(turn, chunk):
+ chunk_type = chunk.get("type")
+ if chunk_type == "model_dispatch":
+ turn.journal.append_operation(
+ turn.run_id,
+ owner=turn.owner,
+ operation_type="model_dispatch",
+ phase="model_inflight",
+ safety="replayable",
+ )
+ elif chunk_type == "thinking":
+ _thinking_evt = build_thinking_event(chunk, turn.chat_id)
+ if _thinking_evt.get("delta"):
+ turn.thinking_parts.append(str(_thinking_evt["delta"]))
+ await turn.emit(_thinking_evt)
+ elif chunk_type in {"ai_message", "content"}:
+ delta = chunk.get("delta", "")
+ if delta:
+ snapshots._flush_thinking(turn)
+ turn.full_response += delta
+ turn.segments.add_text(delta)
+ await turn.emit(
+ {"type": "content", "event": "ai_message", "delta": delta, "chat_id": turn.chat_id}
+ )
+ elif chunk_type == "content_replace":
+ replacement = str(chunk.get("content") or "")
+ turn.thinking_parts.clear()
+ turn.full_response = replacement
+ turn.round_block = None
+ turn.thinking_log.clear()
+ model_steps.replace_final_text(turn.model_steps, replacement)
+ turn.segments.reset(preserve_compaction=True)
+ turn.segments.add_text(replacement)
+ await turn.emit(
+ {
+ "type": "content_replace",
+ "content": replacement,
+ "reason": chunk.get("reason"),
+ "chat_id": turn.chat_id,
+ }
+ )
+ await snapshots._flush_inflight(turn)
+ elif chunk_type == "steer_applied":
+ await steering.apply_steer(turn, chunk)
+ elif chunk_type == "tool_call":
+ snapshots._flush_thinking(turn)
+ turn.round_block = None
+ _tc_evt = build_tool_call_event(chunk, turn.chat_id, turn.tool_calls_log)
+ turn.segments.add_tools(turn.tool_calls_log)
+ await turn.emit(_tc_evt)
+ await snapshots._flush_inflight(turn)
+ elif chunk_type == "tool_call_start":
+ snapshots._flush_thinking(turn)
+ turn.round_block = None
+ await turn.emit(build_tool_call_start_event(chunk, turn.chat_id))
+ elif chunk_type == "tool_call_delta":
+ await turn.emit(build_tool_call_delta_event(chunk, turn.chat_id))
+ elif chunk_type == "tool_progress":
+ # Live status for a running tool card; the result supersedes it, so it is not persisted.
+ await turn.emit({**chunk, "chat_id": turn.chat_id})
+ elif chunk_type == "tool_result":
+ step = chunk.get("model_step")
+ if isinstance(step, dict):
+ turn.model_steps.append(step)
+ _tr_evt = build_tool_result_event(chunk, turn.chat_id, turn.tool_calls_log)
+ turn.segments.add_tools(turn.tool_calls_log)
+ if isinstance(step, dict):
+ if not await snapshots._flush_inflight(turn) and (
+ not await snapshots._flush_inflight(turn)
+ ):
+ raise RuntimeError("Could not persist completed tool result")
+ await turn.emit(_tr_evt)
+ await snapshots._flush_inflight(turn)
+ elif chunk_type == "model_step":
+ step = chunk.get("step")
+ if isinstance(step, dict):
+ turn.model_steps.append(step)
+ if step.get("kind") == "tool_result":
+ if not await snapshots._flush_inflight(turn) and (
+ not await snapshots._flush_inflight(turn)
+ ):
+ raise RuntimeError("Could not persist completed tool result")
+ else:
+ await snapshots._flush_inflight(turn)
+ elif chunk_type == "compaction_progress":
+ turn.segments.add_compaction(chunk)
+ await turn.emit(chunk)
+ elif chunk_type == "context_usage":
+ await turn.emit({**chunk, "chat_id": turn.chat_id})
+ elif chunk_type in ("heartbeat", "model_progress"):
+ return
+ elif chunk_type == "tool_pending":
+ pending_event = {
+ "type": "tool_pending",
+ "chat_id": turn.chat_id,
+ "reason": chunk.get("reason", "llm_buffering"),
+ }
+ if chunk.get("scope"):
+ pending_event["scope"] = chunk["scope"]
+ await turn.emit(pending_event)
+ elif chunk_type == "subagent_event":
+ try:
+ attach_subagent_step(
+ turn.tool_calls_log, str(chunk.get("parent_tool_id", "") or ""), chunk
+ )
+ except Exception:
+ logger.debug("attach_subagent_step failed (ignored)", exc_info=True)
+ chunk["chat_id"] = turn.chat_id
+ await turn.emit(bound_result_for_display(chunk)[0])
+ elif chunk_type == "file_confirm":
+ await turn.emit(
+ {
+ "type": "file_confirm",
+ "chat_id": turn.chat_id,
+ "confirm_id": chunk.get("confirm_id"),
+ "op": chunk.get("op"),
+ "logical_path": chunk.get("logical_path"),
+ "message": chunk.get("message"),
+ "expired": chunk.get("expired", False),
+ }
+ )
+ elif chunk_type == "design_pick":
+ await turn.emit(
+ {
+ "type": "design_pick",
+ "chat_id": turn.chat_id,
+ "confirm_id": chunk.get("confirm_id"),
+ "question": chunk.get("question", ""),
+ "options": chunk.get("options", []),
+ "message": chunk.get("message"),
+ "expired": chunk.get("expired", False),
+ }
+ )
+ elif chunk_type == "user_question":
+ await turn.emit(build_user_question_event(chunk, turn.chat_id))
+ elif chunk_type == "user_question_resolved":
+ await turn.emit(build_user_question_resolved_event(chunk, turn.chat_id))
+ elif chunk_type == "plan_update":
+ await turn.emit(
+ {
+ "type": "plan_update",
+ "chat_id": turn.chat_id,
+ "title": chunk.get("title", ""),
+ "steps": chunk.get("steps", []),
+ }
+ )
+ elif chunk_type in {
+ "ontology_activation",
+ "ontology_gate",
+ "ontology_review",
+ "ontology_repair",
+ "ontology_revision",
+ "ontology_revision_thinking",
+ }:
+ ontology_event = dict(chunk)
+ ontology_event["chat_id"] = turn.chat_id
+ await turn.emit(bound_result_for_display(ontology_event)[0])
+ elif chunk_type == "meta":
+ await completion.complete_turn(turn, chunk)
diff --git a/src/backend/orchestration/post_turn_compaction.py b/src/backend/core/chat/runtime/compaction.py
similarity index 84%
rename from src/backend/orchestration/post_turn_compaction.py
rename to src/backend/core/chat/runtime/compaction.py
index ca07912b..4ffa6ae9 100644
--- a/src/backend/orchestration/post_turn_compaction.py
+++ b/src/backend/core/chat/runtime/compaction.py
@@ -1,8 +1,10 @@
"""Schedule background compression using the completed turn's frozen window."""
from __future__ import annotations
+
import asyncio
from typing import Any, Dict, Optional
+
from core.infra.logging import get_logger
logger = get_logger(__name__)
@@ -20,13 +22,8 @@ def _should_schedule_compaction(
if not _settings.compaction.enabled:
return False
- from core.llm.context_manager import resolve_model_context_window
- from core.services.compaction_service import (
- resolve_active_tokens,
- resolve_token_limit,
- resolve_trigger_ratio,
- should_compact,
- )
+ from core.models.context_window import resolve_model_context_window
+ from core.chat.compaction.service import resolve_active_tokens, resolve_token_limit, resolve_trigger_ratio, should_compact
active_tokens = resolve_active_tokens(usage)
window = int((budget_inputs or {}).get("context_window") or 0)
@@ -54,7 +51,7 @@ def _spawn_compaction_task(
Fire-and-forget; never affects the main conversation. This is *not* a second
compactor — it calls the same
- :func:`core.services.compaction_service.run_compaction` the mid-turn hook
+ :func:`core.chat.compaction.service.run_compaction` the mid-turn hook
does, and normally finds nothing to do: a turn that crossed the threshold
while it still had steps left has already compacted mid-turn and persisted
its checkpoint. It earns its keep only when the threshold is crossed by a
@@ -77,12 +74,8 @@ def _spawn_compaction_task(
async def _bg() -> None:
try:
- from core.llm.context_manager import resolve_model_context_window
- from core.services.compaction_service import (
- resolve_token_limit,
- resolve_trigger_ratio,
- run_post_turn_compaction,
- )
+ from core.models.context_window import resolve_model_context_window
+ from core.chat.compaction.service import resolve_token_limit, resolve_trigger_ratio, run_post_turn_compaction
# Trigger criterion — see resolve_active_tokens: real end-of-turn
# context occupancy, not the cumulative billing value. Gating here
@@ -99,7 +92,7 @@ async def _bg() -> None:
chat_id=chat_id,
limit=limit,
)
- from core.services.compaction_progress import progress_scope
+ from core.chat.compaction.progress import progress_scope
with progress_scope(None, chat_id, run_id, context={"message_id": message_id}):
await run_post_turn_compaction(
diff --git a/src/backend/core/chat/runtime/completion.py b/src/backend/core/chat/runtime/completion.py
new file mode 100644
index 00000000..cae41a44
--- /dev/null
+++ b/src/backend/core/chat/runtime/completion.py
@@ -0,0 +1,231 @@
+"""Commit successful chat output within the run journal transaction."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import time
+import typing as _typing
+
+import core.chat.runtime.compaction as compaction
+import core.chat.runtime.followups as followups
+import core.chat.runtime.handoff as handoff
+import core.runs.events.writer as writer
+import core.chat.service as services_ChatService_module
+from core.chat.context import now_iso
+from core.infra.logging import get_logger
+from core.llm.runtime import workspace as _workspace_mod
+from core.files.artifacts.service import persist_artifacts as _persist_artifacts
+
+
+from core.chat.runtime import snapshots
+
+logger = get_logger(__name__)
+
+
+async def complete_turn(turn, chunk):
+ snapshots._flush_thinking(turn)
+ internal_budget = chunk.get("_compaction_budget_inputs")
+ if isinstance(internal_budget, dict):
+ turn.compaction_budget_inputs = dict(internal_budget)
+ # Strict workspace gate: pinned list is the sole source of
+ # user-visible artifacts. See chats.py:_stream_sse_response.
+ _ws_pinned = _workspace_mod.get_pinned()
+ _ws_files = _workspace_mod.get_pinned_file_ids()
+ usage_payload = chunk.get("usage") or None
+ context_usage_payload = chunk.get("context_usage") or None
+ compaction_pending = not chunk.get(
+ "manual_compaction"
+ ) and compaction._should_schedule_compaction(
+ model_name=turn.model_name,
+ usage=usage_payload,
+ budget_inputs=turn.compaction_budget_inputs,
+ )
+ # 本轮总耗时:同一个值既下发给正在看的这一页,也写进 extra_data 供历史
+ # 加载使用。以前只写库不下发,前端只能自己按「占位气泡创建到现在」估一个,
+ # 把网络往返和排队也算了进去 —— 于是同一条回答,实时看到的耗时和刷新后
+ # 从历史读出来的对不上。
+ _duration_ms = int((time.monotonic() - turn.run_started_monotonic) * 1000)
+ turn.metadata = {
+ "type": "meta",
+ "duration_ms": _duration_ms,
+ "route": chunk.get("route", "main"),
+ "sources": chunk.get("sources", []),
+ "artifacts": _ws_pinned,
+ "warnings": chunk.get("warnings", []),
+ "is_markdown": chunk.get("is_markdown", False),
+ "chat_id": turn.chat_id,
+ "message_id": turn.current_message_id,
+ "citations": chunk.get("citations", []),
+ "workspace_files": _ws_files,
+ "context_usage": context_usage_payload,
+ "compaction_pending": compaction_pending,
+ "ontology_governance": chunk.get("ontology_governance"),
+ # Skeleton marker: settlement runs after the stream closes.
+ "evolution_pending": chunk.get("evolution_pending"),
+ }
+ await turn.emit(turn.metadata)
+
+ # Persist: assistant message + artifacts
+ _persist_extra = {
+ "timestamp": now_iso(),
+ "route": turn.metadata.get("route"),
+ "is_markdown": turn.metadata.get("is_markdown", False),
+ "sources": turn.metadata.get("sources", []),
+ "artifacts": turn.metadata.get("artifacts", []),
+ "warnings": turn.metadata.get("warnings", []),
+ "citations": turn.metadata.get("citations", []),
+ "message_id": turn.current_message_id,
+ "workspace_files": _ws_files,
+ "duration_ms": _duration_ms,
+ "segments": turn.segments.payload(),
+ }
+ if isinstance(context_usage_payload, dict):
+ _persist_extra["context_usage"] = context_usage_payload
+ if turn.metadata.get("ontology_governance"):
+ _persist_extra["ontology_governance"] = turn.metadata["ontology_governance"]
+ if turn.context.get("model_provider_id"):
+ _persist_extra["model_provider_id"] = turn.context.get("model_provider_id")
+ turn.journal.save_snapshot(
+ turn.run_id,
+ owner=turn.owner,
+ phase="model_completed",
+ safety="replayable",
+ snapshot=writer._json_safe(
+ {
+ "assistant_content": turn.full_response,
+ "message_id": turn.current_message_id,
+ "model_name": turn.model_name,
+ "thinking": snapshots._thinking_payload(turn),
+ "tool_calls": turn.tool_calls_log if turn.tool_calls_log else None,
+ "model_steps": snapshots._model_steps_payload(turn),
+ "usage": usage_payload,
+ "extra_data": _persist_extra,
+ "artifacts": _ws_pinned,
+ "context": turn.context,
+ }
+ ),
+ )
+
+ queued_handoff: _typing.Dict[str, _typing.Any] = {}
+
+ def _commit_output(db) -> None:
+ chat_service = services_ChatService_module.ChatService(db)
+ # The row already exists (created at admission, refreshed
+ # while streaming); this is the final write that also drops
+ # the in-flight marker because ``_persist_extra`` never
+ # carries it.
+ chat_service.upsert_message(
+ chat_id=turn.chat_id,
+ role="assistant",
+ content=turn.full_response,
+ model=turn.model_name,
+ thinking=snapshots._thinking_payload(turn),
+ tool_calls=turn.tool_calls_log if turn.tool_calls_log else None,
+ model_steps=snapshots._model_steps_payload(turn),
+ usage=usage_payload,
+ message_id=turn.current_message_id,
+ chat_seq=turn.current_chat_seq,
+ extra_data=_persist_extra,
+ invalidate_mid_turn_checkpoint=compaction_pending,
+ commit=False,
+ )
+ # Build a ProjectScope from the workflow context and pass it
+ # explicitly so pinned files keep their project ownership.
+ from core.projects.edition import project_scope_from_context
+
+ _persist_artifacts(
+ db,
+ turn.user_id,
+ turn.chat_id,
+ _ws_pinned,
+ scope=project_scope_from_context(turn.context),
+ commit=False,
+ )
+ prepared = handoff._commit_queued_handoff_in_session(
+ db,
+ source_run_id=turn.run_id,
+ chat_id=turn.chat_id,
+ user_id=turn.user_id,
+ context=turn.context,
+ model_name=turn.model_name,
+ )
+ if prepared is not None:
+ queued_handoff.update(prepared)
+
+ # Offloaded to a worker thread on purpose. This is the one
+ # journal call that commits the turn's real side effects (the
+ # assistant message, pinned artifacts, a queued handoff), so it
+ # is the one that can sit waiting on a row lock. ``complete``
+ # is synchronous SQLAlchemy; calling it inline blocks the event
+ # loop, and a blocked loop cannot run the coroutine holding the
+ # conflicting transaction — the process deadlocks and every
+ # request, stream and health probe stops answering. In a thread
+ # a lock wait costs this run its latency and nothing else.
+ # ``_commit_output`` and everything it calls are plain sync
+ # code, so they are safe to run off the loop.
+ completed = await asyncio.to_thread(
+ functools.partial(
+ turn.journal.complete,
+ turn.run_id,
+ owner=turn.owner,
+ status="completed",
+ usage=usage_payload,
+ last_event_offset=turn.offsets.last,
+ commit_effect=_commit_output,
+ committed_operation={
+ "operation_type": "message_committed",
+ "phase": "message_committed",
+ "safety": "side_effect_committed",
+ "payload": {"message_id": turn.current_message_id},
+ },
+ )
+ )
+ if not completed:
+ logger.warning(
+ "chat_run_terminal_cas_lost",
+ run_id=turn.run_id,
+ owner=turn.owner,
+ )
+ turn.stopped = True
+ return
+
+ # The DB lease has intentionally ended. Stop the heartbeat
+ # before scheduling the handoff and projecting terminal events;
+ # otherwise its next renew would interpret normal completion as
+ # lease loss and cancel this coroutine mid-handoff.
+ turn.lease_task.cancel()
+
+ if queued_handoff:
+ handoff._schedule_queued_handoff(
+ queued_handoff,
+ chat_id=turn.chat_id,
+ user_id=turn.user_id,
+ )
+ await turn.emit(
+ handoff._queued_run_started_event(turn.run_id, turn.chat_id, queued_handoff),
+ terminal=True,
+ )
+
+ if not chunk.get("manual_compaction"):
+ followups._spawn_followup_task(
+ run_id=turn.run_id,
+ chat_id=turn.chat_id,
+ user_msg=turn.latest_user_message,
+ response=turn.full_response,
+ msg_id=turn.current_message_id,
+ )
+
+ # End-of-turn compaction: when real token usage crosses the
+ # threshold → generate a summary checkpoint in the background;
+ # doesn't block stream close, and failures don't affect the main
+ # conversation.
+ compaction._spawn_compaction_task(
+ run_id=turn.run_id,
+ chat_id=turn.chat_id,
+ model_name=turn.model_name,
+ usage=usage_payload,
+ budget_inputs=turn.compaction_budget_inputs,
+ should_schedule=compaction_pending,
+ message_id=turn.current_message_id,
+ )
diff --git a/src/backend/core/chat/runtime/followups.py b/src/backend/core/chat/runtime/followups.py
new file mode 100644
index 00000000..07943726
--- /dev/null
+++ b/src/backend/core/chat/runtime/followups.py
@@ -0,0 +1,64 @@
+"""Schedule post-turn suggestions and drain admitted follow-up work."""
+
+from __future__ import annotations
+
+import asyncio
+import typing as _typing
+from typing import Any, Dict, List, Optional, Tuple
+
+from core.db.threads import run_in_session
+import core.runs.settings as settings
+import core.chat.service as services_ChatService_module
+from core.infra.logging import get_logger
+from core.runs.events.store import Entry
+
+logger = get_logger(__name__)
+
+
+def _spawn_followup_task(
+ *, run_id: str, chat_id: str, user_msg: str, response: str, msg_id: str
+) -> None:
+ """Equivalent to the existing _generate_followups_bg in chats.py."""
+ from core.chat.followups import get_followup_generator
+
+ async def _bg() -> None:
+ try:
+ questions = await asyncio.wait_for(
+ get_followup_generator().generate(user_msg, response, run_id=run_id),
+ timeout=10,
+ )
+ if questions:
+ await run_in_session(
+ lambda db: services_ChatService_module.ChatService(db).update_message_extra_data(
+ msg_id, {"follow_up_questions": questions}
+ )
+ )
+ except Exception as exc:
+ logger.warning("background_followup_failed", error=str(exc))
+
+ asyncio.create_task(_bg())
+
+
+def _drain(
+ entries: Optional[List[Entry]], from_offset: int, cursor: str
+) -> Tuple[List[Dict[str, Any]], str, bool]:
+ """Split a batch into deliverable events, the new cursor, and "saw terminal".
+
+ Skips ``model_progress`` markers: nothing writes them anymore (the reaper
+ trusts in-process task liveness), but entries from older builds may survive
+ within the log's TTL and must never reach clients — the frontend treats an
+ unknown event type as "pending ended".
+
+ An empty batch may arrive as ``None`` rather than ``[]``; a read that
+ returns nothing must never surface to the client as "流式响应中断".
+ """
+ delivered: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ for entry_cursor, event in entries or ():
+ cursor = entry_cursor
+ if not isinstance(event, dict) or event.get("type") == "model_progress":
+ continue
+ if event.get("type") == settings.TERMINAL_TYPE:
+ return delivered, cursor, True
+ if event.get("_offset", 0) > from_offset:
+ delivered.append(event)
+ return delivered, cursor, False
diff --git a/src/backend/core/chat/runtime/handoff.py b/src/backend/core/chat/runtime/handoff.py
new file mode 100644
index 00000000..13064e89
--- /dev/null
+++ b/src/backend/core/chat/runtime/handoff.py
@@ -0,0 +1,272 @@
+"""Commit queued chat instructions and hand off to the next durable turn."""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Dict, List, Optional
+
+import core.chat.inflight as inflight
+import core.chat.runtime.worker as worker
+import core.db.engine as engine
+import core.db.models as models
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.runtime as runtime
+import core.chat.service as services_ChatService_module
+import core.runs.journal as run_journal
+from core.db.models import ChatRun
+from core.infra.logging import get_logger
+from core.chat.service import ChatService
+
+logger = get_logger(__name__)
+
+
+def _persisted_history(chat_service: ChatService, chat_id: str) -> List[Dict[str, Any]]:
+ """The model-facing history a successor or recovery run starts from.
+
+ Read back from the rows already committed in this transaction — the step
+ record included — never rebuilt from text the source run held in memory.
+ """
+ from core.chat.compaction.service import _load_history
+
+ return writer._json_safe(_load_history(chat_service, chat_id, repair=False))
+
+
+def _commit_queued_handoff_in_session(
+ db,
+ *,
+ source_run_id: str,
+ chat_id: str,
+ user_id: str,
+ context: Dict[str, Any],
+ model_name: Optional[str],
+) -> Optional[Dict[str, Any]]:
+ """Atomically consume one terminal queued item and stage its ChatRun.
+
+ This includes a steer accepted after the source run crossed its final safe
+ model/tool boundary. Such an instruction can no longer enter the source
+ ReAct loop, so completion hands it to a successor instead of stranding it.
+
+ The successor's history is read back from the rows this transaction just
+ committed — the step record included — never rebuilt from the text the
+ source run happened to hold in memory.
+ """
+ from core.db.models.chat import reserve_chat_sequences
+ from core.chat.steer_queue import SteerQueue
+
+ source_run = db.get(models.ChatRun, source_run_id)
+ if source_run is None:
+ return None
+ root_run_id = str(context.get("handoff_root_run_id") or source_run_id)
+ handoff = SteerQueue.consume_next_handoff_in_session(
+ db,
+ source_run_id=source_run_id,
+ root_run_id=root_run_id,
+ chat_id=chat_id,
+ applied_operation_seq=int(source_run.operation_seq or 0) + 1,
+ )
+ if handoff is None or not handoff.applied_run_id:
+ return None
+
+ first_seq = reserve_chat_sequences(
+ db,
+ chat_id,
+ count=2,
+ owner_user_id=user_id,
+ )
+ next_run_id = handoff.applied_run_id
+ next_message_id = (
+ f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:assistant').hex[:16]}"
+ )
+ queued_user_message_id = (
+ f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:user').hex[:16]}"
+ )
+ next_context = writer._json_safe(dict(context or {}))
+ next_context.pop("journal_owner", None)
+ next_context.update(
+ {
+ "run_id": next_run_id,
+ "message_id": next_message_id,
+ "chat_id": chat_id,
+ "user_id": user_id,
+ "handoff_queue_id": handoff.queue_id,
+ "handoff_delivery_mode": handoff.delivery_mode,
+ "handoff_root_run_id": root_run_id,
+ }
+ )
+ next_session_messages = _persisted_history(services_ChatService_module.ChatService(db), chat_id)
+ next_session_messages.append({"role": "user", "content": handoff.message})
+ worker_args = {
+ "session_messages": next_session_messages,
+ "effective_user_message": handoff.message,
+ "raw_user_message": handoff.message,
+ "context": next_context,
+ "model_name": model_name,
+ }
+ request_payload = dict(source_run.request_payload or {})
+ request_payload.update(
+ {
+ "message": handoff.message,
+ "chat_id": chat_id,
+ "attachments": [],
+ "handoff": {
+ "queue_id": handoff.queue_id,
+ "steer_id": handoff.steer_id,
+ "delivery_mode": handoff.delivery_mode,
+ "source_run_id": source_run_id,
+ },
+ }
+ )
+ run_journal.RunJournal.accept_in_session(
+ db,
+ run_id=next_run_id,
+ message_id=next_message_id,
+ chat_id=chat_id,
+ user_id=user_id,
+ request_payload=writer._json_safe(request_payload),
+ recovery_snapshot={"kind": "chat", "worker_args": writer._json_safe(worker_args)},
+ user_message_id=queued_user_message_id,
+ user_chat_seq=first_seq,
+ assistant_chat_seq=first_seq + 1,
+ writer_slot="main",
+ )
+ services_ChatService_module.ChatService(db).upsert_message(
+ chat_id=chat_id,
+ role="user",
+ content=handoff.message,
+ model=model_name,
+ message_id=queued_user_message_id,
+ chat_seq=first_seq,
+ extra_data={
+ "queued_handoff": True,
+ "steer_queue_id": handoff.queue_id,
+ "steer_id": handoff.steer_id,
+ "steer_seq": handoff.steer_seq,
+ "delivery_mode": handoff.delivery_mode,
+ "source_run_id": source_run_id,
+ "run_id": next_run_id,
+ },
+ commit=False,
+ )
+ db.add(
+ inflight.new_assistant_row(
+ message_id=next_message_id,
+ chat_id=chat_id,
+ chat_seq=first_seq + 1,
+ run_id=next_run_id,
+ model=model_name,
+ created_at=clock._utcnow(),
+ )
+ )
+ db.flush()
+ return {
+ "run_id": next_run_id,
+ "message_id": next_message_id,
+ "user_message_id": queued_user_message_id,
+ "assistant_chat_seq": first_seq + 1,
+ "queue_id": handoff.queue_id,
+ "steer_id": handoff.steer_id,
+ "steer_seq": handoff.steer_seq,
+ "delivery_mode": handoff.delivery_mode,
+ **worker_args,
+ }
+
+
+def _schedule_queued_handoff(
+ handoff: Dict[str, Any],
+ *,
+ chat_id: str,
+ user_id: str,
+ task_prefix: str = "chat_run_handoff",
+) -> str:
+ """Start a committed handoff; pending DB state remains recovery-safe on failure."""
+ next_run_id = str(handoff["run_id"])
+ next_owner = ownership.new_worker_owner(next_run_id)
+ runtime.register_run_task(
+ next_run_id,
+ worker._run_workflow(
+ run_id=next_run_id,
+ chat_id=chat_id,
+ user_id=user_id,
+ message_id=str(handoff["message_id"]),
+ assistant_chat_seq=(
+ int(handoff["assistant_chat_seq"])
+ if handoff.get("assistant_chat_seq") is not None
+ else None
+ ),
+ session_messages=list(handoff.get("session_messages") or []),
+ effective_user_message=str(handoff.get("effective_user_message") or ""),
+ raw_user_message=str(handoff.get("raw_user_message") or ""),
+ context=dict(handoff.get("context") or {}),
+ model_name=(str(handoff["model_name"]) if handoff.get("model_name") else None),
+ journal_owner=next_owner,
+ ),
+ name=f"{task_prefix}:{next_run_id}",
+ )
+ return next_run_id
+
+
+def _queued_run_started_event(
+ source_run_id: str,
+ chat_id: str,
+ handoff: Dict[str, Any],
+) -> Dict[str, Any]:
+ return {
+ "type": "queued_run_started",
+ "chat_id": chat_id,
+ "source_run_id": source_run_id,
+ "run_id": str(handoff["run_id"]),
+ "message_id": handoff["message_id"],
+ "user_message_id": handoff["user_message_id"],
+ "message": handoff["raw_user_message"],
+ "queue_id": handoff["queue_id"],
+ "steer_id": handoff["steer_id"],
+ "steer_seq": handoff["steer_seq"],
+ "delivery_mode": handoff["delivery_mode"],
+ }
+
+
+async def _write_queued_run_started_projection(
+ source_run_id: str,
+ *,
+ chat_id: str,
+ handoff: Dict[str, Any],
+) -> None:
+ """Best-effort event projection; the queue API is the durable backfill seam."""
+ try:
+ offset = ownership.journal().allocate_event_offset(source_run_id, terminal=True)
+ await writer.xadd_event(
+ source_run_id,
+ offset,
+ _queued_run_started_event(source_run_id, chat_id, handoff),
+ )
+ except Exception as exc: # Redis projection must not undo the committed handoff
+ logger.warning(
+ "chat_run_handoff_projection_failed",
+ run_id=source_run_id,
+ child_run_id=handoff.get("run_id"),
+ error=str(exc),
+ )
+
+
+def get_successor_run(run_id: str, *, user_id: str) -> Optional[ChatRun]:
+ """Read the successor committed by the durable steering handoff."""
+ from core.db.models import ChatSteerQueueItem
+
+ with engine.SessionLocal() as db:
+ source = db.get(models.ChatRun, run_id)
+ if source is None or source.user_id != user_id:
+ raise runtime.ChatRunPermissionDenied("run does not belong to user")
+ successor_id = (
+ db.query(ChatSteerQueueItem.applied_run_id)
+ .filter(
+ ChatSteerQueueItem.applied_source_run_id == run_id,
+ ChatSteerQueueItem.applied_run_id != run_id,
+ ChatSteerQueueItem.status == "applied",
+ ChatSteerQueueItem.chat_id == source.chat_id,
+ ChatSteerQueueItem.user_id == user_id,
+ )
+ .scalar()
+ )
+ return runtime.get_run(successor_id) if successor_id else None
diff --git a/src/backend/orchestration/job_wakeup.py b/src/backend/core/chat/runtime/job_wakeup.py
similarity index 64%
rename from src/backend/orchestration/job_wakeup.py
rename to src/backend/core/chat/runtime/job_wakeup.py
index ab09d17e..b91630d8 100644
--- a/src/backend/orchestration/job_wakeup.py
+++ b/src/backend/core/chat/runtime/job_wakeup.py
@@ -27,16 +27,12 @@
import logging
from typing import Any, Dict, Optional
+import core.chat.runtime.admission as admission
from core.chat.plan_progress import load_plan_progress, plan_is_unfinished, render_plan_checklist
-from core.db.engine import SessionLocal
+from core.db.threads import run_db, run_in_session
from core.db.models import Job
-from core.llm.context_ir import (
- KIND_REMINDER,
- POLICY_NEVER,
- SESSION_CONTEXT_META_KEY,
- make_text_context_item,
- session_context_metadata,
-)
+from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY
+from core.llm.context.items import make_text_context_item, session_context_metadata
from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
@@ -140,6 +136,46 @@ def _progress_prompt(
return "\n".join(lines)
+def _progress_wake_target(db, job_id: str, stats, budget_left, stalled):
+ """``(chat_id, user_id, model_name, prompt)`` when a running job should report, else None."""
+ job = db.query(Job).filter(Job.job_id == job_id).first()
+ if job is None or not job.chat_id:
+ return None
+ if job.status != "running":
+ return None
+ meta = dict(job.extra_data or {})
+ start_params = dict(meta.get("start_params") or {})
+ if not start_params.get("wake_on_finish"):
+ return None # wait=True 的调用方本来就阻塞在那儿等,不需要播报
+ prompt = _progress_prompt(job, stats, budget_left, stalled)
+ return job.chat_id, job.user_id, start_params.get("model_name"), prompt
+
+
+def _claim_finish_wake(db, job_id: str):
+ """Mark a finished job as woken; ``(chat_id, user_id, model_name, prompt)`` or None."""
+ job = db.query(Job).filter(Job.job_id == job_id).with_for_update().first()
+ if job is None or not job.chat_id:
+ return None
+ if job.status not in ("completed", "failed", "cancelled", "interrupted"):
+ return None
+ meta = dict(job.extra_data or {})
+ if meta.get("woken_at"):
+ return None # 已经叫过了
+ start_params = dict(meta.get("start_params") or {})
+ if not start_params.get("wake_on_finish"):
+ return None # 只有 wait=False 提交的作业才需要唤醒
+ from core.jobs.store import JobService
+
+ stats = JobService(db).stats(job_id)
+ prompt = _wake_prompt(job, stats, load_plan_progress(job.chat_id, db))
+ target = (job.chat_id, job.user_id, start_params.get("model_name"), prompt)
+ meta["woken_at"] = True
+ job.extra_data = meta
+ flag_modified(job, "extra_data")
+ db.commit()
+ return target
+
+
async def wake_on_job_progress(
job_id: str, *, stats: Dict[str, Any], budget_left: Dict[str, Any], stalled: bool
) -> bool:
@@ -148,28 +184,20 @@ async def wake_on_job_progress(
三道闸:作业得还在跑、得是 ``wait=False`` 提交的、会话里不能已有在跑的 run。
最后一道最重要——智能体正忙时再入队一轮,只会让两轮互相打架。
"""
- with SessionLocal() as db:
- job = db.query(Job).filter(Job.job_id == job_id).first()
- if job is None or not job.chat_id:
- return False
- if job.status != "running":
- return False
- meta = dict(job.extra_data or {})
- if not dict(meta.get("start_params") or {}).get("wake_on_finish"):
- return False # wait=True 的调用方本来就阻塞在那儿等,不需要播报
- chat_id = job.chat_id
- user_id = job.user_id
- model_name = dict(meta.get("start_params") or {}).get("model_name")
- prompt = _progress_prompt(job, stats, budget_left, stalled)
+ target = await run_in_session(_progress_wake_target, job_id, stats, budget_left, stalled)
+ if target is None:
+ return False
+ chat_id, user_id, model_name, prompt = target
try:
- from orchestration.chat_run_executor import get_active_run_for_chat
+ from core.runs.runtime import get_active_run_for_chat
- if get_active_run_for_chat(chat_id, user_id) is not None:
+ if await run_db(get_active_run_for_chat, chat_id, user_id) is not None:
logger.info("[job-wake] skip progress wake, chat busy chat=%s job=%s", chat_id, job_id)
return False
- except Exception: # noqa: BLE001 —— 探测失败就当它不忙,宁可多叫一次也别哑掉
- pass
+ except Exception as fallback_error: # noqa: BLE001 —— 探测失败就当它不忙,宁可多叫一次也别哑掉
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.runtime.job_wakeup:wake_on_job_progress', fallback_error)
try:
await _enqueue_followup_run(
@@ -188,31 +216,10 @@ async def wake_on_job_progress(
async def wake_on_job_finish(job_id: str) -> bool:
"""作业终态后叫醒会话。返回是否真的发起了唤醒。"""
- with SessionLocal() as db:
- job = db.query(Job).filter(Job.job_id == job_id).with_for_update().first()
- if job is None or not job.chat_id:
- return False
- if job.status not in ("completed", "failed", "cancelled", "interrupted"):
- return False
- meta = dict(job.extra_data or {})
- if meta.get("woken_at"):
- return False # 已经叫过了
- start_params = dict(meta.get("start_params") or {})
- if not start_params.get("wake_on_finish"):
- return False # 只有 wait=False 提交的作业才需要唤醒
- chat_id = job.chat_id
- user_id = job.user_id
- model_name = start_params.get("model_name")
-
- from core.services.job_service import JobService
-
- stats = JobService(db).stats(job_id)
- prompt = _wake_prompt(job, stats, load_plan_progress(chat_id, db))
-
- meta["woken_at"] = True
- job.extra_data = meta
- flag_modified(job, "extra_data")
- db.commit()
+ target = await run_in_session(_claim_finish_wake, job_id)
+ if target is None:
+ return False
+ chat_id, user_id, model_name, prompt = target
try:
await _enqueue_followup_run(
@@ -229,6 +236,61 @@ async def wake_on_job_finish(job_id: str) -> bool:
return True
+def _accept_wake_run(db, chat_id: str, user_id: str, message: str, wake_kind: str):
+ """Persist the wake message and its accepted run; return what starting it needs."""
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+ from core.chat.sequencer import ChatSequencer
+ from core.chat.service import ChatService
+ from core.chat.operations.session_context import load_session_messages as _load_session_messages
+
+ chat_service = ChatService(db)
+ enabled = resolve_all_runtime_enabled(db, user_id)
+ # 会话模式要跟着唤醒轮一起走。作业只可能诞生在工作流模式里,而 workflow_mode
+ # 决定了 run_job 注不注册、批量提示词注不注入——唤醒轮丢掉它,模型就会看到
+ # 一条"用 run_job(action='resume') 续跑"的系统消息,却在工具面里找不到
+ # run_job,只能弃用作业、把 N 个工作项拖回主循环逐项重做(成本二次方增长,
+ # 前端也再没有后台作业卡片)。会话元数据里的 workflow_chat 是这件事的真源,
+ # 用户消息路径读的也是它(见 chats.py 的 ctx 组装)。
+ _sess = chat_service.get_session(chat_id, user_id)
+ workflow_chat = bool(dict(getattr(_sess, "extra_data", None) or {}).get("workflow_chat"))
+ # 唤醒消息按 user 角色落库:它要能进历史、能被模型看见,走和普通消息完全一样的链路。
+ # 但它是**写给模型的系统指令**,不是用户说的话——`hidden_in_chat` 让消息列表接口
+ # 把它从聊天记录里滤掉(模型侧的历史另走 load_session_history,不受影响)。
+ # 没有这个标记,用户一刷新页面就会看到「[系统] 进度播报:……请只用一两句话转述」
+ # 这种内部提示词以自己的口吻贴在对话里。
+ wake_item = make_text_context_item(
+ message,
+ item_id=f"harness:job_wakeup:{wake_kind}",
+ kind=KIND_REMINDER,
+ origin=f"harness:job_wakeup:{wake_kind}",
+ trust="system",
+ created_seq=0,
+ priority=950,
+ token_budget=4_000,
+ truncation_policy=POLICY_NEVER,
+ )
+ wake_metadata = session_context_metadata(wake_item)
+ accepted = ChatSequencer(db).accept_main_run(
+ chat_id=chat_id,
+ user_id=user_id,
+ user_content=message,
+ user_extra_data={
+ "system_wake": wake_kind,
+ "hidden_in_chat": True,
+ SESSION_CONTEXT_META_KEY: wake_metadata,
+ },
+ request_payload={"chat_id": chat_id, "message": message, "kind": "job_wakeup"},
+ )
+ session_messages = _load_session_messages(chat_service, chat_id, user_id)
+ return accepted.run, session_messages, workflow_chat, enabled
+
+
+def _abandon_wake_run(db, run_id: str, reason: str) -> None:
+ from core.chat.sequencer import ChatSequencer
+
+ ChatSequencer(db).abandon_pending_run(run_id, reason=reason)
+
+
async def _enqueue_followup_run(
*,
chat_id: str,
@@ -241,58 +303,14 @@ async def _enqueue_followup_run(
这样前端不用任何改动:它本来就会在会话里看到新的一轮(断线还能按 run_id 续播)。
"""
- from core.services.session_operations.session_context import _load_session_messages
from core.chat.context import build_runtime_context
- from core.config.catalog_resolver import resolve_all_runtime_enabled
- from core.services.chat_sequencer import ChatSequencer
- from core.services.chat_service import ChatService
- from orchestration import chat_run_executor
accepted_run_id: Optional[str] = None
try:
- with SessionLocal() as db:
- chat_service = ChatService(db)
- skills, agents, mcps = resolve_all_runtime_enabled(db, user_id)
- # 会话模式要跟着唤醒轮一起走。作业只可能诞生在工作流模式里,而 workflow_mode
- # 决定了 run_job 注不注册、批量提示词注不注入——唤醒轮丢掉它,模型就会看到
- # 一条"用 run_job(action='resume') 续跑"的系统消息,却在工具面里找不到
- # run_job,只能弃用作业、把 N 个工作项拖回主循环逐项重做(成本二次方增长,
- # 前端也再没有后台作业卡片)。会话元数据里的 workflow_chat 是这件事的真源,
- # 用户消息路径读的也是它(见 chats.py 的 ctx 组装)。
- _sess = chat_service.get_session(chat_id, user_id)
- workflow_chat = bool(
- dict(getattr(_sess, "extra_data", None) or {}).get("workflow_chat")
- )
- # 唤醒消息按 user 角色落库:它要能进历史、能被模型看见,走和普通消息完全一样的链路。
- # 但它是**写给模型的系统指令**,不是用户说的话——`hidden_in_chat` 让消息列表接口
- # 把它从聊天记录里滤掉(模型侧的历史另走 load_session_history,不受影响)。
- # 没有这个标记,用户一刷新页面就会看到「[系统] 进度播报:……请只用一两句话转述」
- # 这种内部提示词以自己的口吻贴在对话里。
- wake_item = make_text_context_item(
- message,
- item_id=f"harness:job_wakeup:{wake_kind}",
- kind=KIND_REMINDER,
- origin=f"harness:job_wakeup:{wake_kind}",
- trust="system",
- created_seq=0,
- priority=950,
- token_budget=4_000,
- truncation_policy=POLICY_NEVER,
- )
- wake_metadata = session_context_metadata(wake_item)
- accepted = ChatSequencer(db).accept_main_run(
- chat_id=chat_id,
- user_id=user_id,
- user_content=message,
- user_extra_data={
- "system_wake": wake_kind,
- "hidden_in_chat": True,
- SESSION_CONTEXT_META_KEY: wake_metadata,
- },
- request_payload={"chat_id": chat_id, "message": message, "kind": "job_wakeup"},
- )
- accepted_run_id = accepted.run.run_id
- session_messages = _load_session_messages(chat_service, chat_id, user_id)
+ run, session_messages, workflow_chat, (skills, agents, mcps) = await run_in_session(
+ _accept_wake_run, chat_id, user_id, message, wake_kind
+ )
+ accepted_run_id = run.run_id
context = build_runtime_context(
model_name=model_name,
@@ -303,8 +321,8 @@ async def _enqueue_followup_run(
enabled_mcps=mcps,
)
context["workflow_chat"] = workflow_chat
- await chat_run_executor.start_run(
- accepted_run=accepted.run,
+ await admission.start_run(
+ accepted_run=run,
chat_id=chat_id,
user_id=user_id,
session_messages=session_messages,
@@ -315,9 +333,5 @@ async def _enqueue_followup_run(
)
except Exception as exc:
if accepted_run_id:
- with SessionLocal() as cleanup_db:
- ChatSequencer(cleanup_db).abandon_pending_run(
- accepted_run_id,
- reason=str(exc),
- )
+ await run_in_session(_abandon_wake_run, accepted_run_id, str(exc))
raise
diff --git a/src/backend/core/chat/runtime/pauses.py b/src/backend/core/chat/runtime/pauses.py
new file mode 100644
index 00000000..e3b2e628
--- /dev/null
+++ b/src/backend/core/chat/runtime/pauses.py
@@ -0,0 +1,58 @@
+"""Persist tool-outcome and human-hook pauses without reporting success."""
+
+from __future__ import annotations
+
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+from core.harness.hooks import HookPaused
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def _pause_for_tool_outcome(turn, exc: BaseException) -> bool:
+ logger.warning("chat_run_tool_outcome_unknown", run_id=turn.run_id, effect_id=str(exc))
+ if not turn.journal.needs_attention(
+ turn.run_id,
+ owner=turn.owner,
+ reason=f"tool outcome pending policy recovery: {exc}",
+ ):
+ current = runtime.get_run(turn.run_id)
+ if current is None or current.status != "needs_attention":
+ logger.warning("chat_run_tool_unknown_cas_lost", run_id=turn.run_id, owner=turn.owner)
+ return False
+ await turn.emit(
+ {
+ "type": "error",
+ "error": "工具调用结果无法安全确定,任务已暂停等待处理",
+ "delta": "工具调用结果无法安全确定,任务已暂停等待处理",
+ "chat_id": turn.chat_id,
+ },
+ terminal=True,
+ )
+ await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True)
+ return True
+
+
+async def _pause_for_hook(turn, exc: HookPaused) -> bool:
+ logger.info("chat_run_hook_paused", run_id=turn.run_id, reason=str(exc))
+ if not turn.journal.needs_attention(
+ turn.run_id,
+ owner=turn.owner,
+ reason=f"hook requested pause: {exc}",
+ ):
+ current = runtime.get_run(turn.run_id)
+ if current is None or current.status != "needs_attention":
+ logger.warning("chat_run_hook_pause_cas_lost", run_id=turn.run_id, owner=turn.owner)
+ return False
+ await turn.emit(
+ {
+ "type": "error",
+ "error": "任务已按执行策略暂停,等待后续处理",
+ "delta": "任务已按执行策略暂停,等待后续处理",
+ "chat_id": turn.chat_id,
+ },
+ terminal=True,
+ )
+ await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True)
+ return True
diff --git a/src/backend/core/chat/runtime/recovery.py b/src/backend/core/chat/runtime/recovery.py
new file mode 100644
index 00000000..7321d5a1
--- /dev/null
+++ b/src/backend/core/chat/runtime/recovery.py
@@ -0,0 +1,179 @@
+"""Recover chat workers from committed snapshots after process loss."""
+
+from __future__ import annotations
+
+import core.chat.runtime.recovery_snapshot as recovery_snapshot
+import core.chat.runtime.worker as worker
+import core.db.engine as engine
+import core.db.models as models
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+import core.runs.tasks as tasks
+from core.infra.logging import get_logger
+from core.runs.journal import RecoveryDecision
+from core.db.threads import run_in_session
+
+logger = get_logger(__name__)
+
+
+async def recover_orphan_runs() -> int:
+ """Recover claimable runs from their last committed database safe point."""
+ from core.runs.tool_effects.journal import ToolEffectJournal
+ from core.runs.tool_effects.recovery import recover_incomplete_tool_effects
+
+ # Desktop channel runs require a live device lease and a scoped I/O adapter.
+ # Neither survives process restart. Fail closed before any tool-effect recovery.
+ def _orphaned_channel_runs(db):
+ local_channel_runs = [
+ (run.run_id, run.chat_id)
+ for run in db.query(models.ChatRun)
+ .filter(models.ChatRun.status.in_(settings.LIVE_STATUSES))
+ .all()
+ if (run.request_payload or {}).get("source") == "desktop_channel"
+ and run.run_id not in tasks.process_state().active_runs
+ ]
+ return local_channel_runs
+ local_channel_runs = await run_in_session(_orphaned_channel_runs)
+ for run_id, chat_id in local_channel_runs:
+ if ownership.journal().cancel(
+ run_id, reason="desktop channel lease cannot survive process restart"
+ ):
+ await writer._write_terminal_to_stream(
+ run_id,
+ chat_id=chat_id,
+ error_text="本机机器人执行已中断,请检查结果后从渠道重新发起",
+ cancelled=True,
+ )
+ effect_decisions = await recover_incomplete_tool_effects(
+ journal=ToolEffectJournal(engine.SessionLocal)
+ )
+ effect_attention_runs = {
+ item.run_id for item in effect_decisions if item.action == "needs_attention"
+ }
+ for run_id in effect_attention_runs:
+ run = runtime.get_run(run_id)
+ if run is not None:
+ await writer._write_terminal_to_stream(
+ run_id,
+ chat_id=run.chat_id or "",
+ error_text="工具调用结果无法安全确定,任务已暂停等待处理",
+ )
+ decisions = ownership.journal().recover()
+ for decision in decisions:
+ if decision.action == "resume":
+ if not _register_recovered_chat(decision):
+ reason = "run kind or recovery snapshot cannot be resumed automatically"
+ if ownership.journal().needs_attention(decision.run_id, reason=reason):
+ await writer._write_terminal_to_stream(
+ decision.run_id,
+ chat_id=decision.chat_id or "",
+ error_text="任务缺少可恢复快照,已暂停等待处理",
+ )
+ elif decision.action == "resume_from_snapshot":
+ snapshot_result = await recovery_snapshot._commit_recovered_chat_snapshot(decision)
+ if snapshot_result is False:
+ await writer._write_terminal_to_stream(
+ decision.run_id,
+ chat_id=decision.chat_id or "",
+ error_text="已保存的模型结果无法安全提交,任务已暂停等待处理",
+ )
+ elif decision.action == "needs_attention":
+ if decision.phase == "model_inflight":
+ recovery_error = "模型调用结果不确定,任务已暂停等待恢复决策"
+ elif decision.phase == "legacy_unrecoverable":
+ recovery_error = "旧版本任务缺少恢复快照,已暂停等待处理"
+ else:
+ recovery_error = "任务在工具结果不确定的安全边界暂停,等待恢复决策"
+ await writer._write_terminal_to_stream(
+ decision.run_id,
+ chat_id=decision.chat_id or "",
+ error_text=recovery_error,
+ )
+ elif decision.action == "completed":
+ await recovery_snapshot._write_recovered_terminal_marker(
+ decision.run_id,
+ chat_id=decision.chat_id or "",
+ )
+ elif decision.action == "failed":
+ await writer._write_terminal_to_stream(
+ decision.run_id,
+ chat_id=decision.chat_id or "",
+ error_text="服务重启发生在任务准备完成前,请重新发起",
+ )
+ recovered_count = len(
+ {item.run_id for item in decisions}
+ | effect_attention_runs
+ | {run_id for run_id, _ in local_channel_runs}
+ )
+ if recovered_count:
+ logger.info("chat_run_orphan_recovered", count=recovered_count)
+ return recovered_count
+
+
+def _register_recovered_chat(decision: RecoveryDecision) -> bool:
+ snapshot = dict(decision.snapshot or {})
+ if snapshot.get("kind") != "chat":
+ return False
+ args = snapshot.get("worker_args")
+ if not isinstance(args, dict):
+ return False
+ required = {
+ "session_messages",
+ "effective_user_message",
+ "raw_user_message",
+ "context",
+ }
+ if not required.issubset(args):
+ return False
+ owner = ownership.new_worker_owner(decision.run_id)
+ recovered_run = runtime.get_run(decision.run_id)
+ if recovered_run and (recovered_run.request_payload or {}).get("source") == "desktop_channel":
+ return False
+ from core.runs.api_recovery import validate_recovery_context
+ from sqlalchemy.exc import SQLAlchemyError
+
+ try:
+ with engine.SessionLocal() as db:
+ restored_context = validate_recovery_context(
+ db,
+ recovered_run,
+ args.get("context"),
+ user_id=decision.user_id,
+ chat_id=decision.chat_id,
+ )
+ if "context" in snapshot:
+ validate_recovery_context(
+ db,
+ recovered_run,
+ snapshot["context"],
+ user_id=decision.user_id,
+ chat_id=decision.chat_id,
+ )
+ except (ValueError, SQLAlchemyError):
+ logger.warning("chat_run_recovery_scope_rejected", run_id=decision.run_id)
+ return False
+ runtime.register_run_task(
+ decision.run_id,
+ worker._run_workflow(
+ run_id=decision.run_id,
+ chat_id=decision.chat_id,
+ user_id=decision.user_id,
+ message_id=decision.message_id,
+ assistant_chat_seq=(
+ int(recovered_run.assistant_chat_seq)
+ if recovered_run is not None and recovered_run.assistant_chat_seq is not None
+ else None
+ ),
+ session_messages=list(args.get("session_messages") or []),
+ effective_user_message=str(args.get("effective_user_message") or ""),
+ raw_user_message=str(args.get("raw_user_message") or ""),
+ context=restored_context,
+ model_name=(str(args["model_name"]) if args.get("model_name") else None),
+ journal_owner=owner,
+ recovering=True,
+ ),
+ name=f"chat_run_recovery:{decision.run_id}",
+ )
+ return True
diff --git a/src/backend/core/chat/runtime/recovery_snapshot.py b/src/backend/core/chat/runtime/recovery_snapshot.py
new file mode 100644
index 00000000..81edd404
--- /dev/null
+++ b/src/backend/core/chat/runtime/recovery_snapshot.py
@@ -0,0 +1,204 @@
+"""Persist interrupted turn snapshots and recovery terminal markers."""
+
+from __future__ import annotations
+
+import typing as _typing
+from typing import Optional
+
+import core.chat.runtime.handoff as handoff
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+import core.chat.service as services_ChatService_module
+from core.infra.logging import get_logger
+from core.runs.journal import RecoveryDecision
+from core.db.threads import run_in_session
+
+logger = get_logger(__name__)
+
+
+async def _commit_recovered_chat_snapshot(decision: RecoveryDecision) -> Optional[bool]:
+ snapshot = dict(decision.snapshot or {})
+ content = snapshot.get("assistant_content")
+ message_id = str(snapshot.get("message_id") or decision.message_id)
+ if not isinstance(content, str) or not message_id:
+ paused = ownership.journal().needs_attention(
+ decision.run_id,
+ reason="model-completed snapshot is incomplete",
+ )
+ return False if paused else None
+ owner = ownership.new_worker_owner(decision.run_id)
+ recovered_run = runtime.get_run(decision.run_id)
+ journal = ownership.journal()
+ from core.runs.api_recovery import validate_recovery_context
+ from sqlalchemy.exc import SQLAlchemyError
+
+ try:
+ def _restore_context(db):
+ restored_context = validate_recovery_context(
+ db,
+ recovered_run,
+ snapshot.get("context"),
+ user_id=decision.user_id,
+ chat_id=decision.chat_id,
+ )
+ return restored_context
+ restored_context = await run_in_session(_restore_context)
+ worker_args = snapshot.get("worker_args")
+ if isinstance(worker_args, dict) and "context" in worker_args:
+ def _check_worker_context(db):
+ validate_recovery_context(
+ db,
+ recovered_run,
+ worker_args["context"],
+ user_id=decision.user_id,
+ chat_id=decision.chat_id,
+ )
+ await run_in_session(_check_worker_context)
+ except (ValueError, SQLAlchemyError):
+ paused = journal.needs_attention(
+ decision.run_id, reason="agent API recovery scope is unavailable or inconsistent"
+ )
+ return False if paused else None
+ if not journal.claim(
+ decision.run_id,
+ owner=owner,
+ lease_seconds=settings.RUN_LEASE_SECONDS,
+ ):
+ return None
+ try:
+ from core.files.artifacts.service import persist_artifacts as _persist_artifacts
+ from core.projects.edition import project_scope_from_context
+
+ queued_handoff: _typing.Dict[str, _typing.Any] = {}
+
+ def _commit_recovered_output(db) -> None:
+ services_ChatService_module.ChatService(db).upsert_message(
+ chat_id=decision.chat_id,
+ role="assistant",
+ content=content,
+ message_id=message_id,
+ chat_seq=(
+ int(recovered_run.assistant_chat_seq)
+ if recovered_run is not None and recovered_run.assistant_chat_seq is not None
+ else None
+ ),
+ model=(
+ str(snapshot["model_name"]) if snapshot.get("model_name") is not None else None
+ ),
+ thinking=(
+ list(snapshot["thinking"])
+ if isinstance(snapshot.get("thinking"), list)
+ else None
+ ),
+ tool_calls=(
+ list(snapshot["tool_calls"])
+ if isinstance(snapshot.get("tool_calls"), list)
+ else None
+ ),
+ model_steps=(
+ list(snapshot["model_steps"])
+ if isinstance(snapshot.get("model_steps"), list)
+ else None
+ ),
+ usage=(
+ dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None
+ ),
+ extra_data=(
+ dict(snapshot["extra_data"])
+ if isinstance(snapshot.get("extra_data"), dict)
+ else {}
+ ),
+ commit=False,
+ )
+ artifacts = snapshot.get("artifacts")
+ if isinstance(artifacts, list):
+ _persist_artifacts(
+ db,
+ decision.user_id,
+ decision.chat_id,
+ artifacts,
+ scope=project_scope_from_context(restored_context),
+ commit=False,
+ )
+ worker_args = snapshot.get("worker_args")
+ recovery_args = worker_args if isinstance(worker_args, dict) else {}
+ context_snapshot = snapshot.get("context")
+ prepared = handoff._commit_queued_handoff_in_session(
+ db,
+ source_run_id=decision.run_id,
+ chat_id=decision.chat_id,
+ user_id=decision.user_id,
+ context=(
+ restored_context
+ if isinstance(context_snapshot, dict)
+ else dict(recovery_args.get("context") or {})
+ ),
+ model_name=(
+ str(snapshot["model_name"])
+ if snapshot.get("model_name") is not None
+ else (
+ str(recovery_args["model_name"])
+ if recovery_args.get("model_name") is not None
+ else None
+ )
+ ),
+ )
+ if prepared is not None:
+ queued_handoff.update(prepared)
+
+ completed = journal.complete(
+ decision.run_id,
+ owner=owner,
+ status="completed",
+ usage=(dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None),
+ commit_effect=_commit_recovered_output,
+ committed_operation={
+ "operation_type": "message_committed_from_snapshot",
+ "phase": "message_committed",
+ "safety": "side_effect_committed",
+ "payload": {"message_id": message_id},
+ },
+ )
+ if completed:
+ if queued_handoff:
+ handoff._schedule_queued_handoff(
+ queued_handoff,
+ chat_id=decision.chat_id,
+ user_id=decision.user_id,
+ task_prefix="chat_run_handoff_recovery",
+ )
+ await handoff._write_queued_run_started_projection(
+ decision.run_id,
+ chat_id=decision.chat_id,
+ handoff=queued_handoff,
+ )
+ await _write_recovered_terminal_marker(
+ decision.run_id,
+ chat_id=decision.chat_id,
+ )
+ return completed
+ except Exception as exc: # noqa: BLE001 - pause instead of replaying model output
+ logger.error(
+ "chat_run_snapshot_commit_failed",
+ run_id=decision.run_id,
+ error=str(exc),
+ exc_info=True,
+ )
+ paused = journal.needs_attention(
+ decision.run_id,
+ reason=str(exc)[:1000],
+ owner=owner,
+ )
+ return False if paused else None
+
+
+async def _write_recovered_terminal_marker(run_id: str, *, chat_id: str) -> None:
+ event = {"type": settings.TERMINAL_TYPE, "chat_id": chat_id, "_recovered": True}
+ try:
+ offset = ownership.journal().allocate_event_offset(run_id, terminal=True)
+ await writer.xadd_event(run_id, offset, event)
+ await writer.expire_stream(run_id)
+ except Exception as exc: # Redis is only a projection
+ logger.warning("chat_run_recovered_terminal_write_failed", run_id=run_id, error=str(exc))
diff --git a/src/backend/core/chat/runtime/snapshots.py b/src/backend/core/chat/runtime/snapshots.py
new file mode 100644
index 00000000..1924dd8b
--- /dev/null
+++ b/src/backend/core/chat/runtime/snapshots.py
@@ -0,0 +1,201 @@
+"""Checkpoint visible turn output and preserve partial output on cancellation."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import time
+from typing import Any, Dict, List, Optional
+
+import core.chat.inflight as inflight
+import core.chat.message_parser as message_parser
+import core.db.engine as engine
+import core.llm.runtime.model_steps as model_steps
+import core.runs.events.writer as writer
+import core.chat.service as services_ChatService_module
+from core.chat.context import now_iso
+from core.infra.text_bounds import bound_result_for_history
+from core.infra.logging import get_logger
+from core.llm.runtime import workspace as _workspace_mod
+from core.files.artifacts.service import persist_artifacts as _persist_artifacts
+
+logger = get_logger(__name__)
+
+
+def _flush_thinking(turn) -> None:
+ if not turn.thinking_parts:
+ return
+ block = "".join(turn.thinking_parts)
+ turn.thinking_parts.clear()
+ # 结构化 reasoning 的尾部增量可能在正文已开始后才到达(正文首 token
+ # 已出、思考收尾的"。"后到)。实时侧把它并回前一个思考块
+ # (appendThinkingContentBeforeTrailingText),落库遵循同一规则——
+ # 否则思考块会把正文句子从中间切开,刷新后与实时展示不一致。
+ if turn.round_block is not None:
+ turn.round_block["content"] += block
+ return
+ turn.round_block = {"content": block}
+ turn.thinking_log.append(turn.round_block)
+ turn.segments.add_thinking(len(turn.thinking_log) - 1)
+
+
+def _thinking_payload(turn) -> Optional[List[Dict[str, Any]]]:
+ blocks = [b for b in turn.thinking_log if b.get("content")]
+ return blocks or None
+
+
+def _model_steps_payload(turn) -> Optional[List[Dict[str, Any]]]:
+ return list(turn.model_steps) or None
+
+
+def _turn_extra(turn, **more: Any) -> Dict[str, Any]:
+ return {
+ "timestamp": now_iso(),
+ "message_id": turn.current_message_id,
+ "run_id": turn.run_id,
+ "is_markdown": message_parser.looks_markdown(turn.full_response),
+ **more,
+ }
+
+
+def _write_inflight_row(turn, snapshot: Dict[str, Any]) -> bool:
+ with engine.SessionLocal() as db:
+ owned = services_ChatService_module.ChatService(db).refresh_streaming_message(
+ run_id=turn.run_id,
+ owner=turn.owner,
+ message_id=snapshot["message_id"],
+ content=snapshot["content"],
+ thinking=snapshot["thinking"],
+ tool_calls=snapshot["tool_calls"],
+ model_steps=snapshot["model_steps"],
+ extra_data={
+ inflight.IN_FLIGHT_KEY: inflight.mark(
+ turn.run_id, "streaming", event_offset=snapshot["event_offset"]
+ ),
+ "message_id": snapshot["message_id"],
+ "run_id": turn.run_id,
+ "is_markdown": message_parser.looks_markdown(snapshot["content"]),
+ "segments": snapshot["segments"],
+ },
+ )
+ db.commit()
+ return owned
+
+
+async def _flush_inflight(turn) -> bool:
+ _flush_thinking(turn)
+ if not (turn.full_response or turn.tool_calls_log or turn.model_steps or turn.thinking_log):
+ return True
+ async with turn.flush_lock:
+ # Canonical steps can advance without a user-visible SSE event.
+ snapshot_key = (turn.current_message_id, turn.offsets.last, len(turn.model_steps))
+ if snapshot_key == turn.flushed_snapshot_key:
+ return True
+ # Tool results go in at history size; the read path bounds them the
+ # same way, and the final write carries the full payload.
+ snapshot = writer._json_safe(
+ {
+ "content": turn.full_response,
+ "thinking": _thinking_payload(turn),
+ "model_steps": _model_steps_payload(turn),
+ "tool_calls": [
+ (
+ {**tc, "result": bound_result_for_history(tc["result"])[0]}
+ if "result" in tc
+ else tc
+ )
+ for tc in turn.tool_calls_log
+ ]
+ or None,
+ "segments": turn.segments.snapshot(),
+ "message_id": turn.current_message_id,
+ "event_offset": turn.offsets.last,
+ }
+ )
+ try:
+ if not await asyncio.to_thread(functools.partial(_write_inflight_row, turn), snapshot):
+ logger.warning("chat_run_inflight_fenced", run_id=turn.run_id, owner=turn.owner)
+ return False
+ except Exception: # noqa: BLE001 - a missed checkpoint must not fail the run
+ logger.warning("chat_run_inflight_persist_failed", run_id=turn.run_id, exc_info=True)
+ return False # Keep dirty so the periodic checkpoint retries this snapshot.
+ turn.flushed_snapshot_key = snapshot_key
+ return True
+
+
+async def _checkpoint_loop(turn) -> None:
+ while True:
+ await asyncio.sleep(turn.inflight_interval_s)
+ await _flush_inflight(turn)
+
+
+def _commit_cancelled_partial(turn) -> None:
+ """把用户按下停止那一刻已产出的正文与工具卡片落库。
+
+ cancel_run 先写终态并清租约,之后 journal.complete 的 CAS 必然失败,
+ worker 再也借不到 commit_effect 提交本轮输出 —— 所以这里直接写库。
+ message_id 固定,用 upsert 保证重复取消 / 恢复不会留下两条。
+ """
+ _flush_thinking(turn)
+ if not (turn.full_response or turn.tool_calls_log or turn.model_steps or turn.thinking_log):
+ return
+ # 停下来时还没回结果的工具:不标记的话前端按"缺 status 即成功"渲染,
+ # 刷新后一张没跑完的卡片会显示成执行成功。模型侧的记录同样要配平:
+ # 悬空的调用补一条"被中断"的结果,下一轮模型看到的是事实而不是半步。
+ for _tc in turn.tool_calls_log:
+ if "result" not in _tc and not _tc.get("status"):
+ _tc["status"] = "interrupted"
+ model_steps.close_dangling_calls(turn.model_steps)
+ _ws_pinned = _workspace_mod.get_pinned()
+ _extra = {
+ "timestamp": now_iso(),
+ "is_markdown": bool(
+ "\n" in turn.full_response or "```" in turn.full_response or "**" in turn.full_response
+ ),
+ "message_id": turn.current_message_id,
+ "run_id": turn.run_id,
+ "cancelled": True,
+ "artifacts": _ws_pinned,
+ "workspace_files": _workspace_mod.get_pinned_file_ids(),
+ "duration_ms": int((time.monotonic() - turn.run_started_monotonic) * 1000),
+ "segments": turn.segments.payload(),
+ }
+ if turn.context.get("model_provider_id"):
+ _extra["model_provider_id"] = turn.context.get("model_provider_id")
+ from core.projects.edition import project_scope_from_context
+
+ with engine.SessionLocal() as db:
+ chat_service = services_ChatService_module.ChatService(db)
+ chat_service.upsert_message(
+ chat_id=turn.chat_id,
+ role="assistant",
+ content=turn.full_response,
+ model=turn.model_name,
+ thinking=_thinking_payload(turn),
+ tool_calls=turn.tool_calls_log if turn.tool_calls_log else None,
+ model_steps=_model_steps_payload(turn),
+ message_id=turn.current_message_id,
+ chat_seq=turn.current_chat_seq,
+ extra_data=_extra,
+ commit=False,
+ )
+ _persist_artifacts(
+ db,
+ turn.user_id,
+ turn.chat_id,
+ _ws_pinned,
+ scope=project_scope_from_context(turn.context),
+ commit=False,
+ )
+ db.commit()
+
+
+def _persist_cancelled_partial(turn) -> None:
+ try:
+ _commit_cancelled_partial(turn)
+ except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾
+ logger.warning(
+ "chat_run_cancelled_partial_persist_failed",
+ run_id=turn.run_id,
+ exc_info=True,
+ )
diff --git a/src/backend/core/chat/runtime/steering.py b/src/backend/core/chat/runtime/steering.py
new file mode 100644
index 00000000..85ac6b62
--- /dev/null
+++ b/src/backend/core/chat/runtime/steering.py
@@ -0,0 +1,199 @@
+"""Apply queued steering while preserving ordered message and event commits."""
+
+from __future__ import annotations
+
+import time
+import uuid
+
+import core.chat.inflight as inflight
+import core.chat.runtime.handoff as handoff
+import core.db.models as models
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.chat.service as services_ChatService_module
+from core.chat.context import now_iso
+from core.infra.logging import get_logger
+
+
+from core.chat.runtime import snapshots
+
+logger = get_logger(__name__)
+
+
+async def apply_steer(turn, chunk):
+ snapshots._flush_thinking(turn)
+ steer_id = str(chunk.get("steer_id") or "")[:64]
+ steer_queue_id = str(chunk.get("queue_id") or "")[:64]
+ steer_claim_owner = str(chunk.get("claim_owner") or "")[:160]
+ steer_seq = int(chunk.get("steer_seq") or 0)
+ steer_message = str(chunk.get("message") or "").strip()
+ steer_message_id = (
+ f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{turn.run_id}:{steer_id}').hex[:16]}"
+ if steer_id
+ else f"msg_{uuid.uuid4().hex[:16]}"
+ )
+ next_assistant_message_id = (
+ f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{turn.run_id}:{steer_id}:assistant').hex[:16]}"
+ if steer_id
+ else f"msg_{uuid.uuid4().hex[:16]}"
+ )
+ had_assistant_output = bool(turn.full_response or turn.tool_calls_log)
+ if steer_message:
+ # Close the visible assistant segment before inserting the
+ # user's steer. Persisting in this order is what keeps a
+ # refresh from moving every mid-run user message above the
+ # whole assistant response.
+ def _commit_steer(db) -> None:
+ if steer_queue_id and steer_claim_owner:
+ from core.chat.steer_queue import SteerQueue
+
+ owned_run = db.get(models.ChatRun, turn.run_id)
+ SteerQueue.mark_applied_in_session(
+ db,
+ queue_id=steer_queue_id,
+ claim_owner=steer_claim_owner,
+ applied_run_id=turn.run_id,
+ applied_operation_seq=int(owned_run.operation_seq or 0) + 1,
+ )
+ else:
+ owned_run = db.get(models.ChatRun, turn.run_id)
+ chat_service = services_ChatService_module.ChatService(db)
+ if had_assistant_output:
+ chat_service.upsert_message(
+ chat_id=turn.chat_id,
+ role="assistant",
+ content=turn.full_response,
+ model=turn.model_name,
+ thinking=snapshots._thinking_payload(turn),
+ tool_calls=turn.tool_calls_log if turn.tool_calls_log else None,
+ model_steps=snapshots._model_steps_payload(turn),
+ message_id=turn.current_message_id,
+ chat_seq=turn.current_chat_seq,
+ extra_data={
+ "timestamp": now_iso(),
+ "is_markdown": bool(
+ "\n" in turn.full_response
+ or "```" in turn.full_response
+ or "**" in turn.full_response
+ ),
+ "message_id": turn.current_message_id,
+ "run_id": turn.run_id,
+ "steer_segment": True,
+ "duration_ms": int((time.monotonic() - turn.run_started_monotonic) * 1000),
+ "segments": turn.segments.payload(),
+ },
+ commit=False,
+ )
+ # Persist once the middleware has actually injected the
+ # instruction. A deterministic id makes replay safe.
+ chat_service.upsert_message(
+ chat_id=turn.chat_id,
+ role="user",
+ content=steer_message,
+ message_id=steer_message_id,
+ extra_data={
+ "timestamp": now_iso(),
+ "steer": True,
+ "run_id": turn.run_id,
+ "steer_id": steer_id,
+ "steer_queue_id": steer_queue_id or None,
+ "steer_seq": steer_seq or None,
+ },
+ commit=False,
+ )
+ # The next segment's row exists before its first token,
+ # like every other admitted turn.
+ db.add(
+ inflight.new_assistant_row(
+ message_id=next_assistant_message_id,
+ chat_id=turn.chat_id,
+ chat_seq=None,
+ run_id=turn.run_id,
+ model=turn.model_name,
+ created_at=clock._utcnow(),
+ )
+ )
+ # The durable queue acknowledgement and the exact
+ # post-steer prompt must share one transaction. If this
+ # worker disappears before the next model snapshot, a
+ # replacement worker resumes from this context instead
+ # of losing an already-applied instruction.
+ post_steer_messages = handoff._persisted_history(chat_service, turn.chat_id)
+ post_steer_context = writer._json_safe(dict(turn.context or {}))
+ post_steer_context.pop("journal_owner", None)
+ post_steer_context.update(
+ {
+ "run_id": turn.run_id,
+ "message_id": next_assistant_message_id,
+ "chat_id": turn.chat_id,
+ "user_id": turn.user_id,
+ }
+ )
+ owned_run.recovery_snapshot = {
+ "kind": "chat",
+ "worker_args": {
+ "session_messages": post_steer_messages,
+ "effective_user_message": steer_message,
+ "raw_user_message": steer_message,
+ "context": post_steer_context,
+ "model_name": turn.model_name,
+ },
+ }
+ owned_run.snapshot_version = int(owned_run.snapshot_version or 0) + 1
+
+ turn.journal.append_operation(
+ turn.run_id,
+ owner=turn.owner,
+ operation_type="steer_messages_committed",
+ phase="steer_committed",
+ safety="side_effect_committed",
+ payload={
+ "steer_id": steer_id,
+ "queue_id": steer_queue_id or None,
+ "steer_seq": steer_seq or None,
+ "message_id": steer_message_id,
+ "previous_assistant_message_id": (
+ turn.current_message_id if had_assistant_output else None
+ ),
+ },
+ commit_effect=_commit_steer,
+ )
+ turn.latest_user_message = steer_message
+ await turn.emit(
+ {
+ "type": "steer_applied",
+ "chat_id": turn.chat_id,
+ "run_id": turn.run_id,
+ "steer_id": steer_id,
+ "queue_id": steer_queue_id or None,
+ "steer_seq": steer_seq or None,
+ "message": steer_message,
+ "message_id": steer_message_id,
+ "previous_assistant_message_id": (
+ turn.current_message_id if had_assistant_output else None
+ ),
+ "next_assistant_message_id": next_assistant_message_id,
+ "had_assistant_output": had_assistant_output,
+ }
+ )
+ turn.current_message_id = next_assistant_message_id
+ turn.current_chat_seq = None
+ # The workflow reads this dict again when it emits the final
+ # evolution/memory settlement marker, so keep that marker bound
+ # to the post-steer assistant segment too.
+ turn.context["message_id"] = turn.current_message_id
+ turn.full_response = ""
+ turn.tool_calls_log = []
+ turn.model_steps = []
+ turn.thinking_parts.clear()
+ turn.round_block = None
+ turn.thinking_log.clear()
+ turn.segments.reset()
+ turn.metadata = {}
+ try:
+ from core.harness.log_context import set_current_message_id
+
+ set_current_message_id(turn.current_message_id)
+ except Exception as fallback_error: # pragma: no cover - logging must never fail a run
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.runtime.steering:apply_steer', fallback_error)
diff --git a/src/backend/core/chat/runtime/turn_state.py b/src/backend/core/chat/runtime/turn_state.py
new file mode 100644
index 00000000..49eac43a
--- /dev/null
+++ b/src/backend/core/chat/runtime/turn_state.py
@@ -0,0 +1,44 @@
+"""State owned by one chat worker; persistence callbacks share this exact turn."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Awaitable, Callable
+
+if TYPE_CHECKING:
+ from core.chat.tool_log import SegmentRecorder
+ from core.runs.events.writer import EventOffsets
+ from core.runs.journal import RunJournal
+
+
+@dataclass
+class TurnState:
+ chat_id: str
+ context: dict[str, Any]
+ message_id: str
+ model_name: str
+ run_id: str
+ user_id: str
+ stopped: bool = False
+ inflight_interval_s: float = 2.0
+ emit: Callable[[dict[str, Any]], Awaitable[Any]] | None = None
+ flush_lock: asyncio.Lock | None = None
+ flushed_snapshot_key: Any = None
+ model_steps: list[dict[str, Any]] = field(default_factory=list)
+ round_block: dict[str, Any] | None = None
+ run_started_monotonic: float = 0.0
+ segments: SegmentRecorder | None = None
+ thinking_log: list[dict[str, Any]] = field(default_factory=list)
+ thinking_parts: list[str] = field(default_factory=list)
+ compaction_budget_inputs: dict[str, Any] = field(default_factory=dict)
+ current_chat_seq: int | None = None
+ current_message_id: str = ""
+ full_response: str = ""
+ journal: RunJournal | None = None
+ latest_user_message: str = ""
+ lease_task: asyncio.Task | None = None
+ metadata: dict[str, Any] = field(default_factory=dict)
+ offsets: EventOffsets | None = None
+ owner: str = ""
+ tool_calls_log: list[dict[str, Any]] = field(default_factory=list)
diff --git a/src/backend/core/chat/runtime/worker.py b/src/backend/core/chat/runtime/worker.py
new file mode 100644
index 00000000..82259abd
--- /dev/null
+++ b/src/backend/core/chat/runtime/worker.py
@@ -0,0 +1,405 @@
+"""Own one chat worker lifecycle and coordinate its persistence phases."""
+
+from __future__ import annotations
+from core.runs.tool_effects import errors
+
+import asyncio
+import contextlib
+import datetime as _datetime
+import time
+import typing as _typing
+from typing import Any, Dict, List, Optional
+
+from core.harness.registry.chat_execution import stream_chat_workflow
+from core.db.threads import run_in_session
+import core.harness.hooks as hooks
+import core.llm.runtime.model_steps as model_steps
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+import core.runs.watchdog as watchdog
+import core.chat.service as services_ChatService_module
+import core.runs.journal as run_journal
+from core.infra.logging import get_logger
+
+
+from core.chat.runtime import chunks, pauses, snapshots
+from core.chat.runtime.turn_state import TurnState
+
+logger = get_logger(__name__)
+
+
+def _pop_compaction_notice(db, chat_id: str) -> Optional[Dict[str, Any]]:
+ """Consume a pending compaction notice; return the state to show, or None."""
+ from core.chat.compaction.service import pop_compaction_notice
+ from core.chat.compaction.usage_projection import get_chat_context_state
+
+ chat_service = services_ChatService_module.ChatService(db)
+ if not pop_compaction_notice(chat_service, chat_id):
+ return None
+ return get_chat_context_state(chat_service, chat_id)["context_compaction"]
+
+
+async def _run_workflow(
+ *,
+ run_id: str,
+ chat_id: str,
+ user_id: str,
+ message_id: str,
+ session_messages: List[Dict[str, Any]],
+ effective_user_message: str,
+ raw_user_message: str,
+ context: Dict[str, Any],
+ model_name: Optional[str],
+ assistant_chat_seq: Optional[int] = None,
+ journal_owner: Optional[str] = None,
+ recovering: bool = False,
+) -> None:
+ """Consume astream_chat_workflow, forward chunks to the Redis Stream, persist at the end.
+
+ Every chunk is written to ``jx:chat:run:{run_id}:events`` via ``_emit``.
+ The SSE event format stays fully identical to the existing protocol in
+ chats.py:889-1087; the frontend needs no changes to its chunk parsing —
+ only the new ``run_started`` event type is added.
+ """
+ turn = TurnState(
+ chat_id=chat_id,
+ context=context,
+ message_id=message_id,
+ model_name=model_name,
+ run_id=run_id,
+ user_id=user_id,
+ )
+ from core.chat.context import resolve_user_facing_error
+ from core.chat.tool_log import SegmentRecorder
+
+ # 本轮的墙钟起点。认领这一轮的 worker 手里就有它;只有接管别人未完成的
+ # 那一轮时才需要回库里取(见 _run_started_at_ms)。
+ claimed_at: _typing.Optional[_datetime.datetime] = None
+ initial_claimed = False
+ if not recovering:
+ claimed_at = ownership._claim_run_execution(turn.run_id)
+ initial_claimed = claimed_at is not None
+ if not initial_claimed:
+ ownership._acknowledge_never_started_terminal_writer(turn.run_id)
+ return
+
+ turn.owner = journal_owner or ownership.new_worker_owner(turn.run_id)
+ turn.journal = ownership.journal()
+ if not turn.journal.claim(
+ turn.run_id, owner=turn.owner, lease_seconds=settings.RUN_LEASE_SECONDS
+ ):
+ logger.info("chat_run_claim_skipped", run_id=turn.run_id, owner=turn.owner)
+ if initial_claimed:
+ ownership._acknowledge_terminal_writer(turn.run_id)
+ return
+ try:
+ turn.journal.append_operation(
+ turn.run_id,
+ owner=turn.owner,
+ operation_type="worker_started",
+ phase="pre_model",
+ safety="replayable",
+ )
+ except run_journal.RunLeaseLost:
+ logger.warning("chat_run_worker_fenced_before_start", run_id=turn.run_id, owner=turn.owner)
+ return
+ worker_task = asyncio.current_task()
+ if worker_task is None: # pragma: no cover - asyncio always owns this coroutine
+ raise RuntimeError("chat run worker has no asyncio task")
+ lease_lost = asyncio.Event()
+ turn.lease_task = asyncio.create_task(
+ ownership.lease_heartbeat(turn.run_id, turn.owner, worker_task, lease_lost),
+ name=f"chat_run_lease:{turn.run_id}",
+ )
+ # 本轮回答总耗时起点 —— 持久化进 extra_data.duration_ms,历史加载后「用时」不再消失
+ turn.run_started_monotonic = time.monotonic()
+
+ turn.offsets = writer.EventOffsets(
+ turn.journal, turn.run_id, turn.owner, lease_lost=lease_lost
+ )
+
+ turn.emit = turn.offsets.emit
+
+ from core.llm.runtime import workspace as _workspace_mod
+
+ turn.full_response = ""
+ # A steer starts a new visible assistant segment in the same backend run.
+ # Each segment gets its own message id so persisted history stays in the
+ # same chronological order the user saw while streaming.
+ turn.current_message_id = turn.message_id
+ turn.current_chat_seq = assistant_chat_seq
+ turn.latest_user_message = raw_user_message
+ turn.metadata: _typing.Dict[str, _typing.Any] = {}
+ turn.compaction_budget_inputs: _typing.Dict[str, _typing.Any] = {}
+ turn.tool_calls_log: list = []
+ _workspace_mod.init_state()
+ # 结构化 reasoning 通道(deepseek 系 reasoning_content/reasoning)的思考增量。
+ # 落进独立的 thinking 列,不再拼进 full_response —— 拼进正文会把正文顶出
+ # content 的 10 万字上限,截断切掉闭合标签后整段思考会漏成正文。
+ # 每个思考块记录它出现时的正文偏移,历史重建按此原位还原。
+ turn.thinking_parts: _typing.List[str] = []
+ turn.thinking_log: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ # 模型回放用的规范记录:每个模型响应、每个工具结果按发生顺序各记一条
+ # (core/llm/runtime/model_steps.py)。上面的正文 / 思考 / 工具卡是给界面看的投影,
+ # 下一轮发给模型的历史从这一份恢复。
+ turn.model_steps: _typing.List[_typing.Dict[str, _typing.Any]] = []
+ # 正文 / 思考 / 工具卡片的先后,在它们产生的那一刻就记下来,落进
+ # metadata.segments;刷新后照着渲染,不做任何反推。
+ turn.segments = SegmentRecorder()
+ # 本轮已开的思考块;跨轮(工具调用边界)置 None 强制另起一块。不设边界的话,
+ # 正文一旦出现,后面每一轮的思考都会并进同一块里无限膨胀。
+ turn.round_block: _typing.Optional[_typing.Dict[str, _typing.Any]] = None
+ next_cancel_poll = 0.0
+
+ # In-flight persistence. The assistant row was created at admission and is
+ # refreshed with this turn's display state — right away at tool boundaries,
+ # and by the checkpoint task every couple of seconds while text streams —
+ # so the server holds the turn at all times, not only at the end. Each
+ # refresh is one lease-guarded UPDATE off the event loop; the lock keeps a
+ # boundary flush from racing the periodic one with an older snapshot.
+ turn.flush_lock = asyncio.Lock()
+ turn.flushed_snapshot_key = None
+ turn.inflight_interval_s = 2.0
+ checkpoint_task: _typing.Optional[asyncio.Task] = None
+
+ try:
+ if recovering:
+ await writer._reset_stream_projection(turn.run_id)
+ # First frame: run_started — carries run_id / message_id; the frontend uses these to resume / cancel
+ await turn.emit(
+ {
+ "type": "run_started",
+ "run_id": turn.run_id,
+ "message_id": turn.message_id,
+ "chat_id": turn.chat_id,
+ "started_at": (
+ clock._epoch_ms(claimed_at)
+ if claimed_at
+ else clock._run_started_at_ms(turn.run_id)
+ ),
+ }
+ )
+ if recovering:
+ await turn.emit(
+ {
+ "type": "content_replace",
+ "content": "",
+ "reason": "run_recovered",
+ "chat_id": turn.chat_id,
+ }
+ )
+
+ # Compaction notice (mirrors Codex's post-compaction Warning): after the
+ # previous turn's stream closed, a new compaction checkpoint was written
+ # in the background → notify the user once in this turn's first frame.
+ # Failures are silent and must never affect the main conversation.
+ try:
+ _compaction_state = await run_in_session(_pop_compaction_notice, turn.chat_id)
+ if _compaction_state is not None:
+ await turn.emit(
+ {
+ "type": "compaction_notice",
+ "chat_id": turn.chat_id,
+ "context_compaction": _compaction_state,
+ }
+ )
+ except Exception as _cn_exc: # noqa: BLE001
+ logger.debug("compaction_notice_failed", chat_id=turn.chat_id, error=str(_cn_exc))
+
+ # Evidence-plane join keys (GCE ticket 04). Injected here rather than at
+ # every context construction site: this is the one place that owns both
+ # the run and its pre-allocated assistant message id.
+ turn.context["run_id"] = turn.run_id
+ turn.context["message_id"] = turn.message_id
+ turn.context["journal_owner"] = turn.owner
+
+ # Stamp the message id onto tool logging for this run. The plumbing
+ # already existed but had no caller, so every tool call was written with
+ # an empty message_id — which silently broke the join the evidence plane
+ # depends on, and left every episode with no tool sequence to learn from.
+ try:
+ from core.harness.log_context import set_current_message_id
+
+ set_current_message_id(turn.message_id)
+ except Exception as fallback_error: # pragma: no cover - logging must never fail a run
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.runtime.worker:_run_workflow', fallback_error)
+
+ checkpoint_task = asyncio.create_task(snapshots._checkpoint_loop(turn))
+ async for chunk in watchdog._aiter_with_inactivity_timeout(
+ stream_chat_workflow(
+ raw_user_message=raw_user_message,
+ session_messages=session_messages,
+ user_message=effective_user_message,
+ context=turn.context,
+ ),
+ settings._INACTIVITY_TIMEOUT_SEC,
+ is_activity=lambda item: watchdog._is_chat_run_activity(item, turn.chat_id),
+ first_response_timeout=settings._FIRST_RESPONSE_TIMEOUT_SEC,
+ ):
+ now = time.monotonic()
+ if now >= next_cancel_poll:
+ next_cancel_poll = now + settings._CANCEL_VISIBILITY_SEC
+ if runtime.is_run_cancelled(turn.run_id):
+ raise asyncio.CancelledError
+ await chunks.consume_chunk(turn, chunk)
+ if turn.stopped:
+ return
+
+ # AgentScope may absorb a tool adapter exception after ToolEffectMiddleware
+ # has already durably paused the run. Re-read the DB before projecting a
+ # normal terminal marker; otherwise allocate_event_offset correctly
+ # rejects the non-completed run and the outer handler overwrites
+ # ``needs_attention`` with ``failed``.
+ try:
+ current = runtime.get_run(turn.run_id)
+ except AttributeError:
+ # Some narrow unit adapters replace SessionLocal with a commit-only
+ # fake that intentionally has no query surface.
+ current = None
+ if current is not None:
+ if current.status == "needs_attention":
+ await pauses._pause_for_tool_outcome(
+ turn, RuntimeError(current.failure_reason or "tool outcome requires recovery")
+ )
+ return
+ if current.status in settings._TERMINAL_STATUSES and current.status != "completed":
+ raise asyncio.CancelledError
+
+ # Workflow returned normally (with or without a meta event): write the terminal marker
+ await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True)
+
+ except asyncio.CancelledError:
+ if lease_lost.is_set():
+ # Another process owns recovery now. The fenced worker must not
+ # project a user cancellation or mutate the durable row. The one
+ # exception is fencing caused by the user's own cancel: nobody will
+ # take that run over, so the partial answer must be persisted here.
+ logger.warning("chat_run_worker_fenced", run_id=turn.run_id, owner=turn.owner)
+ if ownership.cancelled_by_user(turn.run_id):
+ snapshots._persist_cancelled_partial(turn)
+ else:
+ current = runtime.get_run(turn.run_id)
+ if current is None or current.status != "cancelled":
+ # Reaper/hard-cap/external recovery may cancel the local task
+ # after publishing its own authoritative terminal verdict.
+ logger.warning(
+ "chat_run_worker_stopped_by_external_terminal",
+ run_id=turn.run_id,
+ status=(current.status if current is not None else None),
+ )
+ else:
+ # Triggered by cancel_run: persist what was already produced,
+ # then write a user-cancelled event + terminal marker so
+ # followers exit gracefully.
+ logger.info("chat_run_cancelled", run_id=turn.run_id)
+ snapshots._persist_cancelled_partial(turn)
+ await turn.emit(
+ {
+ "type": "error",
+ "error": "任务已被用户取消",
+ "delta": "任务已被用户取消",
+ "chat_id": turn.chat_id,
+ "_cancelled": True,
+ },
+ terminal=True,
+ )
+ await turn.emit(
+ {"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id, "_cancelled": True},
+ terminal=True,
+ )
+
+ except errors.ToolOutcomeUnknown as exc:
+ await pauses._pause_for_tool_outcome(turn, exc)
+
+ except hooks.HookPaused as exc:
+ await pauses._pause_for_hook(turn, exc)
+
+ except run_journal.RunLeaseLost:
+ # A takeover/cancel may race with any journal append. Once fenced, the
+ # old worker must not persist fallback messages or publish a terminal —
+ # except when the fencing came from the user's own cancel.
+ logger.warning("chat_run_worker_fenced", run_id=turn.run_id, owner=turn.owner)
+ if ownership.cancelled_by_user(turn.run_id):
+ snapshots._persist_cancelled_partial(turn)
+
+ except Exception as exc:
+ nested_pause = hooks.find_hook_paused(exc)
+ if nested_pause is not None:
+ await pauses._pause_for_hook(turn, nested_pause)
+ return
+ nested_unknown = errors.find_tool_outcome_unknown(exc)
+ if nested_unknown is not None:
+ await pauses._pause_for_tool_outcome(turn, nested_unknown)
+ return
+ logger.error("chat_run_failed", run_id=turn.run_id, error=str(exc), exc_info=True)
+ try:
+ user_facing = resolve_user_facing_error(exc)
+ except Exception:
+ user_facing = "请求处理失败,请稍后重试"
+
+ failure_text = str(exc)
+
+ def _commit_failed_output(db) -> None:
+ # Whatever the turn produced before failing stays in the row, with
+ # the error beside it. Runs in the failed-terminal transaction so a
+ # fenced worker can never leave a late message behind.
+ model_steps.close_dangling_calls(turn.model_steps)
+ services_ChatService_module.ChatService(db).upsert_message(
+ chat_id=turn.chat_id,
+ role="assistant",
+ content=turn.full_response,
+ model=turn.model_name,
+ thinking=snapshots._thinking_payload(turn),
+ tool_calls=turn.tool_calls_log if turn.tool_calls_log else None,
+ model_steps=snapshots._model_steps_payload(turn),
+ message_id=turn.current_message_id,
+ chat_seq=turn.current_chat_seq,
+ error={"error": failure_text, "timestamp": clock._utcnow().isoformat()},
+ extra_data=snapshots._turn_extra(
+ turn,
+ duration_ms=int((time.monotonic() - turn.run_started_monotonic) * 1000),
+ segments=turn.segments.payload(),
+ ),
+ commit=False,
+ )
+
+ failed = turn.journal.complete(
+ turn.run_id,
+ owner=turn.owner,
+ status="failed",
+ failure_reason=str(exc)[:1000],
+ last_event_offset=turn.offsets.last,
+ commit_effect=_commit_failed_output,
+ )
+ if not failed:
+ logger.warning(
+ "chat_run_failure_cas_lost",
+ run_id=turn.run_id,
+ owner=turn.owner,
+ )
+ return
+ await turn.emit(
+ {
+ "type": "error",
+ "error": user_facing,
+ "delta": user_facing,
+ "chat_id": turn.chat_id,
+ },
+ terminal=True,
+ )
+ await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True)
+
+ finally:
+ if checkpoint_task is not None:
+ checkpoint_task.cancel()
+ # After the terminal state, keep the event stream for the full human
+ # wait window plus a recovery grace period so late reconnects can replay.
+ await writer.expire_stream(turn.run_id)
+ turn.lease_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await turn.lease_task
+ ownership._acknowledge_terminal_writer(turn.run_id)
diff --git a/src/backend/core/chat/schemas.py b/src/backend/core/chat/schemas.py
index e99bceb1..47bc1896 100644
--- a/src/backend/core/chat/schemas.py
+++ b/src/backend/core/chat/schemas.py
@@ -1,6 +1,6 @@
"""API request/response models."""
-from typing import Any, Dict, List, Literal, Optional
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
from core.config.settings import DEFAULT_CHAT_MODEL_ALIAS
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator
@@ -16,11 +16,32 @@ class AttachmentItem(BaseModel):
file_id: str = Field(..., min_length=1, description="持久化后的文件 ID")
+class QuoteMessageSource(BaseModel):
+ """引用来自本会话的一条消息。"""
+
+ kind: Literal["message"]
+ ts: Optional[int] = Field(None, description="被引用消息的前端时间戳")
+
+
+class QuoteFileSource(BaseModel):
+ """引用来自右侧预览中打开的一份文件。"""
+
+ kind: Literal["file"]
+ file_id: str = Field(..., min_length=1, max_length=128, description="文件 ID")
+ file_name: str = Field(..., min_length=1, max_length=255, description="文件名")
+ locator: Optional[str] = Field(
+ None, max_length=200, description="文件内位置:最近的标题、页码或 Sheet1!A1:C5 这类区域"
+ )
+
+
+QuoteSource = Annotated[Union[QuoteMessageSource, QuoteFileSource], Field(discriminator="kind")]
+
+
class QuotedFollowUpItem(BaseModel):
- """追问引用信息"""
+ """用户为本轮引用的一段原文,及其出处。"""
text: str = Field(..., description="被引用的原始文本", min_length=1, max_length=8000)
- ts: Optional[int] = Field(None, description="前端消息时间戳(可选)")
+ source: Optional[QuoteSource] = Field(None, description="原文出处:会话消息或文件")
@field_validator("text")
@classmethod
@@ -34,7 +55,7 @@ class ReferencedChatItem(BaseModel):
"""用户为本轮显式引用的一段历史会话。
只带 ID:标题、时间、概览等名片信息一律由后端按当前权限现查(见
- ``core.services.chat_reference_service``),前端传什么都不作数——否则任何人都能伪造
+ ``core.chat.references``),前端传什么都不作数——否则任何人都能伪造
一段"来自另一个会话"的内容塞进模型。
"""
@@ -188,7 +209,7 @@ class ChatRequest(BaseModel):
)
quoted_follow_up: Optional[QuotedFollowUpItem] = Field(
default=None,
- description="追问场景下引用的原始文本,用于增强上下文理解",
+ description="本轮引用的原文及出处(会话消息或文件),拼进发给模型的用户消息",
)
project_id: Optional[str] = Field(
default=None,
diff --git a/src/backend/core/services/chat_sequencer.py b/src/backend/core/chat/sequencer.py
similarity index 92%
rename from src/backend/core/services/chat_sequencer.py
rename to src/backend/core/chat/sequencer.py
index 70e27a72..9aefb707 100644
--- a/src/backend/core/services/chat_sequencer.py
+++ b/src/backend/core/chat/sequencer.py
@@ -9,7 +9,6 @@
from __future__ import annotations
import uuid
-from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Dict, Optional
@@ -21,7 +20,7 @@
from core.chat import inflight
from core.db.models import ChatMessage, ChatRun, ChatSession
from core.db.models.chat import reserve_chat_sequences
-from core.services.chat_service import clamp_message_content
+from core.chat.limits import clamp_message_content
LIVE_RUN_STATUSES = ("pending", "running")
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
@@ -82,36 +81,6 @@ def get_active_run(self, chat_id: str) -> Optional[ActiveRunRef]:
return None
return ActiveRunRef(run_id=run.run_id, message_id=run.message_id, status=run.status)
- @contextmanager
- def launching_main_run(
- self,
- *,
- chat_id: str,
- user_id: str,
- user_content: str,
- request_payload: Dict[str, Any],
- model: Optional[str] = None,
- user_extra_data: Optional[Dict[str, Any]] = None,
- ):
- """Admit a main conversation and release its pending writer if launch fails.
-
- Preparation after admission and executor launch belong inside this scope.
- Waiting for the running worker belongs outside it.
- """
- accepted = self.accept_main_run(
- chat_id=chat_id,
- user_id=user_id,
- user_content=user_content,
- request_payload=request_payload,
- model=model,
- user_extra_data=user_extra_data,
- )
- try:
- yield accepted
- except BaseException as exc:
- self.abandon_pending_run(accepted.run.run_id, reason=str(exc) or type(exc).__name__)
- raise
-
def accept_main_run(
self,
*,
@@ -137,7 +106,7 @@ def accept_main_run(
assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}"
try:
- from core.services.api_key_runs import bind_request_key, require_admission_key
+ from core.auth.api_key_runs import bind_request_key, require_admission_key
request_payload = bind_request_key(request_payload, user_id)
require_admission_key(self.db, request_payload, user_id, now)
@@ -148,7 +117,7 @@ def accept_main_run(
chat_id,
count=2,
)
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
access = ChatService(self.db).get_session_with_access(chat_id, user_id)
if access is None or access[1] not in ("admin", "edit"):
raise PermissionError("Conversation is not writable")
@@ -225,7 +194,7 @@ def accept_existing_user_run(
run_id = f"run_{uuid.uuid4().hex[:16]}"
assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}"
try:
- from core.services.api_key_runs import bind_request_key, require_admission_key
+ from core.auth.api_key_runs import bind_request_key, require_admission_key
request_payload = bind_request_key(request_payload, user_id)
require_admission_key(self.db, request_payload, user_id, now)
@@ -331,7 +300,7 @@ def accept_replacement_user_run(
user_message_id = f"msg_{uuid.uuid4().hex[:16]}"
assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}"
try:
- from core.services.api_key_runs import bind_request_key, require_admission_key
+ from core.auth.api_key_runs import bind_request_key, require_admission_key
request_payload = bind_request_key(request_payload, user_id)
require_admission_key(self.db, request_payload, user_id, now)
diff --git a/src/backend/core/chat/service.py b/src/backend/core/chat/service.py
new file mode 100644
index 00000000..0d9349ea
--- /dev/null
+++ b/src/backend/core/chat/service.py
@@ -0,0 +1,256 @@
+"""Chat sessions and messages behind one service facade."""
+from __future__ import annotations
+from core.chat import session_operations
+from core.chat import message_writes
+from core.chat import message_queries
+from core.chat import compaction_snapshots
+from core.chat import compaction_checkpoints
+from dataclasses import replace
+from typing import Any, Dict, List, Optional, Tuple
+from core.auth.edition import ChatAccessLevel
+from core.db.models import ChatCompactionState, ChatMessage, ChatSession
+from core.db.repository import AuditLogRepository, ChatMessageRepository, ChatSessionRepository
+from sqlalchemy.orm import Session
+import logging
+from core.chat import compaction_types
+
+logger = logging.getLogger(__name__)
+
+class ChatService:
+ """Service for chat-related operations."""
+
+ def __init__(self, db: Session):
+ self.db = db
+ self.session_repo = ChatSessionRepository(db)
+ self.message_repo = ChatMessageRepository(db)
+ self.audit_repo = AuditLogRepository(db)
+
+ def create_session(
+ self,
+ user_id: str,
+ title: str = "新对话",
+ extra_data: Dict = None,
+ chat_id: Optional[str] = None,
+ project_id: Optional[str] = None,
+ ) -> ChatSession:
+ return session_operations.create_session(self, user_id, title, extra_data, chat_id, project_id)
+
+ def ensure_session(
+ self,
+ chat_id: str,
+ user_id: str,
+ title: str = "新对话",
+ extra_data: Optional[Dict] = None,
+ project_id: Optional[str] = None,
+ ) -> Optional[ChatSession]:
+ return session_operations.ensure_session(self, chat_id, user_id, title, extra_data, project_id)
+
+ def list_sessions(
+ self,
+ user_id: str,
+ page: int = 1,
+ page_size: int = 20,
+ pinned_only: bool = False,
+ favorite_only: bool = False,
+ exclude_automation: bool = False,
+ ) -> Tuple[List[ChatSession], int, int]:
+ return session_operations.list_sessions(self, user_id, page, page_size, pinned_only, favorite_only, exclude_automation)
+
+ def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]:
+ return session_operations.get_session(self, chat_id, user_id)
+
+ def get_session_with_access(
+ self, chat_id: str, user_id: str
+ ) -> Optional[Tuple[ChatSession, ChatAccessLevel]]:
+ return session_operations.get_session_with_access(self, chat_id, user_id)
+
+ def update_session_fields(
+ self,
+ chat_id: str,
+ fields: Dict[str, Any],
+ *,
+ actor_user_id: Optional[str] = None,
+ ) -> Optional[ChatSession]:
+ return session_operations.update_session_fields(self, chat_id, fields, actor_user_id=actor_user_id)
+
+ def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool:
+ return session_operations.delete_session_force(self, chat_id, actor_user_id=actor_user_id)
+
+ def update_session(
+ self, chat_id: str, user_id: str, update_data: Dict[str, Any]
+ ) -> Optional[ChatSession]:
+ return session_operations.update_session(self, chat_id, user_id, update_data)
+
+ def delete_session(self, chat_id: str, user_id: str) -> bool:
+ return session_operations.delete_session(self, chat_id, user_id)
+
+ # chat_messages.content 有 CHECK 约束 char_length <= 100000。长任务的一轮回复
+ # (反复重试 + 大量思考正文)真的会撞上,而撞上的后果是**整条 INSERT 抛
+ # CheckViolation → 整个 run failed**,一轮的产出全部丢失(实测踩过)。
+ # 与其让约束把一轮工作全毁掉,不如夹逼后落库并明确标注被截断。
+ def add_message(
+ self,
+ chat_id: str,
+ role: str,
+ content: str,
+ model: Optional[str] = None,
+ thinking: Optional[List[Dict[str, Any]]] = None,
+ tool_calls: Optional[List[Dict]] = None,
+ usage: Optional[Dict] = None,
+ error: Optional[Dict] = None,
+ extra_data: Dict = None,
+ message_id: Optional[str] = None,
+ commit: bool = True,
+ chat_seq: Optional[int] = None,
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+ ) -> ChatMessage:
+ return message_writes.add_message(self, chat_id, role, content, model, thinking, tool_calls, usage, error, extra_data, message_id, commit, chat_seq, model_steps)
+
+ def refresh_streaming_message(
+ self,
+ *,
+ run_id: str,
+ owner: str,
+ message_id: str,
+ content: str,
+ thinking: Optional[List[Dict[str, Any]]],
+ tool_calls: Optional[List[Dict]],
+ extra_data: Dict[str, Any],
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+ ) -> bool:
+ return message_writes.refresh_streaming_message(self, run_id=run_id, owner=owner, message_id=message_id, content=content, thinking=thinking, tool_calls=tool_calls, extra_data=extra_data, model_steps=model_steps)
+
+ def upsert_message(
+ self,
+ chat_id: str,
+ role: str,
+ content: str,
+ *,
+ message_id: str,
+ model: Optional[str] = None,
+ thinking: Optional[List[Dict[str, Any]]] = None,
+ tool_calls: Optional[List[Dict]] = None,
+ usage: Optional[Dict] = None,
+ error: Optional[Dict] = None,
+ extra_data: Dict = None,
+ commit: bool = True,
+ chat_seq: Optional[int] = None,
+ model_steps: Optional[List[Dict[str, Any]]] = None,
+ invalidate_mid_turn_checkpoint: bool = False,
+ ) -> ChatMessage:
+ return message_writes.upsert_message(self, chat_id, role, content, message_id=message_id, model=model, thinking=thinking, tool_calls=tool_calls, usage=usage, error=error, extra_data=extra_data, commit=commit, chat_seq=chat_seq, model_steps=model_steps, invalidate_mid_turn_checkpoint=invalidate_mid_turn_checkpoint)
+
+ def list_all_messages(
+ self, chat_id: str, user_id: str
+ ) -> Optional[List[ChatMessage]]:
+ return message_queries.list_all_messages(self, chat_id, user_id)
+
+ def list_messages(
+ self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50
+ ) -> Optional[Tuple[List[ChatMessage], int, int]]:
+ return message_queries.list_messages(self, chat_id, user_id, page, page_size)
+
+ def delete_messages_from(self, chat_id: str, message_id: str) -> int:
+ return message_writes.delete_messages_from(self, chat_id, message_id)
+
+ def _invalidate_compaction_lineage_for_rewrite(
+ self,
+ chat_id: str,
+ *,
+ rewritten_seq: int,
+ ) -> bool:
+ return compaction_checkpoints.invalidate_compaction_lineage_for_rewrite(self, chat_id, rewritten_seq=rewritten_seq)
+
+ def add_compaction_checkpoint(
+ self,
+ chat_id: str,
+ *,
+ summary_text: str,
+ replacement_history: List[Dict],
+ covered_seq: Optional[int] = None,
+ source_hash: Optional[str] = None,
+ base_checkpoint_version: Optional[int] = None,
+ replacement_manifest: Optional[Dict[str, Any]] = None,
+ ) -> ChatMessage:
+ return compaction_checkpoints.add_compaction_checkpoint(self, chat_id, summary_text=summary_text, replacement_history=replacement_history, covered_seq=covered_seq, source_hash=source_hash, base_checkpoint_version=base_checkpoint_version, replacement_manifest=replacement_manifest)
+
+ def _latest_compaction_checkpoint_fallback(
+ self, chat_id: str
+ ) -> Optional[ChatMessage]:
+ return compaction_checkpoints.latest_compaction_checkpoint_fallback(self, chat_id)
+
+ @staticmethod
+ def _checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int:
+ return compaction_checkpoints.checkpoint_covered_seq(checkpoint)
+
+ def _ensure_message_sequences(self, chat_id: str) -> None:
+ return compaction_snapshots.ensure_message_sequences(self, chat_id)
+
+ def _ensure_compaction_state(self, chat_id: str) -> ChatCompactionState:
+ return compaction_snapshots.ensure_compaction_state(self, chat_id)
+
+ def acquire_compaction_snapshot(
+ self,
+ chat_id: str,
+ *,
+ owner: str,
+ lease_seconds: int,
+ ) -> Optional[compaction_types.CompactionSnapshot]:
+ return compaction_snapshots.acquire_compaction_snapshot(self, chat_id, owner=owner, lease_seconds=lease_seconds)
+
+ def _acquire_compaction_snapshot(
+ self,
+ chat_id: str,
+ *,
+ owner: str,
+ lease_seconds: int,
+ ) -> Optional[compaction_types.CompactionSnapshot]:
+ return compaction_snapshots.acquire_compaction_snapshot_impl(self, chat_id, owner=owner, lease_seconds=lease_seconds)
+
+ def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool:
+ return compaction_snapshots.release_compaction_lease(self, chat_id, owner=owner)
+
+ def commit_compaction_checkpoint(
+ self,
+ snapshot: compaction_types.CompactionSnapshot,
+ *,
+ owner: str,
+ summary_text: str,
+ replacement_history: List[Dict[str, Any]],
+ replacement_manifest: Dict[str, Any],
+ ) -> ChatMessage:
+ return compaction_checkpoints.commit_compaction_checkpoint(self, snapshot, owner=owner, summary_text=summary_text, replacement_history=replacement_history, replacement_manifest=replacement_manifest)
+
+ def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]:
+ return compaction_checkpoints.get_latest_compaction_checkpoint(self, chat_id)
+
+ def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]:
+ return message_queries.get_message_by_id(self, message_id)
+
+ def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]:
+ return message_queries.get_message_by_index(self, chat_id, index)
+
+ def get_user_message_before(
+ self, chat_id: str, message_id: str
+ ) -> Optional[ChatMessage]:
+ return message_queries.get_user_message_before(self, chat_id, message_id)
+
+ def update_message_extra_data(
+ self,
+ message_id: str,
+ patch: Dict[str, Any],
+ ) -> bool:
+ return message_writes.update_message_extra_data(self, message_id, patch)
+
+ def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]:
+ return message_writes.accept_ontology_revision(self, message_id)
+
+ def search_sessions(
+ self,
+ user_id: str,
+ query: str,
+ page: int = 1,
+ page_size: int = 20,
+ scope: str = "title",
+ ) -> Tuple[list, int]:
+ return session_operations.search_sessions(self, user_id, query, page, page_size, scope)
diff --git a/src/backend/core/chat/session_operations.py b/src/backend/core/chat/session_operations.py
new file mode 100644
index 00000000..c68c6469
--- /dev/null
+++ b/src/backend/core/chat/session_operations.py
@@ -0,0 +1,284 @@
+"""Create, list, update and delete chat sessions, with ownership and sharing checks."""
+from __future__ import annotations
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+from core.auth.edition import ChatAccessLevel, resolve_chat_access
+from core.db.models import ChatSession
+from core.db.reserved_chats import reserved_chat_owner
+
+def create_session(
+ self,
+ user_id: str,
+ title: str = "新对话",
+ extra_data: Dict = None,
+ chat_id: Optional[str] = None,
+ project_id: Optional[str] = None,
+) -> ChatSession:
+ """Create a new chat session.
+
+ If `chat_id` is provided and belongs to the same user, reuse it.
+ If it belongs to another user, generate a new chat_id.
+ """
+ if chat_id:
+ existing = self.session_repo.get_by_id(chat_id)
+ if existing:
+ if existing.user_id == user_id:
+ return existing
+ chat_id = None
+
+ session_data = {
+ "chat_id": chat_id or f"chat_{uuid.uuid4().hex[:16]}",
+ "user_id": user_id,
+ "title": title,
+ "extra_data": extra_data or {},
+ "project_id": project_id,
+ }
+ session = self.session_repo.create(session_data)
+
+ # Audit log
+ self.audit_repo.create(
+ {
+ "user_id": user_id,
+ "action": "chat.session.created",
+ "resource_type": "chat_session",
+ "resource_id": session.chat_id,
+ "status": "success",
+ }
+ )
+
+ return session
+
+
+def ensure_session(
+ self,
+ chat_id: str,
+ user_id: str,
+ title: str = "新对话",
+ extra_data: Optional[Dict] = None,
+ project_id: Optional[str] = None,
+) -> Optional[ChatSession]:
+ """Ensure a chat session exists for user and chat_id.
+
+ When ``project_id`` is given, attach the session to that project (first write wins; if
+ already attached to a different project it stays unchanged — in-project chats never drift
+ across projects). All session-creation entry points share this attachment rule.
+ """
+ existing = self.session_repo.get_by_id(chat_id)
+ if existing:
+ if existing.user_id != user_id:
+ return None
+ # Merge any missing metadata flags into existing session
+ if extra_data:
+ merged = dict(existing.extra_data or {})
+ changed = False
+ for k, v in extra_data.items():
+ if k not in merged:
+ merged[k] = v
+ changed = True
+ if changed:
+ existing.extra_data = merged
+ self.db.commit()
+ session = existing
+ else:
+ session = self.create_session(
+ user_id=user_id,
+ title=title,
+ extra_data=extra_data or {},
+ chat_id=chat_id,
+ project_id=project_id,
+ )
+ if session is not None and project_id and not session.project_id:
+ session.project_id = project_id
+ self.db.commit()
+ return session
+
+
+def list_sessions(
+ self,
+ user_id: str,
+ page: int = 1,
+ page_size: int = 20,
+ pinned_only: bool = False,
+ favorite_only: bool = False,
+ exclude_automation: bool = False,
+) -> Tuple[List[ChatSession], int, int]:
+ """List chat sessions with pagination."""
+ sessions, total = self.session_repo.list_by_user(
+ user_id,
+ page,
+ page_size,
+ pinned_only,
+ favorite_only,
+ exclude_automation=exclude_automation,
+ )
+
+ total_pages = (total + page_size - 1) // page_size
+
+ return sessions, total, total_pages
+
+
+def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]:
+ """Get chat session with ownership check.
+
+ Historical semantics unchanged: only the owner can get the session. For
+ edition-specific sharing, use :py:meth:`get_session_with_access`.
+ """
+ # Module-owned chats are reachable only through their module, never as ordinary chats.
+ if reserved_chat_owner(chat_id):
+ return None
+ session = self.session_repo.get_by_id(chat_id)
+
+ if session and session.user_id != user_id:
+ # Access denied - user doesn't own this session
+ return None
+
+ return session
+
+
+def get_session_with_access(
+ self, chat_id: str, user_id: str
+) -> Optional[Tuple[ChatSession, ChatAccessLevel]]:
+ """Get a session in a sharing context + compute the access level.
+
+ - Does not exist / soft-deleted → ``None``
+ - No access (``resolve_chat_access`` returns ``'none'``) → ``None``
+ - Module-owned (reserved namespace) → ``None``: not an ordinary chat
+ - Otherwise returns ``(session, 'admin'|'edit'|'read')``
+ """
+ if reserved_chat_owner(chat_id):
+ return None
+ session = self.session_repo.get_by_id(chat_id)
+ if session is None:
+ return None
+ level = resolve_chat_access(self.db, user_id, session)
+ if level == "none":
+ return None
+ return session, level
+
+
+def update_session_fields(
+ self,
+ chat_id: str,
+ fields: Dict[str, Any],
+ *,
+ actor_user_id: Optional[str] = None,
+) -> Optional[ChatSession]:
+ """Field update without ownership check (for sharing contexts).
+
+ The caller is responsible for permission checks beforehand. ``actor_user_id`` is used
+ only for the audit log. The ``extra_data`` field is merged rather than overwritten.
+ """
+ session = self.session_repo.get_by_id(chat_id)
+ if session is None:
+ return None
+ normalized = dict(fields)
+ extra_patch = normalized.get("extra_data")
+ if isinstance(extra_patch, dict):
+ merged = dict(session.extra_data or {})
+ merged.update(extra_patch)
+ normalized["extra_data"] = merged
+ updated = self.session_repo.update(chat_id, normalized)
+ self.audit_repo.create(
+ {
+ "user_id": actor_user_id or session.user_id,
+ "action": "chat.session.updated",
+ "resource_type": "chat_session",
+ "resource_id": chat_id,
+ "details": normalized,
+ "status": "success",
+ }
+ )
+ return updated
+
+
+def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool:
+ """Forced delete in a sharing context (no ownership check). Caller handles permissions."""
+ session = self.session_repo.get_by_id(chat_id)
+ if session is None:
+ return False
+ result = self.session_repo.soft_delete(chat_id)
+ if result:
+ self.audit_repo.create(
+ {
+ "user_id": actor_user_id,
+ "action": "chat.session.deleted",
+ "resource_type": "chat_session",
+ "resource_id": chat_id,
+ "details": {
+ "owner_user_id": session.user_id,
+ "deleted_by_owner": session.user_id == actor_user_id,
+ },
+ "status": "success",
+ }
+ )
+ return result
+
+
+def update_session(
+ self, chat_id: str, user_id: str, update_data: Dict[str, Any]
+) -> Optional[ChatSession]:
+ """Update chat session."""
+ session = self.get_session(chat_id, user_id)
+ if not session:
+ return None
+
+ normalized_update_data = dict(update_data)
+ extra_data_patch = normalized_update_data.get("extra_data")
+ if isinstance(extra_data_patch, dict):
+ merged_extra_data = dict(session.extra_data or {})
+ merged_extra_data.update(extra_data_patch)
+ normalized_update_data["extra_data"] = merged_extra_data
+
+ updated_session = self.session_repo.update(chat_id, normalized_update_data)
+
+ # Audit log
+ self.audit_repo.create(
+ {
+ "user_id": user_id,
+ "action": "chat.session.updated",
+ "resource_type": "chat_session",
+ "resource_id": chat_id,
+ "details": normalized_update_data,
+ "status": "success",
+ }
+ )
+
+ return updated_session
+
+
+def delete_session(self, chat_id: str, user_id: str) -> bool:
+ """Delete chat session (soft delete)."""
+ session = self.get_session(chat_id, user_id)
+ if not session:
+ return False
+
+ result = self.session_repo.soft_delete(chat_id)
+
+ if result:
+ # Audit log
+ self.audit_repo.create(
+ {
+ "user_id": user_id,
+ "action": "chat.session.deleted",
+ "resource_type": "chat_session",
+ "resource_id": chat_id,
+ "status": "success",
+ }
+ )
+
+ return result
+
+
+def search_sessions(
+ self,
+ user_id: str,
+ query: str,
+ page: int = 1,
+ page_size: int = 20,
+ scope: str = "title",
+) -> Tuple[list, int]:
+ """Search chat sessions by title (and optionally message content)."""
+ results, total = self.session_repo.search(
+ user_id, query, page, page_size, scope=scope
+ )
+ return results, total
diff --git a/src/backend/core/services/sidebar_status.py b/src/backend/core/chat/sidebar_status.py
similarity index 100%
rename from src/backend/core/services/sidebar_status.py
rename to src/backend/core/chat/sidebar_status.py
diff --git a/src/backend/core/services/chat_steer_service.py b/src/backend/core/chat/steer.py
similarity index 95%
rename from src/backend/core/services/chat_steer_service.py
rename to src/backend/core/chat/steer.py
index 92cc2d87..5dc37396 100644
--- a/src/backend/core/services/chat_steer_service.py
+++ b/src/backend/core/chat/steer.py
@@ -16,7 +16,7 @@
from core.db.engine import SessionLocal
from core.db.models import ChatRun
from core.infra.ephemeral import get_ephemeral_state
-from core.services.steer_queue import SteerQueue, SteerQueueConflict
+from core.chat.steer_queue import SteerQueue, SteerQueueConflict
_STEER_KEY = "jx:chat:run:{run_id}:steer"
_STEER_TTL_SECONDS = 3600
@@ -127,8 +127,9 @@ async def remove_pending_steer(run_id: str, steer_id: str) -> bool:
if removed:
try:
await get_ephemeral_state().drop(_key(run_id))
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.chat.steer:remove_pending_steer', fallback_error)
return removed
diff --git a/src/backend/core/chat/steer_hook.py b/src/backend/core/chat/steer_hook.py
new file mode 100644
index 00000000..3cd83053
--- /dev/null
+++ b/src/backend/core/chat/steer_hook.py
@@ -0,0 +1,70 @@
+"""Deliver queued user instructions at neutral tool and context boundaries."""
+import asyncio
+from core.harness.events import thaw_value
+from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage
+
+
+class SteeringHook:
+ def __init__(self, take_pending=None, acknowledge=None):
+ self.take_pending = take_pending
+ self.acknowledge = acknowledge
+ self.lock = asyncio.Lock()
+ self.delivery = None
+ self.interrupted = False
+
+ async def poll(self, run_id):
+ if self.delivery is None and run_id:
+ from core.chat.steer import take_pending_steer
+ self.delivery = await (self.take_pending or take_pending_steer)(run_id)
+ return self.delivery
+
+ async def before_tool(self, invocation):
+ async with self.lock:
+ if not await self.poll(invocation.run_id):
+ return Decision()
+ self.interrupted = True
+ notice = "用户追加了新指令;本工具调用已在执行前中止,等待模型按新指令重新规划。"
+ return Decision(DecisionKind.REJECT, patch={"tool_response": {
+ "state": "interrupted", "content": [{"type": "text", "text": notice}]}}, reason=notice)
+
+ async def transform(self, invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ async with self.lock:
+ delivery = await self.poll(invocation.run_id)
+ message = str((delivery or {}).get("message") or "").strip()
+ if not message:
+ self.delivery = None
+ self.interrupted = False
+ return Decision()
+ suffix = ("请立即按这条新指令调整后续计划;不要继续已经被中止的旧工具调用。"
+ if self.interrupted else
+ "请立即按这条新指令调整后续计划;上一轮工具结果已经完成,可按需使用。")
+ item = dict(content="[用户在当前执行中追加的新指令]\n" + message + "\n" + suffix,
+ kind="steer", origin="user:steer", trust="user", priority=950,
+ token_budget=4000, truncation_policy="never")
+ return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), item],
+ "steer_delivery": delivery})
+
+ async def applied(self, invocation):
+ delivery = invocation.data.get("steer_delivery")
+ if delivery is None:
+ return Decision()
+ # Called only after the framework has appended the context successfully.
+ async with self.lock:
+ if self.acknowledge is not None:
+ await self.acknowledge(thaw_value(delivery))
+ self.delivery = None
+ self.interrupted = False
+ return Decision()
+
+ def specs(self):
+ return (
+ HookSpec("user_steering_admission", HookStage.BEFORE_TOOL, self.before_tool,
+ priority=100, timeout_ms=30000, mutable_fields=frozenset({"tool_response"})),
+ HookSpec("user_steering_context", HookStage.TRANSFORM_CONTEXT, self.transform,
+ priority=100, timeout_ms=30000,
+ mutable_fields=frozenset({"context_items", "steer_delivery"})),
+ HookSpec("user_steering_applied", HookStage.AFTER_CONTEXT, self.applied,
+ priority=100, timeout_ms=30000),
+ )
diff --git a/src/backend/core/services/steer_queue.py b/src/backend/core/chat/steer_queue.py
similarity index 100%
rename from src/backend/core/services/steer_queue.py
rename to src/backend/core/chat/steer_queue.py
diff --git a/src/backend/core/services/subagent_routing_service.py b/src/backend/core/chat/subagent_routing.py
similarity index 100%
rename from src/backend/core/services/subagent_routing_service.py
rename to src/backend/core/chat/subagent_routing.py
diff --git a/src/backend/core/services/chat_titles.py b/src/backend/core/chat/titles.py
similarity index 92%
rename from src/backend/core/services/chat_titles.py
rename to src/backend/core/chat/titles.py
index f3fe16cd..cb1b2d89 100644
--- a/src/backend/core/services/chat_titles.py
+++ b/src/backend/core/chat/titles.py
@@ -1,13 +1,14 @@
"""One non-blocking first-question title call, with durable claim and rename fencing."""
+from __future__ import annotations
import asyncio
import logging
+from dataclasses import dataclass, field
from core.db import engine as db_engine
from core.db.models import ChatMessage, ChatSession
-from core.llm.summarizer import get_summarizer
+from core.models.completions.summarizer import get_summarizer
logger = logging.getLogger(__name__)
-_tasks: set[asyncio.Task] = set()
def _claim(chat_id: str, user_id: str, run_id: str) -> str | None:
@@ -99,11 +100,23 @@ def schedule_first_title(chat_id: str, user_id: str, run_id: str) -> asyncio.Tas
if question is None:
return None
task = asyncio.create_task(_generate(chat_id, user_id, run_id, question), name=f"chat-title:{chat_id}")
- _tasks.add(task)
+ process_state().tasks.add(task)
def settled(done: asyncio.Task) -> None:
- _tasks.discard(done)
+ process_state().tasks.discard(done)
if not done.cancelled() and done.exception() is not None:
logger.warning("chat_title_task_failed: %s", type(done.exception()).__name__)
task.add_done_callback(settled)
return task
+
+
+@dataclass
+class TitlesState:
+ tasks: set[asyncio.Task] = field(default_factory=lambda: set())
+
+
+_STATE = TitlesState()
+
+
+def process_state() -> TitlesState:
+ return _STATE
diff --git a/src/backend/core/chat/tool_log.py b/src/backend/core/chat/tool_log.py
index c6c6beaf..0f368009 100644
--- a/src/backend/core/chat/tool_log.py
+++ b/src/backend/core/chat/tool_log.py
@@ -10,7 +10,7 @@
import logging
import time
from typing import Any, Dict, List, Optional
-from core.chat.display_bounds import bound_result_for_display
+from core.infra.text_bounds import bound_result_for_display
logger = logging.getLogger(__name__)
diff --git a/src/backend/orchestration/tool_payloads.py b/src/backend/core/chat/tool_payloads.py
similarity index 92%
rename from src/backend/orchestration/tool_payloads.py
rename to src/backend/core/chat/tool_payloads.py
index f70b74a1..462efd6d 100644
--- a/src/backend/orchestration/tool_payloads.py
+++ b/src/backend/core/chat/tool_payloads.py
@@ -10,7 +10,7 @@
from typing import Any, Dict
-from core.config.catalog_details import get_skill_curated_detail
+from core.capabilities.catalog.details import get_skill_curated_detail
_FILE_PREVIEW_MAX_LINES = 8
_FILE_PREVIEW_MAX_CHARS = 400
@@ -114,8 +114,15 @@ def _build_read_tool_payload(tool_args: Any, result: Any) -> Dict[str, Any]:
# type == "text"
content = result.get("content") if isinstance(result.get("content"), str) else ""
extra2: Dict[str, Any] = {}
- for k in ("start_line", "end_line", "truncated", "parsed_text",
- "recovered_from_artifact", "persistent", "physical_path"):
+ for k in (
+ "start_line",
+ "end_line",
+ "truncated",
+ "parsed_text",
+ "recovered_from_artifact",
+ "persistent",
+ "physical_path",
+ ):
if k in result and result[k] not in (None, "", False):
extra2[k] = result[k]
return {
@@ -146,8 +153,16 @@ def _build_read_artifact_payload(result: Any) -> Dict[str, Any]:
# output structure from the read_artifact tool (see core/llm/tool.py::read_artifact)
content = result.get("content")
extra: Dict[str, Any] = {}
- for k in ("file_id", "offset", "next_offset", "read_chars_so_far",
- "budget_remaining", "sheet_names", "slide_count", "slide_index"):
+ for k in (
+ "file_id",
+ "offset",
+ "next_offset",
+ "read_chars_so_far",
+ "budget_remaining",
+ "sheet_names",
+ "slide_count",
+ "slide_index",
+ ):
if k in result and result[k] not in (None, ""):
extra[k] = result[k]
return {
@@ -198,15 +213,17 @@ def _build_skill_load_payload(skill_id: str) -> Dict[str, Any]:
# LLM a noticeable amount of time to finish writing (e.g. long JSON params),
# otherwise the UI shows only the generic typing placeholder until the tool
# result finally arrives.
-_FAST_EMIT_TOOLS = frozenset({
- "Bash",
- "write_stdin",
- "Read",
- "Edit",
- "Write",
- "Glob",
- "Grep",
-})
+_FAST_EMIT_TOOLS = frozenset(
+ {
+ "Bash",
+ "write_stdin",
+ "Read",
+ "Edit",
+ "Write",
+ "Glob",
+ "Grep",
+ }
+)
def _tool_args_ready(tool_name: str, tool_args: Any) -> bool:
diff --git a/src/backend/core/config/__init__.py b/src/backend/core/config/__init__.py
index b1508569..35218ba3 100644
--- a/src/backend/core/config/__init__.py
+++ b/src/backend/core/config/__init__.py
@@ -1,6 +1,5 @@
-# ``core.config`` holds static config + ``settings`` only (a clean L1 leaf).
-# The DB-backed config *services* live in ``core.services`` so config no longer
-# imports ``core.db`` at module level:
-# from core.services.model_config import ModelConfigService
-# from core.services.system_config import SystemConfigService
-# from core.services.mcp_service import McpServerConfigService
+# ``core.config`` holds static config + ``settings`` only and never imports
+# ``core.db`` at module level. DB-backed config services live elsewhere:
+# from core.models.config import ModelConfigService
+# from core.config.store.system_config import SystemConfigService
+# from core.capabilities.mcp.service import McpServerConfigService
diff --git a/src/backend/core/config/display_names.py b/src/backend/core/config/display_names.py
index f2c595a1..31fa3f74 100644
--- a/src/backend/core/config/display_names.py
+++ b/src/backend/core/config/display_names.py
@@ -8,7 +8,7 @@
from typing import Dict
-from core.config.edition_display_names import (
+from core.config.edition import (
edition_mcp_server_descriptions,
edition_mcp_server_display_names,
edition_tool_display_names,
diff --git a/src/backend/core/config/edition_display_names.py b/src/backend/core/config/edition.py
similarity index 78%
rename from src/backend/core/config/edition_display_names.py
rename to src/backend/core/config/edition.py
index 1ae1d377..1716fd29 100644
--- a/src/backend/core/config/edition_display_names.py
+++ b/src/backend/core/config/edition.py
@@ -28,3 +28,11 @@ def edition_tool_display_names() -> dict[str, str]:
"edition_mcp_user_intros",
"edition_tool_display_names",
]
+
+"""Community edition contributes no enterprise service configuration."""
+
+SEED_CONFIGS = []
+CONFIG_KEY_TO_ENV = {}
+
+
+__all__ += ["CONFIG_KEY_TO_ENV", "SEED_CONFIGS"]
diff --git a/src/backend/core/config/environment.py b/src/backend/core/config/environment.py
index c8fbccae..5f981998 100644
--- a/src/backend/core/config/environment.py
+++ b/src/backend/core/config/environment.py
@@ -8,10 +8,7 @@
from __future__ import annotations
import os
-import tempfile
-from dataclasses import dataclass, field
from pathlib import Path
-from typing import Optional
from dotenv import dotenv_values
diff --git a/src/backend/core/config/identity_settings.py b/src/backend/core/config/identity_settings.py
index 1e81a1f5..50e9e938 100644
--- a/src/backend/core/config/identity_settings.py
+++ b/src/backend/core/config/identity_settings.py
@@ -3,9 +3,8 @@
import os
import tempfile
from dataclasses import dataclass, field
-from pathlib import Path
from typing import Optional
-from .environment import _env, _bool, _int, _float, _REPO_ROOT
+from .environment import _env, _bool, _int
@dataclass(frozen=True)
class AuthSettings:
@@ -135,8 +134,8 @@ class DatabaseSettings:
)
)
echo: bool = field(default_factory=lambda: _bool(_env("DB_ECHO", "false")))
- pool_size: int = field(default_factory=lambda: _int(_env("DB_POOL_SIZE", "20"), 20))
- pool_max_overflow: int = field(
- default_factory=lambda: _int(_env("DB_POOL_MAX_OVERFLOW", "10"), 10)
- )
+ # Threads per process that run blocking work for requests (sync routes and
+ # dependencies, ``core.db.threads.run_db``). The connection pool is derived
+ # from it in ``core.db.engine.thread_budget`` rather than configured apart.
+ worker_threads: int = field(default_factory=lambda: max(1, _int(_env("WORKER_THREADS", "40"), 40)))
pool_timeout: int = field(default_factory=lambda: _int(_env("DB_POOL_TIMEOUT", "30"), 30))
diff --git a/src/backend/core/config/mcp_config.py b/src/backend/core/config/mcp_config.py
index 25c2472b..a6281c46 100755
--- a/src/backend/core/config/mcp_config.py
+++ b/src/backend/core/config/mcp_config.py
@@ -19,10 +19,10 @@
from extensions.mcp_servers._ports import PORTS as _PORTS
-def _mcp_http_url(server_id: str) -> str:
+def mcp_http_url(server_id: str) -> str:
from core.config.settings import settings
- return f"http://{settings.server.mcp_host}:{_PORTS[server_id]}/mcp/"
+ return f"http://{settings.server.mcp_host}:{_PORTS[server_id]}/mcp"
def builtin_mcp_configs(location=None) -> Dict[str, dict]:
@@ -34,7 +34,7 @@ def builtin_mcp_configs(location=None) -> Dict[str, dict]:
return {
server.server_id: {
"transport": "streamable_http",
- "url": _mcp_http_url(server.server_id),
+ "url": mcp_http_url(server.server_id),
"env": {},
}
for server in process_servers(location)
@@ -68,7 +68,7 @@ def builtin_launcher_serves_this_plane() -> bool:
if not settings.deploy.is_local:
return True
- from core.auth.desktop_bridge import bridge_enabled
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
return not bridge_enabled()
diff --git a/src/backend/core/config/product_settings.py b/src/backend/core/config/product_settings.py
index 50eaffb7..217ea86c 100644
--- a/src/backend/core/config/product_settings.py
+++ b/src/backend/core/config/product_settings.py
@@ -1,11 +1,8 @@
"""Environment-backed configuration values."""
from __future__ import annotations
import os
-import tempfile
from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Optional
-from .environment import _env, _bool, _int, _float, _REPO_ROOT
+from .environment import _env, _bool, _int
@dataclass(frozen=True)
class EditionSettings:
diff --git a/src/backend/core/config/runtime_env.py b/src/backend/core/config/runtime_env.py
index ca82ad9f..c6eb252d 100644
--- a/src/backend/core/config/runtime_env.py
+++ b/src/backend/core/config/runtime_env.py
@@ -1,19 +1,6 @@
-"""DB-backed env lookup for MCP servers running in the streamable-http container.
-
-The MCP container's ``os.environ`` is frozen at ``docker-compose up`` time,
-so admin-panel changes never reach it. ``get_runtime_value`` re-routes the
-existing ``os.getenv`` idiom through ``SystemConfigService`` /
-``ModelConfigService`` (both DB-cached for 30s) before falling back to env.
-"""
-
-from __future__ import annotations
-
-import logging
+"""Environment-only paths available before database initialization."""
import os
from pathlib import Path
-from typing import Optional
-
-logger = logging.getLogger(__name__)
def local_data_dir() -> Path:
@@ -24,37 +11,5 @@ def local_data_dir() -> Path:
"""
return Path(os.getenv("HUGAGENT_HOME", str(Path.home() / ".hugagent"))).expanduser()
-# Env vars produced by ``ModelConfigService.get_mcp_env_overlay()``. Looking up
-# a non-model env var should not pay the cost of building that dict.
-_MODEL_CONFIG_ENV_VARS: frozenset[str] = frozenset({
- "MODEL_URL", "API_KEY", "BASE_MODEL_NAME",
- "OPENAI_API_BASE", "OPENAI_BASE_URL", "OPENAI_API_KEY",
- "MEM0_EMBED_URL", "MEM0_EMBED_MODEL", "MEM0_EMBED_API_KEY", "MEM0_EMBED_DIMS",
- "RERANKER_URL", "RERANKER_MODEL", "RERANKER_API_KEY",
-})
-
-
-def get_runtime_value(env_var: str, default: Optional[str] = None) -> Optional[str]:
- """Resolve an env-var name against admin DB configs first, then env."""
- try:
- from core.services.system_config import SystemConfigService, get_config_key_for_env
-
- config_key = get_config_key_for_env(env_var)
- if config_key:
- val = SystemConfigService.get_instance().get(config_key)
- if val:
- return val
- except Exception as exc:
- logger.debug("[runtime_env] system_config lookup failed for %s: %s", env_var, exc)
-
- if env_var in _MODEL_CONFIG_ENV_VARS:
- try:
- from core.services.model_config import ModelConfigService
-
- val = ModelConfigService.get_instance().get_mcp_env_overlay().get(env_var)
- if val:
- return val
- except Exception as exc:
- logger.debug("[runtime_env] model_config lookup failed for %s: %s", env_var, exc)
- return os.getenv(env_var, default)
+CAPS_ROOT_ENV = "HUGAGENT_CAPS_ROOT"
diff --git a/src/backend/core/config/runtime_settings.py b/src/backend/core/config/runtime_settings.py
index 8bf49dc0..117efe56 100644
--- a/src/backend/core/config/runtime_settings.py
+++ b/src/backend/core/config/runtime_settings.py
@@ -1,11 +1,9 @@
"""Environment-backed configuration values."""
from __future__ import annotations
import os
-import tempfile
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Optional
-from .environment import _env, _bool, _int, _float, _REPO_ROOT, DEFAULT_CHAT_MODEL_ALIAS, _load_env_files
+from .environment import _env, _bool, _int, _float
@dataclass(frozen=True)
class LLMSettings:
@@ -187,29 +185,6 @@ class ServerSettings:
def is_prod(self) -> bool:
return self.env in ("prod", "production")
-@dataclass(frozen=True)
-class RateLimitSettings:
- enabled: bool = field(default_factory=lambda: _bool(_env("RATE_LIMIT_ENABLED", "true")))
- storage: str = field(default_factory=lambda: _env("RATE_LIMIT_STORAGE", "memory://"))
- cb_user_center_threshold: int = field(
- default_factory=lambda: _int(_env("CB_USER_CENTER_THRESHOLD", "5"), 5)
- )
- cb_user_center_timeout: int = field(
- default_factory=lambda: _int(_env("CB_USER_CENTER_TIMEOUT", "60"), 60)
- )
- cb_model_api_threshold: int = field(
- default_factory=lambda: _int(_env("CB_MODEL_API_THRESHOLD", "10"), 10)
- )
- cb_model_api_timeout: int = field(
- default_factory=lambda: _int(_env("CB_MODEL_API_TIMEOUT", "30"), 30)
- )
- cb_storage_threshold: int = field(
- default_factory=lambda: _int(_env("CB_STORAGE_THRESHOLD", "5"), 5)
- )
- cb_storage_timeout: int = field(
- default_factory=lambda: _int(_env("CB_STORAGE_TIMEOUT", "60"), 60)
- )
-
@dataclass(frozen=True)
class RoutingSettings:
strategy: str = field(
@@ -219,11 +194,11 @@ class RoutingSettings:
@dataclass(frozen=True)
class CompactionSettings:
- """Context compaction (see core/llm/compaction.py).
+ """Context compaction (see core/llm/context/compaction.py).
Compresses history into "recent user messages + summary" and persists the
summary as a checkpoint. One implementation serves all three phases
- (mid-turn / pre-turn / post-turn, see ``core.llm.compaction.CompactionPhase``),
+ (mid-turn / pre-turn / post-turn, see ``core.llm.context.compaction.CompactionPhase``),
so every knob below governs all of them — there is no separate in-turn ratio
any more.
"""
@@ -260,7 +235,7 @@ class CompactionSettings:
# Per-tool-result cap fed to AgentScope's ContextConfig. This is the
# deterministic, model-free layer that runs before any summarization (the
# counterpart of Codex's output truncation and dsh's toolResultPruner);
- # the overflow is spilled to the sandbox by core/llm/offloader.py.
+ # the overflow is spilled to the sandbox by core/llm/context/offloader.py.
tool_result_limit: int = field(
default_factory=lambda: _int(_env("CHAT_TOOL_RESULT_LIMIT", "20000"), 20000)
)
diff --git a/src/backend/core/config/sandbox_settings.py b/src/backend/core/config/sandbox_settings.py
index f881f28c..d4ab7070 100644
--- a/src/backend/core/config/sandbox_settings.py
+++ b/src/backend/core/config/sandbox_settings.py
@@ -1,11 +1,8 @@
"""Environment-backed configuration values."""
from __future__ import annotations
import os
-import tempfile
from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Optional
-from .environment import _env, _bool, _int, _float, _REPO_ROOT
+from .environment import _env, _bool, _int
@dataclass(frozen=True)
class SandboxSettings:
@@ -118,12 +115,12 @@ class SandboxSettings:
opensandbox_snapshot_wait_timeout_s: int = field(
default_factory=lambda: _int(_env("OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S", "120"), 120)
)
- # Skill dependencies come only from the image bake (docker/Dockerfile.opensandbox); new dependencies go through the admin-panel rebuild.
+ # Skill dependencies come only from the image bake (docker/sandbox/Dockerfile.opensandbox); new dependencies go through the admin-panel rebuild.
# The all-modes code-execution capability switch has moved out of settings:
# the sole control source is sandbox.code_capability_enable in the Config
# admin panel "System Config" (option B, see
- # core.services.system_config.code_capability_enabled). The env is retired.
+ # core.config.store.system_config.code_capability_enabled). The env is retired.
# Hard safety for user confirmation of MySpace write operations (docs §13).
# true (default): Write/Edit/Delete/Move on /myspace must pass out-of-band
# user confirmation; non-interactive mode (sub-agent) rejects writes
@@ -265,7 +262,7 @@ class SandboxSettings:
# then hot-swaps _template with the new template id + writes it back to
# .env. An empty host means "remote rebuild not configured": the rebuild
# endpoint errors out for cube and directs users to the manual docs path.
- # See core/services/cube_template_builder.py for details.
+ # See features/sandbox_ops/cube_template.py for details.
cube_node_ssh_host: str = field(
default_factory=lambda: _env("CUBE_NODE_SSH_HOST", _env("CUBE_NODE_IP", "")).strip()
)
diff --git a/src/backend/core/config/settings.py b/src/backend/core/config/settings.py
index fa6b3fc1..cdc8c4b9 100644
--- a/src/backend/core/config/settings.py
+++ b/src/backend/core/config/settings.py
@@ -1,11 +1,8 @@
"""Environment-backed configuration values."""
from __future__ import annotations
import os
-import tempfile
from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Optional
-from .environment import _env, _bool, _int, _float, _REPO_ROOT, DEFAULT_CHAT_MODEL_ALIAS, _load_env_files
+from .environment import _env as env_value, _bool as env_bool, _int as env_int, DEFAULT_CHAT_MODEL_ALIAS
from .identity_settings import (
AuthSettings,
@@ -21,7 +18,6 @@
KnowledgeBaseSettings,
RedisSettings,
ServerSettings,
- RateLimitSettings,
RoutingSettings,
CompactionSettings,
PromptSettings,
@@ -52,7 +48,6 @@ class AppSettings:
kb: KnowledgeBaseSettings = field(default_factory=KnowledgeBaseSettings)
redis: RedisSettings = field(default_factory=RedisSettings)
server: ServerSettings = field(default_factory=ServerSettings)
- rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings)
routing: RoutingSettings = field(default_factory=RoutingSettings)
compaction: CompactionSettings = field(default_factory=CompactionSettings)
prompt: PromptSettings = field(default_factory=PromptSettings)
diff --git a/src/backend/core/config/store/__init__.py b/src/backend/core/config/store/__init__.py
new file mode 100644
index 00000000..8e4f6e51
--- /dev/null
+++ b/src/backend/core/config/store/__init__.py
@@ -0,0 +1 @@
+"""System configuration stored in the database, with environment fallbacks."""
diff --git a/src/backend/core/config/store/config_keys.py b/src/backend/core/config/store/config_keys.py
new file mode 100644
index 00000000..695edc32
--- /dev/null
+++ b/src/backend/core/config/store/config_keys.py
@@ -0,0 +1,12 @@
+"""Map environment variables to the system-config keys that override them."""
+from __future__ import annotations
+from typing import Optional
+from core.config.store import system_config_constants
+
+def get_config_key_for_env(env_var: str) -> Optional[str]:
+ """Return the system-config key that backs ``env_var``, if any.
+
+ Public lookup helper so callers (e.g. ``runtime_env``) don't need to import
+ the internal reverse map.
+ """
+ return system_config_constants.ENV_TO_CONFIG_KEY.get(env_var)
diff --git a/src/backend/core/config/store/execution_options.py b/src/backend/core/config/store/execution_options.py
new file mode 100644
index 00000000..df1a4220
--- /dev/null
+++ b/src/backend/core/config/store/execution_options.py
@@ -0,0 +1,93 @@
+"""Execution switches read from system config: code capability, turbo mode, planning."""
+from __future__ import annotations
+from core.config.store import system_config
+from core.config.store import system_config_constants
+
+def code_capability_enabled() -> bool:
+ """Master switch for the main agent's code execution capability (single source of truth at runtime).
+
+ Sole control source = the DB value of Config console "System config" (plan B, env fallback retired);
+ seeded explicitly to "true". Saving in the console calls invalidate_cache; takes effect ≤30s without restart.
+ agent_factory and the admin_prompts preview call the same source, ensuring the console display matches runtime.
+ """
+ try:
+ val = system_config.SystemConfigService.get_instance().get("sandbox.code_capability_enable", "false")
+ except Exception: # noqa: BLE001 — conservatively disable on config-layer errors
+ return False
+ return str(val).strip().lower() in ("1", "true", "yes", "on")
+
+
+def turbo_mcp_server_ids() -> frozenset[str]:
+ """极速模式装配的 MCP server 集合(运行时唯一真源)。
+
+ 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.mcp_server_ids``
+ (逗号分隔)。刻意不与能力目录(catalog)的启停求交集——极速模式的工具面
+ 是独立的产品契约。配置异常或解析为空时回退到内置检索三件套。
+ """
+ default = ",".join(sorted(system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS))
+ try:
+ raw = system_config.SystemConfigService.get_instance().get("turbo.mcp_server_ids", default) or default
+ except Exception: # noqa: BLE001 — conservatively fall back to the built-in trio
+ return system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS
+ ids = frozenset(part.strip() for part in str(raw).split(",") if part.strip())
+ return ids or system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS
+
+
+def turbo_skill_ids() -> tuple[str, ...]:
+ """极速模式装配的技能集合(``AdminSkill.skill_id`` / 内置技能 id 列表)。
+
+ 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.skill_ids``(逗号
+ 分隔)。与用户本轮显式呼唤的技能合并去重。默认空 —— 不选就没有技能,
+ 极速模式的系统提示词里也就不出现技能清单。
+ """
+ try:
+ raw = system_config.SystemConfigService.get_instance().get("turbo.skill_ids", "") or ""
+ except Exception: # noqa: BLE001 — 配置层异常时按「没配技能」处理
+ return ()
+ return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip()))
+
+
+def turbo_plugin_ids() -> tuple[str, ...]:
+ """极速模式装配的插件集合(``InstalledPlugin.install_id`` 列表)。
+
+ 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.plugin_ids``(逗号
+ 分隔)。插件是「技能 + 工具」的能力包,装配时由 agent_factory 展开成其组件
+ 技能 / MCP,与 ``turbo.mcp_server_ids`` 合并去重。默认空 —— 不选就没有插件。
+ """
+ try:
+ raw = system_config.SystemConfigService.get_instance().get("turbo.plugin_ids", "") or ""
+ except Exception: # noqa: BLE001 — 配置层异常时按「没配插件」处理
+ return ()
+ return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip()))
+
+
+def turbo_manual_invoke_enabled() -> bool:
+ """极速模式下是否允许显式呼唤(斜杠技能 / @子智能体 / 插件)临时装配。"""
+ try:
+ val = system_config.SystemConfigService.get_instance().get("turbo.manual_invoke_enable", "true")
+ except Exception: # noqa: BLE001
+ return True
+ return str(val).strip().lower() in ("1", "true", "yes", "on")
+
+
+def turbo_max_iters() -> int:
+ """极速模式的 ReAct 迭代硬上限(最小 2,防止配错把回答轮也掐掉)。"""
+ try:
+ val = system_config.SystemConfigService.get_instance().get("turbo.max_iters", "4")
+ return max(2, int(str(val).strip()))
+ except Exception: # noqa: BLE001
+ return 4
+
+
+def auto_plan_entry_enabled() -> bool:
+ """Master switch for the main agent's in-conversation plan tracker (the update_plan tool).
+
+ Control source = the Config console "System config" DB value ``chat.auto_plan_entry_enable``,
+ defaulting to "true" (enabled by default). Ops can set it to false to disable the capability, ≤30s, no restart.
+ Conservatively disable on config-layer errors (don't register the tool, fall back to manually clicking "Plan mode").
+ """
+ try:
+ val = system_config.SystemConfigService.get_instance().get("chat.auto_plan_entry_enable", "true")
+ except Exception: # noqa: BLE001 — conservatively disable on config-layer errors
+ return False
+ return str(val).strip().lower() in ("1", "true", "yes", "on")
diff --git a/src/backend/core/config/store/runtime_env.py b/src/backend/core/config/store/runtime_env.py
new file mode 100644
index 00000000..07a56cba
--- /dev/null
+++ b/src/backend/core/config/store/runtime_env.py
@@ -0,0 +1,52 @@
+"""DB-backed env lookup for MCP servers running in the streamable-http container.
+
+The MCP container's ``os.environ`` is frozen at ``docker-compose up`` time,
+so admin-panel changes never reach it. ``get_runtime_value`` re-routes the
+existing ``os.getenv`` idiom through ``SystemConfigService`` /
+``ModelConfigService`` (both DB-cached for 30s) before falling back to env.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+# Env vars produced by ``ModelConfigService.get_mcp_env_overlay()``. Looking up
+# a non-model env var should not pay the cost of building that dict.
+_MODEL_CONFIG_ENV_VARS: frozenset[str] = frozenset({
+ "MODEL_URL", "API_KEY", "BASE_MODEL_NAME",
+ "OPENAI_API_BASE", "OPENAI_BASE_URL", "OPENAI_API_KEY",
+ "MEM0_EMBED_URL", "MEM0_EMBED_MODEL", "MEM0_EMBED_API_KEY", "MEM0_EMBED_DIMS",
+ "RERANKER_URL", "RERANKER_MODEL", "RERANKER_API_KEY",
+})
+
+
+def get_runtime_value(env_var: str, default: Optional[str] = None) -> Optional[str]:
+ """Resolve an env-var name against admin DB configs first, then env."""
+ try:
+ from core.config.store.system_config import SystemConfigService
+ from core.config.store.config_keys import get_config_key_for_env
+
+ config_key = get_config_key_for_env(env_var)
+ if config_key:
+ val = SystemConfigService.get_instance().get(config_key)
+ if val:
+ return val
+ except Exception as exc:
+ logger.debug("[runtime_env] system_config lookup failed for %s: %s", env_var, exc)
+
+ if env_var in _MODEL_CONFIG_ENV_VARS:
+ try:
+ from core.harness.registry.model_services import model_environment
+
+ val = model_environment().get(env_var)
+ if val:
+ return val
+ except Exception as exc:
+ logger.debug("[runtime_env] model_config lookup failed for %s: %s", env_var, exc)
+
+ return os.getenv(env_var, default)
diff --git a/src/backend/core/config/store/system_config.py b/src/backend/core/config/store/system_config.py
new file mode 100644
index 00000000..0f5fc1e0
--- /dev/null
+++ b/src/backend/core/config/store/system_config.py
@@ -0,0 +1,120 @@
+"""Thread-safe service that resolves service configs from the database with env fallback."""
+from __future__ import annotations
+import os
+import threading
+import time
+from typing import Optional
+from core.harness.registry.system_configuration import system_configuration_store
+import logging
+from core.config.store import system_config_constants
+
+logger = logging.getLogger(__name__)
+
+class SystemConfigService:
+ """Thread-safe singleton that resolves service configs from DB with env fallback."""
+
+ _instance: Optional["SystemConfigService"] = None
+ _lock = threading.Lock()
+
+ def __init__(self) -> None:
+ self._cache: dict[str, Optional[str]] = {}
+ self._cache_meta: dict[str, dict] = {} # full row metadata
+ self._cache_ts: float = 0.0
+ self._cache_lock = threading.Lock()
+
+ @classmethod
+ def get_instance(cls) -> "SystemConfigService":
+ if cls._instance is None:
+ with cls._lock:
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ # ── get / get_group ─────────────────────────────────────────────
+
+ def get(self, key: str, default: str | None = None) -> str | None:
+ """Get a config value. DB first, then env fallback, then default."""
+ self._maybe_refresh()
+ if key in self._cache and self._cache[key] is not None:
+ return self._cache[key]
+ # env fallback
+ env_key = system_config_constants.CONFIG_KEY_TO_ENV.get(key)
+ if env_key:
+ env_val = os.getenv(env_key)
+ if env_val is not None:
+ return env_val.strip()
+ return default
+
+ def get_group(self, group_key: str) -> dict[str, str]:
+ """Return all config key→value pairs for a group."""
+ self._maybe_refresh()
+ result: dict[str, str] = {}
+ for key, meta in self._cache_meta.items():
+ if meta.get("group_key") == group_key:
+ val = self.get(key)
+ if val is not None:
+ result[key] = val
+ return result
+
+ def get_all_configs(self) -> list[dict]:
+ """Return all config rows as dicts (for API responses)."""
+ self._maybe_refresh()
+ return list(self._cache_meta.values())
+
+ def get_group_configs(self, group_key: str) -> list[dict]:
+ """Return config rows for a specific group."""
+ self._maybe_refresh()
+ return [m for m in self._cache_meta.values() if m.get("group_key") == group_key]
+
+ # ── set ─────────────────────────────────────────────────────────
+
+ def set(self, key: str, value: str | None, updated_by: str = "admin") -> None:
+ system_configuration_store().set(key, value, updated_by)
+ self.invalidate_cache()
+
+ def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None:
+ system_configuration_store().bulk_set(items, updated_by)
+ self.invalidate_cache()
+
+ # ── cache management ──────────────────────────────────────────
+
+ def invalidate_cache(self) -> None:
+ with self._cache_lock:
+ self._cache.clear()
+ self._cache_meta.clear()
+ self._cache_ts = 0.0
+
+ def _maybe_refresh(self) -> None:
+ now = time.monotonic()
+ if now - self._cache_ts < system_config_constants.CACHE_TTL_SECONDS and self._cache_meta:
+ return
+ with self._cache_lock:
+ if now - self._cache_ts < system_config_constants.CACHE_TTL_SECONDS and self._cache_meta:
+ return
+ self._load_from_db()
+ self._cache_ts = time.monotonic()
+
+
+ def _load_from_db(self) -> None:
+ try:
+ values, metadata = system_configuration_store().load()
+ except Exception as exc:
+ logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", type(exc).__name__)
+ return
+ self._cache = values
+ self._cache_meta = metadata
+
+ # ── env overlay for MCP sub-processes ─────────────────────────
+
+ def get_service_env_overlay(self) -> dict[str, str]:
+ """Return env-var style dict for injecting into MCP sub-processes.
+
+ Maps DB configs to the env var names that MCP servers already read via os.getenv().
+ Only includes keys that have a non-None value (from DB or env fallback).
+ """
+ overlay: dict[str, str] = {}
+ for config_key, env_key in system_config_constants.CONFIG_KEY_TO_ENV.items():
+ val = self.get(config_key)
+ if val is not None:
+ overlay[env_key] = val
+ return overlay
diff --git a/src/backend/core/config/store/system_config_constants.py b/src/backend/core/config/store/system_config_constants.py
new file mode 100644
index 00000000..698d9458
--- /dev/null
+++ b/src/backend/core/config/store/system_config_constants.py
@@ -0,0 +1,302 @@
+"""Cache lifetime and defaults used by the system config service."""
+from __future__ import annotations
+from core.config.edition import CONFIG_KEY_TO_ENV as EDITION_CONFIG_KEY_TO_ENV
+from core.config.edition import SEED_CONFIGS as EDITION_SEED_CONFIGS
+
+CACHE_TTL_SECONDS = 30.0
+
+
+PROVIDER_SCOPED_KNOWLEDGE_FIELDS = ("url", "api_key", "allowed_dataset_ids")
+
+
+SEED_CONFIGS: list[tuple[str, str | None, str, str, str, bool]] = [
+ # query_database
+ (
+ "query_database.url",
+ None,
+ "数据库查询接口 URL",
+ "NL2SQL 数据库查询服务地址",
+ "query_database",
+ False,
+ ),
+ ("query_database.timeout", "40", "超时时间(秒)", "请求超时时间", "query_database", False),
+ ("query_database.retry_times", "1", "重试次数", "请求失败重试次数", "query_database", False),
+ (
+ "query_database.max_output_tokens",
+ "45000",
+ "最大输出 Token",
+ "数据库查询最大输出 Token 数",
+ "query_database",
+ False,
+ ),
+ (
+ "database_query.capability_enabled",
+ "true",
+ "数据库查询能力总开关",
+ "控制统一「数据库查询」能力是否在能力中心可用。由数据库工具页维护,不写入 catalog.json。",
+ "query_database",
+ False,
+ ),
+ (
+ "knowledge_base.detail_max_chars",
+ "50000",
+ "详情最大字符数",
+ "知识库文档详情最大字符数",
+ "knowledge_base",
+ False,
+ ),
+ # file_parser
+ ("file_parser.api_url", None, "文件解析 API URL", "PDF/文档解析服务地址", "file_parser", False),
+ ("file_parser.timeout", "60", "超时时间(秒)", "文件解析请求超时", "file_parser", False),
+ ("file_parser.lang_list", "ch", "语言列表", "OCR 语言列表", "file_parser", False),
+ ("file_parser.backend", "pipeline", "解析后端", "解析后端引擎", "file_parser", False),
+ ("file_parser.parse_method", "auto", "解析方法", "auto / ocr / txt", "file_parser", False),
+ (
+ "file_parser.formula_enable",
+ "true",
+ "启用公式识别",
+ "是否启用公式识别",
+ "file_parser",
+ False,
+ ),
+ ("file_parser.table_enable", "true", "启用表格识别", "是否启用表格识别", "file_parser", False),
+ # internet_search
+ (
+ "internet_search.engine",
+ "tavily",
+ "搜索引擎",
+ "互联网搜索引擎 (tavily / baidu / langsearch)",
+ "internet_search",
+ False,
+ ),
+ (
+ "internet_search.tavily_api_key",
+ None,
+ "Tavily API Key",
+ "Tavily 互联网搜索服务密钥",
+ "internet_search",
+ True,
+ ),
+ (
+ "internet_search.baidu_api_key",
+ None,
+ "百度搜索 API Key",
+ "百度千帆 AppBuilder 搜索服务密钥",
+ "internet_search",
+ True,
+ ),
+ (
+ "internet_search.langsearch_api_key",
+ None,
+ "LangSearch API Key",
+ "LangSearch Web Search API 服务密钥",
+ "internet_search",
+ True,
+ ),
+ # sandbox / code execution. The admin toggle is the **sole authority** (no env fallback):
+ # explicitly defaults to "true"; admins can turn it off in "System config", which writes "false" and persists.
+ (
+ "sandbox.code_capability_enable",
+ "true",
+ "主智能体代码执行能力",
+ "开启后主/计划/批量/子智能体默认具备文件工具+沙箱+我的空间访问;关闭则仅 Lab "
+ "入口可用。本开关为唯一控制源,保存后 ≤30s 生效、无需重启。",
+ "sandbox",
+ False,
+ ),
+ # ontology —— independent enterprise governance gate for plugin imports.
+ # Unlike the user's ontology preference, this switch cannot be bypassed by
+ # turning off ontology validation in personal settings.
+ (
+ "ontology.force_plugin_import_build_validation",
+ "false",
+ "强制构建时本体校验",
+ "插件导入或安装时必须通过激活 Domain Pack 的构建校验;个人本体开关不能关闭此门禁。",
+ "ontology",
+ False,
+ ),
+ # context —— the single compaction trigger ratio. Read per turn by
+ # compaction_service.resolve_trigger_ratio and applied by every phase (mid-turn at the ReAct step
+ # boundary, pre-turn before a turn samples, post-turn background pre-warm) — it is no longer an
+ # "in-turn only" knob competing with CHAT_COMPACT_TRIGGER_RATIO. Admin config takes precedence over
+ # env; takes effect on new conversations after saving, no restart needed.
+ # ⚠️ Grouped under context, not chat: the chat group is in service_configs._HIDDEN_GROUPS
+ # (chat.user_model_switch_enabled is exclusively managed by the model management panel to prevent a double entry point),
+ # so attaching to the chat group would hide the whole group in the system config UI.
+ (
+ "chat.compress_in_turn_ratio",
+ "0.90",
+ "轮内压缩触发比例",
+ "上下文占用超过「模型窗口 × 该值」时触发压缩(0.5~0.90)。计数优先用模型返回的"
+ "真实 token 用量,拿不到时才退回字节估算,两者都不含模型输出长度——输出余量已由"
+ "「窗口 95% 可用」统一预留。调大=更晚压缩(上下文更完整);调小=更早压缩。"
+ "该比例对轮内、轮前、轮末三个时机统一生效。",
+ "context",
+ False,
+ ),
+ # dingtalk —— DingTalk workbench Custom App (enterprise self-built app) credentials. Once AppKey/AppSecret are filled,
+ # DingTalk login and the dws command inside the sandbox execute as that self-built app, bypassing DingTalk's co-creation-era
+ # "CLI data access" org toggle and self-controlling scope; leave empty to use the dws default shared app (org admin must
+ # enable CLI data access first). Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart.
+ (
+ "dingtalk.client_id",
+ None,
+ "钉钉 AppKey (client-id)",
+ "企业自建钉钉应用的 AppKey。在 open-dev.dingtalk.com 创建企业内部应用后获取。",
+ "dingtalk",
+ True,
+ ),
+ (
+ "dingtalk.client_secret",
+ None,
+ "钉钉 AppSecret (client-secret)",
+ "企业自建钉钉应用的 AppSecret,与 AppKey 配对。",
+ "dingtalk",
+ True,
+ ),
+ (
+ "dingtalk.trusted_domains",
+ "*.dingtalk.com",
+ "受信域名白名单",
+ "dws bearer token 允许发送的域名,逗号分隔,默认锁死 *.dingtalk.com 防 token 外泄。",
+ "dingtalk",
+ False,
+ ),
+ # firecrawl —— deployment-level credentials for the firecrawl web scraping/retrieval CLI. The firecrawl CLI is preinstalled
+ # in the sandbox; at runtime these two items are injected as FIRECRAWL_API_KEY / FIRECRAWL_API_URL env vars into
+ # every user's sandbox commands (one shared config for all users, mirroring the DingTalk dws Custom App approach).
+ # - Cloud version: fill only api_key (starts with fc-).
+ # - Self-hosted: fill api_url pointing to the self-built instance (the CLI skips cloud auth once it sees api_url); if the
+ # self-built instance also needs a key, fill both.
+ # Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart.
+ (
+ "firecrawl.api_key",
+ None,
+ "Firecrawl API Key",
+ "firecrawl 云版 API Key(fc- 开头),在 firecrawl.dev 注册获取。用自托管实例时可留空。",
+ "firecrawl",
+ True,
+ ),
+ (
+ "firecrawl.api_url",
+ None,
+ "Firecrawl 自托管地址",
+ "自托管 firecrawl 实例地址(如 http://firecrawl:3002)。填了即走自建实例、CLI 跳过云端鉴权;用云版时留空。",
+ "firecrawl",
+ False,
+ ),
+ # lark —— Feishu workbench app (enterprise self-built app) credentials. Once AppID/AppSecret are filled, the lark-cli inside
+ # the sandbox uses that self-built app as its OAuth app: admin configures once, and each user completes the device-flow
+ # auth login to operate Feishu as themselves. The app config is seeded by the backend into each user's lark HOME
+ # (config init), not injected as a global env var — this avoids lark-cli defaulting to the tenant/bot token flow when
+ # LARK_APP_* is in the environment, which would break --as user. Sole authority = admin config, ≤30s after saving, no restart.
+ (
+ "lark.app_id",
+ None,
+ "飞书 App ID",
+ "企业自建飞书应用的 App ID。在 open.feishu.cn 创建企业自建应用后获取。",
+ "lark",
+ True,
+ ),
+ (
+ "lark.app_secret",
+ None,
+ "飞书 App Secret",
+ "企业自建飞书应用的 App Secret,与 App ID 配对。",
+ "lark",
+ True,
+ ),
+ # auth —— login and session. Number of days a session is kept after the user checks "remember login state" (checked by
+ # default) on the login page. Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart;
+ # if unchecked, the default session duration (SESSION_TTL_HOURS) is used.
+ (
+ "auth.remember_me_days",
+ "30",
+ "记住登录状态时长(天)",
+ "用户在登录页勾选「记住登录状态」后,登录会话保持的天数(默认 30 天)。"
+ "未勾选则使用默认会话时长。",
+ "auth",
+ False,
+ ),
+ # turbo —— 极速模式(检索速查)。工具集独立于能力目录(catalog)的启停状态:
+ # 极速模式装配的 MCP 以本组配置为唯一来源,管理台改动 ≤30s 生效、无需重启。
+ (
+ "turbo.mcp_server_ids",
+ "retrieve_dataset_content,internet_search,web_fetch",
+ "极速模式可用工具",
+ "极速模式下装配的 MCP 工具集合(逗号分隔的 server id)。"
+ "该集合不受「能力目录」中 MCP/技能/智能体/插件启停的影响,是极速模式工具面的唯一来源。",
+ "turbo",
+ False,
+ ),
+ (
+ "turbo.skill_ids",
+ "",
+ "极速模式可用技能",
+ "极速模式下装配的技能集合(逗号分隔的 skill id)。"
+ "极速模式没有 Bash / 沙箱 / 文件写入,选纯文本类技能(写作模板、话术规范等)效果最好;"
+ "需要代码执行的技能只会被读到说明、无法执行。同样不受「能力目录」启停影响。",
+ "turbo",
+ False,
+ ),
+ (
+ "turbo.plugin_ids",
+ "",
+ "极速模式可用插件",
+ "极速模式下装配的插件集合(逗号分隔的 install_id)。"
+ "插件是「技能 + 工具」的能力包,选中后其全部工具与技能一并在极速模式生效;"
+ "与上面的 MCP 工具集合合并去重,同样不受「能力目录」启停影响。",
+ "turbo",
+ False,
+ ),
+ (
+ "turbo.manual_invoke_enable",
+ "true",
+ "允许手动呼唤能力",
+ "开启后,用户在极速模式对话框中通过斜杠命令呼唤技能/插件、@ 呼唤子智能体时,"
+ "被呼唤的能力将临时装配到本次请求(仅显式呼唤时可用,不会把全量能力清单注入提示词)。"
+ "关闭后极速模式严格只用上面的工具集合。",
+ "turbo",
+ False,
+ ),
+ (
+ "turbo.max_iters",
+ "4",
+ "迭代轮数上限",
+ "极速模式单次回答的最大 ReAct 迭代轮数(含最终作答轮)。"
+ "上限越小响应越快;默认 4 对应「最多 1-2 轮并行检索 + 直接作答」的产品契约。",
+ "turbo",
+ False,
+ ),
+] + EDITION_SEED_CONFIGS
+
+
+CONFIG_KEY_TO_ENV: dict[str, str] = {
+ "query_database.url": "QUERY_DATABASE_URL",
+ "query_database.timeout": "QUERY_DATABASE_TIMEOUT_SECONDS",
+ "query_database.retry_times": "QUERY_DATABASE_RETRY_TIMES",
+ "query_database.max_output_tokens": "QUERY_DATABASE_MAX_OUTPUT_TOKENS",
+ "knowledge_base.detail_max_chars": "KB_DETAIL_CONTENT_MAX_CHARS",
+ "file_parser.api_url": "FILE_PARSER_API_URL",
+ "file_parser.timeout": "FILE_PARSER_TIMEOUT",
+ "file_parser.lang_list": "FILE_PARSER_LANG_LIST",
+ "file_parser.backend": "FILE_PARSER_BACKEND",
+ "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD",
+ "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE",
+ "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE",
+ "internet_search.engine": "INTERNET_SEARCH_ENGINE",
+ "internet_search.tavily_api_key": "TAVILY_API_KEY",
+ "internet_search.baidu_api_key": "BAIDU_API_KEY",
+ "internet_search.langsearch_api_key": "LANGSEARCH_API_KEY",
+ # Note: sandbox.code_capability_enable deliberately does **not** map to an env var —— the admin toggle is the sole
+ # authority (plan B), explicitly seeded to "true", and env CODE_CAPABILITY_ENABLED is retired.
+ **EDITION_CONFIG_KEY_TO_ENV,
+}
+
+
+ENV_TO_CONFIG_KEY: dict[str, str] = {v: k for k, v in CONFIG_KEY_TO_ENV.items()}
+
+
+DEFAULT_TURBO_MCP_SERVER_IDS = frozenset(
+ {"retrieve_dataset_content", "internet_search", "web_fetch"}
+)
diff --git a/src/backend/core/config/user_intros.py b/src/backend/core/config/user_intros.py
index 295dfccf..54e1bb21 100644
--- a/src/backend/core/config/user_intros.py
+++ b/src/backend/core/config/user_intros.py
@@ -20,7 +20,7 @@
from typing import Dict
-from core.config.edition_display_names import edition_mcp_user_intros
+from core.config.edition import edition_mcp_user_intros
# ── Skills ────────────────────────────────────────────────────────────
diff --git a/src/backend/core/content/content_blocks.py b/src/backend/core/content/content_blocks.py
deleted file mode 100644
index 273ff82e..00000000
--- a/src/backend/core/content/content_blocks.py
+++ /dev/null
@@ -1,701 +0,0 @@
-"""Helpers for exporting and importing editable docs content blocks."""
-
-from __future__ import annotations
-
-import re
-from datetime import datetime, timezone
-from typing import Any, Mapping
-
-from core.config.settings import settings
-from core.db.models import ContentBlock
-from sqlalchemy.orm import Session
-
-# Branding source of truth (seam C7): in-code defaults stay neutral; the concrete
-# brand (e.g. "HugAgentOS") is injected via env BRAND_PRODUCT_NAME / the deploy-time
-# content_blocks DB seed.
-_BRAND_NAME = settings.branding.product_name
-
-DOCS_BLOCK_MAP = {
- "updates": "docs_updates",
- "capabilities": "docs_capabilities",
- "prompt_hub": "prompt_hub",
- "page_config": "page_config",
- "app_config": "app_config",
-}
-
-# Aliases retired from DOCS_BLOCK_MAP; ignored (not rejected) when importing an
-# older snapshot that still carries them.
-RETIRED_BLOCK_ALIASES = {"homepage_shortcuts"}
-
-# Blocks whose payload is a dict instead of a list (default is list).
-DICT_PAYLOAD_BLOCKS = {"page_config", "prompt_versions", "app_config"}
-
-# Prompt versions block — managed separately from DOCS_BLOCK_MAP so a routine
-# docs export/import never silently clobbers the system prompt pool. It rides
-# its own opt-in snapshot (build_prompt_snapshot / import_prompt_snapshot).
-PROMPT_VERSIONS_BLOCK_ID = "prompt_versions"
-
-# Opt-in migration set for prompt content: the system prompt version pool
-# (prompt_versions) plus the prompt hub (prompt_hub). Same ContentBlock storage
-# and snapshot format as docs blocks, but a separate map keeps the two apart so
-# a prompts migration never drags page_config / branding fields along with it.
-# Note: prompt_hub also appears in DOCS_BLOCK_MAP — that is intentional and
-# harmless (a docs snapshot and a prompts snapshot are imported independently).
-PROMPT_BLOCK_MAP = {
- "prompt_versions": PROMPT_VERSIONS_BLOCK_ID,
- "prompt_hub": "prompt_hub",
-}
-
-SNAPSHOT_SCHEMA_VERSION = 1
-
-
-class ContentSnapshotError(ValueError):
- """Raised when a docs content snapshot is invalid."""
-
-
-DEFAULT_PAGE_CONFIG: dict[str, Any] = {
- "branding": {
- "product_name": _BRAND_NAME,
- "product_subtitle": "AI 智能助手",
- "logo_url": "/home/header.svg",
- "favicon_url": "/icon.png",
- "page_title": _BRAND_NAME,
- "hero_title": f"你好,我是 {_BRAND_NAME}",
- "hero_subtitle": "今天想从哪里开始?",
- "disclaimer": "本平台生成内容由AI大模型生成,不构成任何建议;涉及业务决策请以权威信息为准。",
- },
- "homepage": {
- # The homepage uses the transparent standalone brand mark rather than
- # the square sidebar logo, mirroring the lighter Qwen Office hero.
- "show_logo": True,
- "logo_url": "/icon.png",
- "show_suggestions": True,
- "suggested_questions": [
- "分析新能源汽车产业链的关键环节与代表企业",
- "对比两份产业政策,提炼支持方向和申报条件",
- "查询一家企业的基本情况、经营风险和产业链位置",
- "从这份材料中提炼核心观点和待办事项",
- "分析这组经营数据的趋势、异常和可能原因",
- "检索相关政策依据,并给出可核验的来源",
- ],
- },
- "navigation": {
- "panel_titles": {
- "ability_center": "能力中心",
- "skills": "技能",
- "agents": "智能体",
- "mcp": "连接器",
- "kb": "知识库",
- "docs": "更新记录",
- "app_center": "应用中心",
- "automation": "定时任务",
- "sites": "站点",
- "projects": "项目",
- "lab": "实验室",
- "settings": "系统设置",
- "my_space": "我的空间",
- },
- "panel_subtitles": {
- "ability_center": "智能体基础能力管理,包含智能体、技能、连接器与插件",
- "skills": "启用/停用技能,并查看详细介绍、输入输出与示例。",
- "agents": "选择与启用智能体,并查看其职责边界与路由提示。",
- "mcp": "管理连接器服务,并查看其作用范围与可靠性影响。",
- "kb": "浏览知识库、查看文档列表,并支持文档内检索。",
- "docs": "查看功能更新、能力中心与平台说明。",
- "app_center": "基于 AI 能力的场景化智能应用",
- "automation": "按计划自动执行任务,也可随时手动触发。在任意对话中描述你想定期做的事,即可快速创建",
- "sites": "在对话里描述需求,AI 生成完整网站并一键发布,由平台托管、凭链接即可访问",
- "projects": "把对话、文件和指令打包成专属工作空间",
- "lab": "AI 能力实验性应用",
- "settings": "",
- "my_space": "",
- },
- "admin_header": {
- "title": f"{_BRAND_NAME} — 后台管理",
- "subtitle": "后台管理",
- },
- # Unified branding + per-page labels for the admin platform (content
- # management / system config / API docs) — product_name drives the header
- # brand and browser tab title of all three admin pages, replacing the old
- # admin_header.
- "admin_platform": {
- "product_name": _BRAND_NAME,
- "content_label": "内容管理",
- "config_label": "系统配置",
- "apidoc_label": "接口文档",
- },
- # 与前端 utils/pageConfigDefaults.ts 的 DEFAULT_SIDEBAR_ITEMS / DEFAULT_MENU_ITEMS 保持一致。
- # 这只是新装环境的出厂值——已存在的部署以 DB 现有取值为准,由 backfill_navigation_entries
- # 补齐新增条目,管理员随后可在 /config「页面配置」里自由编排。
- "sidebar_items": ["ability_center", "automation", "sites", "my_space"],
- "menu_items": ["settings", "app_center", "projects", "lab"],
- },
- "texts": {
- "input_placeholder": "请输入您的问题…",
- "input_placeholder_agent": "请输入您的问题…",
- "search_placeholder": "搜索对话",
- "btn_new_chat": "新建对话",
- "btn_logout": "退出",
- "history_label": "历史对话",
- "sidebar_empty_state": "暂无对话记录",
- "search_no_results": "无匹配结果",
- "dialog_logout_confirm_title": "确认退出登录?",
- "dialog_logout_confirm_content": "退出登录不会丢失任何数据,重新登录后可继续使用。",
- "dialog_logout_confirm_ok": "退出登录",
- "recommend_banner_text": "",
- },
- "defaults": {
- # Initial mode on every user login / new chat:
- # - chat_mode: 'turbo' / 'fast' / 'medium' / 'high' / 'max' (recommended)
- # - thinking_mode: bool (legacy field, fallback compat; true→medium, false→fast)
- "chat_mode": "fast",
- "thinking_mode": False,
- },
- "auth": {
- # Whether the login page shows the "register" entry (tab + register subpage).
- # When off, the login page keeps only login, and backend register submissions
- # are rejected outright (not just hidden on the frontend).
- "allow_register": True,
- },
-}
-
-
-# No external sub-apps are built in by default — the app config is entirely managed
-# by admins via the "app config" panel. (Historically two presets, "enterprise
-# profile / enterprise survey", were seeded here, which caused deleted apps to be
-# re-injected on refresh.)
-DEFAULT_APP_CONFIG: dict[str, Any] = {
- "apps": [],
-}
-
-
-def normalize_app_config(payload: Any) -> dict[str, Any]:
- """Normalize app_config payload to the canonical `{"apps": [...]}` shape.
-
- An empty list (admin deleted all apps) is respected as "no apps" and no longer
- falls back to any default entries.
- """
- if not isinstance(payload, dict):
- return {"apps": list(DEFAULT_APP_CONFIG["apps"])}
-
- apps_raw = payload.get("apps")
- if isinstance(apps_raw, list):
- apps: list[dict[str, Any]] = []
- for item in apps_raw:
- if not isinstance(item, dict):
- continue
- app_id = str(item.get("id") or "").strip()
- if not app_id:
- continue
- apps.append(
- {
- "id": app_id,
- "enabled": bool(item.get("enabled", True)),
- "name": str(item.get("name") or app_id),
- "description": str(item.get("description") or ""),
- "url": str(item.get("url") or ""),
- "icon": str(item.get("icon") or ""),
- }
- )
- return {"apps": apps}
-
- return {"apps": list(DEFAULT_APP_CONFIG["apps"])}
-
-
-# 空的初始负载:哪些 kind 存在、默认版本叫什么,由
-# prompt_version_service.KIND_SPECS 单独持有并在 seed 时写入,这里不再抄一份清单。
-DEFAULT_PROMPT_VERSIONS: dict[str, Any] = {
- "active": {},
- "versions": [],
-}
-
-
-def _default_payload(alias: str) -> Any:
- if alias == "page_config":
- return DEFAULT_PAGE_CONFIG
- if alias == "app_config":
- return DEFAULT_APP_CONFIG
- if alias == "prompt_versions":
- return DEFAULT_PROMPT_VERSIONS
- return {} if alias in DICT_PAYLOAD_BLOCKS else []
-
-
-def seed_page_config_if_missing(db: Session) -> bool:
- """Insert default page_config row if it does not yet exist. Returns True if inserted."""
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- if row:
- return False
- db.add(
- ContentBlock(
- id="page_config",
- payload=DEFAULT_PAGE_CONFIG,
- updated_at=datetime.now(timezone.utc),
- updated_by="system_seed",
- )
- )
- db.commit()
- return True
-
-
-def enforce_ce_branding(db: Session) -> bool:
- """Keep the fixed CE product identity and auth policy consistent.
-
- ``page_config`` survives image upgrades, so merely changing the code/env
- defaults does not repair an already-running CE instance. CE does not expose
- the content-management console and therefore has no supported per-instance
- branding override or self-service registration; normalize both the small
- set of branding fields and the registration switch on startup. EE
- deployments are never touched.
- """
- if settings.edition.edition != "ce":
- return False
-
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- if row is None or not isinstance(row.payload, dict):
- return False
-
- product_name = "HugAgentOS"
- payload = dict(row.payload)
- branding = dict(payload.get("branding") or {})
- navigation = dict(payload.get("navigation") or {})
- auth = dict(payload.get("auth") or {})
- admin_header = dict(navigation.get("admin_header") or {})
- admin_platform = dict(navigation.get("admin_platform") or {})
-
- desired_branding = {
- "product_name": product_name,
- "product_subtitle": "AI 智能助手",
- "page_title": product_name,
- "hero_title": f"你好,我是 {product_name}",
- }
- changed = any(branding.get(key) != value for key, value in desired_branding.items())
- changed = changed or admin_header.get("title") != f"{product_name} — 后台管理"
- changed = changed or admin_platform.get("product_name") != product_name
- changed = changed or auth.get("allow_register") is not False
- if not changed:
- return False
-
- branding.update(desired_branding)
- admin_header["title"] = f"{product_name} — 后台管理"
- admin_platform["product_name"] = product_name
- navigation["admin_header"] = admin_header
- navigation["admin_platform"] = admin_platform
- auth["allow_register"] = False
- payload["branding"] = branding
- payload["navigation"] = navigation
- payload["auth"] = auth
- row.payload = payload
- row.updated_at = datetime.now(timezone.utc)
- row.updated_by = "system_seed"
- from sqlalchemy.orm.attributes import flag_modified
-
- flag_modified(row, "payload")
- db.commit()
- return True
-
-
-# ── Idempotent backfill for navigation entries added after initial deploy ─
-# Lesson learned: when the projects MVP shipped, only the frontend defaults and the
-# backend DEFAULT_PAGE_CONFIG were changed, but **already existing** page_config rows
-# do not upgrade automatically — the frontend's "use DB if present" override
-# semantics pinned the old sidebar_items in place. This repeated for every new
-# entry. The idempotent seeder is the safety net: on startup, backfill missing nav
-# entries from a whitelist.
-#
-# Design principles:
-# - Explicit whitelist; do **not** deep-merge the full defaults — avoids
-# "resurrecting" entries operators deleted on purpose (e.g. someone deliberately
-# turned off app_center) and avoids overwriting branding fields in reverse.
-# - Each entry carries an insert_after anchor; if the anchor is absent, degrade to
-# appending at the end.
-# - ``bucket`` picks which list the entry is seeded into ("sidebar" / "menu").
-# - **Presence is checked across BOTH lists.** An operator who moved an entry from the
-# sidebar into the user menu (or vice versa) has not "lost" it — re-inserting it into
-# the bucket we happen to prefer would silently undo their placement on every restart
-# and leave the entry duplicated in two buckets. Only a genuinely absent key is seeded.
-# (Deliberately hidden entries live in neither list, so they *do* come back — that is
-# the documented trade-off of this safety net; use the admin UI to hide them again.)
-# - Titles/subtitles are **not** repeated here — they are read from DEFAULT_PAGE_CONFIG above,
-# so panel copy has exactly one home and a fresh install can never disagree with a
-# backfilled one.
-# - On failure, quietly return False; never block startup.
-_NAV_BACKFILL_ENTRIES: list[dict[str, Any]] = [
- {"key": "projects", "bucket": "menu", "insert_after": "app_center"},
- {"key": "automation", "bucket": "sidebar", "insert_after": "ability_center"},
- {"key": "sites", "bucket": "sidebar", "insert_after": "automation"},
-]
-
-
-# 能力中心统一表述(技能库→技能 / 子智能体→智能体 / MCP工具库→连接器 / 插件库→插件)之前
-# 建起来的部署,DB 里存的还是旧出厂文案。这里做一次性改写:**仅当**存量取值与旧出厂默认
-# 完全一致时才改,管理员自己改过的标题原样保留。
-_LEGACY_PANEL_TEXT_RENAMES: dict[str, dict[str, tuple[tuple[str, ...], str]]] = {
- "panel_titles": {
- "skills": (("技能库",), "技能"),
- "agents": (("子智能体",), "智能体"),
- "plugins": (("插件库",), "插件"),
- "mcp": (("MCP工具库", "MCP 工具库", "MCP 工具"), "连接器"),
- },
- "panel_subtitles": {
- "agents": (
- ("选择与启用子智能体,并查看其职责边界与路由提示。",),
- "选择与启用智能体,并查看其职责边界与路由提示。",
- ),
- "mcp": (
- (
- "管理 MCP 工具服务,并查看其作用范围与可靠性影响。",
- "管理 MCP 连接器服务,并查看其作用范围与可靠性影响。",
- ),
- "管理连接器服务,并查看其作用范围与可靠性影响。",
- ),
- },
-}
-
-
-def backfill_navigation_entries(db: Session) -> int:
- """Patch existing page_config row to add navigation entries listed in
- ``_NAV_BACKFILL_ENTRIES`` if they are missing.
-
- Returns the number of *individual fields* mutated (across all entries
- and all three locations: sidebar_items / panel_titles / panel_subtitles).
- Zero means nothing changed (idempotent steady state).
- """
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- if not row:
- return 0 # seed_page_config_if_missing handles fresh installs
- payload = row.payload if isinstance(row.payload, dict) else {}
- nav = payload.get("navigation")
- if not isinstance(nav, dict):
- return 0 # malformed payload — leave it alone, admin UI will surface
-
- default_nav = DEFAULT_PAGE_CONFIG["navigation"]
-
- changed = 0
- for entry in _NAV_BACKFILL_ENTRIES:
- key = entry["key"]
- bucket_field = "menu_items" if entry["bucket"] == "menu" else "sidebar_items"
- items = nav.get(bucket_field)
- # Present in *either* list counts as placed: the operator may have moved the entry
- # between buckets on purpose, and re-seeding it would duplicate it across both.
- already_placed = any(
- key in value
- for field in ("sidebar_items", "menu_items")
- if isinstance(value := nav.get(field), list)
- )
- if isinstance(items, list) and not already_placed:
- anchor = entry.get("insert_after")
- new_items = list(items)
- if anchor and anchor in new_items:
- new_items.insert(new_items.index(anchor) + 1, key)
- else:
- new_items.append(key)
- nav[bucket_field] = new_items
- changed += 1
-
- # Copy comes from DEFAULT_PAGE_CONFIG so it is never re-typed here.
- for field in ("panel_titles", "panel_subtitles"):
- target = nav.get(field)
- text = default_nav[field].get(key)
- if isinstance(target, dict) and key not in target and text:
- target[key] = text
- changed += 1
-
- # 旧出厂文案改名(能力中心统一表述),与上面的补齐共用一次提交
- for field, renames in _LEGACY_PANEL_TEXT_RENAMES.items():
- target = nav.get(field)
- if not isinstance(target, dict):
- continue
- for key, (legacy_values, new_text) in renames.items():
- if target.get(key) in legacy_values:
- target[key] = new_text
- changed += 1
-
- if changed:
- payload["navigation"] = nav
- row.payload = payload
- row.updated_at = datetime.now(timezone.utc)
- # Mark updated_by so audit can distinguish automatic backfill vs manual edits
- row.updated_by = "system_seed"
- # SQLAlchemy JSONB does not auto-detect mutation — flag explicitly
- from sqlalchemy.orm.attributes import flag_modified
-
- flag_modified(row, "payload")
- db.commit()
- return changed
-
-
-def get_branding_info() -> dict[str, str]:
- """Read the branding fields from page_config (product_name + logo_url), falling back to defaults on failure.
-
- Reused by server-side rendering paths (e.g. the SSO login page); manages its
- own DB session internally.
- """
- defaults = {
- "product_name": DEFAULT_PAGE_CONFIG["branding"]["product_name"],
- "logo_url": DEFAULT_PAGE_CONFIG["branding"]["logo_url"],
- }
- try:
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- payload = (row.payload if row else {}) or {}
- branding = payload.get("branding", {}) if isinstance(payload, dict) else {}
- return {
- "product_name": branding.get("product_name") or defaults["product_name"],
- "logo_url": branding.get("logo_url") or defaults["logo_url"],
- }
- finally:
- db.close()
- except Exception:
- return defaults
-
-
-def is_register_allowed() -> bool:
- """Return whether self-service local-account registration is enabled.
-
- CE is permanently single-account and therefore always returns ``False``.
- Other editions read ``page_config.auth.allow_register`` and keep the legacy
- default of ``True`` when the setting is missing or cannot be read. The value
- gates both the server-rendered UI and registration submissions.
- """
- if settings.edition.edition == "ce":
- return False
-
- try:
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- payload = (row.payload if row else {}) or {}
- auth = payload.get("auth", {}) if isinstance(payload, dict) else {}
- val = auth.get("allow_register") if isinstance(auth, dict) else None
- return True if val is None else bool(val)
- finally:
- db.close()
- except Exception:
- return True
-
-
-def get_admin_platform_info() -> dict[str, str]:
- """Read page_config.navigation.admin_platform (admin platform brand + per-page labels).
-
- When admin_platform is missing, derive the brand from the legacy
- admin_header.title (shaped like " — 后台管理") as the product_name
- fallback, then fall back to code defaults — consistent with the frontend
- mergeAdminPlatform normalization. Reused by server-side rendering paths
- (Swagger / ReDoc doc page titles etc.); manages its own DB session internally.
- """
- ap_def = DEFAULT_PAGE_CONFIG["navigation"]["admin_platform"]
- defaults = {
- "product_name": ap_def["product_name"],
- "content_label": ap_def["content_label"],
- "config_label": ap_def["config_label"],
- "apidoc_label": ap_def["apidoc_label"],
- }
- try:
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
- payload = (row.payload if row else {}) or {}
- nav = payload.get("navigation", {}) if isinstance(payload, dict) else {}
- ap = nav.get("admin_platform") if isinstance(nav, dict) else None
- ap = ap if isinstance(ap, dict) else {}
- header = nav.get("admin_header") if isinstance(nav, dict) else None
- derived = ""
- if isinstance(header, dict) and isinstance(header.get("title"), str):
- derived = re.split(r"\s+[—-]\s+", header["title"])[0].strip()
-
- def pick(key: str, fallback: str) -> str:
- v = ap.get(key)
- return v.strip() if isinstance(v, str) and v.strip() else fallback
-
- return {
- "product_name": pick("product_name", derived or defaults["product_name"]),
- "content_label": pick("content_label", defaults["content_label"]),
- "config_label": pick("config_label", defaults["config_label"]),
- "apidoc_label": pick("apidoc_label", defaults["apidoc_label"]),
- }
- finally:
- db.close()
- except Exception:
- return defaults
-
-
-def _serialize_block(row: ContentBlock | None, alias: str | None = None) -> dict[str, Any]:
- return {
- "payload": row.payload if row else _default_payload(alias or ""),
- "updated_at": row.updated_at.isoformat() if row and row.updated_at else None,
- "updated_by": row.updated_by if row else None,
- }
-
-
-def build_docs_snapshot(
- db: Session,
- block_map: Mapping[str, str] = DOCS_BLOCK_MAP,
-) -> dict[str, Any]:
- """Build a portable JSON snapshot for content blocks.
-
- Defaults to the docs blocks; pass ``block_map=PROMPT_BLOCK_MAP`` (or use
- ``build_prompt_snapshot``) to snapshot the system prompt pool instead.
- """
- rows = db.query(ContentBlock).filter(ContentBlock.id.in_(list(block_map.values()))).all()
- row_map = {row.id: row for row in rows}
-
- return {
- "schema_version": SNAPSHOT_SCHEMA_VERSION,
- "exported_at": datetime.now(timezone.utc).isoformat(),
- "blocks": {
- alias: _serialize_block(row_map.get(db_id), alias) for alias, db_id in block_map.items()
- },
- }
-
-
-def _parse_snapshot_datetime(value: Any) -> datetime | None:
- if value in (None, ""):
- return None
- if not isinstance(value, str):
- raise ContentSnapshotError("updated_at must be an ISO datetime string")
- try:
- return datetime.fromisoformat(value.replace("Z", "+00:00"))
- except ValueError as exc:
- raise ContentSnapshotError(f"Invalid datetime: {value}") from exc
-
-
-def normalize_docs_snapshot(
- snapshot: Mapping[str, Any],
- block_map: Mapping[str, str] = DOCS_BLOCK_MAP,
-) -> dict[str, Any]:
- """Validate and normalize an incoming content snapshot."""
- if not isinstance(snapshot, Mapping):
- raise ContentSnapshotError("Snapshot body must be a JSON object")
-
- schema_version = snapshot.get("schema_version", SNAPSHOT_SCHEMA_VERSION)
- if schema_version != SNAPSHOT_SCHEMA_VERSION:
- raise ContentSnapshotError(
- f"Unsupported schema_version: {schema_version}. Expected {SNAPSHOT_SCHEMA_VERSION}"
- )
-
- raw_blocks = snapshot.get("blocks")
- if not isinstance(raw_blocks, Mapping):
- raise ContentSnapshotError("Snapshot.blocks must be an object")
-
- unknown_blocks = set(raw_blocks.keys()) - set(block_map.keys()) - RETIRED_BLOCK_ALIASES
- if unknown_blocks:
- raise ContentSnapshotError(
- f"Unknown blocks in snapshot: {', '.join(sorted(unknown_blocks))}"
- )
-
- normalized_blocks: dict[str, dict[str, Any]] = {}
- for alias in block_map:
- raw_block = raw_blocks.get(alias)
- if raw_block is None:
- continue
- if not isinstance(raw_block, Mapping):
- raise ContentSnapshotError(f"Snapshot block '{alias}' must be an object")
-
- payload = raw_block.get("payload", _default_payload(alias))
- expected_type = dict if alias in DICT_PAYLOAD_BLOCKS else list
- if not isinstance(payload, expected_type):
- kind = "object" if expected_type is dict else "list"
- raise ContentSnapshotError(f"Snapshot block '{alias}'.payload must be a {kind}")
-
- normalized_blocks[alias] = {
- "payload": payload,
- "updated_at": _parse_snapshot_datetime(raw_block.get("updated_at")),
- "updated_by": raw_block.get("updated_by"),
- }
-
- if not normalized_blocks:
- raise ContentSnapshotError("Snapshot does not contain any importable content blocks")
-
- exported_at = snapshot.get("exported_at")
- if exported_at not in (None, "") and not isinstance(exported_at, str):
- raise ContentSnapshotError("Snapshot.exported_at must be a string")
-
- return {
- "schema_version": SNAPSHOT_SCHEMA_VERSION,
- "exported_at": exported_at,
- "blocks": normalized_blocks,
- }
-
-
-def import_docs_snapshot(
- db: Session,
- snapshot: Mapping[str, Any],
- *,
- overwrite: bool = True,
- default_updated_by: str | None = None,
- block_map: Mapping[str, str] = DOCS_BLOCK_MAP,
-) -> dict[str, Any]:
- """Import a content snapshot into the current database."""
- normalized = normalize_docs_snapshot(snapshot, block_map=block_map)
- imported: list[str] = []
- skipped: list[str] = []
-
- for alias, block in normalized["blocks"].items():
- db_id = block_map[alias]
- row = db.query(ContentBlock).filter(ContentBlock.id == db_id).first()
- if row and not overwrite:
- skipped.append(alias)
- continue
-
- updated_at = block["updated_at"] or datetime.now(timezone.utc)
- updated_by = block["updated_by"] or default_updated_by
-
- if row:
- row.payload = block["payload"]
- row.updated_at = updated_at
- row.updated_by = updated_by
- else:
- row = ContentBlock(
- id=db_id,
- payload=block["payload"],
- updated_at=updated_at,
- updated_by=updated_by,
- )
- db.add(row)
- imported.append(alias)
-
- db.commit()
-
- return {
- "schema_version": normalized["schema_version"],
- "imported": imported,
- "skipped": skipped,
- "count": len(imported),
- "overwrite": overwrite,
- }
-
-
-# ── System prompt pool snapshot (opt-in migration) ──────────────────────────
-
-
-def build_prompt_snapshot(db: Session) -> dict[str, Any]:
- """Build a portable snapshot of the system prompt pool (prompt_versions).
-
- Same format as the docs snapshot, but scoped to PROMPT_BLOCK_MAP so it can
- be migrated across environments independently of docs/page config.
- """
- return build_docs_snapshot(db, block_map=PROMPT_BLOCK_MAP)
-
-
-def import_prompt_snapshot(
- db: Session,
- snapshot: Mapping[str, Any],
- *,
- overwrite: bool = True,
- default_updated_by: str | None = None,
-) -> dict[str, Any]:
- """Import a system prompt pool snapshot (counterpart of build_prompt_snapshot)."""
- return import_docs_snapshot(
- db,
- snapshot,
- overwrite=overwrite,
- default_updated_by=default_updated_by,
- block_map=PROMPT_BLOCK_MAP,
- )
diff --git a/src/backend/core/content/file_parser.py b/src/backend/core/content/file_parser.py
deleted file mode 100644
index 1e2ebb16..00000000
--- a/src/backend/core/content/file_parser.py
+++ /dev/null
@@ -1,752 +0,0 @@
-"""File parsing utilities for chat attachments.
-
-Supported formats:
- - PDF : calls external file-parser API service
- - DOCX : pandoc (docx → markdown)
- - DOC / WPS : LibreOffice headless → DOCX, then pandoc
- - TXT : UTF-8 / GBK direct decode
- - XLSX : openpyxl → markdown tables
- - XLS : xlrd → markdown tables (fallback: LibreOffice → XLSX → openpyxl)
- - CSV : csv module → markdown table
-
-All public functions return a markdown/plain-text string, or raise RuntimeError
-with a human-readable Chinese error message on failure.
-"""
-
-from __future__ import annotations
-
-import base64
-import binascii
-import csv
-import io
-import json
-import logging
-import os
-import subprocess
-import tempfile
-from dataclasses import dataclass, field
-from typing import Optional
-
-import requests
-
-from core.content.office import find_libreoffice_binary, find_pandoc_binary
-from core.infra.proc import no_window_kwargs
-
-logger = logging.getLogger(__name__)
-
-
-# ── config helpers (DB-first, env fallback) ──────────────────────────────────
-
-def _svc_get(key: str, default: str = "") -> str:
- """Read from SystemConfigService (DB-first), fall back to env."""
- try:
- from core.services.system_config import SystemConfigService
- val = SystemConfigService.get_instance().get(key)
- if val is not None:
- return val.strip()
- except Exception:
- pass
- return (os.getenv(_ENV_MAP.get(key, ""), default) or default).strip()
-
-# config_key → env var fallback mapping
-_ENV_MAP = {
- "file_parser.api_url": "FILE_PARSER_API_URL",
- "file_parser.timeout": "FILE_PARSER_TIMEOUT",
- "file_parser.lang_list": "FILE_PARSER_LANG_LIST",
- "file_parser.backend": "FILE_PARSER_BACKEND",
- "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD",
- "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE",
- "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE",
-}
-
-
-def _cfg_api_url() -> str:
- return _svc_get("file_parser.api_url")
-
-
-def _cfg_timeout() -> int:
- try:
- return int(_svc_get("file_parser.timeout", "60"))
- except ValueError:
- return 60
-
-
-def _cfg_parse_params() -> dict:
- return {
- "lang_list": _svc_get("file_parser.lang_list", "ch"),
- "backend": _svc_get("file_parser.backend", "pipeline"),
- "parse_method": _svc_get("file_parser.parse_method", "auto"),
- "formula_enable": _svc_get("file_parser.formula_enable", "true"),
- "table_enable": _svc_get("file_parser.table_enable", "true"),
- }
-
-
-# ── PDF ───────────────────────────────────────────────────────────────────────
-
-
-@dataclass
-class RichParseResult:
- """Layout-aware parse output: markdown plus the media the layout engine cut out.
-
- ``markdown`` is byte-for-byte what :func:`parse_pdf` returns, so the text path is
- unaffected. ``images`` maps the path used inside the markdown (``images/.jpg``
- in a ```` link) to the decoded bytes, and ``blocks`` is the parsed
- ``content_list`` — one entry per layout block, images carrying ``image_caption`` /
- ``bbox`` / ``page_idx``.
- """
-
- markdown: str
- images: dict[str, bytes] = field(default_factory=dict)
- blocks: list[dict] = field(default_factory=list)
-
-
-def _post_parse(file_bytes: bytes, filename: str, mime: str, *, rich: bool) -> dict:
- """POST to the external parse service and return the single document's result dict.
-
- ``rich`` additionally requests the media and layout payloads. They are opt-in
- because the images come back base64-inlined in the JSON body — several MB for an
- image-heavy PDF — which the chat-attachment path has no use for.
- """
- api_url = _cfg_api_url()
- if not api_url:
- raise RuntimeError("FILE_PARSER_API_URL 未配置,无法解析 PDF 文件")
-
- timeout = _cfg_timeout()
- data = _cfg_parse_params()
- if rich:
- data = {**data, "return_images": "true", "return_content_list": "true"}
-
- try:
- resp = requests.post(
- api_url,
- files={"files": (filename, file_bytes, mime)},
- data=data,
- timeout=timeout,
- )
- resp.raise_for_status()
- except requests.exceptions.Timeout:
- raise RuntimeError(f"PDF 解析服务超时({timeout}s),请稍后重试")
- except requests.exceptions.RequestException as e:
- raise RuntimeError(f"PDF 解析服务请求失败: {e}")
-
- results = (resp.json() or {}).get("results", {})
- if not results:
- raise RuntimeError("PDF 解析服务返回结果为空")
- return results[next(iter(results))] or {}
-
-
-def _decode_data_uri(value: str) -> Optional[bytes]:
- """Decode one ``images`` entry. The service returns ``data:image/jpeg;base64,...``.
-
- A bare base64 payload (no data-URI prefix) is accepted too, so a service variant
- that omits the prefix still works. Undecodable entries are dropped rather than
- failing the whole document.
- """
- if not isinstance(value, str) or not value:
- return None
- payload = value.split(",", 1)[1] if value.startswith("data:") else value
- try:
- return base64.b64decode(payload, validate=False)
- except (binascii.Error, ValueError):
- return None
-
-
-def _parse_content_list(raw) -> list[dict]:
- """Normalise ``content_list`` — the service returns it as a JSON **string**."""
- if isinstance(raw, list):
- return [b for b in raw if isinstance(b, dict)]
- if isinstance(raw, str) and raw.strip():
- try:
- data = json.loads(raw)
- except ValueError:
- return []
- if isinstance(data, list):
- return [b for b in data if isinstance(b, dict)]
- return []
-
-
-def parse_pdf(file_bytes: bytes, filename: str = "file.pdf") -> str:
- """Parse PDF via external file-parser API. Returns markdown text."""
- content = _post_parse(file_bytes, filename, "application/pdf", rich=False).get("md_content", "")
- if not content:
- raise RuntimeError("PDF 解析服务返回内容为空")
- return content
-
-
-def parse_pdf_rich(file_bytes: bytes, filename: str = "file.pdf") -> RichParseResult:
- """Parse PDF keeping the figures the layout engine cut out.
-
- Same call as :func:`parse_pdf` plus ``return_images`` / ``return_content_list``.
- Used by knowledge-base indexing to build image assets; the chat-attachment path
- stays on the text-only call.
- """
- result = _post_parse(file_bytes, filename, "application/pdf", rich=True)
- markdown = result.get("md_content", "")
- if not markdown:
- raise RuntimeError("PDF 解析服务返回内容为空")
-
- images: dict[str, bytes] = {}
- for name, value in (result.get("images") or {}).items():
- decoded = _decode_data_uri(value)
- if decoded:
- # The markdown references the file under an ``images/`` prefix while the
- # payload is keyed by bare filename; index both so lookups by either work.
- images[str(name)] = decoded
- images.setdefault(f"images/{name}", decoded)
-
- return RichParseResult(
- markdown=markdown,
- images=images,
- blocks=_parse_content_list(result.get("content_list")),
- )
-
-
-# ── DOCX (via pandoc) ─────────────────────────────────────────────────────────
-
-def _docx_bytes_to_markdown(docx_bytes: bytes) -> str:
- """Convert DOCX bytes to markdown string via pandoc."""
- with tempfile.TemporaryDirectory() as tmpdir:
- input_path = os.path.join(tmpdir, "input.docx")
- with open(input_path, "wb") as f:
- f.write(docx_bytes)
-
- try:
- result = subprocess.run(
- [find_pandoc_binary() or "pandoc", input_path, "-t", "markdown", "--wrap=none"],
- capture_output=True,
- text=True,
- timeout=60,
- **no_window_kwargs(),
- )
- except FileNotFoundError:
- raise RuntimeError("pandoc 未安装,无法解析 Word 文档")
- except subprocess.TimeoutExpired:
- raise RuntimeError("pandoc 转换超时")
-
- if result.returncode != 0:
- raise RuntimeError(f"pandoc 转换失败: {result.stderr[:300]}")
-
- return result.stdout
-
-
-def parse_docx(file_bytes: bytes) -> str:
- """Parse DOCX bytes → markdown via pandoc."""
- return _docx_bytes_to_markdown(file_bytes)
-
-
-# ── DOC / WPS (LibreOffice → DOCX → pandoc) ──────────────────────────────────
-
-def _convert_to_docx_bytes(file_bytes: bytes, suffix: str) -> bytes:
- """Use LibreOffice headless to convert DOC/WPS bytes → DOCX bytes."""
- with tempfile.TemporaryDirectory() as tmpdir:
- input_path = os.path.join(tmpdir, f"input{suffix}")
- with open(input_path, "wb") as f:
- f.write(file_bytes)
-
- try:
- result = subprocess.run(
- [
- find_libreoffice_binary() or "libreoffice",
- "--headless",
- "--convert-to", "docx",
- "--outdir", tmpdir,
- input_path,
- ],
- capture_output=True,
- text=True,
- timeout=120,
- **no_window_kwargs(),
- )
- except FileNotFoundError:
- raise RuntimeError("LibreOffice 未安装,无法解析 DOC/WPS 文件")
- except subprocess.TimeoutExpired:
- raise RuntimeError("LibreOffice 转换超时")
-
- output_path = os.path.join(tmpdir, "input.docx")
- if not os.path.exists(output_path):
- raise RuntimeError(f"LibreOffice 转换失败: {result.stderr[:300]}")
-
- with open(output_path, "rb") as f:
- return f.read()
-
-
-def parse_doc_wps(file_bytes: bytes, suffix: str) -> str:
- """Parse DOC/WPS via LibreOffice → DOCX → pandoc markdown."""
- docx_bytes = _convert_to_docx_bytes(file_bytes, suffix)
- return _docx_bytes_to_markdown(docx_bytes)
-
-
-# ── TXT ───────────────────────────────────────────────────────────────────────
-
-def parse_txt(file_bytes: bytes) -> str:
- """Decode text file bytes, trying UTF-8 then GBK then latin-1."""
- for enc in ("utf-8", "gbk", "latin-1"):
- try:
- return file_bytes.decode(enc)
- except UnicodeDecodeError:
- continue
- return file_bytes.decode("utf-8", errors="replace")
-
-
-# ── XLSX ──────────────────────────────────────────────────────────────────────
-
-def _deduplicate_headers(header_raw: list[str]) -> list[str]:
- """Disambiguate empty/duplicate header names so {placeholder} rendering
- downstream is unambiguous (e.g., two ``""`` columns become ``col``/``col_1``).
- """
- seen: dict[str, int] = {}
- cleaned: list[str] = []
- for h in header_raw:
- base = h if h else "col"
- if base in seen:
- seen[base] += 1
- cleaned.append(f"{base}_{seen[base]}")
- else:
- seen[base] = 0
- cleaned.append(base)
- return cleaned
-
-
-def _rows_to_markdown(header: list[str], data_rows: list[list[str]],
- sheet_title: str | None = None) -> str:
- """Convert header + data rows into a markdown table string."""
- lines: list[str] = []
- if sheet_title:
- lines.append(f"## Sheet: {sheet_title}")
- lines.append("")
- lines.append("| " + " | ".join(header) + " |")
- lines.append("|" + "|".join("---" for _ in header) + "|")
- for row in data_rows:
- lines.append("| " + " | ".join(row) + " |")
- return "\n".join(lines)
-
-
-def _open_xlsx(file_bytes: bytes):
- """Open a workbook in read-only mode; raise RuntimeError with a Chinese
- install hint if openpyxl is missing."""
- try:
- import openpyxl
- except ImportError:
- raise RuntimeError("openpyxl 未安装,无法解析 XLSX 文件(pip install openpyxl)")
- return openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
-
-
-def _xlsx_sheet_to_markdown(sheet) -> str | None:
- """Render one openpyxl sheet to a markdown table, or None if empty.
- Returns None (not raise) so callers can decide whether emptiness is an
- error (single-sheet mode) or just a skip (whole-workbook mode)."""
- rows = list(sheet.iter_rows(values_only=True))
- if not rows:
- return None
- header = [str(c or "") for c in rows[0]]
- data_rows = [
- [str(c or "") for c in row]
- for row in rows[1:]
- if not all(c is None for c in row)
- ]
- return _rows_to_markdown(header, data_rows, sheet.title)
-
-
-def parse_xlsx(file_bytes: bytes) -> str:
- """Parse XLSX bytes → markdown tables (one per sheet)."""
- wb = _open_xlsx(file_bytes)
- try:
- sections = [md for sheet in wb.worksheets if (md := _xlsx_sheet_to_markdown(sheet))]
- finally:
- wb.close()
- if not sections:
- raise RuntimeError("XLSX 文件无有效数据")
- return "\n\n".join(sections)
-
-
-def parse_xlsx_sheet_names(file_bytes: bytes) -> list[str]:
- """Return the list of sheet names in workbook order. Cheap probe: does
- not iterate any rows, just reads workbook structure."""
- wb = _open_xlsx(file_bytes)
- try:
- return list(wb.sheetnames)
- finally:
- wb.close()
-
-
-def parse_xlsx_single_sheet(file_bytes: bytes, sheet_name: str) -> str:
- """Parse only the named sheet → markdown table. Raises RuntimeError if
- the sheet does not exist or is empty."""
- wb = _open_xlsx(file_bytes)
- try:
- if sheet_name not in wb.sheetnames:
- raise RuntimeError(
- f"sheet 不存在: '{sheet_name}'。可选 sheet: {wb.sheetnames}"
- )
- md = _xlsx_sheet_to_markdown(wb[sheet_name])
- if md is None:
- raise RuntimeError(f"sheet '{sheet_name}' 无数据")
- return md
- finally:
- wb.close()
-
-
-def _default_xlsx_max_cell_chars() -> int:
- """Bound spreadsheet preview cells; the default remains 2000 characters."""
- try:
- return max(50, int(os.getenv("XLSX_PREVIEW_MAX_CELL_CHARS", "2000")))
- except ValueError:
- return 2000
-
-
-def parse_xlsx_preview(
- file_bytes: bytes,
- *,
- char_budget: int = 4000,
- max_columns: int = 40,
- max_cell_chars: Optional[int] = None,
-) -> dict:
- """Parse XLSX into a bounded markdown preview + accurate total counts.
-
- Produces what the main agent's file_context hook should inject for an
- uploaded spreadsheet: a markdown table containing as many rows as fit
- within ``char_budget``, plus the *real* total_rows / total_columns so
- the agent knows the full size without seeing every row.
-
- The total_rows count walks the entire sheet (cheap — read_only mode
- streams the first column only). The preview stops as soon as the
- cumulative markdown length exceeds ``char_budget``.
-
- Returns:
- ``{
- "header": list[str], # column names (post-dedup)
- "preview_md": str, # markdown table, ≤ char_budget chars
- "total_rows": int, # all data rows in sheet 1
- "total_columns": int, # raw column count (before max_columns)
- "preview_rows": int, # rows actually included in preview_md
- "truncated_columns": int, # cols dropped to fit max_columns
- "truncated_cells": int, # cells truncated to max_cell_chars
- }``
-
- Reads only the first sheet.
- """
- if max_cell_chars is None:
- max_cell_chars = _default_xlsx_max_cell_chars()
-
- try:
- import openpyxl
- except ImportError:
- raise RuntimeError("openpyxl 未安装,无法解析 XLSX 文件(pip install openpyxl)")
-
- wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
- try:
- sheet = wb.worksheets[0] if wb.worksheets else None
- if sheet is None:
- raise RuntimeError("XLSX 文件无 sheet")
-
- rows_iter = sheet.iter_rows(values_only=True)
- try:
- header_row = next(rows_iter)
- except StopIteration:
- raise RuntimeError("XLSX 文件无数据行") from None
-
- total_columns = len(header_row)
- truncated_columns = max(0, total_columns - max_columns)
- usable_cols = min(total_columns, max_columns)
-
- header = _deduplicate_headers([
- str(c).strip() if c is not None else f"col{idx + 1}"
- for idx, c in enumerate(header_row[:usable_cols])
- ])
-
- # Markdown header + separator (always present, counts toward budget)
- md_lines: list[str] = [
- "| " + " | ".join(header) + " |",
- "|" + "|".join("---" for _ in header) + "|",
- ]
- running_chars = sum(len(ln) + 1 for ln in md_lines) # +1 for newline
-
- preview_rows = 0
- truncated_cells = 0
- total_rows_seen = 0
- budget_reached = False
-
- for row in rows_iter:
- total_rows_seen += 1
- if all(c is None for c in row):
- continue
-
- if budget_reached:
- continue # keep counting total_rows accurately
-
- cells: list[str] = []
- for val in row[:usable_cols]:
- if val is None:
- cells.append("")
- continue
- s = str(val).replace("|", "\\|").replace("\n", " ")
- if len(s) > max_cell_chars:
- s = s[:max_cell_chars] + "…"
- truncated_cells += 1
- cells.append(s)
-
- # Pad short rows so the markdown table stays well-formed
- while len(cells) < len(header):
- cells.append("")
-
- line = "| " + " | ".join(cells) + " |"
- line_cost = len(line) + 1
- if running_chars + line_cost > char_budget and preview_rows > 0:
- # We have at least one row already; respect the budget.
- budget_reached = True
- continue
- md_lines.append(line)
- running_chars += line_cost
- preview_rows += 1
-
- return {
- "header": header,
- "preview_md": "\n".join(md_lines),
- "total_rows": total_rows_seen,
- "total_columns": total_columns,
- "preview_rows": preview_rows,
- "truncated_columns": truncated_columns,
- "truncated_cells": truncated_cells,
- }
- finally:
- wb.close()
-
-
-def parse_xls(file_bytes: bytes) -> str:
- """Parse legacy XLS bytes → markdown tables.
-
- Tries xlrd first; falls back to LibreOffice → XLSX → openpyxl.
- """
- try:
- import xlrd
- wb = xlrd.open_workbook(file_contents=file_bytes)
- sections: list[str] = []
- for sheet in wb.sheets():
- if sheet.nrows == 0:
- continue
- header = [str(sheet.cell_value(0, c) or "") for c in range(sheet.ncols)]
- data_rows = []
- for r in range(1, sheet.nrows):
- row = [str(sheet.cell_value(r, c) or "") for c in range(sheet.ncols)]
- if all(v == "" for v in row):
- continue
- data_rows.append(row)
- sections.append(_rows_to_markdown(header, data_rows, sheet.name))
- if not sections:
- raise RuntimeError("XLS 文件无有效数据")
- return "\n\n".join(sections)
- except ImportError:
- pass # xlrd not installed, try LibreOffice fallback
-
- # Fallback: LibreOffice → XLSX → openpyxl
- xlsx_bytes = _convert_xls_to_xlsx(file_bytes)
- return parse_xlsx(xlsx_bytes)
-
-
-def _convert_xls_to_xlsx(file_bytes: bytes) -> bytes:
- """Use LibreOffice headless to convert XLS → XLSX."""
- with tempfile.TemporaryDirectory() as tmpdir:
- input_path = os.path.join(tmpdir, "input.xls")
- with open(input_path, "wb") as f:
- f.write(file_bytes)
-
- try:
- result = subprocess.run(
- [
- find_libreoffice_binary() or "libreoffice",
- "--headless",
- "--convert-to", "xlsx",
- "--outdir", tmpdir,
- input_path,
- ],
- capture_output=True,
- text=True,
- timeout=120,
- **no_window_kwargs(),
- )
- except FileNotFoundError:
- raise RuntimeError(
- "xlrd 和 LibreOffice 均不可用,无法解析 XLS 文件"
- "(pip install xlrd 或安装 LibreOffice)"
- )
- except subprocess.TimeoutExpired:
- raise RuntimeError("LibreOffice 转换 XLS → XLSX 超时")
-
- output_path = os.path.join(tmpdir, "input.xlsx")
- if not os.path.exists(output_path):
- raise RuntimeError(f"LibreOffice 转换 XLS 失败: {result.stderr[:300]}")
-
- with open(output_path, "rb") as f:
- return f.read()
-
-
-# ── CSV ──────────────────────────────────────────────────────────────────────
-
-def parse_csv(file_bytes: bytes) -> str:
- """Parse CSV bytes → markdown table."""
- # Decode with encoding detection
- text = parse_txt(file_bytes)
-
- # Sniff dialect
- try:
- sample = text[:8192]
- dialect = csv.Sniffer().sniff(sample)
- except csv.Error:
- dialect = csv.excel
-
- reader = csv.reader(io.StringIO(text), dialect)
- rows = list(reader)
-
- if not rows:
- raise RuntimeError("CSV 文件无有效数据")
-
- header = [c.strip() for c in rows[0]]
- data_rows = []
- for row in rows[1:]:
- if all(c.strip() == "" for c in row):
- continue
- # Pad or trim to match header length
- padded = [c.strip() for c in row]
- while len(padded) < len(header):
- padded.append("")
- data_rows.append(padded[:len(header)])
-
- return _rows_to_markdown(header, data_rows)
-
-
-# ── PPTX ──────────────────────────────────────────────────────────────────────
-
-def _open_pptx(file_bytes: bytes):
- """Open a Presentation from bytes; raise RuntimeError with a Chinese
- install hint if python-pptx is missing."""
- try:
- from pptx import Presentation
- except ImportError:
- raise RuntimeError(
- "python-pptx 未安装,无法解析 PPTX 文件(pip install python-pptx)"
- )
- return Presentation(io.BytesIO(file_bytes))
-
-
-def _extract_slide_text(slide) -> str:
- # Iterate in author/Z order so the rendered output mirrors what a
- # reader sees (title → body → footers), since python-pptx exposes
- # shapes in that order on the slide.shapes collection.
- parts: list[str] = []
- for shape in slide.shapes:
- if not getattr(shape, "has_text_frame", False):
- continue
- try:
- text = shape.text_frame.text
- except Exception:
- continue
- text = (text or "").strip()
- if text:
- parts.append(text)
- return "\n".join(parts)
-
-
-def parse_pptx(file_bytes: bytes) -> str:
- """Parse PPTX bytes → markdown with one ``## Slide N`` section per slide."""
- p = _open_pptx(file_bytes)
- sections: list[str] = []
- for idx, slide in enumerate(p.slides):
- body = _extract_slide_text(slide) or "(此页无文本)"
- sections.append(f"## Slide {idx + 1}\n\n{body}")
- if not sections:
- raise RuntimeError("PPTX 文件无幻灯片")
- return "\n\n".join(sections)
-
-
-def parse_pptx_slide_count(file_bytes: bytes) -> int:
- """Cheap probe: return the slide count without extracting any text."""
- return len(_open_pptx(file_bytes).slides)
-
-
-def parse_pptx_slide(file_bytes: bytes, slide_index: int) -> str:
- """Return only the named slide's text (0-indexed). Raises RuntimeError
- on out-of-range index."""
- slides = _open_pptx(file_bytes).slides
- total = len(slides)
- if total == 0:
- raise RuntimeError("PPTX 文件无幻灯片")
- if slide_index < 0 or slide_index >= total:
- raise RuntimeError(
- f"slide_index 越界: {slide_index}(共 {total} 页,可选 0..{total - 1})"
- )
- return _extract_slide_text(slides[slide_index]) or "(此页无文本)"
-
-
-# ── Dispatcher ────────────────────────────────────────────────────────────────
-
-_EXT_MAP: dict[str, str] = {
- ".pdf": "pdf",
- ".docx": "docx",
- ".doc": "doc_wps",
- ".wps": "doc_wps",
- ".txt": "txt",
- ".xlsx": "xlsx",
- ".xls": "xls",
- ".csv": "csv",
- ".pptx": "pptx",
-}
-
-# Text-like extensions that don't need a real parser — just utf-8 decode.
-# Covers HTML/Markdown/code/config formats the agent commonly produces.
-# Mapping value "raw_text" routes to ``parse_raw_text``.
-_RAW_TEXT_EXTS = {
- ".html", ".htm",
- ".md", ".markdown",
- ".json", ".jsonl", ".ndjson",
- ".yaml", ".yml",
- ".toml", ".ini", ".cfg",
- ".xml", ".svg",
- ".log",
- ".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".bash",
- ".css", ".scss",
- ".tex", ".rst",
- ".sql",
-}
-
-SUPPORTED_EXTENSIONS = list(_EXT_MAP.keys()) + sorted(_RAW_TEXT_EXTS)
-
-
-def parse_raw_text(file_bytes: bytes) -> str:
- """UTF-8 decode for plain-text formats (HTML/MD/code/etc.).
-
- No structural extraction—just the source bytes as a string. Replaces
- undecodable bytes so we always return something usable.
- """
- return file_bytes.decode("utf-8", errors="replace")
-
-
-def parse_file(file_bytes: bytes, filename: str) -> Optional[str]:
- """
- Dispatch to the correct parser by file extension.
-
- Returns extracted text/markdown, or None if the format is unsupported.
- Raises RuntimeError with a Chinese message on parse failure.
- """
- suffix = os.path.splitext(filename.lower())[1]
- kind = _EXT_MAP.get(suffix)
-
- if kind == "pdf":
- return parse_pdf(file_bytes, filename)
- elif kind == "docx":
- return parse_docx(file_bytes)
- elif kind == "doc_wps":
- return parse_doc_wps(file_bytes, suffix)
- elif kind == "txt":
- return parse_txt(file_bytes)
- elif kind == "xlsx":
- return parse_xlsx(file_bytes)
- elif kind == "xls":
- return parse_xls(file_bytes)
- elif kind == "csv":
- return parse_csv(file_bytes)
- elif kind == "pptx":
- return parse_pptx(file_bytes)
- elif suffix in _RAW_TEXT_EXTS:
- return parse_raw_text(file_bytes)
- else:
- return None
diff --git a/src/backend/core/db/__init__.py b/src/backend/core/db/__init__.py
index a8ad7c28..d9283f7e 100644
--- a/src/backend/core/db/__init__.py
+++ b/src/backend/core/db/__init__.py
@@ -1,2 +1,26 @@
-from core.db.engine import get_db, init_db, Base # noqa: F401
-from core.db.models import * # noqa: F401,F403
+"""Database package facade.
+
+Leaf modules (engine, utc_datetime, ...) must stay importable without loading
+every ORM model: edition model extensions import them while ``core.db.models``
+is importing those extensions. Model names resolve lazily on first access.
+"""
+
+import importlib
+import importlib.util
+
+from core.db.engine import Base, get_db, init_db # noqa: F401
+
+
+def __getattr__(name):
+ if name.startswith("__"):
+ raise AttributeError(name)
+ if importlib.util.find_spec(f"{__name__}.{name}") is not None:
+ # ``from core.db import `` must not drag in every model.
+ return importlib.import_module(f"{__name__}.{name}")
+ models = importlib.import_module("core.db.models")
+ try:
+ value = getattr(models, name)
+ except AttributeError:
+ raise AttributeError(f"module 'core.db' has no attribute {name!r}") from None
+ globals()[name] = value
+ return value
diff --git a/src/backend/core/db/artifact_scope.py b/src/backend/core/db/artifact_scope.py
new file mode 100644
index 00000000..fb28ee7c
--- /dev/null
+++ b/src/backend/core/db/artifact_scope.py
@@ -0,0 +1,13 @@
+"""Personal artifact predicates use the installed database edition policy."""
+from typing import Any
+from core.db.edition import personal_artifact_scope_fields
+
+
+def personal_artifact_predicates(artifact_model: Any) -> list[Any]:
+ return [getattr(artifact_model, name).is_(value)
+ for name, value in personal_artifact_scope_fields().items()]
+
+
+def is_personal_artifact(artifact: Any) -> bool:
+ return all(getattr(artifact, name) == value
+ for name, value in personal_artifact_scope_fields().items())
diff --git a/src/backend/core/db/chat_search.py b/src/backend/core/db/chat_search.py
index b3b258b3..d1fa1ac9 100644
--- a/src/backend/core/db/chat_search.py
+++ b/src/backend/core/db/chat_search.py
@@ -6,7 +6,7 @@
until after the authorized page has been selected.
"""
-from sqlalchemy import Text, and_, case, func, or_, select, null
+from sqlalchemy import Text, and_, case, or_, select, null
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.functions import FunctionElement
from core.db.reserved_chats import ordinary_chat_filter
diff --git a/src/backend/core/db/edition.py b/src/backend/core/db/edition.py
new file mode 100644
index 00000000..b258a048
--- /dev/null
+++ b/src/backend/core/db/edition.py
@@ -0,0 +1,27 @@
+"""Community database schema and personal resource ownership."""
+from core.db.personal_model_fields import ProjectEditionFields, project_edition_table_args, ArtifactEditionFields, artifact_edition_table_args, UserAgentEditionFields, user_agent_edition_table_args, ChatSessionEditionFields, chat_session_edition_table_args, MarketplaceListingEditionFields, EDITION_MODEL_EXPORTS
+from core.db.personal_schema import ce_create_all, ce_reconcile_schema
+
+EDITION_MODEL_NAMES = ()
+
+
+def personal_artifact_scope_fields() -> dict[str, None]:
+ return {}
+
+
+def __getattr__(name):
+ if name == "UserAgentRepository":
+ from core.db.repository.personal_agent import UserAgentRepository
+ return UserAgentRepository
+ if name == "AuditLogRepository":
+ from core.db.repository.personal_audit import AuditLogRepository
+ return AuditLogRepository
+ raise AttributeError(name)
+
+
+def personal_index_scope(enterprise=False):
+ return ""
+
+
+def edition_model_exports():
+ return {}
diff --git a/src/backend/core/db/engine.py b/src/backend/core/db/engine.py
index e4cfbcd6..57153719 100644
--- a/src/backend/core/db/engine.py
+++ b/src/backend/core/db/engine.py
@@ -2,6 +2,7 @@
import logging
import os
+from dataclasses import dataclass
from pathlib import Path
from typing import Generator
@@ -32,6 +33,40 @@ def _resolve_database_url() -> str:
return url
+@dataclass(frozen=True)
+class ThreadBudget:
+ """How many threads of one process can hold a database connection at once.
+
+ Database work runs on threads, never on the event loop, and a process has two
+ pools of them: anyio's default limiter (sync routes and dependencies, and
+ ``core.db.threads.run_db``) and the event loop's default executor
+ (``asyncio.to_thread``). The connection pool is one connection per thread of
+ both, plus a little overflow for the few dedicated threads outside them
+ (sub-agent runs, index workers, schedulers), which otherwise queue on
+ ``DB_POOL_TIMEOUT``. A pool larger than this only reserves Postgres
+ connections no thread can use; a smaller one makes threads wait for each
+ other inside a request.
+ """
+
+ request_threads: int
+ background_threads: int
+ overflow: int = 10
+
+ @property
+ def pool_size(self) -> int:
+ return self.request_threads + self.background_threads
+
+
+def thread_budget() -> ThreadBudget:
+ """The one place the per-process thread limits and the pool size come from."""
+ return ThreadBudget(
+ request_threads=settings.db.worker_threads,
+ # CPython's own default for the loop executor, made explicit so the
+ # pool is sized against the number the executor will really use.
+ background_threads=min(32, (os.cpu_count() or 1) + 4),
+ )
+
+
# Database URL from environment variable
DATABASE_URL = _resolve_database_url()
@@ -57,8 +92,8 @@ def _resolve_database_url() -> str:
else:
engine_kwargs["poolclass"] = NullPool
else:
- engine_kwargs["pool_size"] = settings.db.pool_size
- engine_kwargs["max_overflow"] = settings.db.pool_max_overflow
+ engine_kwargs["pool_size"] = thread_budget().pool_size
+ engine_kwargs["max_overflow"] = thread_budget().overflow
engine_kwargs["pool_timeout"] = settings.db.pool_timeout
# 等锁必须有上界:无上界的行锁等待会把持有它的工作线程(或误写成 async 的路由的
# 事件循环)无限期占住,另一侧又要等这条请求收尾才能释放锁,形成互相干等的死结。
@@ -159,12 +194,12 @@ def init_db():
reconcile_audio_model_type(engine)
reconcile_resource_owners(engine)
from core.db.site_storage_retirement import retire_site_storage
- from core.services.application_store import initialize_local_store
+ from core.harness.registry.schema_lifecycle import initialize_local_schema
directory = settings.storage.root / "migration-backups"
directory = Path(os.getenv("SITE_STORAGE_BACKUP_DIR", str(directory)))
retire_site_storage(engine, directory)
- initialize_local_store()
+ initialize_local_schema()
if settings.edition.edition == "ce":
if settings.deploy.is_local:
from core.db.local_schema_upgrade import reconcile_local_chat_sequences
@@ -180,7 +215,7 @@ def init_db():
if any(local_report.values()):
logger.info("Local database compatibility schema reconciled: %s", local_report)
- from core.db.edition_tables import ce_reconcile_schema
+ from core.db.edition import ce_reconcile_schema
report = ce_reconcile_schema(engine)
if any(report.values()):
diff --git a/src/backend/core/db/history_projection.py b/src/backend/core/db/history_projection.py
index c41762b5..1f868b9e 100644
--- a/src/backend/core/db/history_projection.py
+++ b/src/backend/core/db/history_projection.py
@@ -2,7 +2,7 @@
def tool_calls_for_history(raw):
- from core.chat.display_bounds import bound_result_for_history
+ from core.infra.text_bounds import bound_result_for_history
if not isinstance(raw, list):
return raw
diff --git a/src/backend/core/db/models/__init__.py b/src/backend/core/db/models/__init__.py
index bacfc1c7..dc5d9ca0 100644
--- a/src/backend/core/db/models/__init__.py
+++ b/src/backend/core/db/models/__init__.py
@@ -1,70 +1,29 @@
-"""Community-edition ORM model exports."""
+"""SQLAlchemy ORM models.
+
+Split into submodules by domain; this __init__ re-exports all model classes verbatim (plus Base/JSONType/INETType),
+so `from core.db.models import X` and `from core.db.models import *` both stay unchanged.
+"""
from core.db.engine import Base
-from core.db.models.admin import (
- AdminMcpServer,
- AdminPromptPart,
- AdminSkill,
- InstalledPlugin,
- MarketplaceListingState,
- MarketplaceSubmission,
- McpMarketInstallation,
- McpMarketItem,
- McpMarketSubmission,
- McpMarketVersion,
- PluginMarketPackage,
- PluginMarketSkillExclusion,
- SkillDependencyRequest,
-)
-from core.db.models.agent import (
- AgentLoop,
- AgentMarketSubmission,
- LoopIteration,
- Plan,
- PlanStep,
- UserAgent,
-)
+from core.db.models.column_types import INETType, JSONType
+
+from core.db.edition import EDITION_MODEL_EXPORTS
+from core.db.models.chat import ChatCompactionState, ChatMessage, ChatSession
+from core.db.models.chat_runs import ChatRun, ChatRunOperation, ChatSteerQueueItem
+from core.db.models.chat_resources import ChatSandboxSnapshot, ToolMediaBlob, MessageFeedback
+from core.db.models.identity import LocalUser, UserApiKey, UserFolder, UserShadow
+from core.db.models.project import Project, ProjectFavorite
+
+globals().update(EDITION_MODEL_EXPORTS)
+from core.db.models.admin import AdminMcpServer, AdminPromptPart, AdminSkill, InstalledPlugin, SkillDependencyRequest
+from core.db.models.agent import AgentLoop, LoopIteration, Plan, PlanStep, UserAgent
from core.db.models.agent_api import AgentApiCallLog
from core.db.models.artifact import Artifact, ContentBlock
-from core.db.models.automation import PersonaDistillJob, ScheduledTask, ScheduledTaskRun
-from core.db.models.chat import (
- ChatCompactionState,
- ChatMessage,
- ChatRun,
- ChatRunOperation,
- ChatSandboxSnapshot,
- ChatSession,
- ChatSteerQueueItem,
- MessageFeedback,
- ToolMediaBlob,
-)
+
from core.db.models.chat_mode import ChatMode
from core.db.models.config import ModelProvider, ModelRoleAssignment, SystemConfig
-from core.db.models.evolution import (
- EvolutionAgentProfile,
- EvolutionCandidate,
- EvolutionCreditDecision,
- EvolutionEpisode,
- EvolutionEvaluation,
- EvolutionEvidencePack,
- EvolutionMemoryOp,
- EvolutionPromotionLink,
- EvolutionRelease,
- EvolutionTraceEvent,
-)
-from core.db.models.identity import (
- ChannelConnection,
- DingTalkConnection,
- EmailConnection,
- LarkConnection,
- LocalUser,
- UserApiKey,
- UserFolder,
- UserShadow,
-)
-from core.db.models.job import JOB_LIVE_STATUSES, JOB_TERMINAL_STATUSES, Job, JobCall, JobItem
-from core.db.models.kb_wiki import KBWikiFolder, KBWikiJob, KBWikiPage
-from core.db.models.knowledge import CatalogOverride, KBAsset, KBChunk, KBDocument, KBSpace
+
+from core.db.models.knowledge import CatalogOverride
from core.db.models.logs import (
HarnessEventCursor,
HarnessEventLog,
@@ -78,6 +37,18 @@
ToolEffectReceipt,
RemoteToolEffect,
)
+from core.db.models.evolution import (
+ EvolutionAgentProfile,
+ EvolutionCandidate,
+ EvolutionCreditDecision,
+ EvolutionEpisode,
+ EvolutionEvaluation,
+ EvolutionEvidencePack,
+ EvolutionMemoryOp,
+ EvolutionPromotionLink,
+ EvolutionRelease,
+ EvolutionTraceEvent,
+)
from core.db.models.memory import (
MemoryOutbox,
MemoryRefShadow,
@@ -91,39 +62,100 @@
OntologyPackVersion,
OntologyReviewRun,
)
-from core.db.models.project import Project, ProjectFavorite
-from core.db.models.site import Site
-from sqlalchemy import JSON, String
-from sqlalchemy.dialects.postgresql import INET, JSONB
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
-
-# These models use the shared JSONType defined above.
-from core.db.models.capability import (
- DeviceCapabilityComponent,
- DeviceCapabilityInstallation,
- DeviceCapabilityNamePreference,
- DeviceCapabilityTransaction,
-)
+from core.db.models.command import CommandTask
+from core.db.models.job import JOB_LIVE_STATUSES, JOB_TERMINAL_STATUSES, Job, JobCall, JobItem
+from core.db.edition import EDITION_MODEL_NAMES, edition_model_exports
+globals().update(edition_model_exports())
-__all__ = [name for name in globals() if not name.startswith("_")]
-from core.db.models.observability import DesktopOutbox, DesktopRecord, DesktopSyncDevice
+def __getattr__(name):
+ # Edition models are absent above only when the edition model package
+ # itself started this import; it has finished loading by first access.
+ if name in EDITION_MODEL_NAMES:
+ value = edition_model_exports()[name]
+ globals()[name] = value
+ return value
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-from core.db.models.channel_relay import DesktopChannelBinding, ChannelRelayDelivery, ChannelRelayOperation
-from core.db.models.plugin_resource import PluginResource, PluginResourceTicket, PluginResourceCheckpoint
+__all__ = [
+ 'ChatMode',
+ 'Base',
+ 'JSONType',
+ 'INETType',
+ 'UserShadow',
+ 'UserApiKey',
+ 'AgentApiCallLog',
+ 'LocalUser',
+ 'UserFolder',
+ 'Project',
+ 'ProjectFavorite',
+ 'ChatSession',
+ *EDITION_MODEL_EXPORTS.keys(),
+ 'ChatMessage',
+ 'ChatCompactionState',
+ 'ChatRun',
+ 'CommandTask',
+ 'ChatRunOperation',
+ 'ChatSteerQueueItem',
+ 'ToolEffectLedger',
+ 'ToolEffectLease',
+ 'ToolEffectReceipt',
+ 'RemoteToolEffect',
+ 'HarnessUsageCursor',
+ 'HarnessUsageAttempt',
+ 'HarnessEventCursor',
+ 'HarnessEventLog',
+ 'MessageFeedback',
+ 'ChatSandboxSnapshot',
+ 'ToolMediaBlob',
+ 'CatalogOverride',
+ 'Artifact',
+ 'ContentBlock',
+ 'ModelProvider',
+ 'SystemConfig',
+ 'ModelRoleAssignment',
+ 'AdminSkill',
+ 'SkillDependencyRequest',
+ 'AdminPromptPart',
+ 'AdminMcpServer',
+ 'InstalledPlugin',
+ 'UserAgent',
+ 'Plan',
+ 'PlanStep',
+ 'AgentLoop',
+ 'LoopIteration',
+ 'Job',
+ 'JobItem',
+ 'JobCall',
+ 'JOB_TERMINAL_STATUSES',
+ 'JOB_LIVE_STATUSES',
+ 'ToolCallLog',
+ 'SubAgentCallLog',
+ 'SkillCallLog',
+ 'ProfileMemory',
+ 'MemoryOutbox',
+ 'MemorySanitizerRule',
+ 'MemoryRefShadow',
+ 'EvolutionEpisode',
+ 'EvolutionTraceEvent',
+ 'EvolutionCandidate',
+ 'EvolutionRelease',
+ 'EvolutionEvaluation',
+ 'EvolutionAgentProfile',
+ 'EvolutionMemoryOp',
+ 'EvolutionEvidencePack',
+ 'EvolutionCreditDecision',
+ 'EvolutionPromotionLink',
+ 'OntologyPack',
+ 'OntologyPackVersion',
+ 'OntologyEnforcementEvent',
+ 'OntologyReviewRun',
+ 'OntologyDraft',
+ *EDITION_MODEL_NAMES,
+]
-# Shared feature tables exist even when the runtime module is disabled.
-from personal_agent.models import (
- PersonalAgent, PersonalWorkItem, PersonalDevice, PersonalDeviceCommand,
- PersonalMessageTask, PersonalMessageInbox, PersonalSessionOperation,
-)
-__all__ += ["PersonalAgent", "PersonalWorkItem", "PersonalDevice",
- "PersonalDeviceCommand", "PersonalMessageTask", "PersonalMessageInbox",
- "PersonalSessionOperation"]
+from .plugin_resource import PluginResource, PluginResourceTicket, PluginResourceCheckpoint
-from core.db.models.command import CommandTask
-__all__ += ["CommandTask"]
+__all__ = [name for name in __all__ if name in globals() or name in EDITION_MODEL_NAMES]
diff --git a/src/backend/core/db/models/admin.py b/src/backend/core/db/models/admin.py
index 262f7529..20c4fca9 100644
--- a/src/backend/core/db/models/admin.py
+++ b/src/backend/core/db/models/admin.py
@@ -2,29 +2,18 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
-from core.db.model_extensions import MarketplaceListingEditionFields
from sqlalchemy import (
- JSON,
Boolean,
CheckConstraint,
Column,
- ForeignKey,
Index,
Integer,
- Numeric,
String,
Text,
- UniqueConstraint,
- text,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from core.db.models.column_types import JSONType
class AdminSkill(Base):
@@ -145,7 +134,6 @@ class AdminMcpServer(Base):
env_vars = Column(JSONType, default=dict)
env_inherit = Column(JSONType, default=list)
headers = Column(JSONType, default=dict)
- is_stable = Column(Boolean, nullable=False, default=True)
is_enabled = Column(Boolean, nullable=False, default=True)
sort_order = Column(Integer, nullable=False, default=0)
extra_config = Column(JSONType, default=dict)
@@ -172,247 +160,6 @@ class AdminMcpServer(Base):
)
-class McpMarketItem(Base):
- """Installable MCP marketplace entry.
-
- The marketplace stores a credential-free definition. Concrete credentials
- and enablement live on the ``AdminMcpServer`` created for each installation.
- ``latest_version_id`` points at an immutable ``McpMarketVersion`` snapshot.
- """
-
- __tablename__ = "mcp_market_items"
-
- slug = Column(String(128), primary_key=True)
- display_name = Column(String(255), nullable=False)
- description = Column(Text, nullable=False, default="")
- user_intro = Column(Text)
- category = Column(String(64), nullable=False, default="通用工具")
- tags = Column(JSONType, default=list)
- icon = Column(String(500))
- publisher_id = Column(String(64))
- publisher_name = Column(String(255), nullable=False, default="")
- source = Column(String(16), nullable=False, default="community")
- latest_version_id = Column(String(64), nullable=False)
- # active = installable; changed = remote tool drift awaiting review;
- # suspended = security kill switch (derived installations are disabled).
- status = Column(String(16), nullable=False, default="active")
- status_reason = Column(Text)
- last_verified_at = Column(UTCDateTime(timezone=True))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
- updated_at = Column(
- UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
- )
- deleted_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "source IN ('admin', 'community')",
- name="mcp_market_items_source_check",
- ),
- CheckConstraint(
- "status IN ('active', 'changed', 'suspended')",
- name="mcp_market_items_status_check",
- ),
- Index("idx_mcp_market_items_status", "status", "updated_at"),
- Index("idx_mcp_market_items_category", "category"),
- )
-
-
-class McpMarketVersion(Base):
- """Immutable, reviewed MCP marketplace version snapshot."""
-
- __tablename__ = "mcp_market_versions"
-
- version_id = Column(String(64), primary_key=True)
- slug = Column(
- String(128),
- ForeignKey("mcp_market_items.slug", ondelete="CASCADE"),
- nullable=False,
- )
- version = Column(String(50), nullable=False, default="1.0.0")
- transport = Column(String(20), nullable=False, default="streamable_http")
- url = Column(Text, nullable=False)
- # [{key, label, target: "header" | "query" | "url", required, secret, ...}];
- # values are never stored here.
- auth_schema = Column(JSONType, default=list)
- # Provider-neutral authentication contract. ``methods`` may contain
- # none/token/oauth2 options; OAuth endpoint discovery follows MCP metadata.
- auth_config = Column(JSONType, default=dict)
- tools_json = Column(JSONType, default=list)
- tool_hash = Column(String(64), nullable=False)
- listing_notice = Column(JSONType, default=dict)
- source_server_id = Column(String(100))
- approved_by = Column(String(64))
- approved_at = Column(UTCDateTime(timezone=True))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
-
- __table_args__ = (
- CheckConstraint(
- "transport IN ('streamable_http', 'sse')",
- name="mcp_market_versions_transport_check",
- ),
- UniqueConstraint("slug", "version", name="uq_mcp_market_versions_slug_version"),
- Index("idx_mcp_market_versions_slug", "slug", "created_at"),
- )
-
-
-class McpMarketSubmission(Base):
- """Credential-free snapshot submitted by a user for MCP marketplace review."""
-
- __tablename__ = "mcp_market_submissions"
-
- submission_id = Column(String(64), primary_key=True)
- slug = Column(String(128), nullable=False)
- source_server_id = Column(String(100), nullable=False)
- owner_user_id = Column(String(64), nullable=False)
- submitter_name = Column(String(255), nullable=False, default="")
- display_name = Column(String(255), nullable=False)
- description = Column(Text, nullable=False, default="")
- user_intro = Column(Text)
- category = Column(String(64), nullable=False, default="通用工具")
- tags = Column(JSONType, default=list)
- icon = Column(String(500))
- version = Column(String(50), nullable=False, default="1.0.0")
- transport = Column(String(20), nullable=False)
- url = Column(Text, nullable=False)
- auth_schema = Column(JSONType, default=list)
- auth_config = Column(JSONType, default=dict)
- tools_json = Column(JSONType, default=list)
- tool_hash = Column(String(64), nullable=False)
- listing_notice = Column(JSONType, default=dict)
- note = Column(Text, nullable=False, default="")
- status = Column(String(16), nullable=False, default="pending")
- review_note = Column(Text)
- reviewed_by = Column(String(64))
- reviewed_at = Column(UTCDateTime(timezone=True))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
- updated_at = Column(
- UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
- )
- deleted_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('pending', 'approved', 'rejected', 'withdrawn')",
- name="mcp_market_submissions_status_check",
- ),
- CheckConstraint(
- "transport IN ('streamable_http', 'sse')",
- name="mcp_market_submissions_transport_check",
- ),
- Index("idx_mcp_market_submissions_status", "status", "created_at"),
- Index("idx_mcp_market_submissions_owner", "owner_user_id", "created_at"),
- Index("idx_mcp_market_submissions_slug", "slug"),
- )
-
-
-class McpMarketInstallation(Base):
- """Links a reviewed market version to the concrete installed MCP server."""
-
- __tablename__ = "mcp_market_installations"
-
- install_id = Column(String(64), primary_key=True)
- slug = Column(
- String(128),
- ForeignKey("mcp_market_items.slug", ondelete="CASCADE"),
- nullable=False,
- )
- version_id = Column(
- String(64),
- ForeignKey("mcp_market_versions.version_id", ondelete="RESTRICT"),
- nullable=False,
- )
- server_id = Column(
- String(100),
- ForeignKey("admin_mcp_servers.server_id", ondelete="CASCADE"),
- nullable=False,
- unique=True,
- )
- # NULL means the admin installed it globally; otherwise it is private to the user.
- owner_user_id = Column(String(64))
- status = Column(String(16), nullable=False, default="active")
- installed_by = Column(String(64))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
- updated_at = Column(
- UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
- )
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('active', 'suspended')",
- name="mcp_market_installations_status_check",
- ),
- Index("idx_mcp_market_installations_owner", "owner_user_id", "created_at"),
- Index("idx_mcp_market_installations_slug", "slug", "status"),
- Index(
- "uq_mcp_market_installations_user_slug",
- "slug",
- "owner_user_id",
- unique=True,
- postgresql_where=text("owner_user_id IS NOT NULL"),
- sqlite_where=text("owner_user_id IS NOT NULL"),
- ),
- Index(
- "uq_mcp_market_installations_global_slug",
- "slug",
- unique=True,
- postgresql_where=text("owner_user_id IS NULL"),
- sqlite_where=text("owner_user_id IS NULL"),
- ),
- )
-
-
-class MarketplaceSubmission(Base):
- """A user's private skill "apply for listing on the skill marketplace" record (pending admin review).
-
- At submission time a content snapshot of the source ``AdminSkill`` is
- taken (skill_content / extra_files); review and listing are both based
- on the snapshot — the user later modifying or deleting the original
- skill does not affect listed content. Records with
- ``status='approved'`` appear in the skill marketplace list
- (source=community) and can be installed by everyone; the admin can
- reject at any time (including delisting already-listed items).
- """
-
- __tablename__ = "marketplace_submissions"
-
- submission_id = Column(String(64), primary_key=True)
- # Marketplace slug / entry_name after listing (derived at submission time,
- # globally unique, does not clash with the preset marketplace catalog)
- slug = Column(String(128), nullable=False, unique=True)
- skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id
- owner_user_id = Column(String(64), nullable=False) # applicant
- submitter_name = Column(
- String(255), default=""
- ) # applicant display name (denormalized, for display)
-
- display_name = Column(String(255), nullable=False)
- summary = Column(Text, default="")
- user_intro = Column(Text) # explicit user-facing intro snapshot; NULL falls back to SKILL.md
- category = Column(String(64), default="社区共享")
- tags = Column(JSONType, default=list)
- version = Column(String(50), default="1.0.0")
- note = Column(Text, default="") # application note (for the admin)
-
- skill_content = Column(Text, nullable=False) # SKILL.md snapshot
- extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content}
-
- status = Column(String(16), nullable=False, default="pending")
- review_note = Column(Text) # rejection reason / review note
- reviewed_at = Column(UTCDateTime(timezone=True))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('pending', 'approved', 'rejected')",
- name="marketplace_submissions_status_check",
- ),
- Index("idx_marketplace_submissions_status", "status", "created_at"),
- Index("idx_marketplace_submissions_owner", "owner_user_id"),
- )
-
-
class InstalledPlugin(Base):
"""An installed plugin (bundle) — the single source of truth for uninstall/upgrade.
@@ -465,86 +212,3 @@ class InstalledPlugin(Base):
Index("idx_installed_plugins_owner", "owner_user_id"),
Index("idx_installed_plugins_slug", "slug"),
)
-
-
-class PluginMarketPackage(Base):
- """Admin-uploaded plugin marketplace package (stored/listed in the DB).
-
- Counterpart of the skill marketplace's ``MarketplaceSubmission``: a
- plugin zip uploaded by the admin from the /admin console is no longer
- installed globally right away, but persisted as a "market package" —
- storing the original zip's base64 (``package_b64``) + a set of display
- metadata (extracted by a single normalize pass). Once listed it appears
- in the plugin marketplace list for admins/users to install explicitly;
- installation unzips and re-runs the existing ``import_plugin`` chain.
- It coexists with filesystem preset packages
- (``extensions/plugins/{default,marketplace}//``); when resolving by
- slug, **filesystem takes precedence, DB market package is the
- fallback**. Install records still go into ``InstalledPlugin``; this
- table is only an "installable source".
- """
-
- __tablename__ = "plugin_market_packages"
-
- slug = Column(String(100), primary_key=True)
- name = Column(String(255), nullable=False)
- version = Column(String(50), nullable=False, default="1.0.0")
- description = Column(Text, default="")
- category = Column(String(64), default="")
- # Library path / URL / inline data-URI (uploaded custom icons, capped at ~200KB by the service layer)
- icon = Column(Text)
- # Package kind: native / claude / codex (determined by normalize), display only
- kind = Column(String(16), nullable=False, default="native")
- skills_count = Column(Integer, nullable=False, default=0)
- required_secrets = Column(JSONType, default=list)
- has_admin_config = Column(Boolean, nullable=False, default=False)
- package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- created_by = Column(String(64))
-
- __table_args__ = (Index("idx_plugin_market_packages_category", "category"),)
-
-
-class PluginMarketSkillExclusion(Base):
- """A single skill "deleted" by the admin from a plugin in the plugin marketplace (excluded by plugin slug + skill name).
-
- The plugin marketplace originally only supported whole-package deletion;
- this table lets the admin precisely remove a specific skill within a
- plugin. The market list's ``skills_count``, market detail ``skills``,
- and DB persistence at install time are all filtered by it — built-in
- filesystem packages (whose source files cannot be modified at runtime)
- and DB-listed packages are uniformly covered by this exclusion list.
- Installed instances are unaffected (those are install snapshots, managed
- separately via enable/disable and uninstall).
- """
-
- __tablename__ = "plugin_market_skill_exclusions"
-
- slug = Column(String(100), primary_key=True)
- skill_name = Column(String(100), primary_key=True)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- created_by = Column(String(64))
-
-
-class MarketplaceListingState(MarketplaceListingEditionFields, Base):
- """Marketplace listing switch: controls whether a plugin/skill is shown in the (plugin/skill) marketplace.
-
- ``kind`` = ``plugin`` | ``skill``; ``item_id`` = plugin slug or skill
- marketplace slug. **A missing row counts as enabled** (everything listed
- by default); a row with ``enabled=false`` is written only when the admin
- explicitly disables — so built-in/existing items are not wiped after an
- upgrade. The user-facing marketplace shows only enabled items; the admin
- console shows everything with toggles. Physical deletion (only for
- uploaded DB items) still goes through the respective delete endpoints;
- this table only governs "listing visibility".
-
- """
-
- __tablename__ = "marketplace_listing_states"
-
- kind = Column(String(16), primary_key=True) # plugin | skill
- item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug
- enabled = Column(Boolean, nullable=False, default=True)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- updated_by = Column(String(64))
diff --git a/src/backend/core/db/models/agent.py b/src/backend/core/db/models/agent.py
index 20a29eb7..b8bbe71d 100644
--- a/src/backend/core/db/models/agent.py
+++ b/src/backend/core/db/models/agent.py
@@ -2,12 +2,10 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
-from core.db.model_extensions import UserAgentEditionFields, user_agent_edition_table_args
+from core.db.edition import UserAgentEditionFields, user_agent_edition_table_args
from sqlalchemy import (
- JSON,
BigInteger,
Boolean,
CheckConstraint,
@@ -18,13 +16,9 @@
Numeric,
String,
Text,
- UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from sqlalchemy.orm import relationship
+from core.db.models.column_types import JSONType
class UserAgent(UserAgentEditionFields, Base):
@@ -94,67 +88,6 @@ class UserAgent(UserAgentEditionFields, Base):
)
-class AgentMarketSubmission(Base):
- """Record of a user-created sub-agent's "apply for listing on the sub-agent marketplace" (pending admin review).
-
- Mirrors the skill marketplace's ``MarketplaceSubmission``, but snapshots a
- "sub-agent" rather than skill files: on submission, the source ``UserAgent`` is
- content-snapshotted (prompt/welcome message/suggested questions/model config/
- capability bindings); review and listing are both based on the snapshot — later
- edits or deletion of the original agent do not affect listed content.
- ``status='approved'`` records appear in the sub-agent marketplace list
- (source=community) and can be "install-cloned" by everyone as a private sub-agent;
- admins can reject at any time (including delisting already-listed items). Direct
- admin uploads to the marketplace are distinguished by the sentinel owner
- ``__admin_upload__``.
- """
-
- __tablename__ = "agent_market_submissions"
-
- submission_id = Column(String(64), primary_key=True)
- # Marketplace slug once listed (derived from name at submission, globally unique, never colliding with the preset marketplace catalog)
- slug = Column(String(128), nullable=False, unique=True)
- agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id
- owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__)
- submitter_name = Column(
- String(255), default=""
- ) # applicant display name (denormalized for display)
-
- # Marketplace display metadata
- name = Column(String(255), nullable=False)
- avatar = Column(Text)
- description = Column(Text, default="")
- summary = Column(Text, default="")
- category = Column(String(64), default="通用助手")
- tags = Column(JSONType, default=list)
- version = Column(String(50), default="1.0.0")
- note = Column(Text, default="") # application note (for the admin)
-
- # Sub-agent content snapshot (decoupled from the source UserAgent)
- system_prompt = Column(Text, default="")
- welcome_message = Column(Text, default="")
- suggested_questions = Column(JSONType, default=list)
- # {temperature, max_tokens, max_iters, timeout}
- model_config_snapshot = Column(JSONType, default=dict)
- # {skill_ids, mcp_server_ids, plugin_ids, kb_ids}
- bindings_snapshot = Column(JSONType, default=dict)
-
- status = Column(String(16), nullable=False, default="pending")
- review_note = Column(Text) # rejection reason / review note
- reviewed_at = Column(UTCDateTime(timezone=True))
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('pending', 'approved', 'rejected')",
- name="agent_market_submissions_status_check",
- ),
- Index("idx_agent_market_submissions_status", "status", "created_at"),
- Index("idx_agent_market_submissions_owner", "owner_user_id"),
- )
-
-
class Plan(Base):
"""Plan mode - plans table"""
diff --git a/src/backend/core/db/models/artifact.py b/src/backend/core/db/models/artifact.py
index 0c97e645..2a23c8f2 100644
--- a/src/backend/core/db/models/artifact.py
+++ b/src/backend/core/db/models/artifact.py
@@ -2,29 +2,20 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
-from core.db.model_extensions import ArtifactEditionFields, artifact_edition_table_args
+from core.db.edition import ArtifactEditionFields, artifact_edition_table_args
from sqlalchemy import (
- JSON,
BigInteger,
- Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
- Integer,
- Numeric,
String,
Text,
- UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from sqlalchemy.orm import relationship
+from core.db.models.column_types import JSONType
class Artifact(ArtifactEditionFields, Base):
@@ -50,7 +41,7 @@ class Artifact(ArtifactEditionFields, Base):
storage_key = Column(Text, nullable=False)
storage_url = Column(Text)
extra_data = Column("metadata", JSONType, default={})
- # Lazy caches for cross-turn file reading (populated by core/content/artifact_reader.py)
+ # Lazy caches for cross-turn file reading (populated by core/files/artifacts/reader.py)
parsed_text = Column(Text, nullable=True)
summary = Column(Text, nullable=True)
parsed_at = Column(UTCDateTime(timezone=True), nullable=True)
diff --git a/src/backend/core/db/models/channel_relay.py b/src/backend/core/db/models/channel_relay.py
deleted file mode 100644
index cf92575b..00000000
--- a/src/backend/core/db/models/channel_relay.py
+++ /dev/null
@@ -1,40 +0,0 @@
-"""Durable, device-scoped channel delivery. Shared by CE and EE."""
-
-from core.db.engine import Base
-from sqlalchemy import JSON, Boolean, Column, Float, Index, String
-
-
-class DesktopChannelBinding(Base):
- __tablename__ = "desktop_channel_bindings"
- binding_id = Column(String(64), primary_key=True)
- owner_id = Column(String(64), nullable=False, index=True)
- device_id = Column(String(128), nullable=False)
- device_name = Column(String(100), nullable=False)
- consumed = Column(Boolean, nullable=False, default=False)
- created_at = Column(Float, nullable=False)
- last_seen = Column(Float, nullable=False)
-
-
-class ChannelRelayDelivery(Base):
- __tablename__ = "channel_relay_deliveries"
- delivery_id = Column(String(64), primary_key=True)
- channel_id = Column(String(64), nullable=False)
- binding_id = Column(String(64), nullable=False)
- owner_id = Column(String(64), nullable=False)
- device_id = Column(String(128), nullable=False)
- status = Column(String(24), nullable=False)
- payload = Column(JSON, nullable=False)
- lease = Column(String(64), nullable=True)
- created_at = Column(Float, nullable=False)
- deadline = Column(Float, nullable=False)
- error = Column(String(300), nullable=True)
- __table_args__ = (
- Index("ix_channel_relay_queue", "owner_id", "device_id", "status", "created_at"),
- )
-
-
-class ChannelRelayOperation(Base):
- __tablename__ = "channel_relay_operations"
- operation_id = Column(String(128), primary_key=True)
- status = Column(String(24), nullable=False)
- result = Column(JSON, nullable=True)
diff --git a/src/backend/core/db/models/chat.py b/src/backend/core/db/models/chat.py
index 2b092214..36e953d6 100644
--- a/src/backend/core/db/models/chat.py
+++ b/src/backend/core/db/models/chat.py
@@ -1,45 +1,14 @@
-"""SQLAlchemy ORM models — chat sessions/messages."""
+"""ORM models for chat sessions, messages and compaction state, plus sequence allocation."""
+from __future__ import annotations
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-
from datetime import datetime, timezone
-
from core.db.engine import Base
-from core.db.model_extensions import (
- ChatSessionEditionFields,
- chat_session_edition_table_args,
-)
-from sqlalchemy import (
- JSON,
- BigInteger,
- Boolean,
- CheckConstraint,
- Column,
- Computed,
- ForeignKey,
- LargeBinary,
- Index,
- Integer,
- Numeric,
- String,
- Text,
- UniqueConstraint,
- event,
- case,
- func,
- select,
- update,
-)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import deferred, mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
-# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not
-# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite
-# profile. Fall back to Integer on SQLite (still BIGINT on Postgres).
-BigIntPK = BigInteger().with_variant(Integer(), "sqlite")
-
+from core.db.edition import ChatSessionEditionFields, chat_session_edition_table_args
+from sqlalchemy import JSON, BigInteger, Boolean, CheckConstraint, Column, Computed, ForeignKey, Index, Integer, String, Text, UniqueConstraint, event, case, func, select, update
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import deferred, relationship
+from core.db.models.column_types import JSONType
class ChatSession(ChatSessionEditionFields, Base):
"""Chat session table."""
@@ -158,7 +127,7 @@ class ChatMessage(Base):
default=display_default, onupdate=display_default,
))
# 模型回放用的规范记录:按发生顺序的每一步(模型响应块 / 工具结果),见
- # ``core/llm/model_steps.py``。上面三列是给界面看的投影,模型上下文从这一列恢复。
+ # ``core/llm/runtime/model_steps.py``。上面三列是给界面看的投影,模型上下文从这一列恢复。
# NULL = 该行写入时还没有这份记录,回放退回 segments / 历史摘要。
model_steps = Column(JSONType)
usage = Column(JSONType)
@@ -296,7 +265,7 @@ def reserve_chat_message_sequences(
@event.listens_for(ChatMessage, "before_insert")
-def _assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None:
+def assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None:
"""Compatibility guard for legacy/direct message writers.
New main-chat admission reserves its user/reply pair explicitly. Existing
@@ -306,304 +275,3 @@ def _assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None:
if target.chat_seq is None:
target.chat_seq = reserve_chat_sequences(connection, target.chat_id)
-
-
-class ChatRun(Base):
- """Chat Run — decouples AI tasks from the HTTP connection lifecycle.
-
- Every message send creates a run; a background asyncio.Task runs the workflow,
- chunks are written to a Redis Stream, and the SSE endpoint pulls from the
- Stream. After a page refresh, playback resumes via follow_run + offset.
- """
-
- __tablename__ = "chat_runs"
-
- run_id = Column(String(64), primary_key=True)
- chat_id = Column(
- String(64),
- ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
- nullable=False,
- )
- user_id = Column(String(64), nullable=False)
- message_id = Column(
- String(64), nullable=False
- ) # pre-allocated assistant message id
- user_message_id = Column(String(64))
- user_chat_seq = Column(BigInteger)
- assistant_chat_seq = Column(BigInteger)
- # Non-null only while this run owns the chat's main-writer slot. A unique
- # (chat_id, writer_slot) constraint permits unlimited terminal NULL rows but
- # only one live "main" owner, on both PostgreSQL and SQLite.
- writer_slot = Column(String(20))
- status = Column(String(20), nullable=False, default="pending")
- request_payload = Column(
- JSONType
- ) # serialized ChatRequest (for the worker to rebuild the context)
- last_event_offset = Column(Integer, default=0, nullable=False)
- error_message = Column(Text)
- usage = Column(JSONType)
- # Durable run-journal state. Redis remains an event projection only; these
- # columns decide ownership, recovery and terminal CAS after a restart.
- run_phase = Column(String(40), nullable=False, default="accepted")
- lease_owner = Column(String(160))
- lease_expires_at = Column(UTCDateTime(timezone=True))
- operation_seq = Column(Integer, nullable=False, default=0)
- snapshot_version = Column(Integer, nullable=False, default=0)
- # Holds the whole in-flight run payload (megabytes on long chats); loading it
- # with every ChatRun row decodes that JSON again, so it is fetched on access.
- recovery_snapshot = deferred(Column(JSONType))
- last_operation_safety = Column(String(40), nullable=False, default="replayable")
- failure_reason = Column(Text)
- updated_at = Column(
- UTCDateTime(timezone=True),
- default=lambda: datetime.now(timezone.utc),
- onupdate=lambda: datetime.now(timezone.utc),
- nullable=False,
- )
- created_at = Column(
- UTCDateTime(timezone=True),
- default=lambda: datetime.now(timezone.utc),
- nullable=False,
- )
- started_at = Column(UTCDateTime(timezone=True))
- completed_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('pending', 'running', 'needs_attention', 'completed', 'failed', 'cancelled')",
- name="chat_runs_status_check",
- ),
- CheckConstraint(
- "writer_slot IS NULL OR writer_slot = 'main'",
- name="chat_runs_writer_slot_check",
- ),
- UniqueConstraint("chat_id", "writer_slot", name="uq_chat_runs_writer_slot"),
- Index("idx_chat_runs_chat_status", "chat_id", "status"),
- Index("idx_chat_runs_status_lease", "status", "lease_expires_at"),
- Index("idx_chat_runs_user_id", "user_id"),
- Index("idx_chat_runs_created_at", "created_at"),
- )
-
-
-class ChatRunOperation(Base):
- """Append-only, per-run operation journal used for recovery decisions."""
-
- __tablename__ = "chat_run_operations"
-
- operation_id = Column(BigIntPK, primary_key=True, autoincrement=True)
- run_id = Column(
- String(64), ForeignKey("chat_runs.run_id", ondelete="CASCADE"), nullable=False
- )
- operation_seq = Column(Integer, nullable=False)
- operation_type = Column(String(64), nullable=False)
- phase = Column(String(40), nullable=False)
- safety = Column(String(40), nullable=False)
- owner = Column(String(160))
- snapshot_version = Column(Integer, nullable=False, default=0)
- payload = Column(JSONType)
- created_at = Column(
- UTCDateTime(timezone=True),
- default=lambda: datetime.now(timezone.utc),
- nullable=False,
- )
-
- __table_args__ = (
- UniqueConstraint("run_id", "operation_seq", name="uq_chat_run_operation_seq"),
- CheckConstraint("operation_seq > 0", name="chat_run_operations_seq_positive"),
- Index("idx_chat_run_operations_run_seq", "run_id", "operation_seq"),
- )
-
-
-class ChatSteerQueueItem(Base):
- """Durable user input queued around a ChatRun safe boundary."""
-
- __tablename__ = "chat_steer_queue"
-
- queue_id = Column(String(64), primary_key=True)
- steer_id = Column(String(64), nullable=False)
- chat_id = Column(
- String(64),
- ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
- nullable=False,
- )
- user_id = Column(String(64), nullable=False)
- target_run_id = Column(
- String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True
- )
- steer_seq = Column(BigInteger, nullable=False)
- target_operation_seq = Column(Integer)
- delivery_mode = Column(String(20), nullable=False, default="steer")
- status = Column(String(20), nullable=False, default="accepted")
- message = Column(Text, nullable=False)
- lease_owner = Column(String(160))
- lease_expires_at = Column(UTCDateTime(timezone=True))
- delivery_attempt = Column(Integer, nullable=False, default=0)
- superseded_by = Column(String(64))
- applied_run_id = Column(String(64))
- applied_source_run_id = Column(
- String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True
- )
- applied_operation_seq = Column(Integer)
- accepted_at = Column(
- UTCDateTime(timezone=True),
- nullable=False,
- default=lambda: datetime.now(timezone.utc),
- )
- claimed_at = Column(UTCDateTime(timezone=True))
- applied_at = Column(UTCDateTime(timezone=True))
- cancelled_at = Column(UTCDateTime(timezone=True))
- updated_at = Column(
- UTCDateTime(timezone=True),
- nullable=False,
- default=lambda: datetime.now(timezone.utc),
- onupdate=lambda: datetime.now(timezone.utc),
- )
-
- __table_args__ = (
- CheckConstraint(
- "delivery_mode IN ('steer', 'follow_up', 'next_run')",
- name="chat_steer_queue_delivery_mode_check",
- ),
- CheckConstraint(
- "status IN ('accepted', 'claimed', 'applied', 'cancelled', 'superseded')",
- name="chat_steer_queue_status_check",
- ),
- CheckConstraint("steer_seq > 0", name="chat_steer_queue_seq_positive"),
- CheckConstraint(
- "delivery_attempt >= 0", name="chat_steer_queue_attempt_nonnegative"
- ),
- CheckConstraint(
- "length(message) BETWEEN 1 AND 10000",
- name="chat_steer_queue_message_length",
- ),
- UniqueConstraint("chat_id", "steer_id", name="uq_chat_steer_queue_chat_steer"),
- UniqueConstraint("chat_id", "steer_seq", name="uq_chat_steer_queue_chat_seq"),
- Index(
- "idx_chat_steer_queue_run_status_seq",
- "target_run_id",
- "status",
- "steer_seq",
- ),
- Index(
- "idx_chat_steer_queue_chat_status_seq",
- "chat_id",
- "status",
- "steer_seq",
- ),
- Index(
- "idx_chat_steer_queue_applied_source_status_seq",
- "applied_source_run_id",
- "status",
- "steer_seq",
- ),
- Index("idx_chat_steer_queue_lease", "status", "lease_expires_at"),
- )
-
-
-class MessageFeedback(Base):
- """Message feedback table - stores like/dislike ratings and optional comments."""
-
- __tablename__ = "message_feedback"
-
- feedback_id = Column(BigIntPK, primary_key=True, autoincrement=True)
- message_id = Column(
- String(64),
- ForeignKey("chat_messages.message_id", ondelete="CASCADE"),
- nullable=False,
- )
- chat_id = Column(
- String(64),
- ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
- nullable=False,
- )
- user_id = Column(
- String(64),
- ForeignKey("users_shadow.user_id", ondelete="SET NULL"),
- nullable=True,
- )
- rating = Column(String(10), nullable=False) # 'like' or 'dislike'
- comment = Column(Text, nullable=True)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(
- UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now
- )
-
- __table_args__ = (
- CheckConstraint(
- "rating IN ('like', 'dislike')", name="message_feedback_rating_check"
- ),
- Index("idx_message_feedback_message_id", "message_id"),
- Index("idx_message_feedback_chat_id", "chat_id"),
- Index("idx_message_feedback_user_id", "user_id"),
- )
-
-
-class ChatSandboxSnapshot(Base):
- """Per-chat opensandbox snapshot pointer.
-
- A chat keeps at most 1 snapshot at any time (the latest one). The background
- idle worker upserts on snapshot+kill; the next time that chat reconnects,
- _get_or_create_session prefers this snapshot when starting a new sandbox,
- restoring the filesystem state. The GC worker periodically scans rows with
- expires_at < now and deletes remote + DB together. Design in
- internal design docs.
- """
-
- __tablename__ = "chat_sandbox_snapshots"
-
- chat_id = Column(String(64), primary_key=True)
- snapshot_id = Column(String(64), nullable=False, unique=True)
- sandbox_id = Column(
- String(64), nullable=False
- ) # source sandbox id parked at the time, for debugging / reconciliation
- created_at = Column(
- UTCDateTime(timezone=True),
- nullable=False,
- default=lambda: datetime.now(timezone.utc),
- )
- expires_at = Column(
- UTCDateTime(timezone=True), nullable=False
- ) # created_at + SNAPSHOT_RETENTION_DAYS
- size_bytes = Column(BigInteger) # for metrics, nullable
- extra = Column(
- "metadata", JSONType, default=dict
- ) # reserved: image uri / pool kind / notes
-
- __table_args__ = (
- Index("idx_chat_sandbox_snapshots_expires", "expires_at"),
- Index("idx_chat_sandbox_snapshots_created", "created_at"),
- )
-
-
-class ToolMediaBlob(Base):
- """工具返回的媒体字节,按内容寻址,供历史回放取回原件。
-
- ``model_steps`` 是「模型自己的历史」的唯一真源,回放它应当得到模型当时真正看过
- 的那串消息。媒体曾经是这条真源上唯一的缺口:落库时图被换成一行
- ``[image omitted from replay history: …]``,于是图只能活一轮——下一轮重建历史时,
- 模型看到的是一句「这里原本有张图」,而它不会说自己看不见,会拿手上剩下的信息接
- 着编。
-
- 字节单独放一张表、只在 ``model_steps`` 里留 ``sha256`` 指针,是为了不把历史列
- 撑大:拼上下文要把检查点之后每一条消息的 ``model_steps`` 整列拉出来,一页 200dpi
- 的图编码后一两兆,内联进去就意味着**每一轮**都要整份读一遍,哪怕预算马上要把那
- 一步挤掉。主键用内容哈希,所以同一张图读多少次都只存一份。
-
- 与历史行同库,所以备份、迁移、恢复都把两者一起带走,不存在第二个会和它走散的
- 系统。字节先写、引用后写(引用是随那条消息落库的),所以中途崩溃最坏留下一份没人
- 引用的字节,不会出现「引用还在、字节没了」。
- """
-
- __tablename__ = "tool_media_blobs"
-
- sha256 = Column(String(64), primary_key=True)
- media_type = Column(String(128), nullable=False)
- data = Column(LargeBinary, nullable=False)
- size_bytes = Column(Integer, nullable=False)
- created_at = Column(
- UTCDateTime(timezone=True),
- nullable=False,
- default=lambda: datetime.now(timezone.utc),
- )
-
- __table_args__ = (Index("idx_tool_media_blobs_created", "created_at"),)
diff --git a/src/backend/core/db/models/chat_mode.py b/src/backend/core/db/models/chat_mode.py
index 996efc06..6dc193dd 100644
--- a/src/backend/core/db/models/chat_mode.py
+++ b/src/backend/core/db/models/chat_mode.py
@@ -20,11 +20,9 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime
from core.db.engine import Base
from sqlalchemy import (
- JSON,
Boolean,
CheckConstraint,
Column,
@@ -35,9 +33,8 @@
Text,
UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import JSONB
+from core.db.models.column_types import JSONType
-JSONType = JSON().with_variant(JSONB(), "postgresql")
#: 工具面范围。``all`` = 不额外收窄(标准模式:跟随 catalog 与用户自身授权);
#: ``restricted`` = 只装配本行显式列出的 MCP / 技能 / 插件(极速模式与自定义模式)。
diff --git a/src/backend/core/db/models/chat_resources.py b/src/backend/core/db/models/chat_resources.py
new file mode 100644
index 00000000..dff58a8c
--- /dev/null
+++ b/src/backend/core/db/models/chat_resources.py
@@ -0,0 +1,116 @@
+"""ORM models for message feedback, sandbox snapshots and tool media blobs."""
+from __future__ import annotations
+from core.db.utc_datetime import UTCDateTime
+from core.infra.time import utc_now
+from datetime import datetime, timezone
+from core.db.engine import Base
+from sqlalchemy import BigInteger, CheckConstraint, Column, ForeignKey, LargeBinary, Index, Integer, String, Text
+from core.db.models.column_types import BigIntPK, JSONType
+
+class MessageFeedback(Base):
+ """Message feedback table - stores like/dislike ratings and optional comments."""
+
+ __tablename__ = "message_feedback"
+
+ feedback_id = Column(BigIntPK, primary_key=True, autoincrement=True)
+ message_id = Column(
+ String(64),
+ ForeignKey("chat_messages.message_id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ chat_id = Column(
+ String(64),
+ ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ user_id = Column(
+ String(64),
+ ForeignKey("users_shadow.user_id", ondelete="SET NULL"),
+ nullable=True,
+ )
+ rating = Column(String(10), nullable=False) # 'like' or 'dislike'
+ comment = Column(Text, nullable=True)
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(
+ UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now
+ )
+
+ __table_args__ = (
+ CheckConstraint(
+ "rating IN ('like', 'dislike')", name="message_feedback_rating_check"
+ ),
+ Index("idx_message_feedback_message_id", "message_id"),
+ Index("idx_message_feedback_chat_id", "chat_id"),
+ Index("idx_message_feedback_user_id", "user_id"),
+ )
+
+
+class ChatSandboxSnapshot(Base):
+ """Per-chat opensandbox snapshot pointer.
+
+ A chat keeps at most 1 snapshot at any time (the latest one). The background
+ idle worker upserts on snapshot+kill; the next time that chat reconnects,
+ _get_or_create_session prefers this snapshot when starting a new sandbox,
+ restoring the filesystem state. The GC worker periodically scans rows with
+ expires_at < now and deletes remote + DB together. Design in
+ internal design docs.
+ """
+
+ __tablename__ = "chat_sandbox_snapshots"
+
+ chat_id = Column(String(64), primary_key=True)
+ snapshot_id = Column(String(64), nullable=False, unique=True)
+ sandbox_id = Column(
+ String(64), nullable=False
+ ) # source sandbox id parked at the time, for debugging / reconciliation
+ created_at = Column(
+ UTCDateTime(timezone=True),
+ nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ )
+ expires_at = Column(
+ UTCDateTime(timezone=True), nullable=False
+ ) # created_at + SNAPSHOT_RETENTION_DAYS
+ size_bytes = Column(BigInteger) # for metrics, nullable
+ extra = Column(
+ "metadata", JSONType, default=dict
+ ) # reserved: image uri / pool kind / notes
+
+ __table_args__ = (
+ Index("idx_chat_sandbox_snapshots_expires", "expires_at"),
+ Index("idx_chat_sandbox_snapshots_created", "created_at"),
+ )
+
+
+class ToolMediaBlob(Base):
+ """工具返回的媒体字节,按内容寻址,供历史回放取回原件。
+
+ ``model_steps`` 是「模型自己的历史」的唯一真源,回放它应当得到模型当时真正看过
+ 的那串消息。媒体曾经是这条真源上唯一的缺口:落库时图被换成一行
+ ``[image omitted from replay history: …]``,于是图只能活一轮——下一轮重建历史时,
+ 模型看到的是一句「这里原本有张图」,而它不会说自己看不见,会拿手上剩下的信息接
+ 着编。
+
+ 字节单独放一张表、只在 ``model_steps`` 里留 ``sha256`` 指针,是为了不把历史列
+ 撑大:拼上下文要把检查点之后每一条消息的 ``model_steps`` 整列拉出来,一页 200dpi
+ 的图编码后一两兆,内联进去就意味着**每一轮**都要整份读一遍,哪怕预算马上要把那
+ 一步挤掉。主键用内容哈希,所以同一张图读多少次都只存一份。
+
+ 与历史行同库,所以备份、迁移、恢复都把两者一起带走,不存在第二个会和它走散的
+ 系统。字节先写、引用后写(引用是随那条消息落库的),所以中途崩溃最坏留下一份没人
+ 引用的字节,不会出现「引用还在、字节没了」。
+ """
+
+ __tablename__ = "tool_media_blobs"
+
+ sha256 = Column(String(64), primary_key=True)
+ media_type = Column(String(128), nullable=False)
+ data = Column(LargeBinary, nullable=False)
+ size_bytes = Column(Integer, nullable=False)
+ created_at = Column(
+ UTCDateTime(timezone=True),
+ nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ )
+
+ __table_args__ = (Index("idx_tool_media_blobs_created", "created_at"),)
diff --git a/src/backend/core/db/models/chat_runs.py b/src/backend/core/db/models/chat_runs.py
new file mode 100644
index 00000000..e0afa2f3
--- /dev/null
+++ b/src/backend/core/db/models/chat_runs.py
@@ -0,0 +1,199 @@
+"""ORM models for chat runs, their operation journal and the steering queue."""
+from __future__ import annotations
+from core.db.utc_datetime import UTCDateTime
+from datetime import datetime, timezone
+from core.db.engine import Base
+from sqlalchemy import BigInteger, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import deferred
+from core.db.models.column_types import BigIntPK, JSONType
+
+class ChatRun(Base):
+ """Chat Run — decouples AI tasks from the HTTP connection lifecycle.
+
+ Every message send creates a run; a background asyncio.Task runs the workflow,
+ chunks are written to a Redis Stream, and the SSE endpoint pulls from the
+ Stream. After a page refresh, playback resumes via follow_run + offset.
+ """
+
+ __tablename__ = "chat_runs"
+
+ run_id = Column(String(64), primary_key=True)
+ chat_id = Column(
+ String(64),
+ ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ user_id = Column(String(64), nullable=False)
+ message_id = Column(
+ String(64), nullable=False
+ ) # pre-allocated assistant message id
+ user_message_id = Column(String(64))
+ user_chat_seq = Column(BigInteger)
+ assistant_chat_seq = Column(BigInteger)
+ # Non-null only while this run owns the chat's main-writer slot. A unique
+ # (chat_id, writer_slot) constraint permits unlimited terminal NULL rows but
+ # only one live "main" owner, on both PostgreSQL and SQLite.
+ writer_slot = Column(String(20))
+ status = Column(String(20), nullable=False, default="pending")
+ request_payload = Column(
+ JSONType
+ ) # serialized ChatRequest (for the worker to rebuild the context)
+ last_event_offset = Column(Integer, default=0, nullable=False)
+ error_message = Column(Text)
+ usage = Column(JSONType)
+ # Durable run-journal state. Redis remains an event projection only; these
+ # columns decide ownership, recovery and terminal CAS after a restart.
+ run_phase = Column(String(40), nullable=False, default="accepted")
+ lease_owner = Column(String(160))
+ lease_expires_at = Column(UTCDateTime(timezone=True))
+ operation_seq = Column(Integer, nullable=False, default=0)
+ snapshot_version = Column(Integer, nullable=False, default=0)
+ # Holds the whole in-flight run payload (megabytes on long chats); loading it
+ # with every ChatRun row decodes that JSON again, so it is fetched on access.
+ recovery_snapshot = deferred(Column(JSONType))
+ last_operation_safety = Column(String(40), nullable=False, default="replayable")
+ failure_reason = Column(Text)
+ updated_at = Column(
+ UTCDateTime(timezone=True),
+ default=lambda: datetime.now(timezone.utc),
+ onupdate=lambda: datetime.now(timezone.utc),
+ nullable=False,
+ )
+ created_at = Column(
+ UTCDateTime(timezone=True),
+ default=lambda: datetime.now(timezone.utc),
+ nullable=False,
+ )
+ started_at = Column(UTCDateTime(timezone=True))
+ completed_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('pending', 'running', 'needs_attention', 'completed', 'failed', 'cancelled')",
+ name="chat_runs_status_check",
+ ),
+ CheckConstraint(
+ "writer_slot IS NULL OR writer_slot = 'main'",
+ name="chat_runs_writer_slot_check",
+ ),
+ UniqueConstraint("chat_id", "writer_slot", name="uq_chat_runs_writer_slot"),
+ Index("idx_chat_runs_chat_status", "chat_id", "status"),
+ Index("idx_chat_runs_status_lease", "status", "lease_expires_at"),
+ Index("idx_chat_runs_user_id", "user_id"),
+ Index("idx_chat_runs_created_at", "created_at"),
+ )
+
+
+class ChatRunOperation(Base):
+ """Append-only, per-run operation journal used for recovery decisions."""
+
+ __tablename__ = "chat_run_operations"
+
+ operation_id = Column(BigIntPK, primary_key=True, autoincrement=True)
+ run_id = Column(
+ String(64), ForeignKey("chat_runs.run_id", ondelete="CASCADE"), nullable=False
+ )
+ operation_seq = Column(Integer, nullable=False)
+ operation_type = Column(String(64), nullable=False)
+ phase = Column(String(40), nullable=False)
+ safety = Column(String(40), nullable=False)
+ owner = Column(String(160))
+ snapshot_version = Column(Integer, nullable=False, default=0)
+ payload = Column(JSONType)
+ created_at = Column(
+ UTCDateTime(timezone=True),
+ default=lambda: datetime.now(timezone.utc),
+ nullable=False,
+ )
+
+ __table_args__ = (
+ UniqueConstraint("run_id", "operation_seq", name="uq_chat_run_operation_seq"),
+ CheckConstraint("operation_seq > 0", name="chat_run_operations_seq_positive"),
+ Index("idx_chat_run_operations_run_seq", "run_id", "operation_seq"),
+ )
+
+
+class ChatSteerQueueItem(Base):
+ """Durable user input queued around a ChatRun safe boundary."""
+
+ __tablename__ = "chat_steer_queue"
+
+ queue_id = Column(String(64), primary_key=True)
+ steer_id = Column(String(64), nullable=False)
+ chat_id = Column(
+ String(64),
+ ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"),
+ nullable=False,
+ )
+ user_id = Column(String(64), nullable=False)
+ target_run_id = Column(
+ String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True
+ )
+ steer_seq = Column(BigInteger, nullable=False)
+ target_operation_seq = Column(Integer)
+ delivery_mode = Column(String(20), nullable=False, default="steer")
+ status = Column(String(20), nullable=False, default="accepted")
+ message = Column(Text, nullable=False)
+ lease_owner = Column(String(160))
+ lease_expires_at = Column(UTCDateTime(timezone=True))
+ delivery_attempt = Column(Integer, nullable=False, default=0)
+ superseded_by = Column(String(64))
+ applied_run_id = Column(String(64))
+ applied_source_run_id = Column(
+ String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True
+ )
+ applied_operation_seq = Column(Integer)
+ accepted_at = Column(
+ UTCDateTime(timezone=True),
+ nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ )
+ claimed_at = Column(UTCDateTime(timezone=True))
+ applied_at = Column(UTCDateTime(timezone=True))
+ cancelled_at = Column(UTCDateTime(timezone=True))
+ updated_at = Column(
+ UTCDateTime(timezone=True),
+ nullable=False,
+ default=lambda: datetime.now(timezone.utc),
+ onupdate=lambda: datetime.now(timezone.utc),
+ )
+
+ __table_args__ = (
+ CheckConstraint(
+ "delivery_mode IN ('steer', 'follow_up', 'next_run')",
+ name="chat_steer_queue_delivery_mode_check",
+ ),
+ CheckConstraint(
+ "status IN ('accepted', 'claimed', 'applied', 'cancelled', 'superseded')",
+ name="chat_steer_queue_status_check",
+ ),
+ CheckConstraint("steer_seq > 0", name="chat_steer_queue_seq_positive"),
+ CheckConstraint(
+ "delivery_attempt >= 0", name="chat_steer_queue_attempt_nonnegative"
+ ),
+ CheckConstraint(
+ "length(message) BETWEEN 1 AND 10000",
+ name="chat_steer_queue_message_length",
+ ),
+ UniqueConstraint("chat_id", "steer_id", name="uq_chat_steer_queue_chat_steer"),
+ UniqueConstraint("chat_id", "steer_seq", name="uq_chat_steer_queue_chat_seq"),
+ Index(
+ "idx_chat_steer_queue_run_status_seq",
+ "target_run_id",
+ "status",
+ "steer_seq",
+ ),
+ Index(
+ "idx_chat_steer_queue_chat_status_seq",
+ "chat_id",
+ "status",
+ "steer_seq",
+ ),
+ Index(
+ "idx_chat_steer_queue_applied_source_status_seq",
+ "applied_source_run_id",
+ "status",
+ "steer_seq",
+ ),
+ Index("idx_chat_steer_queue_lease", "status", "lease_expires_at"),
+ )
diff --git a/src/backend/core/db/models/column_types.py b/src/backend/core/db/models/column_types.py
new file mode 100644
index 00000000..1306a123
--- /dev/null
+++ b/src/backend/core/db/models/column_types.py
@@ -0,0 +1,9 @@
+"""Portable column types shared by every ORM model: JSONB/INET on PostgreSQL, plain types elsewhere."""
+from __future__ import annotations
+from sqlalchemy import JSON, BigInteger, Integer, String
+from sqlalchemy.dialects.postgresql import INET, JSONB
+
+JSONType = JSON().with_variant(JSONB(), "postgresql")
+INETType = String(45).with_variant(INET(), "postgresql")
+# SQLite only auto-increments the INTEGER rowid, so a BIGINT key would stay NULL there.
+BigIntPK = BigInteger().with_variant(Integer(), "sqlite")
diff --git a/src/backend/core/db/models/config.py b/src/backend/core/db/models/config.py
index 694beb1b..d16b5346 100644
--- a/src/backend/core/db/models/config.py
+++ b/src/backend/core/db/models/config.py
@@ -2,28 +2,20 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
Integer,
- Numeric,
String,
Text,
- UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from sqlalchemy.orm import relationship
+from core.db.models.column_types import JSONType
class ModelProvider(Base):
diff --git a/src/backend/core/db/models/evolution.py b/src/backend/core/db/models/evolution.py
index a82e3330..f27ade76 100644
--- a/src/backend/core/db/models/evolution.py
+++ b/src/backend/core/db/models/evolution.py
@@ -12,12 +12,9 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
@@ -28,10 +25,7 @@
Text,
UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import JSONB
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-BigIntPK = BigInteger().with_variant(Integer(), "sqlite")
+from core.db.models.column_types import JSONType
class EvolutionEpisode(Base):
@@ -282,7 +276,7 @@ class EvolutionEvaluation(Base):
class EvolutionAgentProfile(Base):
- """A published orchestration profile — see :mod:`core.evolution.agent_profile`.
+ """A published orchestration profile — see :mod:`core.evolution.profiles.agent_profile`.
Stored as one JSON payload rather than a column per knob. The profile's
shape is itself under evolution (a new intervention signal, a new routing
diff --git a/src/backend/core/db/models/identity.py b/src/backend/core/db/models/identity.py
index e02a0a64..865bf8dc 100644
--- a/src/backend/core/db/models/identity.py
+++ b/src/backend/core/db/models/identity.py
@@ -2,29 +2,21 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
- Integer,
- Numeric,
String,
Text,
UniqueConstraint,
text,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from sqlalchemy.orm import relationship
+from core.db.models.column_types import JSONType
class UserShadow(Base):
@@ -47,7 +39,6 @@ class UserShadow(Base):
catalog_overrides = relationship(
"CatalogOverride", back_populates="user", cascade="all, delete-orphan"
)
- kb_spaces = relationship("KBSpace", back_populates="user", cascade="all, delete-orphan")
artifacts = relationship("Artifact", back_populates="user", cascade="all, delete-orphan")
user_agents = relationship(
"UserAgent", foreign_keys="[UserAgent.user_id]", back_populates="user"
@@ -102,7 +93,7 @@ class UserFolder(Base):
"""Personal folder — tree structure under My Space; NULL parent means the root directory.
Constraints such as tree depth and name validation are enforced by UserFolderService
- (see core/services/user_folder_service.py).
+ (see core/projects/user_folders.py).
"""
__tablename__ = "user_folders"
@@ -186,271 +177,7 @@ class UserApiKey(Base):
)
-class DingTalkConnection(Base):
- """Per-user DingTalk account connection (used by the dingtalk skill / dws CLI). 1:1 with users_shadow.
-
- The credentials themselves (OAuth token + encrypted keychain) live on the backend
- persistent volume at ``$STORAGE/dws_cache/{user_id}/``, bind-mounted into the sandbox
- dws's ``$HOME`` — they **never go into the DB**. This table only stores the connection
- **status and a DingTalk identity summary**, plus an optional portable credential bundle
- (``auth_bundle``, for plan-B fallback/migration, stored after application-layer
- encryption). See internal design docs.
- """
-
- __tablename__ = "dingtalk_connections"
-
- user_id = Column(
- String(64),
- ForeignKey("users_shadow.user_id", ondelete="CASCADE"),
- primary_key=True,
- )
- # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error
- status = Column(String(16), nullable=False, default="disconnected")
- dingtalk_user_id = Column(
- String(128)
- ) # DingTalk userId (backfilled from get-self after a successful connect)
- dingtalk_name = Column(String(255)) # DingTalk display name
- corp_id = Column(String(128)) # DingTalk corp corpId
- granted_scopes = Column(
- JSONType, default=list
- ) # List of granted scopes (accumulated via PAT grants)
- # Device-flow login state (echoed to the frontend while pending; cleared on success)
- login_verification_url = Column(
- Text
- ) # Plain verification URL (must be used together with user_code)
- login_verification_url_complete = Column(
- Text
- ) # Full URL with the code embedded → target of the QR code
- login_user_code = Column(String(64))
- login_started_at = Column(UTCDateTime(timezone=True))
- auth_bundle = Column(
- Text
- ) # Plan B: dws auth export --base64 (application-layer encrypted), nullable
- last_verified_at = Column(UTCDateTime(timezone=True))
- last_error = Column(Text)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('disconnected', 'pending', 'connected', 'error')",
- name="dingtalk_connections_status_check",
- ),
- Index("idx_dingtalk_connections_status", "status"),
- )
-
-
-class LarkConnection(Base):
- """Per-user Lark (Feishu) account connection (used by the feishu-cli plugin / lark-cli). 1:1 with users_shadow.
-
- Isomorphic to [[DingTalkConnection]]: the credentials themselves (user_access_token +
- file-based encrypted store, including master.key) live on the backend persistent volume at
- ``$STORAGE/lark_cache/{user_id}/``, bind-mounted into the sandbox lark-cli's ``$HOME``
- (~/.lark-cli + ~/.local/share/lark-cli) — they **never go into the DB**. This table only
- stores the connection status, a Lark identity summary, and the device-flow login state.
- The Lark device flow uses ``auth login --no-wait`` to first obtain device_code/
- verification_url (persisted into login_device_code), then completes with ``--device-code``.
- The app's app_id/secret are seeded into the per-user HOME by the backend via config init
- (not injected as env vars, to avoid breaking --as user).
- See internal design docs.
- """
-
- __tablename__ = "lark_connections"
-
- user_id = Column(
- String(64),
- ForeignKey("users_shadow.user_id", ondelete="CASCADE"),
- primary_key=True,
- )
- # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error
- status = Column(String(16), nullable=False, default="disconnected")
- lark_open_id = Column(
- String(128)
- ) # Lark open_id / union_id (backfilled after a successful connect)
- lark_name = Column(String(255)) # Lark display name
- tenant_key = Column(String(128)) # Lark tenant tenant_key (counterpart of DingTalk corp_id)
- granted_scopes = Column(
- JSONType, default=list
- ) # List of granted scopes (accumulated via incremental grants)
- # Device-flow login state (echoed to the frontend while pending; cleared on success)
- login_verification_url = Column(
- Text
- ) # Plain verification URL (must be used together with user_code)
- login_verification_url_complete = Column(
- Text
- ) # Full URL with the code embedded → target of the QR code
- login_user_code = Column(String(64))
- login_device_code = Column(
- Text
- ) # device_code obtained via --no-wait, used when completing with --device-code
- login_started_at = Column(UTCDateTime(timezone=True))
- auth_bundle = Column(
- Text
- ) # Portable credential bundle (for cube cross-machine use, application-layer encrypted), nullable
- last_verified_at = Column(UTCDateTime(timezone=True))
- last_error = Column(Text)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('disconnected', 'pending', 'connected', 'error')",
- name="lark_connections_status_check",
- ),
- Index("idx_lark_connections_status", "status"),
- )
-
-
-class EmailConnection(Base):
- """Per-user email account connection (used by the email plugin / himalaya CLI). 1:1 with users_shadow.
+# Optional modules own their dependent rows; all cleanups share this transaction.
+from core.db.user_cleanup import install_user_cleanup
- Isomorphic to [[DingTalkConnection]]/[[LarkConnection]], but with **no device flow / OAuth /
- QR code**: email binds via per-user IMAP/SMTP + an authorization code (app password),
- covering Gmail/Outlook/Exchange/NetEase enterprise mail/Tencent enterprise mail/self-hosted
- mailboxes. Binding is **synchronous** ("save the form → write himalaya config.toml → run
- ``himalaya folder list`` to verify"), so the status is only
- disconnected/connected/error (no pending).
-
- The credentials themselves (himalaya ``config.toml``, containing plaintext authorization-code
- lines) live on the backend persistent volume at
- ``$STORAGE/email_cache/{user_id}/home/.config/himalaya/`` (0600), bind-mounted into the
- sandbox himalaya's ``~/.config/himalaya``. This table only stores the connection status +
- account/server metadata + the encrypted authorization code (``secret_enc``,
- application-layer Fernet encryption) + a portable config bundle (``config_bundle``, injected
- per-session when cube has no bind-mount, base64(config.toml)).
- See internal design docs.
- """
-
- __tablename__ = "email_connections"
-
- user_id = Column(
- String(64),
- ForeignKey("users_shadow.user_id", ondelete="CASCADE"),
- primary_key=True,
- )
- # disconnected (default / disconnected) | connected | error (no pending — binding is a synchronous check)
- status = Column(String(16), nullable=False, default="disconnected")
- email_address = Column(String(320)) # Bound email address (= IMAP/SMTP login name)
- display_name = Column(String(255)) # Sender display name
- provider = Column(
- String(32)
- ) # Auto-detected provider: gmail/outlook/netease/qq/qiye163/exmail/custom
-
- # Server settings (auto-detected + user-overridable)
- imap_host = Column(String(255))
- imap_port = Column(Integer)
- imap_security = Column(String(16)) # tls | starttls | none
- smtp_host = Column(String(255))
- smtp_port = Column(Integer)
- smtp_security = Column(String(16))
-
- secret_enc = Column(
- Text
- ) # Authorization code (application-layer Fernet encryption, never stored in plaintext)
- config_bundle = Column(Text) # base64(config.toml), for cube cross-session injection, nullable
- last_verified_at = Column(UTCDateTime(timezone=True))
- last_error = Column(Text)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('disconnected', 'connected', 'error')",
- name="email_connections_status_check",
- ),
- Index("idx_email_connections_status", "status"),
- )
-
-
-class ChannelConnection(Base):
- """Inbound channel bot (owner service-account model). One row = one user-created external IM bot.
-
- Orthogonal to the **outbound** plugins [[LarkConnection]]/[[DingTalkConnection]]: those are
- "the agent operates Lark/DingTalk with the user's OAuth identity"; this table is "an
- external IM pushes messages in to trigger the agent" (**inbound**), and the credentials are
- an **application identity** (App ID/Secret), not user OAuth.
-
- Multi-tenancy uses the owner service-account model: the bot is bound to ``owner_user_id``,
- and all inbound messages run with the owner's identity + permissions; people in a group are
- not tied to platform teams and there is no per-sender identity resolution (open_id is only
- recorded for audit). p2p and group share the same orchestration path; the only difference is
- session keying (by sender open_id vs by group chat_id).
-
- Credentials (App Secret / Encrypt Key / Verification Token) are encrypted with
- application-layer Fernet via ``core.infra.crypto`` and stored in the ``config`` JSON —
- **never stored in plaintext, never unioned into the global environment**. ``app_id`` is
- redundantly stored in plaintext in its own column for the unique constraint (token lock:
- the same Lark app cannot be bound by two people).
-
- See internal design docs.
- """
-
- __tablename__ = "channel_connections"
-
- channel_id = Column(String(64), primary_key=True)
- owner_user_id = Column(
- String(64),
- ForeignKey("users_shadow.user_id", ondelete="CASCADE"),
- nullable=False,
- )
- channel_type = Column(String(16), nullable=False) # 'lark' | 'dingtalk' | 'wecom' | ...
- display_name = Column(String(100), nullable=False, default="我的机器人")
- transport = Column(String(16), nullable=False, default="long_conn") # 'long_conn' | 'webhook'
- app_id = Column(
- String(128), nullable=False
- ) # Plaintext redundancy: for the unique constraint + establishing the long connection
- config = Column(
- JSONType, nullable=False, default=dict
- ) # Encrypted credentials {app_secret_enc, encrypt_key_enc, verification_token_enc}
- # Resource allowlist (for when the owner wants precise control over what the bot exposes):
- # {"kb_ids": [...], "skill_ids": [...]}; NULL = expose everything the owner has (default).
- # Retrieval/skill loading narrows down by this on top of the owner's permissions.
- resource_scope = Column(JSONType, nullable=True)
- # Binding to a specific sub-agent: non-NULL → inbound messages are pinned to that
- # [[UserAgent]] (runs with its own prompt/tools/model); written when binding on the
- # "sub-agent page"; NULL → main agent (owner's default capabilities), left empty when
- # binding in the settings "My Bots" page.
- # ondelete=SET NULL: deleting the sub-agent makes the bot fall back to the main agent
- # automatically, without cascading the bot's deletion.
- agent_id = Column(
- String(64), ForeignKey("user_agents.agent_id", ondelete="SET NULL"), nullable=True
- )
- # Group listening: how much of a group conversation the bot may see.
- # mention_only (default) — only messages that @ the bot; anything else is dropped.
- # observe_all — bystander group messages are recorded as background context
- # (never replied to) and folded into the next @bot turn.
- # observe_all is inert unless the channel app also holds the platform's "read all group
- # messages" permission (Lark ``im:message.group_msg`` / DingTalk group-message read) —
- # without it the platform never delivers non-@ messages in the first place.
- # Defaults to mention_only because observing a whole group's chatter is privacy-sensitive
- # and must be an explicit, per-bot decision by the owner.
- group_listen_mode = Column(
- String(16), nullable=False, default="mention_only", server_default="mention_only"
- )
- # disconnected (default / disconnected) | pending (verifying) | connected | error
- status = Column(String(16), nullable=False, default="pending")
- enabled = Column(Boolean, nullable=False, default=True)
- last_event_at = Column(UTCDateTime(timezone=True))
- last_error = Column(Text)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('disconnected', 'pending', 'connected', 'error')",
- name="channel_connections_status_check",
- ),
- CheckConstraint(
- "transport IN ('long_conn', 'webhook')",
- name="channel_connections_transport_check",
- ),
- CheckConstraint(
- "group_listen_mode IN ('mention_only', 'observe_all')",
- name="channel_connections_group_listen_check",
- ),
- # token lock: within the same channel, the same app can only be bound once (prevents multiple people grabbing the same bot credentials)
- UniqueConstraint("channel_type", "app_id", name="uq_channel_connections_type_app"),
- Index("idx_channel_connections_owner", "owner_user_id"),
- Index("idx_channel_connections_enabled", "enabled", "status"),
- Index("idx_channel_connections_agent", "agent_id"),
- )
+install_user_cleanup(UserShadow)
diff --git a/src/backend/core/db/models/job.py b/src/backend/core/db/models/job.py
index 2c4da1a4..bd2f533f 100644
--- a/src/backend/core/db/models/job.py
+++ b/src/backend/core/db/models/job.py
@@ -18,14 +18,18 @@
from datetime import datetime, timezone
from sqlalchemy import (
- Column, String, Integer, BigInteger, Text, TIMESTAMP,
- ForeignKey, CheckConstraint, Index, JSON
+ Column,
+ String,
+ Integer,
+ BigInteger,
+ Text,
+ ForeignKey,
+ CheckConstraint,
+ Index,
)
-from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import relationship
from core.db.engine import Base
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
+from core.db.models.column_types import JSONType
def _utcnow() -> datetime:
diff --git a/src/backend/core/db/models/kb_wiki.py b/src/backend/core/db/models/kb_wiki.py
deleted file mode 100644
index aff8e0e5..00000000
--- a/src/backend/core/db/models/kb_wiki.py
+++ /dev/null
@@ -1,224 +0,0 @@
-"""SQLAlchemy ORM models — 自建知识库的 LLM Wiki 层。
-
-Wiki 是知识库的**结构地图**:管线从原文分块里抽出实体页与概念页,为每页写一段带
-出处的 Markdown,页与页之间用 ``[[slug|显示名]]`` 互链成图。它不是第二个检索源——
-答案与出处始终回到 ``kb_chunks`` 的原文,页上的 ``source_refs`` / ``chunk_refs``
-就是回原文的血缘坐标。
-
-与外接知识库的 Wiki 透传不同,这三张表与 ``kb_documents`` / ``kb_chunks`` 同库,
-血缘可以按 ID 直接 JOIN 取回,不需要「拉整篇文档再本地过滤」。
-"""
-from core.db.utc_datetime import UTCDateTime
-from core.infra.time import utc_now
-
-from datetime import datetime
-
-from core.db.engine import Base
-from sqlalchemy import (
- JSON,
- CheckConstraint,
- Column,
- ForeignKey,
- Index,
- Integer,
- String,
- Text,
- text,
-)
-from sqlalchemy.dialects.postgresql import JSONB
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-
-# 目录树最大层级。提示词只要 2 级,这里留 3 级是防御性上限:模型偶尔多给一层、
-# 或后续 remap 叠了一层时,不至于长出无边界的面包屑。
-WIKI_CATEGORY_MAX_DEPTH = 3
-
-# 页面类型
-WIKI_PAGE_TYPE_SUMMARY = "summary" # 单篇源文档的摘要页,slug = summary/
-WIKI_PAGE_TYPE_ENTITY = "entity" # 实体页(人/组织/产品/地点/技术/事件)
-WIKI_PAGE_TYPE_CONCEPT = "concept" # 概念页(主题/方法/理论)
-WIKI_PAGE_TYPE_INDEX = "index" # 全库索引页
-WIKI_PAGE_TYPE_SYNTHESIS = "synthesis" # 综述页——管线不产,仅由智能体写入
-WIKI_PAGE_TYPE_COMPARISON = "comparison" # 对比页——管线不产,仅由智能体写入
-
-WIKI_PAGE_TYPES = (
- WIKI_PAGE_TYPE_SUMMARY,
- WIKI_PAGE_TYPE_ENTITY,
- WIKI_PAGE_TYPE_CONCEPT,
- WIKI_PAGE_TYPE_INDEX,
- WIKI_PAGE_TYPE_SYNTHESIS,
- WIKI_PAGE_TYPE_COMPARISON,
-)
-
-# 「知识」视图合并展示的页面类型:这四类形态相近,索引页与文档摘要页各自独立成组
-WIKI_KNOWLEDGE_PAGE_TYPES = (
- WIKI_PAGE_TYPE_ENTITY,
- WIKI_PAGE_TYPE_CONCEPT,
- WIKI_PAGE_TYPE_SYNTHESIS,
- WIKI_PAGE_TYPE_COMPARISON,
-)
-
-# 页面类型 → 中文展示名
-WIKI_PAGE_TYPE_LABELS = {
- WIKI_PAGE_TYPE_CONCEPT: "概念",
- WIKI_PAGE_TYPE_ENTITY: "实体",
- WIKI_PAGE_TYPE_SYNTHESIS: "综述",
- WIKI_PAGE_TYPE_COMPARISON: "对比",
- WIKI_PAGE_TYPE_SUMMARY: "文档摘要",
- WIKI_PAGE_TYPE_INDEX: "总索引",
-}
-
-# 编辑来源:区分「这段是管线写的还是人改的」
-WIKI_EDIT_SOURCE_PIPELINE = "pipeline"
-WIKI_EDIT_SOURCE_AGENT = "agent"
-WIKI_EDIT_SOURCE_USER = "user"
-
-
-class KBWikiFolder(Base):
- """Wiki 目录节点(邻接表)。
-
- ``parent_id`` 为空串表示根层。``path`` 是用 ``/`` 连接的名称链(物化路径),
- 只为展示与筛选服务;归属关系的真源永远是 ``parent_id``。空目录允许独立存在,
- 这样用户可以先搭骨架再往里填页。
- """
-
- __tablename__ = "kb_wiki_folders"
-
- folder_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- parent_id = Column(String(64), nullable=False, default="")
- name = Column(String(255), nullable=False)
- path = Column(String(1024), nullable=False, default="")
- depth = Column(Integer, nullable=False, default=0)
- sort_order = Column(Integer, nullable=False, default=0)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- deleted_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- Index("idx_kb_wiki_folders_kb_parent", "kb_id", "parent_id"),
- Index(
- "idx_kb_wiki_folders_unique_name",
- "kb_id",
- "parent_id",
- "name",
- unique=True,
- postgresql_where=Column("deleted_at").is_(None),
- ),
- )
-
-
-class KBWikiPage(Base):
- """一个 Wiki 页面。
-
- ``folder_id`` 是页面归属目录的唯一真源;``category_path`` / ``wiki_path`` /
- ``depth`` / ``sort_order`` 都是从目录链派生出来的缓存投影,重建目录时会一并刷新。
-
- 血缘两列的语义要分清:``source_refs`` 是该页取材自哪些文档
- (``kb_documents.document_id``),``chunk_refs`` 是具体哪几个父块支撑了页面内容
- (``kb_chunks.chunk_id``)。回溯原文按后者直取,前者用于文档删除时定位受影响页。
- """
-
- __tablename__ = "kb_wiki_pages"
-
- page_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- slug = Column(String(255), nullable=False)
- title = Column(String(512), nullable=False, default="")
- page_type = Column(String(32), nullable=False, default=WIKI_PAGE_TYPE_SUMMARY)
- status = Column(String(32), nullable=False, default="published")
- content = Column(Text, nullable=False, default="")
- summary = Column(Text, nullable=False, default="")
-
- folder_id = Column(String(64), nullable=False, default="")
- category_path = Column(JSONType, default=list)
- wiki_path = Column(String(1024), nullable=False, default="")
- depth = Column(Integer, nullable=False, default=0)
- sort_order = Column(Integer, nullable=False, default=0)
-
- source_refs = Column(JSONType, default=list) # [document_id]
- chunk_refs = Column(JSONType, default=list) # [chunk_id]
- in_links = Column(JSONType, default=list) # [slug],反向链接
- out_links = Column(JSONType, default=list) # [slug],正向链接
- aliases = Column(JSONType, default=list)
- page_metadata = Column(JSONType, default=dict)
-
- version = Column(Integer, nullable=False, default=1)
- last_edit_source = Column(String(16), nullable=False, default=WIKI_EDIT_SOURCE_PIPELINE)
- last_editor_id = Column(String(64), nullable=False, default="")
-
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- deleted_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "status IN ('draft', 'published', 'archived')", name="kb_wiki_pages_status_check"
- ),
- # slug 在单个知识库内唯一,跨库可以重复
- Index(
- "idx_kb_wiki_pages_kb_slug",
- "kb_id",
- "slug",
- unique=True,
- postgresql_where=Column("deleted_at").is_(None),
- ),
- Index("idx_kb_wiki_pages_kb_type", "kb_id", "page_type"),
- Index("idx_kb_wiki_pages_folder", "kb_id", "folder_id"),
- Index("idx_kb_wiki_pages_tree", "kb_id", "page_type", "wiki_path", "sort_order", "title"),
- Index("idx_kb_wiki_pages_updated_at", "updated_at"),
- )
-
-
-class KBWikiJob(Base):
- """Wiki 生成作业。
-
- Wiki 生成是分钟级到小时级的长任务,挂在请求生命周期上(BackgroundTasks)会在
- 容器重启时丢一半,所以状态必须落库:worker 认领后写心跳,重启时按心跳超时回收
- 僵尸作业重跑。``progress`` 供前端直接渲染进度,不用另开接口。
- """
-
- __tablename__ = "kb_wiki_jobs"
-
- job_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- job_type = Column(String(20), nullable=False) # ingest | retract | finalize
- payload = Column(JSONType, default=dict) # {"document_ids": [...]}
- status = Column(String(20), nullable=False, default="pending")
- attempt = Column(Integer, nullable=False, default=0)
- max_attempts = Column(Integer, nullable=False, default=3)
- last_error = Column(Text)
- progress = Column(JSONType, default=dict) # {"stage": ..., "done": n, "total": m}
-
- claimed_by = Column(String(64))
- claimed_at = Column(UTCDateTime(timezone=True))
- heartbeat_at = Column(UTCDateTime(timezone=True))
-
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- finished_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "job_type IN ('ingest', 'retract', 'finalize')", name="kb_wiki_jobs_type_check"
- ),
- CheckConstraint(
- "status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')",
- name="kb_wiki_jobs_status_check",
- ),
- Index("idx_kb_wiki_jobs_status", "status", "created_at"),
- Index("idx_kb_wiki_jobs_kb", "kb_id", "status"),
- # 同一知识库只保留一个未完成的 finalize:多篇文档摄入完只需要收尾一次。
- # 条件里必须带 status,否则一次成功的 finalize 会永久占位、再也排不进下一次。
- #
- # 两个方言的 where 都要给:只写 postgresql_where 时,SQLite 会把谓词整个
- # 丢掉、退化成 UNIQUE(kb_id)——那等于「一个知识库只能有一个作业」,本地
- # 开发和测试库上批量摄入会直接撞唯一约束。
- Index(
- "idx_kb_wiki_jobs_pending_finalize",
- "kb_id",
- unique=True,
- postgresql_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"),
- sqlite_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"),
- ),
- )
diff --git a/src/backend/core/db/models/knowledge.py b/src/backend/core/db/models/knowledge.py
index 4fbef84d..17fddd7f 100644
--- a/src/backend/core/db/models/knowledge.py
+++ b/src/backend/core/db/models/knowledge.py
@@ -2,198 +2,20 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
Integer,
- Numeric,
String,
- Text,
UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
-
-
-class KBSpace(Base):
- """Knowledge base space table."""
-
- __tablename__ = "kb_spaces"
-
- kb_id = Column(String(64), primary_key=True)
- user_id = Column(
- String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False
- )
- name = Column(String(255), nullable=False)
- description = Column(Text)
- document_count = Column(Integer, default=0)
- total_size_bytes = Column(BigInteger, default=0)
- visibility = Column(String(16), nullable=False, default="private")
- chunk_method = Column(String(32), nullable=False, default="semantic")
- extra_data = Column("metadata", JSONType, default={})
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
- deleted_at = Column(UTCDateTime(timezone=True))
-
- # Relationships
- user = relationship("UserShadow", back_populates="kb_spaces")
- documents = relationship("KBDocument", back_populates="kb_space", cascade="all, delete-orphan")
-
- __table_args__ = (
- CheckConstraint("length(name) >= 1 AND length(name) <= 255", name="kb_spaces_name_length"),
- CheckConstraint("document_count >= 0", name="kb_spaces_document_count_check"),
- CheckConstraint("total_size_bytes >= 0", name="kb_spaces_total_size_check"),
- CheckConstraint(
- "visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check"
- ),
- Index("idx_kb_spaces_user_id", "user_id"),
- Index("idx_kb_spaces_updated_at", "updated_at"),
- Index(
- "idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)
- ),
- Index("idx_kb_spaces_visibility", "visibility"),
- )
-
-
-class KBDocument(Base):
- """Knowledge base document table."""
-
- __tablename__ = "kb_documents"
-
- document_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- title = Column(String(500), nullable=False)
- filename = Column(String(500), nullable=False)
- size_bytes = Column(BigInteger, nullable=False)
- mime_type = Column(String(100), nullable=False)
- storage_key = Column(Text, nullable=False)
- storage_url = Column(Text)
- checksum = Column(String(64))
- indexing_status = Column(
- String(20), nullable=False, default="processing"
- ) # processing | completed | failed
- extra_data = Column("metadata", JSONType, default={})
- uploaded_at = Column(UTCDateTime(timezone=True), default=utc_now)
- deleted_at = Column(UTCDateTime(timezone=True))
-
- # Relationships
- kb_space = relationship("KBSpace", back_populates="documents")
- chunks = relationship("KBChunk", back_populates="document", cascade="all, delete-orphan")
-
- __table_args__ = (
- CheckConstraint("size_bytes > 0", name="kb_documents_size_check"),
- CheckConstraint("length(filename) >= 1", name="kb_documents_filename_length"),
- Index("idx_kb_documents_kb_id", "kb_id"),
- Index("idx_kb_documents_uploaded_at", "uploaded_at"),
- Index("idx_kb_documents_kb_uploaded", "kb_id", "uploaded_at"),
- Index(
- "idx_kb_documents_deleted",
- "deleted_at",
- postgresql_where=Column("deleted_at").isnot(None),
- ),
- Index("idx_kb_documents_metadata_gin", "metadata", postgresql_using="gin"),
- )
-
-
-class KBChunk(Base):
- """Knowledge base chunk table - stores parent chunks for context retrieval.
-
- Each document is split into parent chunks (stored here) and child chunks
- (vectorised in Milvus hugagent_kb_private collection). Retrieval finds child
- chunks via vector search, then fetches the parent content from this table.
- """
-
- __tablename__ = "kb_chunks"
-
- chunk_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- document_id = Column(
- String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False
- )
- chunk_index = Column(Integer, nullable=False)
- content = Column(
- Text, nullable=False
- ) # parent chunk original text, returned to the LLM on retrieval hit
- tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"]
- questions = Column(JSONType, default=list) # associated question list (array of strings)
- char_start = Column(Integer)
- char_end = Column(Integer)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- # Relationships
- document = relationship("KBDocument", back_populates="chunks")
-
- __table_args__ = (
- Index("idx_kb_chunks_kb_id", "kb_id"),
- Index("idx_kb_chunks_document_id", "document_id"),
- Index("idx_kb_chunks_kb_doc", "kb_id", "document_id"),
- )
-
-
-class KBAsset(Base):
- """Non-text media cut out of a knowledge-base document (图片 / 后续音视频).
-
- One row per figure the layout engine extracted (or per standalone uploaded image).
- The bytes live in object storage under ``storage_key``; this table holds the
- linkage (which document / which parent chunk), where it sits in the source
- (``locator``), and the two text projections retrieval runs on:
-
- ``text_content`` literal text carried by the medium — 图注 / OCR 文字;音视频接入后是 ASR 转写。
- ``caption`` 语义描述,由 VLM 生成(未配置 vlm 角色时为空,检索退化为只用 text_content)。
-
- Deliberately medium-agnostic so audio/video need no migration: ``kind`` widens,
- ``locator`` holds ``{page_idx, bbox}`` for images and would hold ``{start_ms,
- end_ms}`` for a transcript segment.
-
- ``vector_state`` records which vector spaces this asset has been indexed into
- (``{"text": "ok"}`` today). Path B — a true multimodal embedding in its own
- collection — adds a second key here instead of another column.
- """
-
- __tablename__ = "kb_assets"
-
- asset_id = Column(String(64), primary_key=True)
- kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
- document_id = Column(
- String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False
- )
- # Owning parent chunk. Nullable: an asset whose placeholder cannot be located in
- # any chunk is still worth keeping (it stays retrievable on its own row).
- chunk_id = Column(String(64))
- kind = Column(String(16), nullable=False, default="image") # image | audio | video
- mime_type = Column(String(100), nullable=False, default="application/octet-stream")
- storage_key = Column(Text, nullable=False)
- size_bytes = Column(BigInteger, default=0)
- asset_index = Column(Integer, nullable=False, default=0) # order within the document
- locator = Column(JSONType, default=dict) # image: {page_idx, bbox}; av: {start_ms, end_ms}
- caption = Column(Text) # VLM 语义描述
- text_content = Column(Text) # 图注 / OCR / ASR 转写
- caption_status = Column(
- String(20), nullable=False, default="pending"
- ) # pending | completed | skipped | failed
- vector_state = Column(JSONType, default=dict) # {"text": "ok"} —— 路径 B 在此加键
- extra_data = Column("metadata", JSONType, default={})
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
- __table_args__ = (
- CheckConstraint("kind IN ('image', 'audio', 'video')", name="kb_assets_kind_check"),
- Index("idx_kb_assets_kb_id", "kb_id"),
- Index("idx_kb_assets_document_id", "document_id"),
- Index("idx_kb_assets_chunk_id", "chunk_id"),
- )
+from sqlalchemy.orm import relationship
+from core.db.models.column_types import JSONType
class CatalogOverride(Base):
diff --git a/src/backend/core/db/models/logs.py b/src/backend/core/db/models/logs.py
index ff4b90de..f88dcf87 100644
--- a/src/backend/core/db/models/logs.py
+++ b/src/backend/core/db/models/logs.py
@@ -6,28 +6,17 @@
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
Integer,
- Numeric,
String,
Text,
UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
-# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not
-# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite
-# profile. Fall back to Integer on SQLite (still BIGINT on Postgres).
-BigIntPK = BigInteger().with_variant(Integer(), "sqlite")
+from core.db.models.column_types import JSONType, BigIntPK
class ToolCallLog(Base):
diff --git a/src/backend/core/db/models/memory.py b/src/backend/core/db/models/memory.py
index 35134acf..89c33245 100644
--- a/src/backend/core/db/models/memory.py
+++ b/src/backend/core/db/models/memory.py
@@ -2,32 +2,19 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
- ForeignKey,
Index,
Integer,
- Numeric,
String,
Text,
UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
-# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not
-# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite
-# profile. Fall back to Integer on SQLite (still BIGINT on Postgres).
-BigIntPK = BigInteger().with_variant(Integer(), "sqlite")
+from core.db.models.column_types import JSONType, BigIntPK
class ProfileMemory(Base):
diff --git a/src/backend/core/db/models/observability.py b/src/backend/core/db/models/observability.py
deleted file mode 100644
index 6032705c..00000000
--- a/src/backend/core/db/models/observability.py
+++ /dev/null
@@ -1,61 +0,0 @@
-"""Durable desktop delivery references and non-executable cloud management copies."""
-
-from datetime import datetime, timezone
-
-from core.db.engine import Base
-from sqlalchemy import JSON, BigInteger, Boolean, Column, Index, Integer, String, Text
-
-
-def utcnow():
- return datetime.now(timezone.utc).isoformat()
-
-
-class DesktopOutbox(Base):
- __tablename__ = "desktop_observability_outbox"
- id = Column(String(64), primary_key=True)
- cloud_base = Column(String(512), nullable=False)
- subject = Column(String(64), nullable=False)
- device_id = Column(String(128), nullable=False)
- local_user_id = Column(String(64), nullable=False)
- kind = Column(String(24), nullable=False)
- object_id = Column(String(128), nullable=False)
- revision = Column(BigInteger, nullable=False, default=1)
- pending = Column(Boolean, nullable=False, default=True)
- deleted = Column(Boolean, nullable=False, default=False)
- digest = Column(String(64), nullable=False, default="")
- updated_at = Column(String(40), nullable=False, default=utcnow)
- __table_args__ = (
- Index("idx_desktop_outbox_pending", "cloud_base", "subject", "device_id", "pending"),
- )
-
-
-class DesktopRecord(Base):
- __tablename__ = "desktop_observability_records"
- id = Column(String(64), primary_key=True)
- user_id = Column(String(64), nullable=False)
- device_id = Column(String(128), nullable=False)
- kind = Column(String(24), nullable=False)
- object_id = Column(String(128), nullable=False)
- revision = Column(BigInteger, nullable=False)
- chat_id = Column(String(128), nullable=False, default="")
- run_id = Column(String(128), nullable=False, default="")
- observation = Column(String(16), nullable=False, default="desktop")
- payload = Column(JSON, nullable=False)
- received_at = Column(String(40), nullable=False, default=utcnow)
- __table_args__ = (
- Index("idx_desktop_records_identity", "kind", "observation", "user_id", "device_id", "object_id", unique=True),
- Index("idx_desktop_records_user_kind", "user_id", "kind", "received_at"),
- Index("idx_desktop_records_chat", "user_id", "device_id", "chat_id"),
- )
-
-
-class DesktopSyncDevice(Base):
- __tablename__ = "desktop_observability_devices"
- id = Column(String(64), primary_key=True)
- user_id = Column(String(64), nullable=False)
- device_id = Column(String(128), nullable=False)
- last_seen = Column(String(40), nullable=False, default=utcnow)
- pending = Column(Integer, nullable=False, default=0)
- capture_errors = Column(Integer, nullable=False, default=0)
- reconciling = Column(Boolean, nullable=False, default=False)
- __table_args__ = (Index("idx_desktop_devices_user", "user_id"),)
diff --git a/src/backend/core/db/models/ontology.py b/src/backend/core/db/models/ontology.py
index 435d890b..8f85169b 100644
--- a/src/backend/core/db/models/ontology.py
+++ b/src/backend/core/db/models/ontology.py
@@ -2,11 +2,9 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime
from core.db.engine import Base
from sqlalchemy import (
- JSON,
Boolean,
CheckConstraint,
Column,
@@ -18,9 +16,7 @@
UniqueConstraint,
text,
)
-from sqlalchemy.dialects.postgresql import JSONB
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
+from core.db.models.column_types import JSONType
class OntologyPack(Base):
diff --git a/src/backend/core/db/models/project.py b/src/backend/core/db/models/project.py
index 43bbd444..d001185f 100644
--- a/src/backend/core/db/models/project.py
+++ b/src/backend/core/db/models/project.py
@@ -2,29 +2,19 @@
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-from datetime import datetime, timezone
from core.db.engine import Base
-from core.db.model_extensions import ProjectEditionFields, project_edition_table_args
+from core.db.edition import ProjectEditionFields, project_edition_table_args
from sqlalchemy import (
- JSON,
- BigInteger,
Boolean,
CheckConstraint,
Column,
ForeignKey,
Index,
- Integer,
- Numeric,
String,
Text,
- UniqueConstraint,
)
-from sqlalchemy.dialects.postgresql import INET, JSONB
-from sqlalchemy.orm import mapped_column, relationship
-
-JSONType = JSON().with_variant(JSONB(), "postgresql")
-INETType = String(45).with_variant(INET(), "postgresql")
+from core.db.models.column_types import JSONType
class Project(ProjectEditionFields, Base):
diff --git a/src/backend/core/db/personal_file_names.py b/src/backend/core/db/personal_file_names.py
index 38cc4d3a..3e637248 100644
--- a/src/backend/core/db/personal_file_names.py
+++ b/src/backend/core/db/personal_file_names.py
@@ -8,7 +8,9 @@
def personal_file_indexes(*, enterprise=False):
- scope = ""
+ from core.db.edition import personal_index_scope
+
+ scope = personal_index_scope(enterprise)
return tuple(
Index(
name, *columns, unique=True,
@@ -37,7 +39,7 @@ def validate_filename(filename):
def check_name(db, user_id, folder_id, filename, exclude_id=None, exclude_folder=None):
from core.db.models import Artifact, UserFolder
- from core.services.artifact_edition import personal_artifact_predicates
+ from core.db.artifact_scope import personal_artifact_predicates
validate_filename(filename)
with db.no_autoflush:
@@ -63,8 +65,8 @@ def check_name(db, user_id, folder_id, filename, exclude_id=None, exclude_folder
@event.listens_for(Session, "before_flush")
def validate_personal_filenames(db, flush_context, instances):
from core.db.models import Artifact, UserFolder, UserShadow
- from core.services.artifact_edition import is_personal_artifact
- from core.services.folder_batch import lock_folder_owner
+ from core.db.artifact_scope import is_personal_artifact
+ from core.db.repository.folder import lock_folder_owner
candidates = []
for row in db.new.union(db.dirty):
@@ -72,6 +74,9 @@ def validate_personal_filenames(db, flush_context, instances):
fields = ("user_id", "parent_folder_id", "name", "deleted_at")
elif isinstance(row, Artifact) and is_personal_artifact(row):
fields = ("user_id", "user_folder_id", "filename", "deleted_at")
+ from core.db.edition import personal_artifact_scope_fields
+
+ fields += tuple(name for name in personal_artifact_scope_fields() if name in inspect(row).attrs)
else:
continue
if row.deleted_at is None and (row in db.new or any(
diff --git a/src/backend/core/db/model_extensions.py b/src/backend/core/db/personal_model_fields.py
similarity index 100%
rename from src/backend/core/db/model_extensions.py
rename to src/backend/core/db/personal_model_fields.py
diff --git a/src/backend/core/db/edition_tables.py b/src/backend/core/db/personal_schema.py
similarity index 97%
rename from src/backend/core/db/edition_tables.py
rename to src/backend/core/db/personal_schema.py
index a7bd1abf..53848c11 100644
--- a/src/backend/core/db/edition_tables.py
+++ b/src/backend/core/db/personal_schema.py
@@ -19,7 +19,7 @@
}
-def _ce_metadata():
+def ce_metadata():
"""Build metadata with foreign keys to physically omitted tables removed."""
import core.db.models # noqa: F401
from core.db.engine import Base
@@ -44,7 +44,7 @@ def _ce_metadata():
def ce_create_all(bind) -> list[str]:
"""Create the CE metadata and remove foreign keys to omitted edition tables."""
- clone = _ce_metadata()
+ clone = ce_metadata()
clone.create_all(bind=bind)
return sorted(clone.tables)
@@ -55,6 +55,6 @@ def ce_reconcile_schema(bind) -> dict[str, list[str]]:
return reconcile_metadata_schema(
bind,
- _ce_metadata(),
+ ce_metadata(),
bootstrap_server_defaults=_CE_BOOTSTRAP_SERVER_DEFAULTS,
)
diff --git a/src/backend/core/db/repository/__init__.py b/src/backend/core/db/repository/__init__.py
index 32654911..4cda10c1 100644
--- a/src/backend/core/db/repository/__init__.py
+++ b/src/backend/core/db/repository/__init__.py
@@ -1,22 +1,33 @@
-"""Community-edition repository exports."""
+"""Data access layer — Repository pattern.
-from core.db.repository.agent import UserAgentRepository
+Repositories are organised into domain submodules; this package re-exports
+every public class so existing imports (``from core.db.repository import
+XxxRepository``) keep working.
+"""
+
+from core.db.edition import UserAgentRepository
from core.db.repository.artifact import ROOT_FOLDER_SENTINEL, ArtifactRepository
-from core.db.repository.audit import AuditLogRepository
+from core.db.edition import AuditLogRepository
from core.db.repository.catalog import CatalogRepository
-from core.db.repository.channel import ChannelConnectionRepository
+
from core.db.repository.chat import ChatMessageRepository, ChatSessionRepository
-from core.db.repository.kb import KBRepository
+
from core.db.repository.ontology import OntologyRepository
-from core.db.repository.site import SiteRepository
-from core.db.repository.user import (
- DingTalkConnectionRepository,
- EmailConnectionRepository,
- LarkConnectionRepository,
- LocalUserRepository,
- UserRepository,
-)
+from core.db.repository.user import LocalUserRepository, UserRepository
+
+
+
+
__all__ = [
- name for name in globals() if name.endswith("Repository") or name == "ROOT_FOLDER_SENTINEL"
+ "UserRepository",
+ "LocalUserRepository",
+ "ChatSessionRepository",
+ "ChatMessageRepository",
+ "CatalogRepository",
+ "ArtifactRepository",
+ "ROOT_FOLDER_SENTINEL",
+ "AuditLogRepository",
+ "UserAgentRepository",
+ "OntologyRepository",
]
diff --git a/src/backend/core/db/repository/artifact.py b/src/backend/core/db/repository/artifact.py
index c462779a..ad0d45b8 100644
--- a/src/backend/core/db/repository/artifact.py
+++ b/src/backend/core/db/repository/artifact.py
@@ -6,7 +6,6 @@
"""
from core.infra.time import utc_now
-import logging
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -120,10 +119,9 @@ def list_by_user_with_chat(
)
if personal_only:
- # Import lazily: ``core.services`` re-exports repositories, so importing
- # this edition seam while the repository package is initializing would
- # create a package-level cycle.
- from core.services.artifact_edition import personal_artifact_predicates
+ # Import lazily: this edition seam must not load while the repository
+ # package is still initializing.
+ from core.db.artifact_scope import personal_artifact_predicates
query = query.filter(*personal_artifact_predicates(Artifact))
if folder_id == ROOT_FOLDER_SENTINEL:
diff --git a/src/backend/core/db/repository/catalog.py b/src/backend/core/db/repository/catalog.py
index d17f4baf..36d222e4 100644
--- a/src/backend/core/db/repository/catalog.py
+++ b/src/backend/core/db/repository/catalog.py
@@ -6,13 +6,10 @@
"""
from core.infra.time import utc_now
-import logging
from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Optional
-import sqlalchemy as sa
from core.db.models import CatalogOverride
-from sqlalchemy import and_, desc, func, or_, select
from sqlalchemy.orm import Session
diff --git a/src/backend/core/db/repository/chat.py b/src/backend/core/db/repository/chat.py
index 088af4e2..67ea5b28 100644
--- a/src/backend/core/db/repository/chat.py
+++ b/src/backend/core/db/repository/chat.py
@@ -6,7 +6,6 @@
"""
from core.infra.time import utc_now
-import logging
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -14,7 +13,7 @@
from core.db.reserved_chats import ordinary_chat_filter, reserved_chat_error, reserved_chat_owner
from core.db.models import ChatMessage, ChatSession
from core.db.models.chat import reserve_chat_message_sequences
-from sqlalchemy import and_, desc, func, or_, select
+from sqlalchemy import desc, func, or_
from sqlalchemy.orm import Session
@@ -240,18 +239,6 @@ def count_visible_through_seq(self, chat_id: str, covered_seq: int) -> int:
stable sequence boundary in the visible history without exposing the
checkpoint row or its replacement payload.
"""
- return int(
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.role != "system",
- ChatMessage.chat_seq <= covered_seq,
- )
- .count()
- )
-
- def count_visible_through_seq(self, chat_id: str, covered_seq: int) -> int:
- """Count user-facing rows at or below an exact compaction watermark."""
return int(
self.db.query(ChatMessage)
.filter(
diff --git a/src/backend/core/db/repository/folder.py b/src/backend/core/db/repository/folder.py
new file mode 100644
index 00000000..34547f29
--- /dev/null
+++ b/src/backend/core/db/repository/folder.py
@@ -0,0 +1,40 @@
+"""Transaction-scoped owner locks shared by folder and artifact writers."""
+import hashlib
+from threading import Lock
+from sqlalchemy import event
+from sqlalchemy.orm import Session
+
+_SQLITE_LOCKS = [Lock() for _ in range(64)]
+
+
+def _release_sqlite_locks(session, transaction):
+ if transaction.parent is None:
+ for lock in session.info.pop("folder_upload_locks", []):
+ lock.release()
+
+
+def lock_folder_owner(db: Session, owner_model, owner_column: str, owner_id: str) -> None:
+ if db.get_bind().dialect.name == "sqlite":
+ # SQLite ignores FOR UPDATE. Its supported single-process deployment still
+ # serves requests on multiple threads, so retain a bounded process lock
+ # until commit/rollback/close, including early-return and error paths.
+ if not db.in_transaction():
+ db.begin()
+ if not db.info.get("folder_upload_lock_listener"):
+ event.listen(db, "after_transaction_end", _release_sqlite_locks)
+ db.info["folder_upload_lock_listener"] = True
+ digest = hashlib.sha256((owner_model.__tablename__ + ":" + owner_id).encode()).digest()
+ lock = _SQLITE_LOCKS[int.from_bytes(digest[:2], "big") % len(_SQLITE_LOCKS)]
+ held = db.info.setdefault("folder_upload_locks", [])
+ if lock not in held:
+ lock.acquire()
+ held.append(lock)
+ # NO KEY UPDATE does not conflict with FK KEY SHARE taken by artifact inserts.
+ owner = (
+ db.query(owner_model)
+ .filter(getattr(owner_model, owner_column) == owner_id)
+ .with_for_update(key_share=True)
+ .first()
+ )
+ if owner is None:
+ raise ValueError("文件夹所属空间不存在")
diff --git a/src/backend/core/db/repository/agent.py b/src/backend/core/db/repository/personal_agent.py
similarity index 100%
rename from src/backend/core/db/repository/agent.py
rename to src/backend/core/db/repository/personal_agent.py
diff --git a/src/backend/core/db/repository/audit.py b/src/backend/core/db/repository/personal_audit.py
similarity index 96%
rename from src/backend/core/db/repository/audit.py
rename to src/backend/core/db/repository/personal_audit.py
index 98018da9..a82365d6 100644
--- a/src/backend/core/db/repository/audit.py
+++ b/src/backend/core/db/repository/personal_audit.py
@@ -1,6 +1,6 @@
"""No-storage audit repository for the community edition."""
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
class AuditLogRepository:
diff --git a/src/backend/core/db/repository/user.py b/src/backend/core/db/repository/user.py
index 0cedef37..ae2c16ce 100644
--- a/src/backend/core/db/repository/user.py
+++ b/src/backend/core/db/repository/user.py
@@ -6,19 +6,10 @@
"""
from core.infra.time import utc_now
-import logging
from datetime import datetime
-from typing import Any, Dict, List, Optional
-
-import sqlalchemy as sa
-from core.db.models import (
- DingTalkConnection,
- EmailConnection,
- LarkConnection,
- LocalUser,
- UserShadow,
-)
-from sqlalchemy import and_, desc, func, or_, select
+from typing import Any, Dict, Optional
+
+from core.db.models import LocalUser, UserShadow
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -150,103 +141,3 @@ def set_status(self, user_id: str, status: str) -> bool:
record.updated_at = utc_now()
self.db.commit()
return True
-
-
-class DingTalkConnectionRepository:
- """Per-user DingTalk connection (dingtalk_connections) CRUD. 1:1 with users_shadow, no soft delete."""
-
- def __init__(self, db: Session):
- self.db = db
-
- def get(self, user_id: str) -> Optional[DingTalkConnection]:
- return (
- self.db.query(DingTalkConnection).filter(DingTalkConnection.user_id == user_id).first()
- )
-
- def ensure(self, user_id: str) -> DingTalkConnection:
- """Idempotent: return existing record, otherwise create a disconnected record."""
- record = self.get(user_id)
- if record:
- return record
- record = DingTalkConnection(user_id=user_id, status="disconnected")
- self.db.add(record)
- self.db.commit()
- self.db.refresh(record)
- return record
-
- def update(self, user_id: str, data: Dict[str, Any]) -> Optional[DingTalkConnection]:
- record = self.get(user_id)
- if not record:
- return None
- for k, v in data.items():
- setattr(record, k, v)
- record.updated_at = utc_now()
- self.db.commit()
- self.db.refresh(record)
- return record
-
-
-class LarkConnectionRepository:
- """Per-user Lark connection (lark_connections) CRUD. 1:1 with users_shadow, no soft delete.
- Structurally identical to [[DingTalkConnectionRepository]]."""
-
- def __init__(self, db: Session):
- self.db = db
-
- def get(self, user_id: str) -> Optional[LarkConnection]:
- return self.db.query(LarkConnection).filter(LarkConnection.user_id == user_id).first()
-
- def ensure(self, user_id: str) -> LarkConnection:
- """Idempotent: return existing record, otherwise create a disconnected record."""
- record = self.get(user_id)
- if record:
- return record
- record = LarkConnection(user_id=user_id, status="disconnected")
- self.db.add(record)
- self.db.commit()
- self.db.refresh(record)
- return record
-
- def update(self, user_id: str, data: Dict[str, Any]) -> Optional[LarkConnection]:
- record = self.get(user_id)
- if not record:
- return None
- for k, v in data.items():
- setattr(record, k, v)
- record.updated_at = utc_now()
- self.db.commit()
- self.db.refresh(record)
- return record
-
-
-class EmailConnectionRepository:
- """Per-user email connection (email_connections) CRUD. 1:1 with users_shadow, no soft delete.
- Structurally identical to [[LarkConnectionRepository]]."""
-
- def __init__(self, db: Session):
- self.db = db
-
- def get(self, user_id: str) -> Optional[EmailConnection]:
- return self.db.query(EmailConnection).filter(EmailConnection.user_id == user_id).first()
-
- def ensure(self, user_id: str) -> EmailConnection:
- """Idempotent: return existing record, otherwise create a disconnected record."""
- record = self.get(user_id)
- if record:
- return record
- record = EmailConnection(user_id=user_id, status="disconnected")
- self.db.add(record)
- self.db.commit()
- self.db.refresh(record)
- return record
-
- def update(self, user_id: str, data: Dict[str, Any]) -> Optional[EmailConnection]:
- record = self.get(user_id)
- if not record:
- return None
- for k, v in data.items():
- setattr(record, k, v)
- record.updated_at = utc_now()
- self.db.commit()
- self.db.refresh(record)
- return record
diff --git a/src/backend/core/db/site_storage_retirement.py b/src/backend/core/db/site_storage_retirement.py
index 2769e9eb..51d463b3 100644
--- a/src/backend/core/db/site_storage_retirement.py
+++ b/src/backend/core/db/site_storage_retirement.py
@@ -20,7 +20,7 @@ def retire_site_storage(engine, directory: Path):
return retire_connection(connection, directory, strict=False)
-def _digest(payload):
+def digest(payload):
return hashlib.sha256(
json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
@@ -50,7 +50,7 @@ def retire_connection(connection, directory: Path, *, strict=True):
required = {row["site_id"] for rows in payload.values() for row in rows}
if (
receipt.get("verified") is True
- and receipt.get("legacy_sha256") == _digest(payload)
+ and receipt.get("legacy_sha256") == digest(payload)
and required <= covered
):
verified = True
@@ -66,7 +66,7 @@ def retire_connection(connection, directory: Path, *, strict=True):
archive = directory / f"site-storage-{uuid4().hex}.json"
manifest = {
"revision": revision,
- "sha256": _digest(payload),
+ "sha256": digest(payload),
"tables": payload,
"created_at": datetime.now(timezone.utc).isoformat(),
}
@@ -76,7 +76,7 @@ def retire_connection(connection, directory: Path, *, strict=True):
handle.flush()
os.fsync(handle.fileno())
verified = json.loads(archive.read_text(encoding="utf-8"))
- if verified["sha256"] != _digest(verified["tables"]) or verified["tables"] != payload:
+ if verified["sha256"] != digest(verified["tables"]) or verified["tables"] != payload:
raise RuntimeError("Retired storage backup verification failed")
for name in existing:
sa.Table(name, sa.MetaData(), autoload_with=connection).drop(connection)
diff --git a/src/backend/core/db/system_configuration.py b/src/backend/core/db/system_configuration.py
new file mode 100644
index 00000000..1e988a97
--- /dev/null
+++ b/src/backend/core/db/system_configuration.py
@@ -0,0 +1,146 @@
+"""Database persistence and seed migration for system configuration."""
+from __future__ import annotations
+import logging
+from typing import Optional
+from core.db.engine import SessionLocal
+from core.db.models import SystemConfig
+from core.config.store import system_config_constants
+
+logger = logging.getLogger(__name__)
+
+
+class SystemConfigurationStore:
+ def set(self, key: str, value: str | None, updated_by: str = "admin") -> None:
+ """Update a config value in DB."""
+ try:
+ db = SessionLocal()
+ try:
+ row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first()
+ if row is None:
+ logger.warning("[SystemConfigService] key %s not found, skipping set", key)
+ return
+ row.config_value = value
+ row.updated_by = updated_by
+ db.commit()
+ finally:
+ db.close()
+ except Exception as exc:
+ logger.error("[SystemConfigService] set(%s) failed: %s", key, exc)
+ raise
+ def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None:
+ """Batch update multiple config values. Each item: {key, value}.
+
+ For secret fields, masked values (containing '****') are skipped to
+ prevent the frontend from accidentally overwriting real secrets with
+ their masked representation.
+ """
+ try:
+ db = SessionLocal()
+ try:
+ for item in items:
+ key = item.get("key", "").strip()
+ value = item.get("value")
+ if not key:
+ continue
+ row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first()
+ if row is None:
+ continue
+ # Skip masked values for secret fields
+ if row.is_secret and isinstance(value, str) and "****" in value:
+ continue
+ row.config_value = value if value != "" else None
+ row.updated_by = updated_by
+ db.commit()
+ finally:
+ db.close()
+ except Exception as exc:
+ logger.error("[SystemConfigService] bulk_set failed: %s", exc)
+ raise
+ def _ensure_seed_rows(self, db) -> None:
+ """Insert any missing seed config rows. Idempotent."""
+ rows_by_key = {row.config_key: row for row in db.query(SystemConfig).all()}
+ existing_keys = set(rows_by_key)
+ inserted_keys: set[str] = set()
+ inserted = 0
+ for key, val, display, desc, group, secret in system_config_constants.SEED_CONFIGS:
+ if key not in existing_keys:
+ row = SystemConfig(
+ config_key=key,
+ config_value=val,
+ display_name=display,
+ description=desc,
+ group_key=group,
+ is_secret=secret,
+ )
+ db.add(row)
+ rows_by_key[key] = row
+ inserted_keys.add(key)
+ inserted += 1
+ migrated = self._migrate_legacy_knowledge_config(rows_by_key, inserted_keys)
+ if inserted or migrated:
+ db.commit()
+ logger.info(
+ "[SystemConfigService] Inserted %d missing seed row(s), migrated %d legacy KB value(s)",
+ inserted,
+ migrated,
+ )
+ @staticmethod
+ def _migrate_legacy_knowledge_config(
+ rows_by_key: dict[str, SystemConfig],
+ inserted_keys: set[str],
+ ) -> int:
+ """Move the old shared KB connection into one provider exactly once.
+
+ Older releases stored every external backend in the shared
+ knowledge_base.url/api_key/allowed_dataset_ids rows. Those rows cannot
+ prove which dropdown option they belonged to, and copying them into the
+ currently selected provider can reproduce the original cross-provider
+ leak. Treat them as Dify compatibility values and leave FastGPT/WeKnora
+ empty until each provider is configured explicitly.
+
+ inserted_keys is the one-time guard: after scoped rows exist, later
+ legacy edits can never overwrite provider-specific values.
+ """
+ migrated = 0
+ for field in system_config_constants.PROVIDER_SCOPED_KNOWLEDGE_FIELDS:
+ destination_key = f"knowledge_base.dify.{field}"
+ if destination_key not in inserted_keys:
+ continue
+ destination = rows_by_key.get(destination_key)
+ legacy = rows_by_key.get(f"knowledge_base.{field}")
+ legacy_value = getattr(legacy, "config_value", None)
+ if (
+ destination is not None
+ and not getattr(destination, "config_value", None)
+ and legacy_value not in (None, "")
+ ):
+ destination.config_value = legacy_value
+ migrated += 1
+ return migrated
+ def load(self):
+ new_cache: dict[str, Optional[str]] = {}
+ new_meta: dict[str, dict] = {}
+ try:
+ db = SessionLocal()
+ try:
+ self._ensure_seed_rows(db)
+ rows = db.query(SystemConfig).all()
+ for row in rows:
+ new_cache[row.config_key] = row.config_value
+ new_meta[row.config_key] = {
+ "config_key": row.config_key,
+ "config_value": row.config_value,
+ "display_name": row.display_name,
+ "description": row.description,
+ "group_key": row.group_key,
+ "is_secret": row.is_secret,
+ "updated_at": row.updated_at.isoformat() if row.updated_at else None,
+ "updated_by": row.updated_by,
+ }
+ finally:
+ db.close()
+ except Exception as exc:
+ logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", exc)
+ raise
+
+ return new_cache, new_meta
diff --git a/src/backend/core/db/threads.py b/src/backend/core/db/threads.py
new file mode 100644
index 00000000..ea16595a
--- /dev/null
+++ b/src/backend/core/db/threads.py
@@ -0,0 +1,99 @@
+"""Run database work off the event loop.
+
+Every query is blocking I/O, so async code never touches a ``Session`` itself:
+it hands a sync function to :func:`run_db` (when it already holds a session,
+such as FastAPI's request session) or to :func:`run_in_session` (which opens and
+closes a session inside the worker thread). Network waits happen between those
+calls, never inside them, so no pooled connection is ever held across an
+``await``.
+
+Both run on anyio's default thread limiter, which :func:`apply_thread_budget`
+sizes from :func:`core.db.engine.thread_budget` — the same number the
+connection pool is derived from.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import functools
+from typing import Any, Awaitable, Callable, TypeVar
+
+import anyio.to_thread
+
+from core.db import engine
+
+T = TypeVar("T")
+
+
+async def run_db(fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> T:
+ """Run blocking database code in a worker thread."""
+ return await anyio.to_thread.run_sync(functools.partial(fn, *args, **kwargs))
+
+
+def off_loop(fn: Callable[..., T]) -> Callable[..., Awaitable[T]]:
+ """Expose a blocking function as a coroutine function that runs it in a worker thread.
+
+ For interfaces that must be async (agent tools, startup steps, registry
+ hooks) but whose body is plain database work: the wrapper keeps the
+ signature and docstring, so tool schemas built from it are unchanged.
+ """
+
+ @functools.wraps(fn)
+ async def wrapper(*args: Any, **kwargs: Any) -> T:
+ return await run_db(fn, *args, **kwargs)
+
+ return wrapper
+
+
+def _in_session(fn: Callable[..., T], args: tuple, kwargs: dict) -> T:
+ with engine.SessionLocal() as db:
+ return fn(db, *args, **kwargs)
+
+
+async def run_in_session(fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> T:
+ """Run ``fn(db, *args, **kwargs)`` on a fresh session, closed before returning."""
+ return await run_db(_in_session, fn, args, kwargs)
+
+
+def _detached(db, value):
+ if value is not None and value in db:
+ db.refresh(value)
+ db.expunge(value)
+ return value
+
+
+def _call_repository(db, factory, name, args, kwargs):
+ return _detached(db, getattr(factory(db), name)(*args, **kwargs))
+
+
+class AsyncRepository:
+ """Await a session-bound repository's methods, each on a session of its own.
+
+ ``AsyncRepository(Repo)`` exposes ``await repo.method(...)`` for every
+ ``Repo(db).method(...)``; an ORM row it returns comes back loaded and
+ detached, so async services read it without holding a connection.
+ """
+
+ def __init__(self, factory: Callable[[Any], Any]):
+ self._factory = factory
+
+ def __getattr__(self, name: str) -> Callable[..., Awaitable[Any]]:
+ async def call(*args: Any, **kwargs: Any) -> Any:
+ return await run_in_session(_call_repository, self._factory, name, args, kwargs)
+
+ return call
+
+
+def apply_thread_budget() -> None:
+ """Size this process's thread pools to the budget the connection pool assumes.
+
+ Called once per worker on its own event loop, before it serves anything.
+ """
+ budget = engine.thread_budget()
+ anyio.to_thread.current_default_thread_limiter().total_tokens = budget.request_threads
+ asyncio.get_running_loop().set_default_executor(
+ concurrent.futures.ThreadPoolExecutor(
+ max_workers=budget.background_threads, thread_name_prefix="asyncio"
+ )
+ )
diff --git a/src/backend/core/db/user_cleanup.py b/src/backend/core/db/user_cleanup.py
new file mode 100644
index 00000000..867cf070
--- /dev/null
+++ b/src/backend/core/db/user_cleanup.py
@@ -0,0 +1,13 @@
+"""Run module-owned cleanup within the deleting session's transaction."""
+from sqlalchemy import event
+from sqlalchemy.orm import Session
+
+from core.harness.registry.user_cleanup import cleanup_user
+
+
+def install_user_cleanup(user_model):
+ @event.listens_for(Session, "before_flush")
+ def clean_deleted_users(session, flush_context, instances):
+ for instance in tuple(session.deleted):
+ if isinstance(instance, user_model):
+ cleanup_user(session, instance)
diff --git a/src/backend/core/evolution/__init__.py b/src/backend/core/evolution/__init__.py
index 383ada1c..dbdbc548 100644
--- a/src/backend/core/evolution/__init__.py
+++ b/src/backend/core/evolution/__init__.py
@@ -18,11 +18,7 @@
AssetKind,
AssetRef,
)
-from core.evolution.runtime_binding import (
- bind_runtime_assets,
- get_bundle,
- resolve_bundle_for_run,
-)
+from core.evolution.cycle.runtime_binding import bind_runtime_assets, get_bundle, resolve_bundle_for_run
__all__ = [
"ASSET_KINDS",
diff --git a/src/backend/core/evolution/activation.py b/src/backend/core/evolution/activation.py
deleted file mode 100644
index 09b70bb6..00000000
--- a/src/backend/core/evolution/activation.py
+++ /dev/null
@@ -1,1502 +0,0 @@
-"""Materialise an approved candidate into a runtime capability.
-
-This is the step the loop was missing. Everything upstream — discovery,
-attribution, promotion, the IR, replay — produced a *candidate*, and a candidate
-is a proposal. Nothing turned it into an asset the agent could actually load,
-so measured end to end the system learned nothing: the benchmark's "after" arm
-saw exactly zero new skills.
-
-Materialisation closes that gap, and it is deliberately the **only** way an
-evolution artefact reaches the runtime. It sits behind the release controller
-so every governance property upstream still holds:
-
-* the candidate must have cleared the verification its risk tier demands;
-* the approver must not be the proposer;
-* the previous state is captured **before** the write, so rollback has somewhere
- to go rather than having to reconstruct one afterwards.
-
-Materialised skills are tagged so they can always be told apart from
-hand-authored ones. A capability the system wrote about itself should never be
-indistinguishable from one a person wrote.
-"""
-
-from __future__ import annotations
-
-import logging
-import uuid
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-from core.db.engine import SessionLocal
-
-logger = logging.getLogger(__name__)
-
-# Marks a skill that the evolution loop wrote. Never remove this on update: it
-# is what lets an operator see at a glance which capabilities were authored by
-# the system rather than by a person.
-EVOLUTION_TAG = "evolved"
-EVOLUTION_PREFIX = "evo-"
-
-# Operator stop switch, in ``system_configs`` so an incident does not need a
-# redeploy to halt every layer.
-FREEZE_CONFIG_KEY = "evolution_frozen"
-
-
-class ActivationError(Exception):
- def __init__(self, code: str, message: str):
- super().__init__(message)
- self.code = code
- self.message = message
-
-
-def _skill_label(tools: Sequence[str], *, rules_only: bool = False) -> str:
- """A human- and model-readable skill name.
-
- The name and description are the *only* things the model sees before it
- decides whether to open a skill, so they have to describe the job. The
- candidate's ``hypothesis`` describes the evidence ("「…」类请求在 N 次独立
- 会话中以同一序列成功完成") — true, and useless for that decision, since it
- says nothing about when the skill applies. It belongs in the provenance
- section, not the title.
- """
- head = " → ".join(tools[:4]) + ("…" if len(tools) > 4 else "")
- return ("工具顺序规则:" if rules_only else "固定调用顺序:") + head
-
-
-def _skill_description(
- tools: Sequence[str], constraints: Sequence[Dict[str, Any]], *, rules_only: bool
-) -> str:
- """One line, single-line by necessity — the frontmatter parser is line-based."""
- joined = "、".join(f"`{t}`" for t in tools[:6])
- if rules_only:
- return (
- f"当任务用到 {joined} 时,按已验证的先后顺序约束安排调用次序"
- f"({len(constraints)} 条约束,来自历史成功执行)。"
- )
- return (
- f"当任务需要 {joined} 且执行顺序会影响结果时使用;"
- f"按 {' → '.join(tools)} 的顺序调用。"
- )
-
-
-def _rules_consistent_with(
- constraints: Sequence[Dict[str, Any]], tools: Sequence[str]
-) -> List[Dict[str, Any]]:
- """Drop rules whose direction the skill's own step list violates.
-
- The loop attaches every rule touching a skill's tools, and scoping makes that
- formally sound — a rule valid in ``finance`` and a sequence distilled from
- ``trap`` genuinely can point opposite ways. But the rendered document then
- tells the model to call A before B in its steps and B before A two sections
- later, with nothing saying which wins. Observed on a real materialisation.
-
- For this skill's own tool set the step list is the governing statement, so a
- rule contradicting it is describing some other context and does not belong in
- this document. It remains attached to the skills whose sequences agree with
- it, and in ``ordering_constraints.json`` for any consumer that reasons about
- scope itself.
- """
- if not tools:
- return list(constraints)
- position = {tool: index for index, tool in enumerate(tools)}
- kept: List[Dict[str, Any]] = []
- for rule in constraints:
- before, after = rule.get("before"), rule.get("after")
- if before in position and after in position and position[before] > position[after]:
- logger.info(
- "[activation] rule %s<%s omitted: contradicts this skill's own order",
- before,
- after,
- )
- continue
- kept.append(rule)
- return kept
-
-
-def _ordering_rules_section(constraints: Sequence[Dict[str, Any]]) -> str:
- """Render the learned ordering constraints into the body the model reads.
-
- Without this the transferable half of what was learned never reaches the
- runtime. ``ordering_constraints.json`` is written alongside, but nothing in
- the agent path reads extra files as policy — the model reads SKILL.md, so a
- rule that is not written here is a rule the model never receives.
-
- Each rule carries its own scope. Stating where it was contradicted matters
- as much as stating where it held: a rule applied in the context that
- disproved it actively breaks that context.
- """
- shippable = [r for r in constraints if not (r.get("contradicted_in") or [])]
- withheld = [r for r in constraints if (r.get("contradicted_in") or [])]
-
- lines: List[str] = []
- if shippable:
- lines += ["", "## 顺序约束(可迁移)", ""]
- lines.append("下列约束在历史成功执行中无反例,适用于本技能覆盖的工具:")
- lines.append("")
- for rule in shippable:
- before, after = rule.get("before"), rule.get("after")
- validated = [str(x) for x in (rule.get("validated_in") or [])]
- line = f"- 先 `{before}` 再 `{after}`"
- if validated:
- line += f"——已在 {'、'.join(validated)} 类任务中验证"
- line += f"(证据 {rule.get('support', 0)} 条)"
- lines.append(line)
-
- if withheld:
- # Scoped rules are deliberately kept out of the model's view.
- #
- # Measured twice against a real model, and both framings failed:
- #
- # * exception as a trailing qualifier ("在 X 类任务中不适用") — the model
- # applied the rule in X anyway, 5 of 5 times;
- # * exception first and imperative ("若属于 X 则整条作废,那里顺序相反")
- # — the model then applied the *reversed* order everywhere, dropping the
- # family the rule was valid in from 50% to 17% and disturbing an
- # unrelated control family.
- #
- # So a text annotation cannot make a model route on task type: it either
- # ignores the exception or over-applies it. Handing it a rule that is
- # wrong somewhere makes the system worse where the rule is wrong, which
- # is a poor trade for a benefit available only where it is right.
- #
- # The rule is not discarded — it stays in ``ordering_constraints.json``
- # for a runtime applier that knows the task type and can decide before
- # injection, which is where a scope decision belongs. Until that exists,
- # withholding is the conservative and measurably better option.
- lines += ["", "## 已扣留的上下文相关约束", ""]
- lines.append(
- f"另有 {len(withheld)} 条约束在部分任务类型中被证据推翻,"
- "因此**不在本技能中给出**——它们只在特定任务类型下成立,"
- "由运行时按任务类型判定后注入,不交由模型自行取舍。"
- )
- for rule in withheld:
- lines.append(
- f"- (已扣留)`{rule.get('before')}` / `{rule.get('after')}`:"
- f"在 {'、'.join(str(x) for x in (rule.get('validated_in') or []))} 成立,"
- f"在 {'、'.join(str(x) for x in (rule.get('contradicted_in') or []))} 被推翻"
- )
- return "\n".join(lines)
-
-
-def _skill_markdown(
- *,
- skill_id: str,
- label: str,
- description: str,
- tools: Sequence[str],
- evidence: int,
- constraints: Sequence[Dict[str, Any]] = (),
- rationale: str = "",
- rules_only: bool = False,
-) -> str:
- """Render the SKILL.md body for a distilled skill.
-
- ``name`` is the skill **id** — the loader treats that field as the id and
- rejects anything outside ``[a-z0-9_-]``. Writing the human label there made
- every materialised skill unparseable: it still showed up in the capability
- list (that metadata comes from table columns) while ``load_skill_full``
- returned ``None``, so the skill was present and inert.
-
- States its provenance in the document itself, so anyone reading the skill —
- not just anyone reading the console — knows where it came from and on how
- much evidence it rests.
- """
- # Applicability is declared for the **whole skill**, not per rule. Measured:
- # withholding a contradicted rule from the text was not enough, because the
- # step list generalises on its own — the model applied this skill's order to
- # a family where the reverse is correct, 5 of 5 times, with the rule absent.
- # So the exclusion has to bind the steps too, which means stating it once, at
- # the top, before anything the model could copy.
- excluded = sorted(
- {
- str(name)
- for rule in constraints
- for name in (rule.get("contradicted_in") or [])
- if str(name)
- }
- )
- covered = sorted(
- {
- str(name)
- for rule in constraints
- for name in (rule.get("validated_in") or [])
- if str(name) and str(name) not in excluded
- }
- )
- scope_block = ""
- if excluded:
- scope_block = (
- "## 不适用范围(先读这一段)\n\n"
- f"**本技能不适用于以下任务类型:{'、'.join(excluded)}。**"
- "在这些场景中,本技能给出的顺序已被历史证据推翻——"
- "遇到它们请完全忽略本技能,按任务本身的依赖关系自行规划。\n\n"
- + (
- f"本技能仅适用于:{'、'.join(covered)}。\n\n"
- if covered
- else ""
- )
- )
-
- if rules_only:
- body_head = (
- scope_block
- + "## 适用条件\n\n"
- "当任务用到下列工具、且它们的先后顺序会影响结果时,按「顺序约束」小节排序。\n\n"
- "## 涉及工具\n\n"
- + "\n".join(f"- `{tool}`" for tool in tools)
- )
- else:
- body_head = (
- scope_block
- + "## 适用条件\n\n"
- "当任务需要按固定顺序完成下列工具调用时使用本技能。\n\n"
- "## 步骤\n\n"
- + "\n".join(f"{i}. `{tool}`" for i, tool in enumerate(tools, start=1))
- )
-
- # A rules-only skill has no governing step list, so nothing there can be
- # contradicted; for a sequence skill the steps win and conflicting rules are
- # about other contexts.
- shown_rules = constraints if rules_only else _rules_consistent_with(constraints, tools)
-
- provenance = (
- f"本技能由能力进化流水线从 {evidence} 条历史执行证据中沉淀而成,非人工撰写。"
- )
- if rationale:
- provenance += f"\n\n沉淀依据:{rationale}"
-
- return f"""---
-name: {skill_id}
-display_name: {label}
-description: {description}
----
-
-# {label}
-
-{description}
-
-{body_head}
-{_ordering_rules_section(shown_rules)}
-
-## 来源
-
-{provenance}
-"""
-
-
-def _body_of(rendered: str) -> str:
- """The body of a rendered SKILL.md, without frontmatter or provenance.
-
- Re-rendering from the body rather than storing the generator's output
- verbatim is what lets the id change between drafting and activation (an
- evolution prefix, an owner suffix) without the document's ``name`` field
- disagreeing with the row it is stored under — a mismatch that makes the
- skill unloadable while still listing it as available.
- """
- text = rendered or ""
- if text.startswith("---"):
- end = text.find("\n---", 3)
- if end != -1:
- text = text[end + 4 :]
- # Drop the rendered title/description pair and the provenance section; both
- # are regenerated, and keeping them would double on every re-render.
- marker = text.find("## ")
- if marker != -1:
- text = text[marker:]
- source = text.find("\n## 来源")
- if source != -1:
- text = text[:source]
- return text.strip()
-
-
-def _capture_previous(db, skill_id: str) -> Optional[Dict[str, Any]]:
- """Snapshot the current state so rollback has a concrete target."""
- from core.db.models import AdminSkill
-
- row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
- if row is None:
- return None
- return {
- "skill_id": row.skill_id,
- "skill_content": row.skill_content,
- "display_name": row.display_name,
- "description": row.description,
- "version": row.version,
- "tags": list(row.tags or []),
- "allowed_tools": list(row.allowed_tools or []),
- "extra_files": dict(row.extra_files or {}),
- "is_enabled": bool(row.is_enabled),
- }
-
-
-def _is_frozen(db) -> bool:
- """Global stop switch, readable without a redeploy.
-
- An incident needs a way to stop every layer activating *now*, so it lives in
- ``system_configs`` rather than in an environment variable.
- """
- try:
- from core.db.models import SystemConfig
-
- row = (
- db.query(SystemConfig.config_value)
- .filter(SystemConfig.config_key == FREEZE_CONFIG_KEY)
- .first()
- )
- return bool(row) and str(row[0]).strip().lower() in ("1", "true", "yes", "on")
- except Exception: # noqa: BLE001 - absence of the row means "not frozen"
- return False
-
-
-def _skill_layer_state(db) -> Tuple[Optional[datetime], int]:
- """When a skill last went live, and how many releases are mid-flight.
-
- Read from the release ledger rather than kept in memory: a per-process
- limiter resets on every restart and multiplies by the number of replicas,
- which is exactly how a cooldown quietly stops existing.
- """
- from core.db.models.evolution import EvolutionRelease
- from core.evolution.release import ACTIVE, IN_FLIGHT_STAGES
-
- last_row = (
- db.query(EvolutionRelease.created_at)
- .filter(
- EvolutionRelease.target_kind == "skill",
- EvolutionRelease.stage == ACTIVE,
- )
- .order_by(EvolutionRelease.created_at.desc())
- .first()
- )
- in_flight = (
- db.query(EvolutionRelease)
- .filter(EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)))
- .count()
- )
- last = last_row[0] if last_row else None
- if last is not None and last.tzinfo is None:
- last = last.replace(tzinfo=timezone.utc)
- return last, int(in_flight or 0)
-
-
-def _check_pace(db, *, tenant_id: str, layer: str = "skill") -> None:
- """Enforce the multi-timescale gate the design declares.
-
- Skills sit on an hours-to-days cadence so the system does not become
- non-stationary. That was a documented number with no enforcement: nothing
- consulted ``timescales`` before writing a live asset, so ten activations
- could land in one minute and no metric movement afterwards would be
- attributable to any of them.
- """
- from core.evolution.timescales import RateLimiter
-
- last, in_flight = _skill_layer_state(db)
- limiter = RateLimiter(frozen=_is_frozen(db))
- if last is not None:
- limiter.last_activation[layer] = last
- allowed, reason = limiter.may_activate(
- layer, tenant=tenant_id or "default", concurrent_activations=in_flight
- )
- if not allowed:
- raise ActivationError("pace_limited", f"节奏闸门拒绝本次激活:{reason}")
-
-
-def _entry_stage(
- candidate,
- *,
- approver: str,
- force: bool,
- owner_user_id: Optional[str],
-) -> Tuple[str, int]:
- """The stage this candidate has actually earned, and its traffic share.
-
- Previously every activation wrote ``active`` at 100%, whatever the risk tier
- said. ``ir.MIN_VERIFICATION`` declares that a high-risk change owes replay,
- shadow, canary *and* human approval before going live, and a brand-new
- skill is exactly that tier — so the old behaviour skipped two of the four
- stages its own IR demanded.
-
- A personal activation is the one case that legitimately goes straight live:
- its blast radius is one consenting user, and the ramp exists to bound blast
- radius. It is recorded as scoped rather than as full traffic.
- """
- from core.evolution.ir import RISK_LOW, required_verification
- from core.evolution.release import ACTIVE, REPLAY_PASSED, SHADOW
-
- risk = str(candidate.risk_tier or "").strip() or RISK_LOW
- replayed = candidate.status in (REPLAY_PASSED, ACTIVE)
-
- if not replayed:
- # ``force`` waives the replay *stage* — and only for the tier whose
- # verification requirement is replay alone. Waiving it for a tier that
- # also owes shadow and canary is not a shortcut, it is the whole gate.
- if not force:
- raise ActivationError(
- "verification_incomplete",
- f"候选尚未通过回放(当前 {candidate.status})",
- )
- if risk != RISK_LOW:
- raise ActivationError(
- "force_forbidden_for_risk",
- f"force 只允许用于 low 风险候选,本候选为 {risk};"
- f"需完成 {list(required_verification(risk))}",
- )
-
- if owner_user_id:
- return ACTIVE, 0
-
- if set(required_verification(risk)) <= {"replay"}:
- return ACTIVE, 100
-
- # Higher tiers enter at shadow: the asset exists and is measurable, and
- # reaches nobody until an operator ramps it.
- return SHADOW, 0
-
-
-def materialise_skill_candidate(
- candidate_id: str,
- *,
- approver: str,
- force: bool = False,
- owner_user_id: Optional[str] = None,
- tenant_id: str = "default",
-) -> Dict[str, Any]:
- """Materialise an approved skill candidate, at the stage it has earned.
-
- This is the only path by which an evolution artefact reaches the agent, and
- it now enters the release pipeline rather than bypassing it:
-
- * ``low`` risk (verification = replay only) goes live at 100%;
- * ``medium`` / ``high`` enter **shadow**, reaching nobody, and must be ramped
- through canary by :func:`advance_skill_release`;
- * a personal activation (``owner_user_id``) goes live scoped to that user.
-
- ``force`` waives the replay *stage* and only for ``low`` risk. It never
- waives separation of duties — an override is exactly what an attacker or an
- over-eager script would reach for.
-
- ``owner_user_id`` scopes the resulting skill to one person. A personal
- activation cannot change what anyone else's agent does, which is what lets a
- user accept a capability distilled from their own conversations without an
- administrator standing in the way.
- """
- from core.db.models import AdminSkill
- from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
- from core.evolution.release import (
- ACTIVE,
- IN_FLIGHT_STAGES,
- REPLAY_PASSED,
- ROLLBACK_MANUAL,
- new_release_id,
- )
-
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- if candidate is None:
- raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
- if candidate.target_kind != "skill":
- raise ActivationError(
- "unsupported_kind", f"暂不支持物化 {candidate.target_kind} 候选"
- )
-
- # Separation of duties. No force path — see the docstring.
- if approver and approver == (candidate.proposer or ""):
- raise ActivationError(
- "self_approval_forbidden", "生成者不能激活自己提出的候选"
- )
- if not approver:
- raise ActivationError("approver_required", "激活必须署名")
-
- stage, traffic = _entry_stage(
- candidate, approver=approver, force=force, owner_user_id=owner_user_id
- )
- # Pace and freeze apply to anything that becomes live immediately. A
- # shadow entry changes nothing users can see, so holding it back would
- # only delay measurement.
- if stage == ACTIVE and not owner_user_id:
- _check_pace(db, tenant_id=tenant_id)
-
- ir = candidate.ir or {}
- changes = ir.get("changes") or []
- tools: List[str] = []
- constraints: List[Dict[str, Any]] = []
- document: Optional[Dict[str, Any]] = None
- manifest: Optional[Dict[str, Any]] = None
- # ref → owning user. Demotion happens inside one person's store, so a
- # skill compiled from several people's memories needs the owner of each.
- source_memories: Dict[str, str] = {}
- for change in changes:
- sequence = change.get("tool_sequence")
- if isinstance(sequence, list) and sequence and not tools:
- tools = [str(t) for t in sequence]
- rules = change.get("ordering_constraints")
- if isinstance(rules, list) and rules:
- constraints = rules
- written = change.get("document")
- if isinstance(written, dict) and written.get("content"):
- document = written
- declared = change.get("manifest")
- if isinstance(declared, dict) and declared:
- manifest = declared
- for item in change.get("source_memories") or []:
- if isinstance(item, dict) and item.get("ref"):
- source_memories[str(item["ref"])] = str(item.get("user_id") or "")
-
- # 图谱证据自相矛盾的候选可以存在、可以进 shadow 观测,但不允许自动
- # 走到 ACTIVE:冲突意味着「适用条件」本身不可信,必须先由人裁决。
- if manifest and manifest.get("graph_conflict") and stage == ACTIVE:
- raise ActivationError(
- "graph_conflict_requires_review",
- f"L3 图谱证据存在冲突,禁止直接激活:{manifest.get('graph_conflicts')}",
- )
-
- skill_id = candidate.target_asset_id
- if not skill_id.startswith(EVOLUTION_PREFIX):
- skill_id = EVOLUTION_PREFIX + skill_id.replace("auto-", "")
- if owner_user_id:
- # Distinct id per owner: a personal activation must never overwrite
- # the global asset, nor another user's copy of the same candidate.
- skill_id = f"{skill_id}-u{owner_user_id[-8:]}"
-
- if document is not None:
- # A written skill: the generator produced the body and the machine
- # validator already refused everything it was told to refuse. It is
- # re-checked here against this candidate's id, because the id gains
- # a prefix (and, for a personal activation, an owner suffix) after
- # the draft was validated — and an id the loader rejects yields a
- # skill that shows up in the capability list and loads as nothing.
- from core.evolution.skill_gen import render_skill_markdown, validate_draft
-
- tools = [str(t) for t in (document.get("allowed_tools") or [])]
- display_name = str(document.get("display_name") or "")
- description = str(document.get("description") or "")
- body = _body_of(str(document.get("content") or ""))
- violations = validate_draft(
- skill_id=skill_id,
- display_name=display_name,
- description=description,
- body=body,
- allowed_tools=tools,
- )
- if violations:
- raise ActivationError(
- "draft_failed_validation",
- f"技能正文未通过机器校验,拒绝物化: {violations}",
- )
- rules_only = False
- content = render_skill_markdown(
- skill_id=skill_id,
- display_name=display_name,
- description=description,
- body=body,
- support=len(candidate.evidence_refs or []),
- rationale=candidate.hypothesis or "",
- )
- else:
- # A candidate carrying only ordering rules is still a capability,
- # and on real traffic it is the *common* case: five runs repeating
- # one identical tool sequence is rare, while the same pairwise
- # ordering recurring across different tool sets is not.
- rules_only = not tools and bool(constraints)
- if rules_only:
- tools = _tools_from_constraints(constraints)
- if not tools:
- raise ActivationError(
- "no_tool_sequence", "候选既未声明工具顺序、也未携带顺序约束,无法物化"
- )
- display_name = _skill_label(tools, rules_only=rules_only)
- description = _skill_description(tools, constraints, rules_only=rules_only)
- content = _skill_markdown(
- skill_id=skill_id,
- label=display_name,
- description=description,
- tools=tools,
- evidence=len(candidate.evidence_refs or []),
- constraints=constraints,
- rationale=candidate.hypothesis or "",
- rules_only=rules_only,
- )
-
- previous = _capture_previous(db, skill_id)
-
- # One release in flight per asset. Two concurrent ramps on the same skill
- # make any measured effect unattributable and leave rollback without a
- # single target.
- conflicting = (
- db.query(EvolutionRelease)
- .filter(
- EvolutionRelease.target_asset_id == skill_id,
- EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)),
- )
- .first()
- )
- if conflicting is not None:
- raise ActivationError(
- "release_in_flight",
- f"该资产已有进行中的发布 {conflicting.release_id}({conflicting.stage})",
- )
-
- row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
- if row is None:
- row = AdminSkill(skill_id=skill_id, created_by=f"evolution:{approver}")
- db.add(row)
- row.owner_user_id = owner_user_id or None
-
- row.skill_content = content
- row.display_name = display_name
- row.description = description
- row.version = _next_version(previous)
- row.tags = sorted(set((previous or {}).get("tags", [])) | {EVOLUTION_TAG})
- # The distilled ordering, in the shape the runtime reads.
- extra_files = {
- **(previous or {}).get("extra_files", {}),
- "tool_sequence.json": _json(tools),
- # The transferable part. A consumer that only understands sequences
- # still works; one that understands rules gets the wider benefit.
- "ordering_constraints.json": _json(constraints),
- }
- if manifest:
- # 机器读 manifest(边界/依赖/证据引用),Agent 读 SKILL.md —— 两者
- # 职责分离。作用域以本次激活实际发生的为准,而不是候选申报的。
- extra_files["evolution_manifest.json"] = _json(
- {
- **manifest,
- "scope": "personal" if owner_user_id else str(
- manifest.get("scope") or "workspace"
- ),
- }
- )
- row.extra_files = extra_files
- row.allowed_tools = tools
- row.is_enabled = True
-
- # The scope the IR asked for, now actually enforced at load time by
- # core.evolution.exposure. Recording it here rather than deriving it
- # later means a scope decision is auditable against the release it
- # governs.
- scope_filter: Dict[str, Any] = {"level": (ir.get("scope") or {}).get("level") or "workspace"}
- if owner_user_id:
- scope_filter["owner_user_id"] = owner_user_id
- else:
- scope_filter["tenant_id"] = tenant_id or "default"
-
- release = EvolutionRelease(
- release_id=new_release_id(),
- candidate_id=candidate_id,
- target_kind="skill",
- target_asset_id=skill_id,
- stage=stage,
- # A personal activation reaches exactly one user, so recording it as
- # full traffic would misreport the blast radius on the release board.
- traffic_percent=traffic,
- scope_filter=scope_filter,
- # Written now, not derived later: rollback needs a target that
- # existed before anything changed.
- rollback_version_id=(previous or {}).get("version") or "",
- guardrails={
- "previous": previous,
- # Which verification stages are behind us. Read by
- # advance_skill_release before every hop.
- "completed_verification": (
- ["replay"] if candidate.status in (REPLAY_PASSED, ACTIVE) else []
- ),
- "proposer": candidate.proposer or "",
- },
- approved_by=approver,
- )
- db.add(release)
-
- # A personal activation leaves the candidate available for others to
- # accept, and for an administrator to consider globally. Marking it
- # ACTIVE would end its life for everyone on one person's decision.
- # A shadow entry is likewise not the end of the candidate's life.
- if not owner_user_id and stage == ACTIVE:
- candidate.status = ACTIVE
- candidate.approved_by = approver
- candidate.approved_at = datetime.now(timezone.utc)
- db.commit()
-
- release_id = release.release_id
- new_version = row.version
- candidate_owner = str((ir.get("scope") or {}).get("user_id") or "")
- logger.info(
- "[activation] materialised %s from %s at %s/%d%% (tools=%s written=%s)",
- skill_id,
- candidate_id,
- stage,
- traffic,
- tools,
- document is not None,
- )
-
- # Close the promotion chain: the memories this skill was compiled from now
- # have a cheaper carrier, so they stop consuming the retrieval budget that
- # made compiling them worthwhile. Outside the transaction above because it
- # writes to its own ledger, and down-weighted rather than removed, so
- # rolling the skill back restores them exactly.
- demoted: List[str] = []
- if source_memories:
- from core.evolution.memory_apply import demote_promoted_memories
-
- by_owner: Dict[str, List[str]] = {}
- for ref, owner in source_memories.items():
- # ``owner_user_id`` for a personal activation, the ref's recorded
- # owner otherwise. A ref with neither is skipped rather than guessed
- # at: applying a change to the wrong person's memories is worse than
- # not applying it.
- resolved = owner_user_id or owner or candidate_owner
- if resolved:
- by_owner.setdefault(resolved, []).append(ref)
-
- for owner, refs in by_owner.items():
- outcome = demote_promoted_memories(
- candidate_id=candidate_id,
- user_id=owner,
- memory_refs=refs,
- skill_id=skill_id,
- )
- demoted.extend(op["memory_ref"] for op in outcome.get("applied", []))
-
- # 血缘落账:这一版技能由哪些 L2 记忆编译(compiled_from)、被哪些 L3 关系
- # 约束(constrained_by)、由哪些 Episode 验证(validated_by)。尽力而为——
- # 账本失败绝不回滚一次已经成立的激活。
- lineage_written = 0
- try:
- from core.evolution import lineage
-
- with SessionLocal() as db:
- refreshed = db.get(EvolutionCandidate, candidate_id)
- pack_id = str(getattr(refreshed, "evidence_pack_id", "") or "")
- pack = lineage.load_evidence_pack(pack_id) if pack_id else None
- if pack is not None:
- lineage_written = len(
- lineage.record_links(
- lineage.links_for_skill(
- pack, skill_id=skill_id, skill_version=new_version
- ),
- candidate_id=candidate_id,
- )
- )
- except Exception as exc: # noqa: BLE001
- logger.debug("[activation] lineage recording skipped: %s", exc)
-
- return {
- "skill_id": skill_id,
- "release_id": release_id,
- "stage": stage,
- "traffic_percent": traffic,
- "scope": scope_filter,
- "demoted_memories": demoted,
- "lineage_links": lineage_written,
- # Spelled out because "activated" previously implied "live for
- # everyone", and for anything above low risk it no longer does.
- "exposed_to": (
- "all users in scope"
- if stage == ACTIVE and not owner_user_id
- else (f"only {owner_user_id}" if owner_user_id else "nobody (shadow)")
- ),
- "tools": tools,
- "version": new_version,
- "replaced_previous": previous is not None,
- }
-
-
-def advance_skill_release(
- release_id: str,
- *,
- actor: str,
- observed_metrics: Optional[Dict[str, float]] = None,
- guardrail_thresholds: Optional[Dict[str, float]] = None,
-) -> Dict[str, Any]:
- """Ramp a release one step: shadow → canary 5 → 25 → 100 → active.
-
- One step per call, deliberately. A single "go live" button that walked the
- whole ladder would reproduce the behaviour this replaces — the point of the
- ladder is that somebody looks at the metrics between rungs.
-
- Guardrails are checked when metrics are supplied, and a breach rolls back
- rather than pausing: the recorded threshold already *is* the level at which
- the release is unacceptable.
- """
- from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
- from core.evolution.release import (
- ACTIVE,
- CANARY,
- CANARY_STEPS,
- SHADOW,
- can_promote,
- evaluate_guardrails,
- next_traffic,
- should_auto_rollback,
- )
-
- if not actor:
- raise ActivationError("actor_required", "推进发布必须署名")
-
- with SessionLocal() as db:
- release = db.get(EvolutionRelease, release_id)
- if release is None:
- raise ActivationError("release_not_found", f"发布记录不存在: {release_id}")
- if _is_frozen(db):
- raise ActivationError("globally_frozen", "全局冻结开关已打开,禁止推进发布")
-
- candidate = db.get(EvolutionCandidate, release.candidate_id)
- risk = str(getattr(candidate, "risk_tier", "") or "low")
- guardrails = dict(release.guardrails or {})
- completed: List[str] = list(guardrails.get("completed_verification") or [])
- proposer = str(guardrails.get("proposer") or getattr(candidate, "proposer", "") or "")
-
- if observed_metrics:
- thresholds = guardrail_thresholds or guardrails.get("thresholds") or {}
- breaches = evaluate_guardrails(observed_metrics, thresholds, window="canary")
- if should_auto_rollback(breaches):
- detail = ", ".join(
- f"{b.metric}={b.observed}>{b.threshold}" for b in breaches
- )
- db.commit()
- rollback_skill_activation(
- release_id,
- actor=actor,
- reason=f"guardrail breach: {detail}",
- kind="guardrail",
- )
- return {
- "release_id": release_id,
- "stage": "rolled_back",
- "guardrail_breaches": [b.to_dict() for b in breaches],
- }
-
- stage = str(release.stage or "")
- traffic = int(release.traffic_percent or 0)
-
- if stage == SHADOW:
- target_stage, target_traffic = CANARY, CANARY_STEPS[0]
- completed = sorted(set(completed) | {"shadow"})
- elif stage == CANARY and traffic < CANARY_STEPS[-1]:
- target_stage, target_traffic = CANARY, next_traffic(traffic)
- elif stage == CANARY:
- target_stage, target_traffic = ACTIVE, 100
- completed = sorted(set(completed) | {"canary"})
- else:
- raise ActivationError(
- "not_rampable", f"当前阶段 {stage} 无法继续放量"
- )
-
- allowed, reason = can_promote(
- current_stage=stage,
- target_stage=target_stage,
- risk_tier=risk,
- # ``human_approval`` is this call: a named actor who is not the
- # proposer. Recording it before the check keeps the requirement
- # satisfied by the act itself rather than by a stored flag.
- completed_verification=sorted(set(completed) | {"human_approval"})
- if target_stage == ACTIVE
- else completed,
- proposer=proposer,
- approver=actor,
- )
- # A same-stage traffic bump is not a transition; can_promote would reject
- # canary→canary, which is correct for stages and wrong for ramp steps.
- if not allowed and not (stage == target_stage == CANARY):
- raise ActivationError("promotion_refused", reason)
-
- if target_stage == ACTIVE:
- # 冲突的 L3 证据在 shadow/canary 里观测没问题,但升到全量必须先由
- # 人裁决冲突本身——适用条件不可信的技能不配自动放大到所有人。
- if _has_graph_conflict(candidate):
- raise ActivationError(
- "graph_conflict_requires_review",
- "候选的 L3 图谱证据存在未裁决的冲突,禁止放量到 active",
- )
- _check_pace(db, tenant_id=str((release.scope_filter or {}).get("tenant_id") or "default"))
-
- release.stage = target_stage
- release.traffic_percent = target_traffic
- guardrails["completed_verification"] = completed
- if target_stage == ACTIVE:
- guardrails["approved_by"] = actor
- release.guardrails = guardrails
- release.approved_by = actor
- if candidate is not None and target_stage == ACTIVE:
- candidate.status = ACTIVE
- db.commit()
-
- logger.info(
- "[activation] release %s → %s/%d%% by %s",
- release_id,
- target_stage,
- target_traffic,
- actor,
- )
- return {
- "release_id": release_id,
- "stage": target_stage,
- "traffic_percent": target_traffic,
- "completed_verification": completed,
- "exposed_to": (
- "all users in scope"
- if target_stage == ACTIVE
- else f"{target_traffic}% of users in scope"
- ),
- }
-
-
-def _has_graph_conflict(candidate) -> bool:
- """Whether the candidate's manifest flags contradictory L3 evidence."""
- if candidate is None:
- return False
- for change in (getattr(candidate, "ir", None) or {}).get("changes") or []:
- manifest = change.get("manifest") if isinstance(change, dict) else None
- if isinstance(manifest, dict) and manifest.get("graph_conflict"):
- return True
- return False
-
-
-def rollback_skill_activation(
- release_id: str, *, actor: str, reason: str, kind: str = "manual"
-) -> Dict[str, Any]:
- """Undo a materialisation, restoring the snapshot taken at activation time.
-
- With no prior version the skill is disabled rather than deleted: history
- referencing it must stay resolvable, and a disabled row is both reversible
- and visible in a way a missing row is not.
-
- ``kind`` separates "a guardrail caught it" from "a person caught it".
- Collapsing the two would hide how well the evaluation stage is working,
- which is the number that tells you whether the ramp is worth its cost.
- """
- from core.db.models import AdminSkill
- from core.db.models.evolution import (
- EvolutionAgentProfile,
- EvolutionCandidate,
- EvolutionRelease,
- )
- from core.evolution.release import ROLLBACK_GUARDRAIL, ROLLBACK_MANUAL, ROLLED_BACK
-
- if not actor or not reason:
- raise ActivationError("audit_required", "回滚必须署名并填写理由")
-
- with SessionLocal() as db:
- release = db.get(EvolutionRelease, release_id)
- if release is None:
- raise ActivationError("release_not_found", f"发布记录不存在: {release_id}")
-
- previous = (release.guardrails or {}).get("previous")
- skill_id = release.target_asset_id
- row = (
- db.query(AdminSkill)
- .filter(AdminSkill.skill_id == skill_id)
- .first()
- )
- invalidated: List[str] = []
- if row is not None:
- if previous:
- row.skill_content = previous["skill_content"]
- row.display_name = previous["display_name"]
- row.description = previous["description"]
- row.version = previous["version"]
- row.tags = previous["tags"]
- row.allowed_tools = previous["allowed_tools"]
- row.extra_files = previous["extra_files"]
- row.is_enabled = previous["is_enabled"]
- else:
- row.is_enabled = False
- # No prior version means the skill just stopped being loadable,
- # so a profile assembled on top of it is now promising a
- # capability that fails silently — same cross-layer rule as
- # retirement. With a restored prior version the skill stays
- # loadable and the profiles stay valid.
- for profile_row in (
- db.query(EvolutionAgentProfile)
- .filter(EvolutionAgentProfile.is_active.is_(True))
- .all()
- ):
- payload = profile_row.payload or {}
- if skill_id in (payload.get("skill_ids") or []):
- profile_row.is_active = False
- invalidated.append(profile_row.profile_id)
-
- release.stage = ROLLED_BACK
- release.rolled_back_at = datetime.now(timezone.utc)
- release.rollback_reason = reason
- release.rollback_kind = (
- ROLLBACK_GUARDRAIL if kind == "guardrail" else ROLLBACK_MANUAL
- )
-
- candidate = db.get(EvolutionCandidate, release.candidate_id)
- if candidate is not None:
- candidate.status = ROLLED_BACK
- candidate_id = release.candidate_id
- db.commit()
-
- # The reverse edge of the promotion chain. Materialisation down-weighted the
- # memories this skill was compiled from; rolling the skill back must restore
- # them in the same motion, or the knowledge stays half-priced with no
- # carrier — retrieval skips it because a skill supposedly covers it, and the
- # skill is gone. Same ledger the retire path already replays.
- from core.evolution.memory_apply import revert_memory_ops
-
- restored = revert_memory_ops(candidate_id=candidate_id) if candidate_id else {}
-
- # 血缘:这一版回滚到了哪一版。审计问「技能 v2 后来怎么了」时,答案在链上。
- try:
- from core.evolution import lineage
-
- current_version = str((previous or {}).get("version") or "")
- lineage.record_links(
- [
- (
- "skill",
- f"{skill_id}@rolled_back",
- lineage.REL_ROLLED_BACK_TO,
- "skill",
- f"{skill_id}@{current_version or 'disabled'}",
- )
- ],
- candidate_id=candidate_id,
- )
- except Exception as exc: # noqa: BLE001
- logger.debug("[activation] rollback lineage skipped: %s", exc)
-
- logger.info(
- "[activation] rolled back %s (release=%s memories restored=%d profiles invalidated=%s)",
- skill_id,
- release_id,
- len(restored.get("reverted", [])),
- invalidated,
- )
- return {
- "release_id": release_id,
- "restored_previous": bool(previous),
- "restored_memory_ops": restored.get("reverted", []),
- "invalidated_profiles": invalidated,
- "reason": reason,
- }
-
-
-def _tools_from_constraints(constraints: Sequence[Dict[str, Any]]) -> List[str]:
- """The tools a set of ordering rules talks about, in an order they permit.
-
- Not a claimed execution plan — a rules-only skill has no fixed sequence. It
- is the tool list the skill declares, listed in an order that does not
- contradict its own rules so the document does not read as self-refuting.
- """
- tools: List[str] = []
- for rule in constraints:
- for key in ("before", "after"):
- name = str(rule.get(key) or "")
- if name and name not in tools:
- tools.append(name)
- if not tools:
- return []
- from core.evolution.promotion import OrderingConstraint, apply_ordering_constraints
-
- parsed = [
- OrderingConstraint(
- before=str(rule.get("before") or ""),
- after=str(rule.get("after") or ""),
- support=int(rule.get("support") or 0),
- contexts=int(rule.get("contexts") or 0),
- validated_in=tuple(rule.get("validated_in") or ()),
- # Scope is deliberately dropped for this one purpose: we are only
- # ordering the *document's* tool list, not deciding applicability.
- )
- for rule in constraints
- if rule.get("before") and rule.get("after")
- ]
- return apply_ordering_constraints(tools, parsed)
-
-
-def _next_version(previous: Optional[Dict[str, Any]]) -> str:
- if not previous:
- return "1.0.0"
- parts = str(previous.get("version") or "1.0.0").split(".")
- try:
- parts[-1] = str(int(parts[-1]) + 1)
- return ".".join(parts)
- except Exception:
- return "1.0.1"
-
-
-def _json(value: Any) -> str:
- import json
-
- return json.dumps(value, ensure_ascii=False)
-
-
-# ── Orchestration assets ─────────────────────────────────────────────────────
-
-
-def materialise_profile_candidate(
- candidate_id: str, *, approver: str, tenant_id: str = "default"
-) -> Dict[str, Any]:
- """Publish an approved agent profile.
-
- Profiles do not ride the shadow → canary ladder that skills do, and the
- reason is not leniency. A profile *is* the assembly, so a canary would mean
- two different assemblies running concurrently under one profile id, with
- every downstream measurement of either one confounded by the other. Instead
- a profile is scoped: it governs one task type, and its blast radius is that
- task type's traffic, recorded on the release row like any other change.
-
- Rolling back is switching ``is_active`` off, which restores the built-in
- assembly on the next run — no restart, because profiles are read per run.
- """
- from core.db.models.evolution import EvolutionAgentProfile
- from core.evolution.agent_profile import AgentProfile, TARGET_KIND, validate_profile
- from core.evolution.release import ACTIVE as _ACTIVE, new_release_id
-
- if not approver:
- raise ActivationError("approver_required", "激活必须署名")
-
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- if candidate is None:
- raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
- if candidate.target_kind != TARGET_KIND:
- raise ActivationError(
- "unsupported_kind", f"该入口只处理编排候选,收到 {candidate.target_kind}"
- )
- if approver == (candidate.proposer or ""):
- raise ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选")
- if _is_frozen(db):
- raise ActivationError("layer_frozen", "进化已被全局冻结")
-
- payload: Dict[str, Any] = {}
- for change in candidate.ir.get("changes") or []:
- if isinstance(change.get("profile"), dict):
- payload = change["profile"]
- break
- if not payload:
- raise ActivationError("no_profile_payload", "候选未携带 profile 定义")
-
- profile = AgentProfile.from_dict(payload)
- profile.scope = {**(profile.scope or {}), "tenant_id": tenant_id or "default"}
-
- # Re-validated at activation against the *current* tool grant, not the
- # one that existed when the candidate was written. A grant narrowed in
- # the meantime must not be widened again by approving an older proposal.
- from core.config.catalog import get_enabled_ids
-
- base_tools = [str(t) for t in get_enabled_ids("tools")] + [
- str(t) for t in get_enabled_ids("mcp")
- ]
- ok, problems = validate_profile(profile, base_tools=base_tools)
- if not ok:
- raise ActivationError("profile_invalid", f"编排定义未通过校验: {problems}")
-
- previous = db.get(EvolutionAgentProfile, profile.profile_id)
- previous_payload = dict(previous.payload) if previous is not None else None
-
- # One active profile per (task type, scope). Two would make the
- # resolution order the thing that decides behaviour, which is not a
- # decision anyone reviewed.
- for row in (
- db.query(EvolutionAgentProfile)
- .filter(EvolutionAgentProfile.is_active.is_(True))
- .all()
- ):
- if row.profile_id == profile.profile_id:
- continue
- same_types = set(row.task_types or []) == set(profile.task_types)
- same_scope = dict(row.scope or {}) == dict(profile.scope)
- if same_types and same_scope:
- row.is_active = False
-
- row = previous or EvolutionAgentProfile(profile_id=profile.profile_id)
- row.version = profile.version
- row.task_types = list(profile.task_types)
- row.scope = dict(profile.scope)
- row.payload = profile.to_dict()
- row.candidate_id = candidate_id
- row.is_active = True
- row.created_by = f"evolution:{approver}"
- if previous is None:
- db.add(row)
-
- release = EvolutionRelease(
- release_id=new_release_id(),
- candidate_id=candidate_id,
- target_kind=TARGET_KIND,
- target_asset_id=profile.profile_id,
- stage=_ACTIVE,
- traffic_percent=100,
- scope_filter=dict(profile.scope),
- guardrails={"previous": previous_payload, "proposer": candidate.proposer or ""},
- rollback_version_id=(previous_payload or {}).get("version") or "builtin",
- approved_by=approver,
- )
- db.add(release)
-
- candidate.status = _ACTIVE
- candidate.approved_by = approver
- candidate.approved_at = datetime.now(timezone.utc)
- db.commit()
-
- # 血缘:哪些技能被装配进了这个 profile。
- try:
- from core.evolution import lineage
-
- lineage.record_links(
- [
- (
- "skill",
- str(skill_id),
- lineage.REL_ASSEMBLED_INTO,
- "agent_profile",
- f"{profile.profile_id}@{profile.version}",
- )
- for skill_id in profile.skill_ids
- ],
- candidate_id=candidate_id,
- )
- except Exception as exc: # noqa: BLE001
- logger.debug("[activation] profile lineage skipped: %s", exc)
-
- logger.info(
- "[activation] profile %s active for %s (tools=%s skills=%s)",
- profile.profile_id,
- profile.task_types or ["*"],
- "narrowed" if profile.tool_allowlist is not None else "unchanged",
- len(profile.skill_ids),
- )
- return {
- "profile_id": profile.profile_id,
- "release_id": release.release_id,
- "task_types": list(profile.task_types),
- "scope": dict(profile.scope),
- "tool_allowlist": profile.tool_allowlist,
- "skill_ids": list(profile.skill_ids),
- "replaced_previous": previous_payload is not None,
- }
-
-
-def materialise_subagent_candidate(
- candidate_id: str, *, approver: str, owner_user_id: Optional[str] = None
-) -> Dict[str, Any]:
- """Create an approved sub-agent, and nothing else.
-
- Deliberately does **not** install the route that sends work to it. The route
- is a sentence in the parent's prompt, and a sentence in the parent's prompt
- changes every request the parent handles — so it goes through prompt review
- as its own candidate. Creating the agent here and routing to it there means
- approving the agent cannot silently also approve a behaviour change.
-
- The agent is created disabled for the same reason: an agent nobody routes to
- does nothing, and one that is enabled before its route is reviewed is a
- capability sitting in the product that no reviewer has seen used.
- """
- from core.db.models import UserAgent
- from core.evolution.release import ACTIVE as _ACTIVE, SHADOW, new_release_id
-
- if not approver:
- raise ActivationError("approver_required", "激活必须署名")
-
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- if candidate is None:
- raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
- if candidate.target_kind != "subagent":
- raise ActivationError(
- "unsupported_kind", f"该入口只处理子智能体候选,收到 {candidate.target_kind}"
- )
- if approver == (candidate.proposer or ""):
- raise ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选")
- if _is_frozen(db):
- raise ActivationError("layer_frozen", "进化已被全局冻结")
-
- spec: Dict[str, Any] = {}
- for change in candidate.ir.get("changes") or []:
- if change.get("agent_id"):
- spec = change
- break
- if not spec:
- raise ActivationError("no_subagent_payload", "候选未携带子智能体定义")
-
- # The subset check runs again at activation against the current grant.
- from core.config.catalog import get_enabled_ids
-
- base_tools = {str(t) for t in get_enabled_ids("tools")} | {
- str(t) for t in get_enabled_ids("mcp")
- }
- requested = {str(t) for t in (spec.get("tools_allowlist") or [])}
- widened = sorted(requested - base_tools)
- if widened:
- raise ActivationError(
- "tool_escalation", f"子智能体工具集超出主体授权: {widened}"
- )
- if spec.get("may_delegate"):
- raise ActivationError("subagent_nesting", "子智能体不得再派生子智能体")
-
- agent_id = str(spec.get("agent_id"))
- existing = db.get(UserAgent, agent_id)
- if existing is not None:
- raise ActivationError("agent_exists", f"子智能体已存在: {agent_id}")
-
- agent = UserAgent(
- agent_id=agent_id,
- owner_type="user" if owner_user_id else "admin",
- user_id=owner_user_id,
- name=str(spec.get("name") or agent_id)[:255],
- description=str(spec.get("responsibility") or "")[:2000],
- system_prompt=str(spec.get("responsibility") or ""),
- skill_ids=[str(s) for s in (spec.get("required_skills") or [])],
- mcp_server_ids=sorted(requested),
- kb_ids=[],
- plugin_ids=[],
- max_iters=12,
- timeout=int(spec.get("timeout") or 180),
- # Disabled until its route is approved — see the docstring.
- is_enabled=False,
- extra_config={
- "origin": "evolution",
- "candidate_id": candidate_id,
- "route_hint": str(spec.get("route_hint") or ""),
- "task_types": [str(t) for t in (spec.get("task_types") or [])],
- "tools_allowlist": sorted(requested),
- "may_delegate": False,
- },
- )
- db.add(agent)
-
- release = EvolutionRelease(
- release_id=new_release_id(),
- candidate_id=candidate_id,
- target_kind="subagent",
- target_asset_id=agent_id,
- # Reaches nobody until the route exists, which is what shadow means.
- stage=SHADOW,
- traffic_percent=0,
- scope_filter={"owner_user_id": owner_user_id} if owner_user_id else {},
- guardrails={"proposer": candidate.proposer or "", "route_pending": True},
- rollback_version_id="v0",
- approved_by=approver,
- )
- db.add(release)
-
- candidate.approved_by = approver
- candidate.approved_at = datetime.now(timezone.utc)
- db.commit()
-
- logger.info(
- "[activation] subagent %s created (disabled, route pending) tools=%s",
- agent_id,
- sorted(requested),
- )
- return {
- "agent_id": agent_id,
- "release_id": release.release_id,
- "stage": SHADOW,
- "is_enabled": False,
- "tools_allowlist": sorted(requested),
- "route_hint": str(spec.get("route_hint") or ""),
- "next_step": "路由句需作为提示词候选单独审批后,子智能体才会被启用",
- }
-
-
-def retire_skill_candidate(candidate_id: str, *, approver: str) -> Dict[str, Any]:
- """Take an approved retirement into effect.
-
- Retirement is disabling plus a release row, never a delete. Three reasons it
- is done this way: the evidence behind the decision stays inspectable, the
- skill can be brought back in one operation if the decision was wrong, and
- anything built on top of it — a profile naming it, a sub-agent requiring it
- — can be found and invalidated rather than silently pointing at nothing.
- """
- from core.db.models import AdminSkill
- from core.db.models.evolution import EvolutionAgentProfile
- from core.evolution.release import RETIRED, new_release_id
-
- if not approver:
- raise ActivationError("approver_required", "退役必须署名")
-
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- if candidate is None:
- raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
- if candidate.operation != "deprecate":
- raise ActivationError("not_a_retirement", "该候选不是退役候选")
- if approver == (candidate.proposer or ""):
- raise ActivationError("self_approval_forbidden", "生成者不能批准自己提出的退役")
-
- skill_id = candidate.target_asset_id
- row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
- if row is None:
- raise ActivationError("skill_not_found", f"技能不存在: {skill_id}")
-
- previous_enabled = bool(row.is_enabled)
- row.is_enabled = False
-
- # Cross-layer: anything assembled on top of this skill stops being
- # applied. Leaving them active would leave a profile promising a
- # capability that is no longer loadable, which fails silently.
- invalidated: List[str] = []
- for profile_row in (
- db.query(EvolutionAgentProfile)
- .filter(EvolutionAgentProfile.is_active.is_(True))
- .all()
- ):
- payload = profile_row.payload or {}
- if skill_id in (payload.get("skill_ids") or []):
- profile_row.is_active = False
- invalidated.append(profile_row.profile_id)
-
- release = EvolutionRelease(
- release_id=new_release_id(),
- candidate_id=candidate_id,
- target_kind="skill",
- target_asset_id=skill_id,
- stage=RETIRED,
- traffic_percent=0,
- scope_filter={},
- guardrails={
- "previous_enabled": previous_enabled,
- "invalidated_profiles": invalidated,
- "proposer": candidate.proposer or "",
- },
- rollback_version_id=row.version or "",
- approved_by=approver,
- )
- db.add(release)
-
- candidate.status = "active"
- candidate.approved_by = approver
- candidate.approved_at = datetime.now(timezone.utc)
- db.commit()
-
- # Restore the memories this skill was compiled from: the knowledge lost
- # its cheaper carrier, so it goes back to being paid for by retrieval.
- from core.evolution.memory_apply import revert_memory_ops
-
- source_candidate = _source_candidate_for(db, skill_id)
- restored = revert_memory_ops(candidate_id=source_candidate) if source_candidate else {}
-
- logger.info(
- "[activation] retired %s (profiles invalidated=%s memories restored=%d)",
- skill_id,
- invalidated,
- len(restored.get("reverted", [])),
- )
- return {
- "skill_id": skill_id,
- "release_id": release.release_id,
- "invalidated_profiles": invalidated,
- "restored_memory_ops": restored.get("reverted", []),
- }
-
-
-def _source_candidate_for(db, skill_id: str) -> str:
- """The candidate that first materialised this skill.
-
- Memory demotions are recorded against that candidate, so it is the key that
- finds them again when the skill is retired.
- """
- row = (
- db.query(EvolutionRelease)
- .filter(
- EvolutionRelease.target_kind == "skill",
- EvolutionRelease.target_asset_id == skill_id,
- )
- .order_by(EvolutionRelease.created_at.asc())
- .first()
- )
- return str(row.candidate_id) if row is not None else ""
diff --git a/src/backend/core/evolution/agent_profile.py b/src/backend/core/evolution/agent_profile.py
deleted file mode 100644
index 7d73981f..00000000
--- a/src/backend/core/evolution/agent_profile.py
+++ /dev/null
@@ -1,505 +0,0 @@
-"""The orchestration asset: what one ReAct assembly is made of.
-
-Orchestration evolution used to target :class:`~core.evolution.policies.WorkflowPolicy`,
-a set of retry/budget knobs read by ``autonomous_loop``. That was the wrong
-target, for a reason that has nothing to do with how the knobs were written: the
-product's main axis is a **ReAct loop**, and a ReAct loop has no DAG to tune. It
-re-decides every turn from whatever was assembled in front of it. So the
-orchestration of a ReAct system *is* the assembly, and it has six parts:
-
-1. **prompt fragments** — what standing instructions this task type carries;
-2. **tool allowlist** — which tools are even visible (a narrower set is a better
- set: every irrelevant tool is a wrong turn the model can take);
-3. **skill set** — which capabilities are offered, i.e. "the combination of
- skills", which is what a recurring skill co-occurrence should promote *into*;
-4. **subagent routes** — what gets handed to a separate context entirely;
-5. **retrieval and budget** — memory budget, max turns, reviewer checkpoints;
-6. **intervention rules** — what to do when the loop stops making progress,
- which applies to a ReAct loop exactly as it applied to the autonomous one.
-
-A profile is **declarative only**. It selects, orders, budgets and routes; it
-never carries executable content. ``OP_WHITELIST["agent_profile"]`` has no
-operation resembling ``edit_source``, and the tool allowlist can only ever
-narrow: :func:`validate_profile` refuses a profile requesting a tool outside the
-base grant, so an evolved profile cannot become a privilege-escalation path.
-
-Scope resolution is explicit and ordered — user, then tenant, then deployment
-default — and a profile belonging to *another* tenant is never a fallback.
-Inheriting someone else's orchestration is invisible when it happens and shows
-up as "the system got slower last week", which is the hardest class of problem
-to attribute.
-"""
-
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-logger = logging.getLogger(__name__)
-
-# ── Intervention vocabulary (signals a ReAct loop can actually observe) ───────
-
-SIGNAL_NO_PROGRESS = "no_progress"
-SIGNAL_REPEATED_ACTIONS = "repeated_actions"
-SIGNAL_NO_DIFF = "no_diff"
-SIGNAL_REVIEWER_FLAT = "reviewer_score_flat"
-SIGNAL_BUDGET_LOW = "budget_low"
-SIGNAL_TOOL_ERROR_STREAK = "tool_error_streak"
-ALLOWED_SIGNALS = (
- SIGNAL_NO_PROGRESS,
- SIGNAL_REPEATED_ACTIONS,
- SIGNAL_NO_DIFF,
- SIGNAL_REVIEWER_FLAT,
- SIGNAL_BUDGET_LOW,
- SIGNAL_TOOL_ERROR_STREAK,
-)
-
-# What a single ReAct turn can actually observe about itself. The rest need a
-# file tree and a reviewer verdict — things a long-running loop has and one
-# answer does not — so a rule keyed on them is inert on the main axis. Stated
-# here rather than discovered later: a rule that can never fire looks identical,
-# in the console, to one that simply has not fired yet.
-REACT_OBSERVABLE_SIGNALS = frozenset(
- {SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_TOOL_ERROR_STREAK}
-)
-LOOP_ONLY_SIGNALS = frozenset(
- {SIGNAL_NO_DIFF, SIGNAL_REVIEWER_FLAT, SIGNAL_BUDGET_LOW}
-)
-
-ACTION_RETRY = "retry"
-ACTION_CHANGE_STRATEGY = "change_strategy"
-ACTION_ROLLBACK_AND_FORK = "rollback_and_fork"
-ACTION_NARROW_SCOPE = "narrow_scope"
-ACTION_DELEGATE = "delegate"
-ACTION_ESCALATE = "escalate"
-ACTION_STOP = "stop"
-ALLOWED_ACTIONS = (
- ACTION_RETRY,
- ACTION_CHANGE_STRATEGY,
- ACTION_ROLLBACK_AND_FORK,
- ACTION_NARROW_SCOPE,
- ACTION_DELEGATE,
- ACTION_ESCALATE,
- ACTION_STOP,
-)
-
-# Budget multipliers outside this band stop being tuning and start being a
-# different product: a 10× budget is a cost incident, a 0.05× budget is an
-# outage. The band is validated, not clamped — silently correcting an
-# out-of-range value hides the fact that something proposed it.
-MIN_BUDGET_MULTIPLIER = 0.25
-MAX_BUDGET_MULTIPLIER = 3.0
-# 0 is the default and means "no turn cap" — the main agent is unbounded unless
-# a published profile deliberately fences it. Any other value must land in the
-# band below; the band starts at 3 because a 1-2 turn main agent cannot finish a
-# single tool round-trip.
-MIN_REACT_TURNS = 3
-MAX_REACT_TURNS = 80
-UNBOUNDED_REACT_TURNS = 0
-MIN_MEMORY_BUDGET_MS = 100
-MAX_MEMORY_BUDGET_MS = 5000
-MIN_LOOP_ATTEMPTS = 2
-MAX_LOOP_ATTEMPTS = 20
-
-TARGET_KIND = "agent_profile"
-
-
-@dataclass
-class InterventionRule:
- """signal → threshold → action."""
-
- signal: str
- threshold: int
- action: str
- note: str = ""
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "signal": self.signal,
- "threshold": self.threshold,
- "action": self.action,
- "note": self.note,
- }
-
- @classmethod
- def from_dict(cls, raw: Dict[str, Any]) -> "InterventionRule":
- return cls(
- signal=str(raw.get("signal") or ""),
- threshold=int(raw.get("threshold") or 0),
- action=str(raw.get("action") or ""),
- note=str(raw.get("note") or ""),
- )
-
-
-@dataclass
-class SubagentRoute:
- """Hand this shape of sub-task to a separate context.
-
- ``agent_id`` points at a real sub-agent; the route is only the *decision to
- delegate*. Creating the sub-agent is a separate, separately-approved change
- (see :mod:`core.evolution.orchestration_gen`), because a route to an agent
- that does not exist is a dead end and an agent nobody routes to is dead
- weight — they have to be reviewed as one change but recorded as two.
- """
-
- agent_id: str
- when: str = ""
- task_types: List[str] = field(default_factory=list)
-
- def to_dict(self) -> Dict[str, Any]:
- return {"agent_id": self.agent_id, "when": self.when, "task_types": list(self.task_types)}
-
- @classmethod
- def from_dict(cls, raw: Dict[str, Any]) -> "SubagentRoute":
- return cls(
- agent_id=str(raw.get("agent_id") or ""),
- when=str(raw.get("when") or ""),
- task_types=[str(t) for t in (raw.get("task_types") or [])],
- )
-
-
-@dataclass
-class AgentProfile:
- """One assembled ReAct configuration, as data."""
-
- profile_id: str = "builtin"
- version: str = "v0"
- # Which tasks this profile governs. Empty means "every task in scope" — the
- # deployment default. A profile with task types is only consulted for those,
- # which is how a capability learned in one family stops leaking into others.
- task_types: List[str] = field(default_factory=list)
- scope: Dict[str, str] = field(default_factory=dict)
-
- prompt_fragments: List[str] = field(default_factory=list)
- # ``None`` means "no restriction beyond the base grant". An empty list means
- # "no tools", which is a legitimate profile for a pure-reasoning task type
- # and must not be confused with the former.
- tool_allowlist: Optional[List[str]] = None
- # Which skills this task type may load at all. Empty means "no restriction"
- # — skill loading keeps its own logic and this only ever narrows it, and
- # only for task types an approved profile actually covers.
- skill_ids: List[str] = field(default_factory=list)
- subagent_routes: List[SubagentRoute] = field(default_factory=list)
-
- memory_budget_ms: int = 600
- # 0 = 不限(默认)。See UNBOUNDED_REACT_TURNS: the main agent's round ceiling
- # was removed, so a turn budget here is an opt-in narrowing a published
- # profile performs deliberately, not a floor every profile inherits.
- max_react_turns: int = UNBOUNDED_REACT_TURNS
- budget_multiplier: float = 1.0
- reviewer_checkpoints: List[str] = field(default_factory=list)
- intervention_rules: List[InterventionRule] = field(default_factory=list)
-
- # The autonomous loop's own caps. Held as their own fields rather than
- # derived from ``max_react_turns``: the two loops bound different things —
- # turns inside one answer versus attempts at one requirement across many
- # answers — and a ratio between them would be an invented relationship that
- # quietly changes one whenever the other is tuned.
- loop_max_attempts_per_requirement: int = 6
- loop_strategy_change_after: int = 2
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "profile_id": self.profile_id,
- "version": self.version,
- "task_types": list(self.task_types),
- "scope": dict(self.scope),
- "prompt_fragments": list(self.prompt_fragments),
- "tool_allowlist": None if self.tool_allowlist is None else list(self.tool_allowlist),
- "skill_ids": list(self.skill_ids),
- "subagent_routes": [r.to_dict() for r in self.subagent_routes],
- "memory_budget_ms": self.memory_budget_ms,
- "max_react_turns": self.max_react_turns,
- "budget_multiplier": self.budget_multiplier,
- "reviewer_checkpoints": list(self.reviewer_checkpoints),
- "intervention_rules": [r.to_dict() for r in self.intervention_rules],
- "loop_max_attempts_per_requirement": self.loop_max_attempts_per_requirement,
- "loop_strategy_change_after": self.loop_strategy_change_after,
- }
-
- @classmethod
- def from_dict(cls, raw: Dict[str, Any]) -> "AgentProfile":
- allowlist = raw.get("tool_allowlist", None)
- return cls(
- profile_id=str(raw.get("profile_id") or "builtin"),
- version=str(raw.get("version") or "v0"),
- task_types=[str(t) for t in (raw.get("task_types") or [])],
- scope=dict(raw.get("scope") or {}),
- prompt_fragments=[str(f) for f in (raw.get("prompt_fragments") or [])],
- tool_allowlist=None if allowlist is None else [str(t) for t in allowlist],
- skill_ids=[str(s) for s in (raw.get("skill_ids") or [])],
- subagent_routes=[
- SubagentRoute.from_dict(r) for r in (raw.get("subagent_routes") or [])
- ],
- memory_budget_ms=int(raw.get("memory_budget_ms") or 600),
- # Missing key → unbounded, not an invented ceiling: a profile
- # published for unrelated reasons (tool allowlist, memory budget)
- # must not silently re-fence the main loop.
- max_react_turns=int(raw.get("max_react_turns") or UNBOUNDED_REACT_TURNS),
- budget_multiplier=float(raw.get("budget_multiplier") or 1.0),
- reviewer_checkpoints=[str(c) for c in (raw.get("reviewer_checkpoints") or [])],
- intervention_rules=[
- InterventionRule.from_dict(r) for r in (raw.get("intervention_rules") or [])
- ],
- loop_max_attempts_per_requirement=int(
- raw.get("loop_max_attempts_per_requirement") or 6
- ),
- loop_strategy_change_after=int(raw.get("loop_strategy_change_after") or 2),
- )
-
- def applies_to(self, task_type: str) -> bool:
- return not self.task_types or str(task_type or "chat") in self.task_types
-
-
-def builtin_profile() -> AgentProfile:
- """The deployment default.
-
- Reproduces today's behaviour exactly — no tool restriction, no forced skill
- set, the retrieval budget and turn cap the runtime already uses — so
- installing profile resolution changes nothing until a profile is published.
- """
- return AgentProfile(
- profile_id="builtin",
- version="v0",
- tool_allowlist=None,
- memory_budget_ms=600,
- # The main agent's runtime default, reproduced exactly — which is now
- # "no cap". This field only bounds the loop when a published profile
- # sets it on purpose.
- max_react_turns=UNBOUNDED_REACT_TURNS,
- budget_multiplier=1.0,
- loop_max_attempts_per_requirement=6,
- loop_strategy_change_after=2,
- intervention_rules=[
- InterventionRule(SIGNAL_NO_PROGRESS, 2, ACTION_CHANGE_STRATEGY),
- InterventionRule(SIGNAL_REPEATED_ACTIONS, 3, ACTION_NARROW_SCOPE),
- InterventionRule(SIGNAL_TOOL_ERROR_STREAK, 3, ACTION_CHANGE_STRATEGY),
- InterventionRule(SIGNAL_BUDGET_LOW, 1, ACTION_ESCALATE),
- ],
- )
-
-
-def validate_profile(
- profile: AgentProfile,
- *,
- base_tools: Optional[Sequence[str]] = None,
- known_skill_ids: Optional[Sequence[str]] = None,
- known_agent_ids: Optional[Sequence[str]] = None,
-) -> Tuple[bool, List[str]]:
- """Structural and security validation.
-
- ``base_tools`` is the authority this profile is allowed to draw from. A
- profile naming a tool outside it is refused — orchestration evolution may
- narrow authority and may never widen it, and this is the check that makes
- "narrow only" a property of the system rather than an intention.
- """
- problems: List[str] = []
-
- if not profile.profile_id:
- problems.append("profile_id 不能为空")
-
- if profile.tool_allowlist is not None and base_tools is not None:
- widened = sorted(set(profile.tool_allowlist) - set(base_tools))
- if widened:
- problems.append(f"工具授权越界(只允许收敛,不允许扩权): {widened}")
-
- if known_skill_ids is not None:
- unknown = sorted(set(profile.skill_ids) - set(known_skill_ids))
- if unknown:
- problems.append(f"引用了不存在的技能: {unknown}")
-
- seen_agents = set()
- for route in profile.subagent_routes:
- if not route.agent_id:
- problems.append("子智能体路由缺少 agent_id")
- continue
- if route.agent_id in seen_agents:
- problems.append(f"重复的子智能体路由: {route.agent_id}")
- seen_agents.add(route.agent_id)
- if known_agent_ids is not None and route.agent_id not in set(known_agent_ids):
- problems.append(f"路由指向不存在的子智能体: {route.agent_id}")
-
- for rule in profile.intervention_rules:
- if rule.signal not in ALLOWED_SIGNALS:
- problems.append(f"未知信号 {rule.signal}")
- if rule.action not in ALLOWED_ACTIONS:
- problems.append(f"未知动作 {rule.action}")
- if rule.threshold < 1:
- problems.append(f"{rule.signal} 阈值必须 ≥1(0 会让干预立刻触发)")
-
- if not (MIN_BUDGET_MULTIPLIER <= profile.budget_multiplier <= MAX_BUDGET_MULTIPLIER):
- problems.append(
- f"预算倍数 {profile.budget_multiplier} 超出 "
- f"[{MIN_BUDGET_MULTIPLIER}, {MAX_BUDGET_MULTIPLIER}]"
- )
- if profile.max_react_turns != UNBOUNDED_REACT_TURNS and not (
- MIN_REACT_TURNS <= profile.max_react_turns <= MAX_REACT_TURNS
- ):
- problems.append(
- f"最大轮数 {profile.max_react_turns} 超出 "
- f"[{MIN_REACT_TURNS}, {MAX_REACT_TURNS}]({UNBOUNDED_REACT_TURNS} 表示不限)"
- )
- if not (MIN_MEMORY_BUDGET_MS <= profile.memory_budget_ms <= MAX_MEMORY_BUDGET_MS):
- problems.append(
- f"记忆检索预算 {profile.memory_budget_ms}ms 超出 "
- f"[{MIN_MEMORY_BUDGET_MS}, {MAX_MEMORY_BUDGET_MS}]"
- )
- if not (
- MIN_LOOP_ATTEMPTS
- <= profile.loop_max_attempts_per_requirement
- <= MAX_LOOP_ATTEMPTS
- ):
- problems.append(
- f"单需求最大尝试次数 {profile.loop_max_attempts_per_requirement} 超出 "
- f"[{MIN_LOOP_ATTEMPTS}, {MAX_LOOP_ATTEMPTS}]"
- )
- if profile.loop_strategy_change_after < 1:
- problems.append("换策略阈值必须 ≥1(0 会让每一轮都换策略)")
-
- return (not problems), problems
-
-
-# ── Resolution ───────────────────────────────────────────────────────────────
-
-
-def _scope_rank(scope: Dict[str, str], *, user_id: str, tenant_id: str) -> Optional[int]:
- """How specific this profile is to the subject, or ``None`` if it is not.
-
- Lower is more specific. Returning ``None`` for a profile owned by another
- subject is the whole point: it must not act as a fallback.
- """
- owner = str((scope or {}).get("user_id") or "")
- tenant = str((scope or {}).get("tenant_id") or "")
- if owner:
- return 0 if owner == user_id else None
- if tenant:
- return 1 if tenant == (tenant_id or "default") else None
- return 2
-
-
-def pick_profile(
- profiles: Sequence[AgentProfile],
- *,
- task_type: str,
- user_id: str = "",
- tenant_id: str = "default",
-) -> Optional[AgentProfile]:
- """The one profile that governs this run, or ``None``.
-
- A task-type-specific profile beats a general one at the same scope level:
- the narrower statement is the more deliberate one.
- """
- ranked: List[Tuple[int, int, AgentProfile]] = []
- for profile in profiles:
- if not profile.applies_to(task_type):
- continue
- rank = _scope_rank(profile.scope, user_id=user_id, tenant_id=tenant_id)
- if rank is None:
- continue
- ranked.append((rank, 0 if profile.task_types else 1, profile))
- if not ranked:
- return None
- ranked.sort(key=lambda item: (item[0], item[1]))
- return ranked[0][2]
-
-
-def _profile_from_row(row: Any) -> AgentProfile:
- """The profile a stored row describes, with the columns believed over payload.
-
- ``profile_id`` / ``version`` / ``task_types`` / ``scope`` exist as real
- columns *and* inside ``payload`` (:meth:`AgentProfile.to_dict` writes them),
- so for anything published through :mod:`core.evolution.activation` the two
- agree and this is a no-op. They stop agreeing for a row written by hand or
- by a seeding script, and reading only ``payload`` then fails open in the
- worst possible direction: a missing ``task_types`` parses as ``[]``, which
- :meth:`AgentProfile.applies_to` treats as *every* task type, and a missing
- ``scope`` parses as ``{}``, which :func:`_scope_rank` treats as *every*
- subject. A profile published for one task type silently governs every run —
- reported under the id ``builtin``, because that is what
- :meth:`AgentProfile.from_dict` falls back to, so the log line naming the
- culprit names the deployment default instead.
-
- The two narrowing fields resolve to whichever side is non-empty rather than
- to the column unconditionally: an empty column against a populated payload
- is the same widening failure with the sides swapped. When both are populated
- and disagree the column wins — it is what the activation uniqueness check and
- the console read — and the disagreement is logged, because it means something
- wrote this row outside the activation path.
- """
- raw = dict(row.payload or {})
-
- # Identity: the column is the primary key and NOT NULL, so it is always the
- # more trustworthy of the two, and a wrong id here is what makes a bad
- # profile untraceable in the logs.
- raw["profile_id"] = row.profile_id
- raw["version"] = row.version
-
- col_types = [str(t) for t in (row.task_types or [])]
- payload_types = [str(t) for t in (raw.get("task_types") or [])]
- if col_types and payload_types and set(col_types) != set(payload_types):
- logger.warning(
- "[agent-profile] %s: task_types column %s != payload %s, using column",
- row.profile_id, col_types, payload_types,
- )
- raw["task_types"] = col_types or payload_types
-
- col_scope = dict(row.scope or {})
- payload_scope = dict(raw.get("scope") or {})
- if col_scope and payload_scope and col_scope != payload_scope:
- logger.warning(
- "[agent-profile] %s: scope column %s != payload %s, using column",
- row.profile_id, col_scope, payload_scope,
- )
- raw["scope"] = col_scope or payload_scope
-
- return AgentProfile.from_dict(raw)
-
-
-def load_active_profile(
- *,
- task_type: str = "chat",
- user_id: str = "",
- tenant_id: str = "default",
-) -> AgentProfile:
- """The profile in force for this run.
-
- Read fresh on every call. Resolving once at import — which is what the old
- orchestration policy did — means a published change needs a process restart
- to take effect and different replicas disagree in the meantime, neither of
- which is acceptable for an asset whose entire purpose is to change behaviour
- between runs.
- """
- from core.db.engine import SessionLocal
- from core.db.models.evolution import EvolutionAgentProfile
-
- try:
- with SessionLocal() as db:
- rows = (
- db.query(EvolutionAgentProfile)
- .filter(EvolutionAgentProfile.is_active.is_(True))
- .order_by(EvolutionAgentProfile.updated_at.desc())
- .all()
- )
- profiles = [_profile_from_row(row) for row in rows]
- except Exception as exc: # noqa: BLE001
- logger.warning("[agent-profile] load failed, using built-in: %s", exc)
- return builtin_profile()
-
- chosen = pick_profile(
- profiles, task_type=task_type, user_id=user_id, tenant_id=tenant_id
- )
- if chosen is None:
- return builtin_profile()
- ok, problems = validate_profile(chosen)
- if not ok:
- # A stored profile that no longer validates is not applied. The
- # alternative — applying it partially — produces a configuration nobody
- # reviewed and nobody can reproduce.
- logger.warning(
- "[agent-profile] stored profile %s rejected at load: %s",
- chosen.profile_id,
- problems,
- )
- return builtin_profile()
- return chosen
diff --git a/src/backend/core/evolution/assessment/__init__.py b/src/backend/core/evolution/assessment/__init__.py
new file mode 100644
index 00000000..c3a8e199
--- /dev/null
+++ b/src/backend/core/evolution/assessment/__init__.py
@@ -0,0 +1 @@
+"""Evaluate evolution candidates: replay benchmarks, metrics and similarity."""
diff --git a/src/backend/core/evolution/applicability.py b/src/backend/core/evolution/assessment/applicability.py
similarity index 98%
rename from src/backend/core/evolution/applicability.py
rename to src/backend/core/evolution/assessment/applicability.py
index 37351eae..754a2e51 100644
--- a/src/backend/core/evolution/applicability.py
+++ b/src/backend/core/evolution/assessment/applicability.py
@@ -107,7 +107,7 @@ def assess_replay_coverage(
task_ids: Sequence[str],
) -> Dict[str, Any]:
"""Whether a replay set covers what the pack claims. ``pack`` is a
- :class:`~core.evolution.evidence_contract.CapabilityEvidencePackV2`.
+ :class:`~core.evolution.evidence.evidence_contract.CapabilityEvidencePackV2`.
Required, each only when the pack actually contains that class of evidence:
diff --git a/src/backend/core/evolution/benchmark.py b/src/backend/core/evolution/assessment/benchmark.py
similarity index 98%
rename from src/backend/core/evolution/benchmark.py
rename to src/backend/core/evolution/assessment/benchmark.py
index ac65aee5..936ee3ad 100644
--- a/src/backend/core/evolution/benchmark.py
+++ b/src/backend/core/evolution/assessment/benchmark.py
@@ -41,7 +41,7 @@
import hashlib
import logging
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
logger = logging.getLogger(__name__)
@@ -211,7 +211,7 @@ def plan(
proposed, replans = self._unaided(task)
if ordering_rules:
- from core.evolution.promotion import apply_ordering_constraints
+ from core.evolution.cycle.ordering import apply_ordering_constraints
applicable = [
r for r in ordering_rules if r.applies_to(proposed, task.family)
@@ -314,7 +314,7 @@ def to_dict(self) -> Dict[str, Any]:
def compare(before: ArmSummary, after: ArmSummary) -> Dict[str, Any]:
"""Compare arms, including the checks that can invalidate the result."""
- from core.evolution.replay import mcnemar_p_value
+ from core.evolution.evidence.replay import mcnemar_p_value
by_before = {r.task_id: r for r in before.results}
by_after = {r.task_id: r for r in after.results}
diff --git a/src/backend/core/evolution/evaluator.py b/src/backend/core/evolution/assessment/evaluator.py
similarity index 100%
rename from src/backend/core/evolution/evaluator.py
rename to src/backend/core/evolution/assessment/evaluator.py
diff --git a/src/backend/core/evolution/experiments.py b/src/backend/core/evolution/assessment/experiments.py
similarity index 99%
rename from src/backend/core/evolution/experiments.py
rename to src/backend/core/evolution/assessment/experiments.py
index cc6af77c..21eb841d 100644
--- a/src/backend/core/evolution/experiments.py
+++ b/src/backend/core/evolution/assessment/experiments.py
@@ -26,7 +26,7 @@
import logging
from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/fault_injection.py b/src/backend/core/evolution/assessment/fault_injection.py
similarity index 98%
rename from src/backend/core/evolution/fault_injection.py
rename to src/backend/core/evolution/assessment/fault_injection.py
index 2c37de26..6535060c 100644
--- a/src/backend/core/evolution/fault_injection.py
+++ b/src/backend/core/evolution/assessment/fault_injection.py
@@ -38,20 +38,7 @@
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Tuple
-from core.evolution.credit import (
- ALL_TARGETS,
- NON_ASSET_TARGETS,
- T_ENVIRONMENT,
- T_MEMORY,
- T_MODEL,
- T_NO_UPDATE,
- T_ONTOLOGY,
- T_RETRIEVAL,
- T_SKILL,
- T_ORCHESTRATION,
- CreditFeatures,
- assign_credit,
-)
+from core.evolution.evidence.credit import NON_ASSET_TARGETS, T_ENVIRONMENT, T_MEMORY, T_MODEL, T_NO_UPDATE, T_ONTOLOGY, T_RETRIEVAL, T_SKILL, T_ORCHESTRATION, CreditFeatures, assign_credit
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/metrics.py b/src/backend/core/evolution/assessment/metrics.py
similarity index 100%
rename from src/backend/core/evolution/metrics.py
rename to src/backend/core/evolution/assessment/metrics.py
diff --git a/src/backend/core/evolution/similarity.py b/src/backend/core/evolution/assessment/similarity.py
similarity index 98%
rename from src/backend/core/evolution/similarity.py
rename to src/backend/core/evolution/assessment/similarity.py
index e49d34c3..8e033e94 100644
--- a/src/backend/core/evolution/similarity.py
+++ b/src/backend/core/evolution/assessment/similarity.py
@@ -30,9 +30,8 @@
import logging
import math
import re
-from collections import defaultdict
from dataclasses import dataclass, field
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
logger = logging.getLogger(__name__)
@@ -219,7 +218,7 @@ def _normalise(text: str) -> str:
return _PUNCT.sub("", str(text or "")).lower()
-def _ngrams(text: str, n: int = 3) -> set:
+def ngrams(text: str, n: int = 3) -> set:
"""Character n-grams.
Character-level on purpose: word segmentation for Chinese needs a tokeniser
@@ -239,7 +238,7 @@ def lexical_similarity(a: str, b: str, n: int = 3) -> float:
and concept names rather than whole questions — want ``n=2``: trigrams over a
four-character name leave too few grams to overlap on.
"""
- ga, gb = _ngrams(a, n), _ngrams(b, n)
+ ga, gb = ngrams(a, n), ngrams(b, n)
if not ga or not gb:
return 0.0
intersection = len(ga & gb)
@@ -267,7 +266,7 @@ def _embed_all(texts: Sequence[str]) -> Optional[List[List[float]]]:
artefact of which calls happened to succeed.
"""
try:
- from core.kb.kb_vector import embed_batch
+ from core.models.embedding import embed_batch
vectors = embed_batch(list(texts))
except Exception as exc: # noqa: BLE001
diff --git a/src/backend/core/evolution/sop_judge.py b/src/backend/core/evolution/assessment/sop_judge.py
similarity index 98%
rename from src/backend/core/evolution/sop_judge.py
rename to src/backend/core/evolution/assessment/sop_judge.py
index 28268fe0..f83dc8fb 100644
--- a/src/backend/core/evolution/sop_judge.py
+++ b/src/backend/core/evolution/assessment/sop_judge.py
@@ -10,7 +10,7 @@
* 把这套做法固化下来照做,可能有规则看不见的**明显风险**。
所以在规则幸存者上加一道 LLM 裁决。边界与技能编译的「两段式生成」一致
-(见 :mod:`core.evolution.skill_gen`):模型只回答「这值不值得沉淀为 SOP」
+(见 :mod:`core.evolution.generators.skill_gen`):模型只回答「这值不值得沉淀为 SOP」
并给出理由和触发条件,它不能扩权、不能改工具序列、不能凭空造资产——那些
仍由代码与 IR 不变量把守。裁决结论随候选落库,评审员能看到机器放行的理由。
diff --git a/src/backend/core/evolution/backfill.py b/src/backend/core/evolution/backfill.py
index 0a51bd02..c831e481 100644
--- a/src/backend/core/evolution/backfill.py
+++ b/src/backend/core/evolution/backfill.py
@@ -22,7 +22,7 @@
from core.db.engine import SessionLocal
from core.db.usage_filters import model_usage_message_filter
-from core.evolution.trace_assembler import assemble_episode
+from core.evolution.evidence.trace_assembler import assemble_episode
logger = logging.getLogger(__name__)
@@ -108,7 +108,7 @@ def _outcome_from_logs(db, message_id: str, chat_id: str) -> Optional[Dict[str,
apart from an explicit user judgement and weight it accordingly.
"""
from core.db.models import ChatMessage
- from core.evolution.trace_assembler import _tool_rows_for
+ from core.evolution.evidence.trace_assembler import _tool_rows_for
rows = _tool_rows_for(db, message_id, chat_id)
if not rows:
@@ -154,7 +154,6 @@ def backfill_episodes(
start = end - timedelta(days=days)
stats = {"scanned": 0, "created": 0, "skipped": 0, "batches": 0}
- offset = 0
while True:
if max_batches is not None and stats["batches"] >= max_batches:
@@ -174,8 +173,8 @@ def backfill_episodes(
# Historical runs have tool logs but no trace events;
# bridge them so the backfill is actually usable for mining.
- from core.evolution.events import TraceSink
- from core.evolution.trace_assembler import emit_tool_events
+ from core.evolution.evidence.events import TraceSink
+ from core.evolution.evidence.trace_assembler import emit_tool_events
sink = TraceSink(message_id=message_id, chat_id=chat_id or "", user_id=user_id)
if emit_tool_events(sink, message_id, chat_id or ""):
diff --git a/src/backend/core/evolution/contract.py b/src/backend/core/evolution/contract.py
index b56518d0..231117c9 100644
--- a/src/backend/core/evolution/contract.py
+++ b/src/backend/core/evolution/contract.py
@@ -20,10 +20,10 @@
import hashlib
import json
-from dataclasses import asdict, dataclass, field
+from dataclasses import dataclass, field
from typing import Any, Dict, List, Mapping, Optional, Tuple
-from core.immutable import FrozenDict, freeze_json, thaw_json
+from core.infra.immutable import FrozenDict, freeze_json, thaw_json
# The four governed asset classes, plus the ones that turned out to matter in a
# multi-provider / multi-KB deployment. Prompt is included because this project
diff --git a/src/backend/core/evolution/cycle/__init__.py b/src/backend/core/evolution/cycle/__init__.py
new file mode 100644
index 00000000..fbdf0bdd
--- /dev/null
+++ b/src/backend/core/evolution/cycle/__init__.py
@@ -0,0 +1 @@
+"""The evolution cycle: mine episodes, propose candidates, evaluate and release them."""
diff --git a/src/backend/core/evolution/cycle/activation.py b/src/backend/core/evolution/cycle/activation.py
new file mode 100644
index 00000000..b92c3f66
--- /dev/null
+++ b/src/backend/core/evolution/cycle/activation.py
@@ -0,0 +1,353 @@
+"""Materialise an approved skill candidate at the release stage it has earned."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+from core.db.engine import SessionLocal
+import logging
+from core.evolution.cycle import activation_policy
+from core.evolution.cycle import activation_constants
+from core.evolution.cycle import activation_types
+from core.evolution.cycle import skill_document
+
+logger = logging.getLogger(__name__)
+
+def materialise_skill_candidate(
+ candidate_id: str,
+ *,
+ approver: str,
+ force: bool = False,
+ owner_user_id: Optional[str] = None,
+ tenant_id: str = "default",
+) -> Dict[str, Any]:
+ """Materialise an approved skill candidate, at the stage it has earned.
+
+ This is the only path by which an evolution artefact reaches the agent, and
+ it now enters the release pipeline rather than bypassing it:
+
+ * ``low`` risk (verification = replay only) goes live at 100%;
+ * ``medium`` / ``high`` enter **shadow**, reaching nobody, and must be ramped
+ through canary by :func:`advance_skill_release`;
+ * a personal activation (``owner_user_id``) goes live scoped to that user.
+
+ ``force`` waives the replay *stage* and only for ``low`` risk. It never
+ waives separation of duties — an override is exactly what an attacker or an
+ over-eager script would reach for.
+
+ ``owner_user_id`` scopes the resulting skill to one person. A personal
+ activation cannot change what anyone else's agent does, which is what lets a
+ user accept a capability distilled from their own conversations without an
+ administrator standing in the way.
+ """
+ from core.db.models import AdminSkill
+ from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
+ from core.evolution.cycle.release import ACTIVE, IN_FLIGHT_STAGES, REPLAY_PASSED, new_release_id
+
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ if candidate is None:
+ raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
+ if candidate.target_kind != "skill":
+ raise activation_types.ActivationError(
+ "unsupported_kind", f"暂不支持物化 {candidate.target_kind} 候选"
+ )
+
+ # Separation of duties. No force path — see the docstring.
+ if approver and approver == (candidate.proposer or ""):
+ raise activation_types.ActivationError(
+ "self_approval_forbidden", "生成者不能激活自己提出的候选"
+ )
+ if not approver:
+ raise activation_types.ActivationError("approver_required", "激活必须署名")
+
+ stage, traffic = activation_policy.entry_stage(
+ candidate, approver=approver, force=force, owner_user_id=owner_user_id
+ )
+ # Pace and freeze apply to anything that becomes live immediately. A
+ # shadow entry changes nothing users can see, so holding it back would
+ # only delay measurement.
+ if stage == ACTIVE and not owner_user_id:
+ activation_policy.check_pace(db, tenant_id=tenant_id)
+
+ ir = candidate.ir or {}
+ changes = ir.get("changes") or []
+ tools: List[str] = []
+ constraints: List[Dict[str, Any]] = []
+ document: Optional[Dict[str, Any]] = None
+ manifest: Optional[Dict[str, Any]] = None
+ # ref → owning user. Demotion happens inside one person's store, so a
+ # skill compiled from several people's memories needs the owner of each.
+ source_memories: Dict[str, str] = {}
+ for change in changes:
+ sequence = change.get("tool_sequence")
+ if isinstance(sequence, list) and sequence and not tools:
+ tools = [str(t) for t in sequence]
+ rules = change.get("ordering_constraints")
+ if isinstance(rules, list) and rules:
+ constraints = rules
+ written = change.get("document")
+ if isinstance(written, dict) and written.get("content"):
+ document = written
+ declared = change.get("manifest")
+ if isinstance(declared, dict) and declared:
+ manifest = declared
+ for item in change.get("source_memories") or []:
+ if isinstance(item, dict) and item.get("ref"):
+ source_memories[str(item["ref"])] = str(item.get("user_id") or "")
+
+ # 图谱证据自相矛盾的候选可以存在、可以进 shadow 观测,但不允许自动
+ # 走到 ACTIVE:冲突意味着「适用条件」本身不可信,必须先由人裁决。
+ if manifest and manifest.get("graph_conflict") and stage == ACTIVE:
+ raise activation_types.ActivationError(
+ "graph_conflict_requires_review",
+ f"L3 图谱证据存在冲突,禁止直接激活:{manifest.get('graph_conflicts')}",
+ )
+
+ skill_id = candidate.target_asset_id
+ if not skill_id.startswith(activation_constants.EVOLUTION_PREFIX):
+ skill_id = activation_constants.EVOLUTION_PREFIX + skill_id.replace("auto-", "")
+ if owner_user_id:
+ # Distinct id per owner: a personal activation must never overwrite
+ # the global asset, nor another user's copy of the same candidate.
+ skill_id = f"{skill_id}-u{owner_user_id[-8:]}"
+
+ if document is not None:
+ # A written skill: the generator produced the body and the machine
+ # validator already refused everything it was told to refuse. It is
+ # re-checked here against this candidate's id, because the id gains
+ # a prefix (and, for a personal activation, an owner suffix) after
+ # the draft was validated — and an id the loader rejects yields a
+ # skill that shows up in the capability list and loads as nothing.
+ from core.evolution.generators.skill_drafting import render_skill_markdown, validate_draft
+
+ tools = [str(t) for t in (document.get("allowed_tools") or [])]
+ display_name = str(document.get("display_name") or "")
+ description = str(document.get("description") or "")
+ body = skill_document.body_of(str(document.get("content") or ""))
+ violations = validate_draft(
+ skill_id=skill_id,
+ display_name=display_name,
+ description=description,
+ body=body,
+ allowed_tools=tools,
+ )
+ if violations:
+ raise activation_types.ActivationError(
+ "draft_failed_validation",
+ f"技能正文未通过机器校验,拒绝物化: {violations}",
+ )
+ rules_only = False
+ content = render_skill_markdown(
+ skill_id=skill_id,
+ display_name=display_name,
+ description=description,
+ body=body,
+ support=len(candidate.evidence_refs or []),
+ rationale=candidate.hypothesis or "",
+ )
+ else:
+ # A candidate carrying only ordering rules is still a capability,
+ # and on real traffic it is the *common* case: five runs repeating
+ # one identical tool sequence is rare, while the same pairwise
+ # ordering recurring across different tool sets is not.
+ rules_only = not tools and bool(constraints)
+ if rules_only:
+ tools = skill_document.tools_from_constraints(constraints)
+ if not tools:
+ raise activation_types.ActivationError(
+ "no_tool_sequence", "候选既未声明工具顺序、也未携带顺序约束,无法物化"
+ )
+ display_name = skill_document.skill_label(tools, rules_only=rules_only)
+ description = skill_document.skill_description(tools, constraints, rules_only=rules_only)
+ content = skill_document.skill_markdown(
+ skill_id=skill_id,
+ label=display_name,
+ description=description,
+ tools=tools,
+ evidence=len(candidate.evidence_refs or []),
+ constraints=constraints,
+ rationale=candidate.hypothesis or "",
+ rules_only=rules_only,
+ )
+
+ previous = activation_policy.capture_previous(db, skill_id)
+
+ # One release in flight per asset. Two concurrent ramps on the same skill
+ # make any measured effect unattributable and leave rollback without a
+ # single target.
+ conflicting = (
+ db.query(EvolutionRelease)
+ .filter(
+ EvolutionRelease.target_asset_id == skill_id,
+ EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)),
+ )
+ .first()
+ )
+ if conflicting is not None:
+ raise activation_types.ActivationError(
+ "release_in_flight",
+ f"该资产已有进行中的发布 {conflicting.release_id}({conflicting.stage})",
+ )
+
+ row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
+ if row is None:
+ row = AdminSkill(skill_id=skill_id, created_by=f"evolution:{approver}")
+ db.add(row)
+ row.owner_user_id = owner_user_id or None
+
+ row.skill_content = content
+ row.display_name = display_name
+ row.description = description
+ row.version = activation_policy.next_version(previous)
+ row.tags = sorted(set((previous or {}).get("tags", [])) | {activation_constants.EVOLUTION_TAG})
+ # The distilled ordering, in the shape the runtime reads.
+ extra_files = {
+ **(previous or {}).get("extra_files", {}),
+ "tool_sequence.json": activation_policy.json(tools),
+ # The transferable part. A consumer that only understands sequences
+ # still works; one that understands rules gets the wider benefit.
+ "ordering_constraints.json": activation_policy.json(constraints),
+ }
+ if manifest:
+ # 机器读 manifest(边界/依赖/证据引用),Agent 读 SKILL.md —— 两者
+ # 职责分离。作用域以本次激活实际发生的为准,而不是候选申报的。
+ extra_files["evolution_manifest.json"] = activation_policy.json(
+ {
+ **manifest,
+ "scope": "personal" if owner_user_id else str(
+ manifest.get("scope") or "workspace"
+ ),
+ }
+ )
+ row.extra_files = extra_files
+ row.allowed_tools = tools
+ row.is_enabled = True
+
+ # The scope the IR asked for, now actually enforced at load time by
+ # core.evolution.evidence.exposure. Recording it here rather than deriving it
+ # later means a scope decision is auditable against the release it
+ # governs.
+ scope_filter: Dict[str, Any] = {"level": (ir.get("scope") or {}).get("level") or "workspace"}
+ if owner_user_id:
+ scope_filter["owner_user_id"] = owner_user_id
+ else:
+ scope_filter["tenant_id"] = tenant_id or "default"
+
+ release = EvolutionRelease(
+ release_id=new_release_id(),
+ candidate_id=candidate_id,
+ target_kind="skill",
+ target_asset_id=skill_id,
+ stage=stage,
+ # A personal activation reaches exactly one user, so recording it as
+ # full traffic would misreport the blast radius on the release board.
+ traffic_percent=traffic,
+ scope_filter=scope_filter,
+ # Written now, not derived later: rollback needs a target that
+ # existed before anything changed.
+ rollback_version_id=(previous or {}).get("version") or "",
+ guardrails={
+ "previous": previous,
+ # Which verification stages are behind us. Read by
+ # advance_skill_release before every hop.
+ "completed_verification": (
+ ["replay"] if candidate.status in (REPLAY_PASSED, ACTIVE) else []
+ ),
+ "proposer": candidate.proposer or "",
+ },
+ approved_by=approver,
+ )
+ db.add(release)
+
+ # A personal activation leaves the candidate available for others to
+ # accept, and for an administrator to consider globally. Marking it
+ # ACTIVE would end its life for everyone on one person's decision.
+ # A shadow entry is likewise not the end of the candidate's life.
+ if not owner_user_id and stage == ACTIVE:
+ candidate.status = ACTIVE
+ candidate.approved_by = approver
+ candidate.approved_at = datetime.now(timezone.utc)
+ db.commit()
+
+ release_id = release.release_id
+ new_version = row.version
+ candidate_owner = str((ir.get("scope") or {}).get("user_id") or "")
+ logger.info(
+ "[activation] materialised %s from %s at %s/%d%% (tools=%s written=%s)",
+ skill_id,
+ candidate_id,
+ stage,
+ traffic,
+ tools,
+ document is not None,
+ )
+
+ # Close the promotion chain: the memories this skill was compiled from now
+ # have a cheaper carrier, so they stop consuming the retrieval budget that
+ # made compiling them worthwhile. Outside the transaction above because it
+ # writes to its own ledger, and down-weighted rather than removed, so
+ # rolling the skill back restores them exactly.
+ demoted: List[str] = []
+ if source_memories:
+ from core.evolution.generators.memory_apply import demote_promoted_memories
+
+ by_owner: Dict[str, List[str]] = {}
+ for ref, owner in source_memories.items():
+ # ``owner_user_id`` for a personal activation, the ref's recorded
+ # owner otherwise. A ref with neither is skipped rather than guessed
+ # at: applying a change to the wrong person's memories is worse than
+ # not applying it.
+ resolved = owner_user_id or owner or candidate_owner
+ if resolved:
+ by_owner.setdefault(resolved, []).append(ref)
+
+ for owner, refs in by_owner.items():
+ outcome = demote_promoted_memories(
+ candidate_id=candidate_id,
+ user_id=owner,
+ memory_refs=refs,
+ skill_id=skill_id,
+ )
+ demoted.extend(op["memory_ref"] for op in outcome.get("applied", []))
+
+ # 血缘落账:这一版技能由哪些 L2 记忆编译(compiled_from)、被哪些 L3 关系
+ # 约束(constrained_by)、由哪些 Episode 验证(validated_by)。尽力而为——
+ # 账本失败绝不回滚一次已经成立的激活。
+ lineage_written = 0
+ try:
+ from core.evolution.evidence import lineage
+
+ with SessionLocal() as db:
+ refreshed = db.get(EvolutionCandidate, candidate_id)
+ pack_id = str(getattr(refreshed, "evidence_pack_id", "") or "")
+ pack = lineage.load_evidence_pack(pack_id) if pack_id else None
+ if pack is not None:
+ lineage_written = len(
+ lineage.record_links(
+ lineage.links_for_skill(
+ pack, skill_id=skill_id, skill_version=new_version
+ ),
+ candidate_id=candidate_id,
+ )
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[activation] lineage recording skipped: %s", exc)
+
+ return {
+ "skill_id": skill_id,
+ "release_id": release_id,
+ "stage": stage,
+ "traffic_percent": traffic,
+ "scope": scope_filter,
+ "demoted_memories": demoted,
+ "lineage_links": lineage_written,
+ # Spelled out because "activated" previously implied "live for
+ # everyone", and for anything above low risk it no longer does.
+ "exposed_to": (
+ "all users in scope"
+ if stage == ACTIVE and not owner_user_id
+ else (f"only {owner_user_id}" if owner_user_id else "nobody (shadow)")
+ ),
+ "tools": tools,
+ "version": new_version,
+ "replaced_previous": previous is not None,
+ }
diff --git a/src/backend/core/evolution/cycle/activation_constants.py b/src/backend/core/evolution/cycle/activation_constants.py
new file mode 100644
index 00000000..1831d53a
--- /dev/null
+++ b/src/backend/core/evolution/cycle/activation_constants.py
@@ -0,0 +1,11 @@
+"""Tags and config keys that mark evolution-generated assets."""
+from __future__ import annotations
+
+
+EVOLUTION_TAG = "evolved"
+
+
+EVOLUTION_PREFIX = "evo-"
+
+
+FREEZE_CONFIG_KEY = "evolution_frozen"
diff --git a/src/backend/core/evolution/cycle/activation_policy.py b/src/backend/core/evolution/cycle/activation_policy.py
new file mode 100644
index 00000000..c04bec0a
--- /dev/null
+++ b/src/backend/core/evolution/cycle/activation_policy.py
@@ -0,0 +1,196 @@
+"""Gates on activation: freeze switch, pacing, entry stage and rollback snapshots."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, Tuple
+from core.evolution.cycle import activation_constants
+from core.evolution.cycle import activation_types
+
+def capture_previous(db, skill_id: str) -> Optional[Dict[str, Any]]:
+ """Snapshot the current state so rollback has a concrete target."""
+ from core.db.models import AdminSkill
+
+ row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
+ if row is None:
+ return None
+ return {
+ "skill_id": row.skill_id,
+ "skill_content": row.skill_content,
+ "display_name": row.display_name,
+ "description": row.description,
+ "version": row.version,
+ "tags": list(row.tags or []),
+ "allowed_tools": list(row.allowed_tools or []),
+ "extra_files": dict(row.extra_files or {}),
+ "is_enabled": bool(row.is_enabled),
+ }
+
+
+def is_frozen(db) -> bool:
+ """Global stop switch, readable without a redeploy.
+
+ An incident needs a way to stop every layer activating *now*, so it lives in
+ ``system_configs`` rather than in an environment variable.
+ """
+ try:
+ from core.db.models import SystemConfig
+
+ row = (
+ db.query(SystemConfig.config_value)
+ .filter(SystemConfig.config_key == activation_constants.FREEZE_CONFIG_KEY)
+ .first()
+ )
+ return bool(row) and str(row[0]).strip().lower() in ("1", "true", "yes", "on")
+ except Exception: # noqa: BLE001 - absence of the row means "not frozen"
+ return False
+
+
+def skill_layer_state(db) -> Tuple[Optional[datetime], int]:
+ """When a skill last went live, and how many releases are mid-flight.
+
+ Read from the release ledger rather than kept in memory: a per-process
+ limiter resets on every restart and multiplies by the number of replicas,
+ which is exactly how a cooldown quietly stops existing.
+ """
+ from core.db.models.evolution import EvolutionRelease
+ from core.evolution.cycle.release import ACTIVE, IN_FLIGHT_STAGES
+
+ last_row = (
+ db.query(EvolutionRelease.created_at)
+ .filter(
+ EvolutionRelease.target_kind == "skill",
+ EvolutionRelease.stage == ACTIVE,
+ )
+ .order_by(EvolutionRelease.created_at.desc())
+ .first()
+ )
+ in_flight = (
+ db.query(EvolutionRelease)
+ .filter(EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)))
+ .count()
+ )
+ last = last_row[0] if last_row else None
+ if last is not None and last.tzinfo is None:
+ last = last.replace(tzinfo=timezone.utc)
+ return last, int(in_flight or 0)
+
+
+def check_pace(db, *, tenant_id: str, layer: str = "skill") -> None:
+ """Enforce the multi-timescale gate the design declares.
+
+ Skills sit on an hours-to-days cadence so the system does not become
+ non-stationary. That was a documented number with no enforcement: nothing
+ consulted ``timescales`` before writing a live asset, so ten activations
+ could land in one minute and no metric movement afterwards would be
+ attributable to any of them.
+ """
+ from core.evolution.cycle.timescales import RateLimiter
+
+ last, in_flight = skill_layer_state(db)
+ limiter = RateLimiter(frozen=is_frozen(db))
+ if last is not None:
+ limiter.last_activation[layer] = last
+ allowed, reason = limiter.may_activate(
+ layer, tenant=tenant_id or "default", concurrent_activations=in_flight
+ )
+ if not allowed:
+ raise activation_types.ActivationError("pace_limited", f"节奏闸门拒绝本次激活:{reason}")
+
+
+def entry_stage(
+ candidate,
+ *,
+ approver: str,
+ force: bool,
+ owner_user_id: Optional[str],
+) -> Tuple[str, int]:
+ """The stage this candidate has actually earned, and its traffic share.
+
+ Previously every activation wrote ``active`` at 100%, whatever the risk tier
+ said. ``ir.MIN_VERIFICATION`` declares that a high-risk change owes replay,
+ shadow, canary *and* human approval before going live, and a brand-new
+ skill is exactly that tier — so the old behaviour skipped two of the four
+ stages its own IR demanded.
+
+ A personal activation is the one case that legitimately goes straight live:
+ its blast radius is one consenting user, and the ramp exists to bound blast
+ radius. It is recorded as scoped rather than as full traffic.
+ """
+ from core.evolution.ir import RISK_LOW, required_verification
+ from core.evolution.cycle.release import ACTIVE, REPLAY_PASSED, SHADOW
+
+ risk = str(candidate.risk_tier or "").strip() or RISK_LOW
+ replayed = candidate.status in (REPLAY_PASSED, ACTIVE)
+
+ if not replayed:
+ # ``force`` waives the replay *stage* — and only for the tier whose
+ # verification requirement is replay alone. Waiving it for a tier that
+ # also owes shadow and canary is not a shortcut, it is the whole gate.
+ if not force:
+ raise activation_types.ActivationError(
+ "verification_incomplete",
+ f"候选尚未通过回放(当前 {candidate.status})",
+ )
+ if risk != RISK_LOW:
+ raise activation_types.ActivationError(
+ "force_forbidden_for_risk",
+ f"force 只允许用于 low 风险候选,本候选为 {risk};"
+ f"需完成 {list(required_verification(risk))}",
+ )
+
+ if owner_user_id:
+ return ACTIVE, 0
+
+ if set(required_verification(risk)) <= {"replay"}:
+ return ACTIVE, 100
+
+ # Higher tiers enter at shadow: the asset exists and is measurable, and
+ # reaches nobody until an operator ramps it.
+ return SHADOW, 0
+
+
+def has_graph_conflict(candidate) -> bool:
+ """Whether the candidate's manifest flags contradictory L3 evidence."""
+ if candidate is None:
+ return False
+ for change in (getattr(candidate, "ir", None) or {}).get("changes") or []:
+ manifest = change.get("manifest") if isinstance(change, dict) else None
+ if isinstance(manifest, dict) and manifest.get("graph_conflict"):
+ return True
+ return False
+
+
+def next_version(previous: Optional[Dict[str, Any]]) -> str:
+ if not previous:
+ return "1.0.0"
+ parts = str(previous.get("version") or "1.0.0").split(".")
+ try:
+ parts[-1] = str(int(parts[-1]) + 1)
+ return ".".join(parts)
+ except Exception:
+ return "1.0.1"
+
+
+def json(value: Any) -> str:
+ import json
+
+ return json.dumps(value, ensure_ascii=False)
+
+
+def source_candidate_for(db, skill_id: str) -> str:
+ """The candidate that first materialised this skill.
+
+ Memory demotions are recorded against that candidate, so it is the key that
+ finds them again when the skill is retired.
+ """
+ from core.db.models.evolution import EvolutionRelease
+
+ row = (
+ db.query(EvolutionRelease)
+ .filter(
+ EvolutionRelease.target_kind == "skill",
+ EvolutionRelease.target_asset_id == skill_id,
+ )
+ .order_by(EvolutionRelease.created_at.asc())
+ .first()
+ )
+ return str(row.candidate_id) if row is not None else ""
diff --git a/src/backend/core/evolution/cycle/activation_types.py b/src/backend/core/evolution/cycle/activation_types.py
new file mode 100644
index 00000000..f9f6f31a
--- /dev/null
+++ b/src/backend/core/evolution/cycle/activation_types.py
@@ -0,0 +1,9 @@
+"""Error raised when a candidate cannot be activated."""
+from __future__ import annotations
+
+
+class ActivationError(Exception):
+ def __init__(self, code: str, message: str):
+ super().__init__(message)
+ self.code = code
+ self.message = message
diff --git a/src/backend/core/evolution/cycle/candidate_records.py b/src/backend/core/evolution/cycle/candidate_records.py
new file mode 100644
index 00000000..ee136888
--- /dev/null
+++ b/src/backend/core/evolution/cycle/candidate_records.py
@@ -0,0 +1,197 @@
+"""Persist candidates and derive their attribution features."""
+from __future__ import annotations
+from core.evolution.cycle import patterns
+from core.evolution.cycle import promotion_constants
+from typing import Any, Dict, List, Optional
+from core.db.engine import SessionLocal
+from core.db.models.evolution import EvolutionCandidate
+from core.evolution.evidence.credit import CreditFeatures
+from core.evolution.ir import EvolutionIR, new_candidate_id
+from core.evolution.cycle.release import DRAFT
+import logging
+logger = logging.getLogger(__name__)
+
+def features_for(pattern: patterns.Pattern) -> CreditFeatures:
+ """Translate a discovered pattern into attribution features.
+
+ Success-subsequence tool patterns no longer come through here: they are an
+ aggregate observation over successful runs, and inventing a ``failed``
+ verdict to route them through the failure attributor is the exact move
+ :func:`core.evolution.evidence.credit.aggregate_finding` exists to replace.
+ """
+ if pattern.kind == promotion_constants.PATTERN_REPEATED_FAILURE:
+ return CreditFeatures(
+ outcome_verdict="failed",
+ outcome_confidence=0.8,
+ loop_stagnation=True,
+ no_diff_iterations=2,
+ )
+ return CreditFeatures(outcome_verdict="unknown", outcome_confidence=0.3)
+
+
+def features_for_ordering_rules(constraints: List[Any]) -> CreditFeatures:
+ """Attribution features for "a stable ordering keeps being re-derived".
+
+ The mapping, stated so it can be argued with:
+
+ * ``repeated_tool_subsequence`` — a pairwise ordering recurring across
+ different tool sets is a repeated successful subsequence that is
+ re-planned on every occurrence, which is what the feature means.
+ * ``outcome_verdict="failed"`` — the runs this would fix are the ones that
+ ordered the calls wrongly; the successful ones are the evidence, not the
+ target.
+
+ ``step_order_suspect`` is deliberately *not* set even though ordering is the
+ subject. That feature says "this pipeline's step order looks wrong", a
+ workflow-level claim, and asserting it ties workflow with skill at 0.65 —
+ the assigner then correctly reports ``unidentifiable`` and nothing is
+ produced. The evidence here does separate them: an ordering that holds
+ across several different tool sets and task types is reusable knowledge, not
+ one fixed pipeline, and "the workflow is fixed" is the reading it rules out.
+
+ Confidence sits just above the ``MIN_CONFIDENCE`` floor on purpose: with
+ support and context breadth at their minimum it lands below the floor and
+ resolves to No-Update, so thin evidence cannot mint a permanent asset.
+ """
+ support = max((int(getattr(c, "support", 0) or 0) for c in constraints), default=0)
+ contexts = max((int(getattr(c, "contexts", 0) or 0) for c in constraints), default=0)
+ return CreditFeatures(
+ outcome_verdict="failed",
+ outcome_confidence=min(0.9, 0.6 + 0.05 * support + 0.05 * contexts),
+ repeated_tool_subsequence=True,
+ )
+
+
+def uncovered_constraints(constraints: List[Any], covered_tool_sets: List[Any]) -> List[Any]:
+ """Rules that no sequence candidate would carry into the runtime.
+
+ A rule attached to a materialised sequence already reaches the agent through
+ that skill. The ones worth a candidate of their own are the rest — and on
+ real traffic that is most of them, because a rule earns its scope by
+ appearing in *several* tool-set shapes and no single sequence covers them
+ all.
+ """
+ out: List[Any] = []
+ for constraint in constraints:
+ pair = {constraint.before, constraint.after}
+ if any(pair <= set(tools) for tools in covered_tool_sets):
+ continue
+ out.append(constraint)
+ return out
+
+
+def persist_candidate(
+ ir: EvolutionIR,
+ *,
+ credit: Dict[str, Any],
+ proposer: str,
+ evidence_refs: List[str],
+) -> Optional[str]:
+ """Store a candidate, collapsing duplicates rather than failing on them."""
+ from core.evolution.evidence import lineage
+
+ # Attribution outlives the candidate that happened to carry it: every
+ # decision that reaches persistence gets its own ledger row, and the stored
+ # copy inside the candidate becomes a cache keyed by ``decision_id``.
+ decision_id = lineage.record_credit_decision(
+ credit, episode_id=(evidence_refs[0] if evidence_refs else "")
+ )
+ credit = {**credit, "decision_id": decision_id}
+
+ checksum = ir.change_checksum()
+ try:
+ with SessionLocal() as db:
+ existing = (
+ db.query(EvolutionCandidate)
+ .filter(
+ EvolutionCandidate.target_kind == ir.target_kind,
+ EvolutionCandidate.target_asset_id == ir.target_asset_id,
+ EvolutionCandidate.base_version_id == (ir.base_version or None),
+ EvolutionCandidate.change_checksum == checksum,
+ )
+ .first()
+ )
+ if existing is not None:
+ # Same edit proposed again from fresh evidence: one review item,
+ # not two.
+ return existing.candidate_id
+
+ candidate_id = new_candidate_id()
+ db.add(
+ EvolutionCandidate(
+ candidate_id=candidate_id,
+ target_kind=ir.target_kind,
+ target_asset_id=ir.target_asset_id,
+ base_version_id=ir.base_version or None,
+ operation=ir.operation,
+ ir=ir.to_dict(),
+ hypothesis=ir.hypothesis,
+ change_checksum=checksum,
+ evidence_refs=evidence_refs,
+ credit_decision=credit,
+ risk_tier=ir.risk_tier,
+ scope=ir.scope,
+ status=DRAFT,
+ proposer=proposer,
+ )
+ )
+ db.commit()
+ except Exception as exc:
+ logger.warning("[evolution-loop] persisting candidate failed: %s", exc)
+ return None
+
+ # No backfill onto the turn cards. A skill candidate is a batch product
+ # reviewed in the console; pushing it back into a conversation from days ago
+ # put capability claims on a card whose only subject is what that turn
+ # remembered. The candidate's own evidence_refs still record which episodes
+ # produced it, which is where that link belongs.
+ return candidate_id
+
+
+def cross_user_mining_available() -> bool:
+ """Whether fleet-wide mining may run in this edition.
+
+ Personal evolution runs everywhere — it needs no aggregation across people
+ and its output reaches only its owner. Mining *across* users and releasing
+ to everyone is a governance surface, so it is enterprise-gated. The
+ community edition keeps a complete personal evolution path rather than a
+ degraded one.
+ """
+ try:
+ # Resolved by name rather than a literal enterprise-tree import, so the
+ # community build's boundary scan does not read this file as reaching
+ # across the edition boundary. The behaviour is unchanged: the module is
+ # physically absent in CE and the resulting ImportError still lands in
+ # the handler below.
+ from importlib import import_module
+
+ features = import_module("edition_ee.licensing.features")
+ manager = import_module("edition_ee.licensing.manager")
+
+ return bool(manager.license_manager.has(features.Feature.EVOLUTION_CONTROL))
+ except Exception:
+ # The community tree physically excludes edition_ee, so an import error
+ # here is the expected CE signal rather than a fault: personal only.
+ return False
+
+
+def run_async(coro) -> Any:
+ """Run one coroutine from the cycle's synchronous body.
+
+ The cycle is a background job and must run in a worker thread. Calling it
+ from inside a running loop is a programming error and is raised as one:
+ degrading to "skip the generators" would produce a cycle that silently
+ stopped writing skills, which is the failure mode this whole rework exists
+ to eliminate.
+ """
+ import asyncio
+
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(coro)
+ coro.close()
+ raise RuntimeError(
+ "run_evolution_cycle must not be called from a running event loop; "
+ "dispatch it with run_in_executor"
+ )
diff --git a/src/backend/core/evolution/cycle/composition_candidates.py b/src/backend/core/evolution/cycle/composition_candidates.py
new file mode 100644
index 00000000..c3793835
--- /dev/null
+++ b/src/backend/core/evolution/cycle/composition_candidates.py
@@ -0,0 +1,221 @@
+"""Retirement, merge, profile and sub-agent candidates."""
+from __future__ import annotations
+from typing import Any, Dict, List
+from core.evolution.evidence.credit import T_ORCHESTRATION, T_SKILL, aggregate_finding, confidence_from_evidence
+from core.evolution.ir import RISK_HIGH, RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir
+import logging
+from core.evolution.cycle import candidate_records
+
+logger = logging.getLogger(__name__)
+
+def decremental_candidates(extras: Dict[str, Any], *, dry_run: bool) -> List[Dict[str, Any]]:
+ """Retirement and merge findings, as candidates rather than as a list.
+
+ Retiring goes through the same approval path as creating. Removing a
+ capability produces no error — only a slow decline — so it is precisely the
+ direction that must not be automatic.
+ """
+ from core.evolution.cycle.promotion_constants import ACTION_REWRITE
+
+ decremental = extras.get("decremental") or {}
+ created: List[Dict[str, Any]] = []
+
+ for item in decremental.get("retirements") or []:
+ asset_id = str(item.get("asset_id") or "")
+ if not asset_id:
+ continue
+ action = str(item.get("action") or "")
+ if action == ACTION_REWRITE:
+ # "Opened and ignored" is a content problem. Proposing retirement
+ # would remove a capability the model wanted; the fix is a rewrite,
+ # and that is a different candidate written from failure evidence.
+ created.append({"kind": "needs_rewrite", "asset_id": asset_id, **item})
+ continue
+
+ decision = aggregate_finding(
+ target=T_SKILL,
+ explanation=str(item.get("explanation") or item.get("reason") or ""),
+ confidence=confidence_from_evidence(int(item.get("offers", 0) or 0), saturates_at=20),
+ evidence_count=int(item.get("offers", 0) or 0),
+ )
+ if not decision.may_produce_candidate:
+ continue
+
+ ir = EvolutionIR(
+ target_kind="skill",
+ target_asset_id=asset_id,
+ base_version="current",
+ operation="deprecate",
+ hypothesis=str(item.get("explanation") or item.get("reason") or ""),
+ changes=[
+ {
+ "retire_reason": item.get("reason"),
+ "stats": {
+ k: v for k, v in item.items() if k not in ("asset_id", "explanation")
+ },
+ }
+ ],
+ evidence_refs=[asset_id],
+ risk_tier=RISK_MEDIUM,
+ scope={"level": "workspace"},
+ rollback=Rollback(version_id="current", triggers=["manual"]),
+ evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(ir, credit_decision=decision.to_dict())
+ except InvariantViolation as violation:
+ logger.info("[evolution-loop] retirement rejected: %s", violation.code)
+ continue
+ if dry_run:
+ created.append({"dry_run": True, "kind": "retirement", "asset_id": asset_id})
+ continue
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="decremental_engine",
+ evidence_refs=[asset_id],
+ )
+ if candidate_id:
+ created.append(
+ {
+ "candidate_id": candidate_id,
+ "kind": "retirement",
+ "asset_id": asset_id,
+ "reason": item.get("reason"),
+ }
+ )
+ return created
+
+
+def orchestration_candidates(
+ views: List[Dict[str, Any]], *, dry_run: bool
+) -> List[Dict[str, Any]]:
+ """Profile and sub-agent candidates.
+
+ The tool authority every candidate here is measured against is the *current*
+ grant, read once. A profile may only ever narrow it, and a derived sub-agent
+ may only ever receive a subset — both enforced by the IR gate rather than by
+ the generator, so the check holds even for a candidate that arrived by some
+ other route.
+ """
+ from core.capabilities.catalog.api import get_enabled_ids
+ from core.evolution.profiles.agent_profile_constants import TARGET_KIND
+ from core.evolution.profiles.agent_profile import builtin_profile, validate_profile
+ from core.evolution.generators.orchestration_gen import generate_orchestration_proposals, profile_from_proposals
+
+ base_tools = [str(t) for t in get_enabled_ids("tools")] + [
+ str(t) for t in get_enabled_ids("mcp")
+ ]
+ offered_tools = {str(view.get("task_type") or "chat"): base_tools for view in views}
+ proposals, subagents = generate_orchestration_proposals(
+ views, offered_tools=offered_tools, parent_tools=base_tools
+ )
+ created: List[Dict[str, Any]] = []
+
+ task_types = sorted({p.task_type for p in proposals})
+ for task_type in task_types:
+ task_proposals = [p for p in proposals if p.task_type == task_type]
+ profile = profile_from_proposals(
+ task_proposals, task_type=task_type, base=builtin_profile()
+ )
+ ok, problems = validate_profile(profile, base_tools=base_tools)
+ if not ok:
+ logger.info("[evolution-loop] profile %s rejected: %s", profile.profile_id, problems)
+ continue
+
+ evidence = [ref for p in task_proposals for ref in p.evidence_refs][:20]
+ # An observation across the corpus, not a failure attributed to one run.
+ # Inventing failure features to make the attributor return "workflow"
+ # is what put "过程停滞:重复动作、无产出增量" in front of reviewers for a
+ # candidate whose actual reason was "eight tools were never used".
+ decision = aggregate_finding(
+ target=T_ORCHESTRATION,
+ explanation=";".join(p.rationale for p in task_proposals),
+ confidence=confidence_from_evidence(sum(p.support for p in task_proposals)),
+ evidence_count=len(evidence),
+ )
+ if not decision.may_produce_candidate:
+ continue
+ ir = EvolutionIR(
+ target_kind=TARGET_KIND,
+ target_asset_id=profile.profile_id,
+ base_version="builtin",
+ operation="new",
+ hypothesis=";".join(p.rationale for p in task_proposals),
+ changes=[
+ {
+ "profile": profile.to_dict(),
+ "tool_allowlist": profile.tool_allowlist or [],
+ }
+ ],
+ evidence_refs=evidence,
+ requested_tools=list(profile.tool_allowlist or []),
+ risk_tier=RISK_MEDIUM,
+ scope={"level": "workspace"},
+ rollback=Rollback(version_id="builtin", triggers=["success -3pp"]),
+ evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools)
+ except InvariantViolation as violation:
+ logger.info("[evolution-loop] profile candidate rejected: %s", violation.code)
+ continue
+ if dry_run:
+ created.append({"dry_run": True, "kind": "agent_profile", "task_type": task_type})
+ continue
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="orchestration_engine",
+ evidence_refs=evidence,
+ )
+ if candidate_id:
+ created.append(
+ {"candidate_id": candidate_id, "kind": "agent_profile", "task_type": task_type}
+ )
+
+ for subagent in subagents:
+ decision = aggregate_finding(
+ target=T_ORCHESTRATION,
+ explanation=subagent.rationale,
+ confidence=confidence_from_evidence(subagent.support),
+ evidence_count=len(subagent.evidence_refs),
+ )
+ if not decision.may_produce_candidate:
+ continue
+ payload = subagent.to_dict()
+ ir = EvolutionIR(
+ target_kind="subagent",
+ target_asset_id=subagent.agent_id,
+ base_version="v0",
+ operation="new",
+ hypothesis=subagent.rationale,
+ changes=[payload],
+ evidence_refs=subagent.evidence_refs,
+ requested_tools=list(subagent.tools_allowlist),
+ # Creating an agent is the widest-reaching change this system can
+ # propose, and the route into it changes the parent's prompt.
+ risk_tier=RISK_HIGH,
+ scope={"level": "workspace"},
+ rollback=Rollback(version_id="v0", triggers=["manual"]),
+ evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools)
+ except InvariantViolation as violation:
+ logger.info("[evolution-loop] subagent candidate rejected: %s", violation.code)
+ continue
+ if dry_run:
+ created.append({"dry_run": True, "kind": "subagent", "name": subagent.name})
+ continue
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="orchestration_engine",
+ evidence_refs=subagent.evidence_refs,
+ )
+ if candidate_id:
+ created.append(
+ {"candidate_id": candidate_id, "kind": "subagent", "name": subagent.name}
+ )
+ return created
diff --git a/src/backend/core/evolution/cycle/cycle_extras.py b/src/backend/core/evolution/cycle/cycle_extras.py
new file mode 100644
index 00000000..9852f995
--- /dev/null
+++ b/src/backend/core/evolution/cycle/cycle_extras.py
@@ -0,0 +1,32 @@
+"""Run the cycle's secondary engines in one call."""
+from __future__ import annotations
+from typing import Any, Dict, Optional, Sequence
+import logging
+from core.evolution.cycle import extra_findings
+from core.evolution.cycle import memory_findings
+
+logger = logging.getLogger(__name__)
+
+def run_extras(
+ views: Sequence[Dict[str, Any]],
+ *,
+ scan_findings: Optional[Sequence[Any]] = None,
+) -> Dict[str, Any]:
+ """Everything the cycle was missing, in one call.
+
+ Each section is independently guarded: a failure in the decremental pass
+ must not cost the intent pass its findings, because they answer different
+ questions and an operator needs whichever ones succeeded.
+ """
+ report: Dict[str, Any] = {}
+ for name, fn in (
+ ("intent", lambda: extra_findings.intent_findings(views)),
+ ("decremental", lambda: extra_findings.decremental_findings(views)),
+ ("memory", lambda: memory_findings.memory_findings(views, scan_findings=scan_findings)),
+ ):
+ try:
+ report[name] = fn()
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[cycle-extras] %s pass failed: %s", name, exc)
+ report[name] = {"error": str(exc)}
+ return report
diff --git a/src/backend/core/evolution/cycle/cycle_extras_constants.py b/src/backend/core/evolution/cycle/cycle_extras_constants.py
new file mode 100644
index 00000000..967d9902
--- /dev/null
+++ b/src/backend/core/evolution/cycle/cycle_extras_constants.py
@@ -0,0 +1,20 @@
+"""Thresholds for the intent, demotion and uplift engines."""
+from __future__ import annotations
+
+
+MIN_INTENT_OCCASIONS = 3
+
+
+INTENT_MIN_SUCCESS_RATE = 0.6
+
+
+MIN_PLAN_VARIETY = 2
+
+
+DEMOTE_MIN_HITS = 4
+
+
+DEMOTE_SUCCESS_CEILING = 0.34
+
+
+MIN_UPLIFT_SAMPLES = 10
diff --git a/src/backend/core/evolution/cycle/episode_views.py b/src/backend/core/evolution/cycle/episode_views.py
new file mode 100644
index 00000000..25798316
--- /dev/null
+++ b/src/backend/core/evolution/cycle/episode_views.py
@@ -0,0 +1,178 @@
+"""Flatten episodes and their trace events into the views the engines read."""
+from __future__ import annotations
+from typing import Any, Dict, List
+from core.db.engine import SessionLocal
+from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent
+from core.evolution.profiles.user_settings_constants import PRIVACY_PRIVATE
+import logging
+logger = logging.getLogger(__name__)
+
+def view_for(episode, events) -> Dict[str, Any]:
+ """One Episode plus its events, flattened into what the engines read.
+
+ Three fields here did not exist and each blocked an engine outright:
+
+ ``memory_refs`` now carries the **stable content ref**, not the raw content
+ hash. The two look alike and are not: the ref is what the overlay keys on,
+ so an op proposed against a hash could never be applied to anything.
+
+ ``skills_opened`` is what the model chose to read, as opposed to
+ ``skill_sequence``, which is what it was offered. Every decrement decision
+ depends on the difference.
+
+ ``selection_degraded`` marks turns where ranking could not run and
+ everything loaded. Counting those as selections is what made every skill's
+ offer count identical, and therefore useless.
+ """
+ from core.evolution.evidence.events import EV_MEMORY_RETRIEVED, EV_SKILL_OPENED, EV_SKILL_SELECTED, EV_TOOL_CALLED
+ from core.memory.ref_shadow import build_ref_id
+ from core.memory.retrieval_types import LAYER_FACT, LAYER_GRAPH
+
+ tools: List[str] = []
+ skills: List[str] = []
+ memory_refs: List[str] = []
+ graph_refs: List[str] = []
+ graph_relations: List[Dict[str, Any]] = []
+ skills_opened: List[Dict[str, Any]] = []
+ selection_degraded = False
+ error_count = 0
+
+ user_id = episode.user_id or ""
+ for event in events:
+ payload = event.payload or {}
+ if event.event_type == EV_TOOL_CALLED:
+ name = payload.get("tool_name")
+ if name:
+ tools.append(str(name))
+ if str(payload.get("status") or "") == "error":
+ error_count += 1
+ elif event.event_type == EV_SKILL_SELECTED:
+ if payload.get("degraded"):
+ selection_degraded = True
+ skills.extend(str(s) for s in (payload.get("selected") or []))
+ elif event.event_type == EV_SKILL_OPENED:
+ skills_opened.append(
+ {
+ "skill_id": str(payload.get("skill_id") or ""),
+ "tools_after_open": [str(t) for t in (payload.get("tools_after_open") or [])],
+ }
+ )
+ elif event.event_type == EV_MEMORY_RETRIEVED:
+ for item in payload.get("items") or []:
+ content_hash = str(item.get("content_hash") or "")
+ if not content_hash:
+ continue
+ memory_refs.append(
+ build_ref_id(
+ layer=str(item.get("layer") or LAYER_FACT),
+ user_id=user_id,
+ workspace_id=str(item.get("workspace_id") or "default"),
+ content_hash=content_hash,
+ )
+ )
+ # L3 is deliberately tracked separately from L2. A graph relation
+ # is declarative context, not a procedure, and must never enter the
+ # memory→skill compiler as though it were an instruction.
+ for relation in payload.get("graph") or []:
+ content_hash = str(relation.get("content_hash") or "")
+ if not content_hash:
+ continue
+ graph_refs.append(
+ build_ref_id(
+ layer=LAYER_GRAPH,
+ user_id=user_id,
+ workspace_id=str(relation.get("workspace_id") or "default"),
+ content_hash=content_hash,
+ )
+ )
+ # The full reference, kept for the evidence resolver: a pack's
+ # L3 entry must carry the store's stable relation_id so an
+ # activated skill's graph evidence can be resolved back to the
+ # exact Neo4j edge — a bare ref id cannot do that.
+ graph_relations.append(
+ {
+ "relation_id": str(relation.get("relation_id") or ""),
+ "content_hash": content_hash,
+ "predicate": str(relation.get("predicate") or ""),
+ "confidence": relation.get("confidence") or 0.0,
+ "workspace_id": str(relation.get("workspace_id") or "default"),
+ }
+ )
+
+ outcome = episode.outcome or {}
+ return {
+ "episode_id": episode.episode_id,
+ "chat_id": episode.chat_id or "",
+ "user_id": user_id,
+ "tenant_id": episode.tenant_id or "default",
+ "verdict": outcome.get("verdict")
+ or ("success" if (episode.quality_score or 0) >= 0.65 else "unknown"),
+ "tool_sequence": tools,
+ "skill_sequence": skills,
+ "skills_opened": skills_opened,
+ "selection_degraded": selection_degraded,
+ "memory_refs": memory_refs,
+ "graph_refs": graph_refs,
+ "graph_relations": graph_relations,
+ "task_type": episode.task_type or "chat",
+ "backfilled": bool(episode.backfilled),
+ # Repeated tool errors are the observable form of "this run was stuck",
+ # which sub-agent derivation needs and nothing else recorded.
+ "stalled": error_count >= 3,
+ # What the user actually asked. Clustering only ever saw tool sequences
+ # before this, so "the same question keeps coming back" was not a
+ # detectable event. Sanitised at write time, so this is the preview
+ # rather than the raw text.
+ "objective": episode.objective_preview or "",
+ "objective_hash": episode.objective_hash or "",
+ "attributed_to": str(outcome.get("attributed_to") or ""),
+ # Needed to place an episode before or after an activation. Without it
+ # negative-transfer detection has no way to split the corpus and returns
+ # nothing on every corpus — wired, running, and structurally incapable
+ # of finding anything.
+ "created_at": episode.created_at,
+ }
+
+
+def load_episode_views(*, limit: int = 500, since_days: int = 30) -> List[Dict[str, Any]]:
+ """Flatten Episodes plus their trace events into the shape pattern mining wants.
+
+ Backfilled Episodes are included on purpose: they cannot be replayed, but
+ they are perfectly good for discovering what recurs — which is exactly why
+ the backfill was worth doing.
+ """
+ views: List[Dict[str, Any]] = []
+ try:
+ with SessionLocal() as db:
+ # The single enforcement point for per-user opt-out. Episodes from
+ # users who declined to contribute are still recorded (they power
+ # that user's own card and history) but never feed cross-user
+ # pattern mining. Filtering here rather than at each call site is
+ # what makes the promise checkable.
+ episodes = (
+ db.query(EvolutionEpisode)
+ .filter(EvolutionEpisode.privacy_class != PRIVACY_PRIVATE)
+ .order_by(EvolutionEpisode.created_at.desc())
+ .limit(limit)
+ .all()
+ )
+ if not episodes:
+ return []
+
+ ids = [e.episode_id for e in episodes]
+ events = (
+ db.query(EvolutionTraceEvent)
+ .filter(EvolutionTraceEvent.episode_id.in_(ids))
+ .order_by(EvolutionTraceEvent.seq.asc())
+ .all()
+ )
+ by_episode: Dict[str, List[EvolutionTraceEvent]] = {}
+ for event in events:
+ by_episode.setdefault(event.episode_id, []).append(event)
+
+ for episode in episodes:
+ views.append(view_for(episode, by_episode.get(episode.episode_id, [])))
+ except Exception as exc:
+ logger.warning("[evolution-loop] loading episodes failed: %s", exc)
+ return []
+ return views
diff --git a/src/backend/core/evolution/cycle/evaluation.py b/src/backend/core/evolution/cycle/evaluation.py
new file mode 100644
index 00000000..8bccae99
--- /dev/null
+++ b/src/backend/core/evolution/cycle/evaluation.py
@@ -0,0 +1,153 @@
+"""Replay-evaluate candidates and record the results."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+from core.db.engine import SessionLocal
+from core.db.models.evolution import EvolutionCandidate
+import logging
+logger = logging.getLogger(__name__)
+
+def record_evaluation(candidate_id: str, report: Any, eval_type: str) -> None:
+ """Persist an evaluation result against its candidate."""
+ from core.db.models.evolution import EvolutionEvaluation
+
+ try:
+ payload = report.to_dict()
+ with SessionLocal() as db:
+ db.add(
+ EvolutionEvaluation(
+ evaluation_id=payload["evaluation_id"],
+ candidate_id=candidate_id,
+ eval_type=eval_type,
+ # Frozen: a dataset edited after the fact invalidates every
+ # conclusion computed from it.
+ dataset_snapshot=payload["dataset_snapshot"],
+ baseline_metrics=payload["baseline"],
+ candidate_metrics=payload["candidate"],
+ guardrails={"breaches": payload["guardrail_breaches"]},
+ verdict=payload["verdict"],
+ effect_size=payload["effect_size"],
+ p_value=payload["p_value"],
+ sample_size=payload["sample_size"],
+ )
+ )
+ db.commit()
+ except Exception as exc:
+ logger.warning("[evolution-loop] recording evaluation failed: %s", exc)
+
+
+def replay_coverage_for(candidate_id: str, task_ids: List[str]) -> Optional[Dict[str, Any]]:
+ """Coverage assessment for candidates that carry an evidence pack.
+
+ Returns ``None`` for pre-pack candidates — the gate governs only what the
+ unified engine produced, so legacy candidates keep their old semantics.
+ """
+ try:
+ from core.evolution.evidence import lineage
+ from core.evolution.assessment.applicability import assess_replay_coverage
+
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ pack_id = str(getattr(candidate, "evidence_pack_id", "") or "")
+ if not pack_id:
+ return None
+ pack = lineage.load_evidence_pack(pack_id)
+ if pack is None:
+ return None
+ return assess_replay_coverage(pack, task_ids)
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[evolution-loop] replay coverage check skipped: %s", exc)
+ return None
+
+
+def evaluate_candidate(
+ candidate_id: str,
+ run_arm: Any,
+ task_ids: List[str],
+) -> Optional[Dict[str, Any]]:
+ """Run tiered replay for a candidate and advance it if it earns the step.
+
+ ``run_arm(task_id, is_candidate)`` is injected: the caller owns isolation
+ (frozen bundle, cassette, sandbox snapshot). Only ``replay_passed`` is
+ reachable from here — shadow and canary need real traffic, and activation
+ needs a human, so this can never quietly ship anything.
+ """
+ from core.evolution.cycle.release import DRAFT, REJECTED, REPLAY_PASSED, is_legal_transition
+ from core.evolution.evidence.replay import VERDICT_IMPROVED, VERDICT_REGRESSED, run_tiered_replay
+
+ tiered = run_tiered_replay(task_ids, run_arm)
+ report = tiered.report or tiered.sentinel_report
+ if report is None:
+ return None
+
+ record_evaluation(candidate_id, report, "replay" if tiered.passed_sentinel else "sentinel")
+
+ # Replay 覆盖闸门:带证据 pack 的候选,其回放集必须覆盖 pack 声称的行为面
+ # (成功 / 失败恢复 / 负样本 / 图谱依赖两类场景)。只用成功案例喂出来的
+ # "improved" 是单面证据,不允许换来 replay_passed;回归照常生效——安全
+ # 方向的结论不需要覆盖完整也成立。
+ coverage: Optional[Dict[str, Any]] = None
+ if report.verdict == VERDICT_IMPROVED:
+ coverage = replay_coverage_for(candidate_id, task_ids)
+ if coverage is not None and not coverage.get("covered", True):
+ return {
+ "candidate_id": candidate_id,
+ "verdict": report.verdict,
+ "status": DRAFT,
+ "transition": "held_insufficient_coverage",
+ "coverage": coverage,
+ "tasks_run": tiered.tasks_run,
+ "effect_size": report.effect_size,
+ "p_value": report.p_value,
+ }
+
+ try:
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ if candidate is None:
+ return None
+ current = candidate.status
+
+ # Re-evaluating a candidate that has already moved on is a no-op,
+ # not a failure. Collapsing the two into None left callers unable to
+ # tell "already advanced" from "something broke".
+ target = (
+ REJECTED
+ if report.verdict == VERDICT_REGRESSED
+ else (REPLAY_PASSED if report.verdict == VERDICT_IMPROVED else current)
+ )
+ if target != current and not is_legal_transition(current, target):
+ return {
+ "candidate_id": candidate_id,
+ "verdict": report.verdict,
+ "status": current,
+ "transition": "skipped_already_advanced",
+ "tasks_run": tiered.tasks_run,
+ "effect_size": report.effect_size,
+ "p_value": report.p_value,
+ }
+
+ if report.verdict == VERDICT_REGRESSED:
+ candidate.status = REJECTED
+ candidate.reject_reason = (
+ f"replay regression: effect={report.effect_size:.3f} "
+ f"p={report.p_value:.4f} {report.guardrail_breaches}"
+ )
+ elif report.verdict == VERDICT_IMPROVED:
+ candidate.status = REPLAY_PASSED
+ else:
+ # Neutral or insufficient: stays a draft. Not advancing is the
+ # correct outcome — "we could not show it helps" is not "it helps".
+ pass
+ db.commit()
+ return {
+ "candidate_id": candidate_id,
+ "verdict": report.verdict,
+ "status": candidate.status,
+ "transition": "applied" if candidate.status != current else "held",
+ "tasks_run": tiered.tasks_run,
+ "effect_size": report.effect_size,
+ "p_value": report.p_value,
+ }
+ except Exception as exc:
+ logger.warning("[evolution-loop] advancing candidate failed: %s", exc)
+ return None
diff --git a/src/backend/core/evolution/cycle/extra_findings.py b/src/backend/core/evolution/cycle/extra_findings.py
new file mode 100644
index 00000000..214feb8c
--- /dev/null
+++ b/src/backend/core/evolution/cycle/extra_findings.py
@@ -0,0 +1,166 @@
+"""Intent, decremental and negative-transfer findings."""
+from __future__ import annotations
+from core.evolution.cycle import promotion
+from core.evolution.cycle import retirement
+import hashlib
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+from core.evolution.cycle import cycle_extras_constants
+from core.evolution.cycle import usage_evidence
+
+def asset_id(prefix: str, seed: str) -> str:
+ return f"{prefix}-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12]
+
+
+def intent_findings(views: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
+ """Recurring requests whose handling never settled.
+
+ The pairing matters. Recurrence alone says the question is common;
+ recurrence **with plan variety** says the system has not found a stable way
+ to answer it and is paying the planning cost fresh every time. That second
+ condition is what distinguishes a capability gap from a popular question.
+ """
+ from core.evolution.assessment.similarity import cluster_intents
+
+ clusters, method = cluster_intents(views, min_support=cycle_extras_constants.MIN_INTENT_OCCASIONS)
+ findings: List[Dict[str, Any]] = []
+ for cluster in clusters:
+ if cluster.occasions < cycle_extras_constants.MIN_INTENT_OCCASIONS:
+ # Recurrence inside one conversation is a user rephrasing after a
+ # bad answer, which is evidence about that answer, not about demand.
+ continue
+ if cluster.plan_variety < cycle_extras_constants.MIN_PLAN_VARIETY:
+ # Same question, same plan, every time. Already effectively a skill;
+ # writing one down adds an asset to maintain and changes nothing.
+ continue
+ if cluster.success_rate < cycle_extras_constants.INTENT_MIN_SUCCESS_RATE:
+ # Recurring *and* failing. Distilling the current plan would encode
+ # the failure. This belongs to attribution, not to promotion.
+ findings.append(
+ {
+ "kind": "recurring_failure_intent",
+ "cluster": cluster.to_dict(),
+ "action": "no_skill_candidate",
+ "reason": (
+ f"「{cluster.representative[:30]}」类请求出现 {cluster.occasions} 次,"
+ f"但成功率只有 {cluster.success_rate:.0%}——"
+ "把当前做法蒸馏出来等于把失败固化"
+ ),
+ }
+ )
+ continue
+ findings.append(
+ {
+ "kind": "recurring_intent",
+ "cluster": cluster.to_dict(),
+ "action": "skill_candidate",
+ "asset_id": asset_id("auto-intent", cluster.representative),
+ "reason": (
+ f"同一意图在 {cluster.occasions} 次独立会话中出现,却用了 "
+ f"{cluster.plan_variety} 种不同的工具序列——每次都在重新规划"
+ ),
+ }
+ )
+ return {"method": method, "clusters": len(clusters), "findings": findings}
+
+
+def decremental_findings(
+ views: Sequence[Dict[str, Any]],
+ *,
+ skill_stats: Optional[Dict[str, Dict[str, Any]]] = None,
+ declared_tools: Optional[Dict[str, List[str]]] = None,
+) -> Dict[str, Any]:
+ """Merge, retirement and negative-transfer findings.
+
+ ``skill_stats`` is derived from the episode views when not supplied.
+ ``declared_tools`` maps skill id → the tools its own document names; overlap
+ is computed from that and never from the tools observed in episodes where
+ the skill happened to be loaded, which would pair every co-enabled skill.
+ """
+
+ stats = skill_stats if skill_stats is not None else usage_evidence.skill_usage_from_views(views)
+
+ overlaps: List[Tuple[str, str, int]] = []
+ if declared_tools is None:
+ declared_tools = usage_evidence.declared_tools_for(sorted(stats))
+ if declared_tools:
+ overlaps = promotion.find_overlaps(declared_tools)
+
+ retirements = retirement.find_retirement_candidates(stats)
+ return {
+ "overlaps": [
+ {"a": a, "b": b, "shared_tools": shared} for a, b, shared in overlaps
+ ],
+ "retirements": retirements,
+ "asset_count": len(stats),
+ "negative_transfer": negative_transfer_for_activations(views),
+ }
+
+
+def negative_transfer_for_activations(
+ views: Sequence[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+ """Check each recent activation for collateral damage.
+
+ The detector needs a corpus split into before and after *something*. Calling
+ it on views where nothing carries that marker returns nothing on every
+ corpus, which is a detector that runs and cannot find anything — the exact
+ shape of defect this rework exists to remove. So the split is computed here,
+ from the release ledger, once per activated asset.
+ """
+ activations = usage_evidence.recent_activations()
+ if not activations:
+ return []
+
+ findings: List[Dict[str, Any]] = []
+ for asset_id, activated_at, related in activations:
+ split = [
+ {**view, "after": usage_evidence.is_after(view.get("created_at"), activated_at)}
+ for view in views
+ if view.get("created_at") is not None
+ ]
+ if not any(v["after"] for v in split) or not any(not v["after"] for v in split):
+ # One-sided evidence cannot show a change. Reporting nothing is the
+ # honest output; reporting a drop from an empty baseline is not.
+ continue
+ findings.extend(
+ negative_transfer_findings(split, asset_id=asset_id, related_task_types=related)
+ )
+ return findings
+
+
+def negative_transfer_findings(
+ views: Sequence[Dict[str, Any]],
+ *,
+ asset_id: str = "",
+ related_task_types: Sequence[str] = (),
+ before_key: str = "before",
+ after_key: str = "after",
+) -> List[Dict[str, Any]]:
+ """Task types that got worse after some activation, unrelated to it.
+
+ The one failure mode of joint evolution that no single release's guardrail
+ can catch: each change looks fine on the metric it was measured against, and
+ the damage shows up somewhere nobody was looking.
+ """
+
+ by_type_before: Dict[str, List[bool]] = {}
+ by_type_after: Dict[str, List[bool]] = {}
+ for view in views:
+ target = by_type_after if view.get(after_key) else by_type_before
+ target.setdefault(str(view.get("task_type") or "chat"), []).append(
+ view.get("verdict") == "success"
+ )
+
+ def rates(source: Dict[str, List[bool]]) -> Dict[str, float]:
+ return {k: (sum(v) / len(v) if v else 0.0) for k, v in source.items()}
+
+ findings = retirement.detect_negative_transfer(
+ asset_id=asset_id or "unspecified",
+ # Only *unrelated* task types are examined. A dip on the task the change
+ # targeted is an ordinary regression its own guardrail catches; the
+ # dangerous case is collateral damage where nobody is looking.
+ related_task_types=list(related_task_types),
+ before=rates(by_type_before),
+ after=rates(by_type_after),
+ )
+ return [f.to_dict() for f in findings]
diff --git a/src/backend/core/evolution/cycle/graph_candidates.py b/src/backend/core/evolution/cycle/graph_candidates.py
new file mode 100644
index 00000000..78c8369b
--- /dev/null
+++ b/src/backend/core/evolution/cycle/graph_candidates.py
@@ -0,0 +1,156 @@
+"""L3 graph evolution candidates and the dispatcher for the other engines."""
+from __future__ import annotations
+import hashlib
+from typing import Any, Dict, List
+from core.evolution.evidence.credit import T_MEMORY, aggregate_finding, confidence_from_evidence
+from core.evolution.ir import RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir
+import logging
+from core.evolution.cycle import candidate_records
+from core.evolution.cycle import composition_candidates
+from core.evolution.cycle import memory_candidates
+from core.evolution.cycle import prompt_candidates
+
+logger = logging.getLogger(__name__)
+
+def graph_evolution_candidates(
+ views: List[Dict[str, Any]], *, dry_run: bool
+) -> List[Dict[str, Any]]:
+ """L3 的演进:强化/降权自动执行,语义变更走人工候选。
+
+ Per user, because the graph is per user scope: a contradiction between two
+ people's graphs is not a contradiction at all.
+ """
+ from core.config.settings import settings
+
+ if not (settings.memory.enabled and settings.memory.graph_enabled):
+ return []
+
+ from core.evolution.generators.graph_scan import apply_graph_ops, scan_graph_relations
+
+ observed_by_user: Dict[str, Dict[str, int]] = {}
+ for view in views:
+ user_id = str(view.get("user_id") or "")
+ if not user_id:
+ continue
+ bucket = observed_by_user.setdefault(user_id, {})
+ for relation in view.get("graph_relations") or []:
+ relation_id = str(relation.get("relation_id") or "")
+ if relation_id:
+ bucket[relation_id] = bucket.get(relation_id, 0) + 1
+
+ created: List[Dict[str, Any]] = []
+ for user_id, observed in observed_by_user.items():
+ try:
+ from core.memory.graph import list_graph_relations
+
+ relations = candidate_records.run_async(list_graph_relations(user_id, limit=200))
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[evolution-loop] graph listing failed for %s: %s", user_id, exc)
+ continue
+ if not relations:
+ continue
+
+ ops = scan_graph_relations(relations, observed_relation_ids=observed)
+ if not ops:
+ continue
+ if dry_run:
+ created.append(
+ {"dry_run": True, "kind": "graph", "user_id": user_id, "ops": len(ops)}
+ )
+ continue
+
+ outcome = apply_graph_ops(ops, scope_user_id=user_id)
+ manual = outcome.get("manual") or []
+ entry: Dict[str, Any] = {
+ "kind": "graph",
+ "user_id": user_id,
+ "auto_applied": len(outcome.get("applied") or []),
+ "manual_proposed": len(manual),
+ }
+ if manual:
+ decision = aggregate_finding(
+ target=T_MEMORY,
+ explanation=";".join(str(op.get("reason") or "") for op in manual[:3]),
+ confidence=confidence_from_evidence(len(manual), saturates_at=6),
+ evidence_count=len(manual),
+ )
+ if decision.may_produce_candidate:
+ asset_id = "graph-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12]
+ ir = EvolutionIR(
+ target_kind="memory",
+ target_asset_id=asset_id,
+ base_version="current",
+ operation="update",
+ hypothesis=(
+ f"图谱有 {len(manual)} 处语义变更需要人工裁决:"
+ + ";".join(str(op.get("reason") or "") for op in manual[:3])
+ ),
+ changes=[{"user_id": user_id, "graph_ops": manual}],
+ evidence_refs=[
+ str(op.get("relation_id") or "") for op in manual if op.get("relation_id")
+ ][:20],
+ risk_tier=RISK_MEDIUM,
+ scope={"level": "user", "user_id": user_id},
+ rollback=Rollback(version_id="current", triggers=["manual"]),
+ evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(
+ ir,
+ credit_decision=decision.to_dict(),
+ allowed_scopes=["user", "workspace"],
+ )
+ except InvariantViolation as violation:
+ logger.info(
+ "[evolution-loop] graph candidate rejected: %s", violation.code
+ )
+ else:
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="graph_engine",
+ evidence_refs=list(ir.evidence_refs),
+ )
+ if candidate_id:
+ entry["candidate_id"] = candidate_id
+ created.append(entry)
+ return created
+
+
+def run_other_engines(views: List[Dict[str, Any]], *, dry_run: bool) -> Dict[str, Any]:
+ """Memory, decremental, intent, orchestration, prompt and graph — each with an exit.
+
+ Independently guarded: a failure in one engine must not cost the others
+ their findings, because they answer different questions and an operator
+ needs whichever ones succeeded.
+ """
+ from core.evolution.cycle.cycle_extras import run_extras
+
+ scan_findings = memory_candidates.memory_scan_findings(views)
+ extras = run_extras(views, scan_findings=scan_findings)
+
+ produced: List[Dict[str, Any]] = []
+ for name, fn in (
+ ("memory", lambda: memory_candidates.memory_candidates(extras, views, dry_run=dry_run)),
+ ("intent", lambda: memory_candidates.intent_skill_candidates(extras, views, dry_run=dry_run)),
+ ("decremental", lambda: composition_candidates.decremental_candidates(extras, dry_run=dry_run)),
+ ("orchestration", lambda: composition_candidates.orchestration_candidates(views, dry_run=dry_run)),
+ ("prompt", lambda: prompt_candidates.prompt_candidates(views, dry_run=dry_run)),
+ ("graph", lambda: graph_evolution_candidates(views, dry_run=dry_run)),
+ ):
+ try:
+ produced.extend(fn())
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[evolution-loop] %s engine failed: %s", name, exc)
+ produced.append({"kind": name, "error": str(exc)})
+
+ # ``ops`` holds MemoryOp objects for the candidate builder; the report is
+ # serialised into the console, so it must not carry them.
+ memory_report = dict(extras.get("memory") or {})
+ memory_report.pop("ops", None)
+ extras["memory"] = memory_report
+ extras["memory"]["scan_findings"] = [
+ f.to_dict() if hasattr(f, "to_dict") else f for f in scan_findings
+ ][:50]
+
+ return {"extras": extras, "engine_candidates": produced}
diff --git a/src/backend/core/evolution/cycle/loop.py b/src/backend/core/evolution/cycle/loop.py
new file mode 100644
index 00000000..90cca13b
--- /dev/null
+++ b/src/backend/core/evolution/cycle/loop.py
@@ -0,0 +1,292 @@
+"""One fleet-wide evolution pass: discover, attribute, promote, validate, persist."""
+from __future__ import annotations
+from core.evolution.cycle import ordering
+from core.evolution.cycle import patterns
+from core.evolution.cycle import promotion
+from core.evolution.cycle import promotion_constants
+import hashlib
+from typing import Any, Dict, List
+from core.evolution.evidence.credit import T_SKILL, aggregate_finding, assign_credit, confidence_from_evidence
+from core.evolution.ir import RISK_HIGH, RISK_MEDIUM
+import logging
+from core.evolution.cycle import candidate_records
+from core.evolution.cycle import episode_views
+from core.evolution.cycle import graph_candidates
+
+logger = logging.getLogger(__name__)
+
+def run_evolution_cycle(*, limit: int = 500, dry_run: bool = False) -> Dict[str, Any]:
+ """One fleet-wide pass: discover → attribute → promote → validate → persist.
+
+ Returns a report rather than raising: this runs as a background job and a
+ bad cycle must leave the product untouched.
+ """
+ if not candidate_records.cross_user_mining_available():
+ logger.info("[evolution-loop] cross-user mining unavailable; personal only")
+ return {
+ "episodes": 0,
+ "patterns": 0,
+ "candidates": [],
+ "rejected": [],
+ "no_update": 0,
+ "skipped": "cross_user_mining_requires_enterprise",
+ }
+ # Refuse rather than degrade. Without an embedding service two of the three
+ # engines lose their primary path and the cycle reports success having found
+ # nothing — which is indistinguishable from "there was nothing to find", and
+ # is the state nobody investigates.
+ from core.evolution.cycle.readiness import check_readiness
+
+ readiness = check_readiness()
+ if not readiness.ready:
+ logger.warning("[evolution-loop] not ready, refusing to run: %s", readiness.blocking)
+ return {
+ "episodes": 0,
+ "patterns": 0,
+ "candidates": [],
+ "rejected": [],
+ "no_update": 0,
+ "skipped": "not_ready",
+ "readiness": readiness.to_dict(),
+ }
+
+ report: Dict[str, Any] = {
+ "episodes": 0,
+ "patterns": 0,
+ "proposals": 0,
+ "candidates": [],
+ "rejected": [],
+ "no_update": 0,
+ "readiness": readiness.to_dict(),
+ }
+
+ views = episode_views.load_episode_views(limit=limit)
+ report["episodes"] = len(views)
+ if not views:
+ return report
+
+ discovered = patterns.discover_patterns(views)
+ report["patterns"] = len(discovered)
+
+ # Transferable ordering rules, extracted once across the whole corpus rather
+ # than per pattern. A rigid sequence only helps tasks whose tool set it
+ # happens to cover; the ordering rule behind it helps every task making the
+ # same mistake, which is the difference between memorising and learning.
+ constraints = ordering.extract_ordering_constraints(views)
+ report["ordering_constraints"] = [c.to_dict() for c in constraints]
+
+ from core.evolution.generators.skill_candidate_engine import STATUS_CREATED, STATUS_DRY_RUN, STATUS_REJECTED, SkillCandidateSpec, create_skill_candidate
+
+ existing_signatures: Dict[str, str] = {}
+ covered_tool_sets: List[List[str]] = []
+ by_episode_id = {str(v.get("episode_id") or ""): v for v in views}
+
+ for pattern in discovered:
+ if pattern.kind == promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE and pattern.tool_sequence:
+ # An observation across many *successful* runs — aggregate decision,
+ # not failure attribution with invented features.
+ decision = aggregate_finding(
+ target=T_SKILL,
+ explanation=(
+ f"{pattern.support} 个成功 Episode 复现同一 "
+ f"{len(pattern.tool_sequence)} 步工具序列"
+ f"(成功率 {pattern.success_rate:.0%})"
+ ),
+ confidence=confidence_from_evidence(pattern.support, saturates_at=10),
+ evidence_count=pattern.support,
+ )
+ else:
+ decision = assign_credit(candidate_records.features_for(pattern))
+ if not decision.may_produce_candidate:
+ # No-update / unidentifiable / a non-asset cause. Refusing here is
+ # the mechanism that stops a KB gap being "fixed" by editing a skill.
+ report["no_update"] += 1
+ continue
+
+ proposal = promotion.promote_tool_sequence_to_skill(
+ pattern,
+ episodes=views,
+ skill_credit=decision.scores.get("skill", 0.0),
+ workflow_credit=decision.scores.get("workflow", 0.0),
+ existing_skill_signatures=existing_signatures,
+ )
+ if proposal is None:
+ continue
+ report["proposals"] += 1
+
+ # Stable, content-derived id. Python's built-in hash() is randomised per
+ # process, so using it here minted a fresh asset id on every cycle and the
+ # de-duplication constraint never matched — one candidate per run, forever.
+ asset_id = proposal.patch_target or (
+ "auto-" + hashlib.sha256(proposal.signature.encode("utf-8")).hexdigest()[:12]
+ )
+ sequence = [str(t) for t in proposal.payload.get("tool_sequence", [])]
+ supporting = [
+ by_episode_id[e]
+ for e in (proposal.payload.get("episode_ids") or [])
+ if e in by_episode_id
+ ]
+
+ # 规则门测统计,测不了语义——意图簇可能是巧合归并,序列可能是对任何
+ # 任务都成立的通用组合。规则幸存者再过一道 LLM 裁决,拒绝即不产候选。
+ from core.evolution.assessment.sop_judge import adjudicate_sequence_sop
+
+ verdict = candidate_records.run_async(
+ adjudicate_sequence_sop(
+ representative=str(proposal.payload.get("intent") or ""),
+ objectives=[str(v.get("objective") or "") for v in supporting],
+ tool_sequence=sequence,
+ occasions=int(proposal.payload.get("occasions") or 0),
+ support=proposal.support,
+ success_rate=pattern.success_rate,
+ )
+ )
+ if not verdict.worthy:
+ report["rejected"].append(
+ {
+ "signature": proposal.signature,
+ "code": (
+ "sop_judge_unavailable"
+ if verdict.judge_unavailable
+ else "sop_judge_refused"
+ ),
+ "reason": verdict.reason,
+ }
+ )
+ continue
+ spec = SkillCandidateSpec(
+ asset_id=asset_id,
+ proposer="promotion_chain",
+ hypothesis=proposal.rationale,
+ views=supporting,
+ tool_sequence=sequence,
+ # Attach every rule whose tools this skill touches, and carry its
+ # scope along with it. Attaching must not apply the usability
+ # policy: doing so dropped exactly the context-dependent rules the
+ # scoping exists to preserve — a rule contradicted in one task type
+ # is still valid in the ones where it held, and that decision
+ # belongs at use time, where the context is known.
+ ordering_constraints=[
+ c.to_dict()
+ for c in constraints
+ if c.before in sequence and c.after in sequence
+ ],
+ operation="patch" if proposal.patch_target else "new",
+ base_version="current" if proposal.patch_target else "v0",
+ # A brand-new capability is riskier than tweaking an existing one.
+ risk_tier=RISK_MEDIUM if proposal.patch_target else RISK_HIGH,
+ scope={"level": "workspace"},
+ tenant_id=str(
+ (supporting[0].get("tenant_id") if supporting else "") or "default"
+ ),
+ write_document=False,
+ )
+ # 裁决结论随候选进 credit_decision 落库,评审员能看到机器放行的理由。
+ outcome = create_skill_candidate(
+ spec,
+ credit={**decision.to_dict(), "sop_judge": verdict.to_dict()},
+ dry_run=dry_run,
+ )
+ status = outcome.get("status")
+ if status == STATUS_REJECTED:
+ report["rejected"].append(
+ {"signature": proposal.signature, "code": outcome.get("code")}
+ )
+ elif status == STATUS_DRY_RUN:
+ report["candidates"].append({"dry_run": True, "signature": proposal.signature})
+ elif status == STATUS_CREATED:
+ covered_tool_sets.append(sequence)
+ report["candidates"].append(
+ {
+ "candidate_id": outcome["candidate_id"],
+ "signature": proposal.signature,
+ "evidence_pack_id": outcome.get("pack_id", ""),
+ }
+ )
+
+ # ── Ordering rules that no sequence candidate carries ────────────────────
+ #
+ # Without this the only shape of learning that can become an asset is "five
+ # runs repeated one identical tool sequence", which on real traffic is close
+ # to unobservable — measured on this deployment's history, every episode's
+ # tool sequence was unique. The rules are the part that generalises, so they
+ # get to stand on their own rather than only riding along on a sequence that
+ # happens to qualify.
+ orphan_rules = candidate_records.uncovered_constraints(constraints, covered_tool_sets)
+ if orphan_rules:
+ rule_decision = assign_credit(candidate_records.features_for_ordering_rules(orphan_rules))
+ if not rule_decision.may_produce_candidate:
+ report["no_update"] += 1
+ else:
+ rule_payload = [c.to_dict() for c in orphan_rules]
+ signature = "|".join(sorted(f"{c.before}<{c.after}" for c in orphan_rules))
+ asset_id = "auto-rules-" + hashlib.sha256(signature.encode("utf-8")).hexdigest()[:12]
+ supporting_views = [
+ view
+ for view in views
+ if view.get("verdict") == "success"
+ and any(
+ c.before in (view.get("tool_sequence") or [])
+ and c.after in (view.get("tool_sequence") or [])
+ for c in orphan_rules
+ )
+ ]
+ rule_spec = SkillCandidateSpec(
+ asset_id=asset_id,
+ proposer="promotion_chain",
+ hypothesis=(
+ f"{len(orphan_rules)} 条工具顺序约束在 {len(supporting_views)} 条成功执行中"
+ "稳定成立且跨多种工具组合复现,但规划阶段每次仍可能排错"
+ ),
+ views=supporting_views,
+ # Deliberately no tool sequence: this asset claims no fixed
+ # order of its own, only constraints. Materialisation renders
+ # it as rules.
+ tool_sequence=[],
+ ordering_constraints=rule_payload,
+ operation="new",
+ base_version="v0",
+ risk_tier=RISK_HIGH,
+ scope={"level": "workspace"},
+ tenant_id=str(
+ (supporting_views[0].get("tenant_id") if supporting_views else "")
+ or "default"
+ ),
+ write_document=False,
+ )
+ outcome = create_skill_candidate(
+ rule_spec, credit=rule_decision.to_dict(), dry_run=dry_run
+ )
+ status = outcome.get("status")
+ if status == STATUS_REJECTED:
+ report["rejected"].append({"signature": signature, "code": outcome.get("code")})
+ else:
+ report["proposals"] += 1
+ if status == STATUS_DRY_RUN:
+ report["candidates"].append(
+ {"dry_run": True, "signature": signature, "kind": "ordering_rules"}
+ )
+ elif status == STATUS_CREATED:
+ report["candidates"].append(
+ {
+ "candidate_id": outcome["candidate_id"],
+ "signature": signature,
+ "kind": "ordering_rules",
+ "evidence_pack_id": outcome.get("pack_id", ""),
+ }
+ )
+
+ # The other two engines, plus orchestration. Every finding below becomes a
+ # candidate in the same table, behind the same security gate and the same
+ # review queue as the skill candidates above — a finding that stops at a
+ # report is a log line, not an engine.
+ report.update(graph_candidates.run_other_engines(views, dry_run=dry_run))
+
+ logger.info(
+ "[evolution-loop] cycle done: episodes=%d patterns=%d candidates=%d no_update=%d",
+ report["episodes"],
+ report["patterns"],
+ len(report["candidates"]),
+ report["no_update"],
+ )
+ return report
diff --git a/src/backend/core/evolution/cycle/memory_candidates.py b/src/backend/core/evolution/cycle/memory_candidates.py
new file mode 100644
index 00000000..9570e4ad
--- /dev/null
+++ b/src/backend/core/evolution/cycle/memory_candidates.py
@@ -0,0 +1,286 @@
+"""Memory and intent candidates built from per-user memory scans."""
+from __future__ import annotations
+import hashlib
+from typing import Any, Dict, List
+from core.evolution.evidence.credit import T_MEMORY, T_SKILL, aggregate_finding, confidence_from_evidence
+from core.evolution.ir import RISK_HIGH, RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir
+import logging
+from core.evolution.cycle import candidate_records
+
+logger = logging.getLogger(__name__)
+
+def memory_scan_findings(views: List[Dict[str, Any]]) -> List[Any]:
+ """Scan every contributing user's store.
+
+ Per user, because the store is per user: near-duplicates and contradictions
+ only mean anything within one person's memories, and a cross-user scan would
+ both mean nothing and read what it has no business reading.
+ """
+ from core.evolution.generators.memory_scan import scan_user_memories
+
+ retrieved: Dict[str, Dict[str, int]] = {}
+ for view in views:
+ user_id = str(view.get("user_id") or "")
+ if not user_id:
+ continue
+ bucket = retrieved.setdefault(user_id, {})
+ for ref in view.get("memory_refs") or []:
+ bucket[str(ref)] = bucket.get(str(ref), 0) + 1
+
+ findings: List[Any] = []
+ for user_id, refs in retrieved.items():
+ try:
+ _, user_findings = candidate_records.run_async(scan_user_memories(user_id, retrieved_refs=refs))
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[evolution-loop] memory scan failed for %s: %s", user_id, exc)
+ continue
+ findings.extend(user_findings)
+ return findings
+
+
+def memory_candidates(
+ extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool
+) -> List[Dict[str, Any]]:
+ """One memory candidate per user, carrying that user's operations.
+
+ Grouped per user rather than per operation because the approval question is
+ "may the system reorganise *my* memories", and answering it forty times for
+ forty entries is not review, it is attrition.
+ """
+ memory = extras.get("memory") or {}
+ ops = memory.get("ops") or []
+ if not ops:
+ return []
+
+ owner_by_ref: Dict[str, str] = {}
+ for view in views:
+ user_id = str(view.get("user_id") or "")
+ for ref in view.get("memory_refs") or []:
+ owner_by_ref.setdefault(str(ref), user_id)
+
+ by_user: Dict[str, List[Any]] = {}
+ for op in ops:
+ owner = owner_by_ref.get(op.memory_ref, "")
+ if not owner:
+ # An operation whose subject cannot be identified must not be
+ # proposed: applying it would need a user id, and guessing one is
+ # how a change lands on the wrong person's memories.
+ continue
+ by_user.setdefault(owner, []).append(op)
+
+ created: List[Dict[str, Any]] = []
+ for user_id, user_ops in by_user.items():
+ # Duplicates, contradictions and stale entries are properties of the
+ # store, found by scanning it — not a conflict with any one request.
+ decision = aggregate_finding(
+ target=T_MEMORY,
+ explanation=";".join(op.reason for op in user_ops[:3]),
+ confidence=confidence_from_evidence(len(user_ops), saturates_at=8),
+ evidence_count=len(user_ops),
+ )
+ if not decision.may_produce_candidate:
+ continue
+
+ asset_id = "mem-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12]
+ ir = EvolutionIR(
+ target_kind="memory",
+ target_asset_id=asset_id,
+ base_version="current",
+ operation="merge" if any(op.operation == "merge" for op in user_ops) else "reweight",
+ hypothesis=(
+ f"记忆库有 {len(user_ops)} 处需要重组:"
+ + ";".join(op.reason for op in user_ops[:3])
+ ),
+ changes=[
+ {
+ "user_id": user_id,
+ "operations": [op.to_dict() for op in user_ops],
+ }
+ ],
+ evidence_refs=[op.memory_ref for op in user_ops][:20],
+ # Changing what a person's agent remembers is never low risk, and
+ # the scope is that person alone.
+ risk_tier=RISK_MEDIUM,
+ scope={"level": "user", "user_id": user_id},
+ rollback=Rollback(version_id="current", triggers=["user_disable"]),
+ evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(
+ ir, credit_decision=decision.to_dict(), allowed_scopes=["user", "workspace"]
+ )
+ except InvariantViolation as violation:
+ logger.info("[evolution-loop] memory candidate rejected: %s", violation.code)
+ continue
+ if dry_run:
+ created.append({"dry_run": True, "kind": "memory", "user_id": user_id})
+ continue
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="memory_engine",
+ evidence_refs=[op.memory_ref for op in user_ops][:20],
+ )
+ if candidate_id:
+ created.append(
+ {
+ "candidate_id": candidate_id,
+ "kind": "memory",
+ "user_id": user_id,
+ "operations": len(user_ops),
+ }
+ )
+ return created
+
+
+def intent_skill_candidates(
+ extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool
+) -> List[Dict[str, Any]]:
+ """Recurring intents, compiled through the unified skill candidate engine.
+
+ The intent finding says a capability is missing; it does not say what the
+ capability is. The engine resolves the cluster's evidence into an immutable
+ pack, derives the deterministic structure (tools, boundaries, dependencies),
+ has the model write only the document, and validates the layered contract —
+ the same chain the tool-sequence path uses, so identical evidence can no
+ longer produce structurally different skills depending on which engine
+ noticed it first.
+ """
+ from core.evolution.generators.skill_candidate_engine import STATUS_CREATED, STATUS_DRY_RUN, STATUS_NOT_WRITTEN, STATUS_REJECTED, SkillCandidateSpec, create_skill_candidate
+
+ findings = [
+ f
+ for f in (extras.get("intent") or {}).get("findings", [])
+ if f.get("action") == "skill_candidate"
+ ]
+ if not findings:
+ return []
+
+ by_episode = {str(v.get("episode_id") or ""): v for v in views}
+ created: List[Dict[str, Any]] = []
+
+ for finding in findings:
+ cluster = finding.get("cluster") or {}
+ episode_ids = [str(e) for e in (cluster.get("episode_ids") or [])]
+ episodes = [by_episode[e] for e in episode_ids if e in by_episode]
+ if not episodes:
+ continue
+
+ # The cluster mostly *succeeded* — that is a precondition for
+ # distilling it. Declaring the turns failed to satisfy the failure
+ # attributor would contradict the very check that let this through.
+ decision = aggregate_finding(
+ target=T_SKILL,
+ explanation=str(finding.get("reason") or ""),
+ confidence=confidence_from_evidence(len(episode_ids), saturates_at=10),
+ evidence_count=len(episode_ids),
+ )
+ if not decision.may_produce_candidate:
+ continue
+
+ asset_id = str(finding.get("asset_id") or "")
+ spec = SkillCandidateSpec(
+ asset_id=asset_id,
+ proposer="intent_engine",
+ hypothesis=str(finding.get("reason") or ""),
+ views=episodes,
+ intent_cluster=cluster,
+ procedures=procedures_for_cluster(cluster, views),
+ operation="new",
+ base_version="v0",
+ risk_tier=RISK_HIGH,
+ scope={"level": "workspace"},
+ tenant_id=str(episodes[0].get("tenant_id") or "default"),
+ write_document=True,
+ )
+ outcome = create_skill_candidate(
+ spec, credit=decision.to_dict(), dry_run=dry_run, run_async=candidate_records.run_async
+ )
+ status = outcome.get("status")
+ if status == STATUS_NOT_WRITTEN:
+ created.append(
+ {
+ "kind": "recurring_intent",
+ "asset_id": asset_id,
+ "written": False,
+ "why_not": outcome.get("why_not") or {},
+ }
+ )
+ elif status == STATUS_REJECTED:
+ logger.info(
+ "[evolution-loop] intent candidate rejected: %s %s",
+ outcome.get("code"),
+ outcome.get("violations") or "",
+ )
+ elif status == STATUS_DRY_RUN:
+ created.append({"dry_run": True, "kind": "recurring_intent", "asset_id": asset_id})
+ elif status == STATUS_CREATED:
+ created.append(
+ {
+ "candidate_id": outcome["candidate_id"],
+ "kind": "recurring_intent",
+ "asset_id": asset_id,
+ "written": True,
+ "attempts": outcome.get("attempts", 0),
+ "evidence_pack_id": outcome.get("pack_id", ""),
+ }
+ )
+ return created
+
+
+def procedures_for_cluster(
+ cluster: Dict[str, Any], views: List[Dict[str, Any]]
+) -> List[Dict[str, str]]:
+ """The procedural memories retrieved during this intent's episodes.
+
+ These are what makes the resulting document a *compilation* of memory rather
+ than a restatement of a tool list — and their refs are what gets demoted
+ once the skill is live.
+ """
+ from core.evolution.generators.memory_scan import procedural_records, to_records
+ from core.memory.ref_shadow import build_ref_id
+ from core.memory.retrieval_types import LAYER_FACT
+
+ wanted = {str(ref) for ref in (cluster.get("memory_refs") or [])}
+ if not wanted:
+ return []
+
+ episode_ids = {str(e) for e in (cluster.get("episode_ids") or [])}
+ users = {
+ str(v.get("user_id") or "")
+ for v in views
+ if str(v.get("episode_id") or "") in episode_ids and v.get("user_id")
+ }
+
+ procedures: List[Dict[str, str]] = []
+ for user_id in users:
+ try:
+ from core.memory.service import get_all_memories
+
+ raw = candidate_records.run_async(get_all_memories(user_id, top_k=500))
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[evolution-loop] procedure lookup failed for %s: %s", user_id, exc)
+ continue
+ for record in procedural_records(to_records(raw)):
+ ref = build_ref_id(
+ layer=LAYER_FACT,
+ user_id=user_id,
+ workspace_id=record.workspace_id,
+ content_hash=record.content_hash,
+ )
+ if ref in wanted:
+ procedures.append(
+ {
+ "ref": ref,
+ # Carried because a fleet-wide skill can be compiled from
+ # several people's memories, and demotion is per-store: a
+ # bare list of refs has no owner to apply them against, so
+ # the reverse edge would silently do nothing for exactly
+ # the candidates that need it most.
+ "user_id": user_id,
+ "rule": record.content,
+ "why": record.why,
+ "applies_to": record.applies_to,
+ }
+ )
+ return procedures
diff --git a/src/backend/core/evolution/cycle/memory_findings.py b/src/backend/core/evolution/cycle/memory_findings.py
new file mode 100644
index 00000000..8ddaf33f
--- /dev/null
+++ b/src/backend/core/evolution/cycle/memory_findings.py
@@ -0,0 +1,109 @@
+"""Reorganisation proposals for the memory layer."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, Sequence
+from core.evolution.cycle import cycle_extras_constants
+
+def memory_findings(
+ views: Sequence[Dict[str, Any]],
+ *,
+ scan_findings: Optional[Sequence[Any]] = None,
+) -> Dict[str, Any]:
+ """Reorganisation proposals for the memory layer.
+
+ Two independent sources, because they answer different questions and only
+ one of them is answerable from a run's evidence:
+
+ * **inventory** (``scan_findings``) — duplicates, contradictions and stale
+ entries, which require reading the store;
+ * **behaviour** (``views``) — memories that keep being injected into turns
+ that fail, which requires reading the runs.
+
+ The behavioural signal is deliberately *not* framed as duplication. A
+ memory injected repeatedly is either helping or not, and the difference is
+ the outcome of the turns it landed in — never the count on its own.
+ """
+ from core.evolution.generators.memory_ops import OP_MERGE, OP_REWEIGHT, SCOPE_USER, MemoryOp, partition_by_approval
+ from core.evolution.generators.memory_scan import FINDING_CONTRADICTION, FINDING_DUPLICATE, FINDING_STALE
+
+ retrieved: Dict[str, int] = {}
+ helped: Dict[str, int] = {}
+ for view in views:
+ for ref in view.get("memory_refs") or []:
+ key = str(ref)
+ retrieved[key] = retrieved.get(key, 0) + 1
+ if view.get("verdict") == "success":
+ helped[key] = helped.get(key, 0) + 1
+
+ ops: List[MemoryOp] = []
+ contradictions: List[Dict[str, Any]] = []
+
+ # 1. Inventory findings, from an actual scan of the store.
+ for finding in scan_findings or []:
+ payload = finding.to_dict() if hasattr(finding, "to_dict") else dict(finding)
+ kind = payload.get("kind")
+ refs = [str(r) for r in (payload.get("refs") or [])]
+ if kind == FINDING_DUPLICATE and len(refs) >= 2:
+ # The second entry loses. Content is not rewritten and nothing is
+ # removed — the survivor already says the same thing.
+ ops.append(
+ MemoryOp(
+ operation=OP_MERGE,
+ memory_ref=refs[1],
+ scope=SCOPE_USER,
+ reason=payload.get("reason", ""),
+ before={"injected": True, "duplicate_of": refs[0]},
+ after={"injected": False, "superseded_by": refs[0]},
+ )
+ )
+ elif kind == FINDING_STALE and refs:
+ ops.append(
+ MemoryOp(
+ operation=OP_REWEIGHT,
+ memory_ref=refs[0],
+ scope=SCOPE_USER,
+ reason=payload.get("reason", ""),
+ before={"weight": 1.0},
+ after={"weight": 0.7},
+ )
+ )
+ elif kind == FINDING_CONTRADICTION:
+ # Neither side is automatically right, and picking by recency is
+ # how a corrected mistake becomes the new truth. This is reported
+ # for a human, with both sides and their dates.
+ contradictions.append(payload)
+
+ # 2. Behavioural: injected often, associated with failure. Down-weighted,
+ # never deleted — a wrong reweight is recoverable and a wrong deletion is
+ # not.
+ for ref, count in retrieved.items():
+ if count < cycle_extras_constants.DEMOTE_MIN_HITS:
+ continue
+ rate = helped.get(ref, 0) / count
+ if rate <= cycle_extras_constants.DEMOTE_SUCCESS_CEILING:
+ ops.append(
+ MemoryOp(
+ operation=OP_REWEIGHT,
+ memory_ref=ref,
+ scope=SCOPE_USER,
+ reason=(
+ f"被注入 {count} 次,其中仅 {helped.get(ref, 0)} 次所在回答成功"
+ "——检索到它并没有帮上忙"
+ ),
+ before={"weight": 1.0},
+ after={"weight": 0.5},
+ )
+ )
+
+ auto, needs_review = partition_by_approval(ops)
+ return {
+ "distinct_memories_retrieved": len(retrieved),
+ "merge_proposals": sum(1 for op in ops if op.operation == OP_MERGE),
+ "reweight_proposals": sum(1 for op in ops if op.operation == OP_REWEIGHT),
+ "contradictions": contradictions,
+ # Reported separately because the split *is* the governance boundary:
+ # reweighting is reversible and may proceed, merging changes what gets
+ # injected and may not.
+ "auto_applicable": [op.to_dict() for op in auto],
+ "needs_review": [op.to_dict() for op in needs_review],
+ "ops": ops,
+ }
diff --git a/src/backend/core/evolution/cycle/ordering.py b/src/backend/core/evolution/cycle/ordering.py
new file mode 100644
index 00000000..2df58c89
--- /dev/null
+++ b/src/backend/core/evolution/cycle/ordering.py
@@ -0,0 +1,103 @@
+"""Learn tool orderings that hold across successful episodes and apply them."""
+from __future__ import annotations
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
+from core.evolution.cycle import patterns
+from core.evolution.cycle import promotion_constants
+from core.evolution.cycle import promotion_types
+
+def extract_ordering_constraints(
+ episodes: Sequence[Dict[str, Any]],
+ *,
+ min_support: int = promotion_constants.MIN_SUPPORT,
+ min_contexts: int = promotion_constants.MIN_CONSTRAINT_CONTEXTS,
+) -> List[promotion_types.OrderingConstraint]:
+ """Find orderings that hold *invariantly* across successful episodes.
+
+ Conflicting evidence vetoes a constraint outright rather than letting the
+ majority win. A pair that appears both ways round is not a rule — it is a
+ signal that ordering depends on context the system cannot yet see, and
+ promoting the popular direction would actively break the minority case.
+ """
+ forward: Dict[Tuple[str, str], Set[str]] = defaultdict(set)
+ by_task_type: Dict[Tuple[str, str], Set[str]] = defaultdict(set)
+
+ for episode in episodes:
+ if episode.get("verdict") != "success":
+ continue
+ tools = [str(t) for t in (episode.get("tool_sequence") or [])]
+ if len(tools) < 2:
+ continue
+ shape = patterns.pattern_signature(sorted(set(tools)))
+ task_type = str(episode.get("task_type") or "unknown")
+ for i, earlier in enumerate(tools):
+ for later in tools[i + 1 :]:
+ if earlier == later:
+ continue
+ forward[(earlier, later)].add(shape)
+ by_task_type[(earlier, later)].add(task_type)
+
+ constraints: List[promotion_types.OrderingConstraint] = []
+ for (a, b), shapes in forward.items():
+ validated = by_task_type[(a, b)]
+ contradicted = by_task_type.get((b, a), set())
+
+ # A pair that flips *within the same task type* is genuinely ambiguous
+ # there — no amount of context resolves it, so the rule is dropped for
+ # that type rather than resolved by majority.
+ ambiguous = validated & contradicted
+ validated = validated - ambiguous
+ if not validated:
+ continue
+
+ support = sum(
+ 1
+ for e in episodes
+ if e.get("verdict") == "success"
+ and a in (e.get("tool_sequence") or [])
+ and b in (e.get("tool_sequence") or [])
+ and str(e.get("task_type") or "unknown") in validated
+ )
+ if support < min_support or len(shapes) < min_contexts:
+ continue
+ constraints.append(
+ promotion_types.OrderingConstraint(
+ before=a,
+ after=b,
+ support=support,
+ contexts=len(shapes),
+ validated_in=tuple(sorted(validated)),
+ contradicted_in=tuple(sorted(contradicted | ambiguous)),
+ )
+ )
+
+ constraints.sort(key=lambda c: (c.contexts, c.support), reverse=True)
+ return constraints
+
+
+def apply_ordering_constraints(
+ tools: Sequence[str],
+ constraints: Sequence[promotion_types.OrderingConstraint],
+ task_type: Optional[str] = None,
+) -> List[str]:
+ """Reorder a task's tools to satisfy every applicable constraint.
+
+ A stable topological sort: tools with no constraint between them keep their
+ original relative order, so applying a rule about one pair never silently
+ reshuffles everything else.
+ """
+ items = list(tools)
+ applicable = [c for c in constraints if c.applies_to(items, task_type)]
+ if not applicable:
+ return items
+
+ for _ in range(len(items)):
+ changed = False
+ for constraint in applicable:
+ i, j = items.index(constraint.before), items.index(constraint.after)
+ if i > j:
+ items.insert(j, items.pop(i))
+ changed = True
+ if not changed:
+ break
+ return items
diff --git a/src/backend/core/evolution/cycle/patterns.py b/src/backend/core/evolution/cycle/patterns.py
new file mode 100644
index 00000000..56ffa95f
--- /dev/null
+++ b/src/backend/core/evolution/cycle/patterns.py
@@ -0,0 +1,113 @@
+"""Cluster episodes into recurring patterns."""
+from __future__ import annotations
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Sequence
+from core.evolution.cycle import promotion_constants
+
+@dataclass
+class Pattern:
+ """A recurring shape found across episodes."""
+
+ kind: str
+ signature: str
+ support: int
+ episode_ids: List[str] = field(default_factory=list)
+ success_rate: float = 0.0
+ tool_sequence: List[str] = field(default_factory=list)
+ skill_sequence: List[str] = field(default_factory=list)
+ memory_refs: List[str] = field(default_factory=list)
+ single_shot: bool = False
+
+ @property
+ def meets_support(self) -> bool:
+ return self.support >= promotion_constants.MIN_SUPPORT or self.single_shot
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "kind": self.kind,
+ "signature": self.signature,
+ "support": self.support,
+ "success_rate": round(self.success_rate, 3),
+ "tool_sequence": self.tool_sequence,
+ "skill_sequence": self.skill_sequence,
+ "memory_refs": self.memory_refs,
+ "single_shot": self.single_shot,
+ "episode_ids": self.episode_ids[:20],
+ }
+
+
+def pattern_signature(items: Sequence[str]) -> str:
+ return "→".join(items)
+
+
+def discover_patterns(episodes: List[Dict[str, Any]]) -> List[Pattern]:
+ """Cluster episodes into recurring shapes.
+
+ ``episodes`` are plain dicts so this stays testable without a database and
+ usable over backfilled history (which has no asset bundle and therefore
+ cannot be replayed, but is perfectly good for finding patterns).
+ """
+ by_tool_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+ by_skill_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+
+ for episode in episodes:
+ tools = [str(t) for t in (episode.get("tool_sequence") or [])]
+ skills = [str(s) for s in (episode.get("skill_sequence") or [])]
+ if tools:
+ by_tool_seq[pattern_signature(tools)].append(episode)
+ if len(skills) >= 2:
+ by_skill_seq[pattern_signature(skills)].append(episode)
+
+ patterns: List[Pattern] = []
+
+ for signature, group in by_tool_seq.items():
+ successes = [e for e in group if e.get("verdict") == "success"]
+ failures = [e for e in group if e.get("verdict") == "failed"]
+ rate = len(successes) / len(group) if group else 0.0
+
+ if len(successes) >= promotion_constants.MIN_SUPPORT and rate >= promotion_constants.SKILL_MIN_SUCCESS_RATE:
+ patterns.append(
+ Pattern(
+ kind=promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE,
+ signature=signature,
+ support=len(successes),
+ episode_ids=[e.get("episode_id", "") for e in successes],
+ success_rate=rate,
+ tool_sequence=signature.split("→"),
+ memory_refs=sorted(
+ {
+ ref
+ for e in successes
+ for ref in (e.get("memory_refs") or [])
+ }
+ ),
+ )
+ )
+ if len(failures) >= promotion_constants.MIN_SUPPORT:
+ patterns.append(
+ Pattern(
+ kind=promotion_constants.PATTERN_REPEATED_FAILURE,
+ signature=signature,
+ support=len(failures),
+ episode_ids=[e.get("episode_id", "") for e in failures],
+ success_rate=rate,
+ tool_sequence=signature.split("→"),
+ )
+ )
+
+ for signature, group in by_skill_seq.items():
+ if len(group) >= promotion_constants.WORKFLOW_MIN_COMBO_FREQUENCY:
+ successes = [e for e in group if e.get("verdict") == "success"]
+ patterns.append(
+ Pattern(
+ kind=promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE,
+ signature=signature,
+ support=len(group),
+ episode_ids=[e.get("episode_id", "") for e in group],
+ success_rate=len(successes) / len(group),
+ skill_sequence=signature.split("→"),
+ )
+ )
+
+ return [p for p in patterns if p.meets_support]
diff --git a/src/backend/core/evolution/cycle/profile_release.py b/src/backend/core/evolution/cycle/profile_release.py
new file mode 100644
index 00000000..795ca5b0
--- /dev/null
+++ b/src/backend/core/evolution/cycle/profile_release.py
@@ -0,0 +1,281 @@
+"""Publish approved agent profiles and sub-agents."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+from core.db.engine import SessionLocal
+import logging
+from core.evolution.cycle import activation_policy
+from core.evolution.cycle import activation_types
+
+logger = logging.getLogger(__name__)
+
+def materialise_profile_candidate(
+ candidate_id: str, *, approver: str, tenant_id: str = "default"
+) -> Dict[str, Any]:
+ """Publish an approved agent profile.
+
+ Profiles do not ride the shadow → canary ladder that skills do, and the
+ reason is not leniency. A profile *is* the assembly, so a canary would mean
+ two different assemblies running concurrently under one profile id, with
+ every downstream measurement of either one confounded by the other. Instead
+ a profile is scoped: it governs one task type, and its blast radius is that
+ task type's traffic, recorded on the release row like any other change.
+
+ Rolling back is switching ``is_active`` off, which restores the built-in
+ assembly on the next run — no restart, because profiles are read per run.
+ """
+ from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
+
+ from core.db.models.evolution import EvolutionAgentProfile
+ from core.evolution.profiles.agent_profile import AgentProfile, validate_profile
+ from core.evolution.profiles.agent_profile_constants import TARGET_KIND
+ from core.evolution.cycle.release import ACTIVE as _ACTIVE, new_release_id
+
+ if not approver:
+ raise activation_types.ActivationError("approver_required", "激活必须署名")
+
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ if candidate is None:
+ raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
+ if candidate.target_kind != TARGET_KIND:
+ raise activation_types.ActivationError(
+ "unsupported_kind", f"该入口只处理编排候选,收到 {candidate.target_kind}"
+ )
+ if approver == (candidate.proposer or ""):
+ raise activation_types.ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选")
+ if activation_policy.is_frozen(db):
+ raise activation_types.ActivationError("layer_frozen", "进化已被全局冻结")
+
+ payload: Dict[str, Any] = {}
+ for change in candidate.ir.get("changes") or []:
+ if isinstance(change.get("profile"), dict):
+ payload = change["profile"]
+ break
+ if not payload:
+ raise activation_types.ActivationError("no_profile_payload", "候选未携带 profile 定义")
+
+ profile = AgentProfile.from_dict(payload)
+ profile.scope = {**(profile.scope or {}), "tenant_id": tenant_id or "default"}
+
+ # Re-validated at activation against the *current* tool grant, not the
+ # one that existed when the candidate was written. A grant narrowed in
+ # the meantime must not be widened again by approving an older proposal.
+ from core.capabilities.catalog.api import get_enabled_ids
+
+ base_tools = [str(t) for t in get_enabled_ids("tools")] + [
+ str(t) for t in get_enabled_ids("mcp")
+ ]
+ ok, problems = validate_profile(profile, base_tools=base_tools)
+ if not ok:
+ raise activation_types.ActivationError("profile_invalid", f"编排定义未通过校验: {problems}")
+
+ previous = db.get(EvolutionAgentProfile, profile.profile_id)
+ previous_payload = dict(previous.payload) if previous is not None else None
+
+ # One active profile per (task type, scope). Two would make the
+ # resolution order the thing that decides behaviour, which is not a
+ # decision anyone reviewed.
+ for row in (
+ db.query(EvolutionAgentProfile)
+ .filter(EvolutionAgentProfile.is_active.is_(True))
+ .all()
+ ):
+ if row.profile_id == profile.profile_id:
+ continue
+ same_types = set(row.task_types or []) == set(profile.task_types)
+ same_scope = dict(row.scope or {}) == dict(profile.scope)
+ if same_types and same_scope:
+ row.is_active = False
+
+ row = previous or EvolutionAgentProfile(profile_id=profile.profile_id)
+ row.version = profile.version
+ row.task_types = list(profile.task_types)
+ row.scope = dict(profile.scope)
+ row.payload = profile.to_dict()
+ row.candidate_id = candidate_id
+ row.is_active = True
+ row.created_by = f"evolution:{approver}"
+ if previous is None:
+ db.add(row)
+
+ release = EvolutionRelease(
+ release_id=new_release_id(),
+ candidate_id=candidate_id,
+ target_kind=TARGET_KIND,
+ target_asset_id=profile.profile_id,
+ stage=_ACTIVE,
+ traffic_percent=100,
+ scope_filter=dict(profile.scope),
+ guardrails={"previous": previous_payload, "proposer": candidate.proposer or ""},
+ rollback_version_id=(previous_payload or {}).get("version") or "builtin",
+ approved_by=approver,
+ )
+ db.add(release)
+
+ candidate.status = _ACTIVE
+ candidate.approved_by = approver
+ candidate.approved_at = datetime.now(timezone.utc)
+ db.commit()
+
+ # 血缘:哪些技能被装配进了这个 profile。
+ try:
+ from core.evolution.evidence import lineage
+
+ lineage.record_links(
+ [
+ (
+ "skill",
+ str(skill_id),
+ lineage.REL_ASSEMBLED_INTO,
+ "agent_profile",
+ f"{profile.profile_id}@{profile.version}",
+ )
+ for skill_id in profile.skill_ids
+ ],
+ candidate_id=candidate_id,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[activation] profile lineage skipped: %s", exc)
+
+ logger.info(
+ "[activation] profile %s active for %s (tools=%s skills=%s)",
+ profile.profile_id,
+ profile.task_types or ["*"],
+ "narrowed" if profile.tool_allowlist is not None else "unchanged",
+ len(profile.skill_ids),
+ )
+ return {
+ "profile_id": profile.profile_id,
+ "release_id": release.release_id,
+ "task_types": list(profile.task_types),
+ "scope": dict(profile.scope),
+ "tool_allowlist": profile.tool_allowlist,
+ "skill_ids": list(profile.skill_ids),
+ "replaced_previous": previous_payload is not None,
+ }
+
+
+def materialise_subagent_candidate(
+ candidate_id: str, *, approver: str, owner_user_id: Optional[str] = None
+) -> Dict[str, Any]:
+ """Create an approved sub-agent, and nothing else.
+
+ Deliberately does **not** install the route that sends work to it. The route
+ is a sentence in the parent's prompt, and a sentence in the parent's prompt
+ changes every request the parent handles — so it goes through prompt review
+ as its own candidate. Creating the agent here and routing to it there means
+ approving the agent cannot silently also approve a behaviour change.
+
+ The agent is created disabled for the same reason: an agent nobody routes to
+ does nothing, and one that is enabled before its route is reviewed is a
+ capability sitting in the product that no reviewer has seen used.
+ """
+ from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
+
+ from core.db.models import UserAgent
+ from core.evolution.cycle.release import SHADOW, new_release_id
+
+ if not approver:
+ raise activation_types.ActivationError("approver_required", "激活必须署名")
+
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ if candidate is None:
+ raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
+ if candidate.target_kind != "subagent":
+ raise activation_types.ActivationError(
+ "unsupported_kind", f"该入口只处理子智能体候选,收到 {candidate.target_kind}"
+ )
+ if approver == (candidate.proposer or ""):
+ raise activation_types.ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选")
+ if activation_policy.is_frozen(db):
+ raise activation_types.ActivationError("layer_frozen", "进化已被全局冻结")
+
+ spec: Dict[str, Any] = {}
+ for change in candidate.ir.get("changes") or []:
+ if change.get("agent_id"):
+ spec = change
+ break
+ if not spec:
+ raise activation_types.ActivationError("no_subagent_payload", "候选未携带子智能体定义")
+
+ # The subset check runs again at activation against the current grant.
+ from core.capabilities.catalog.api import get_enabled_ids
+
+ base_tools = {str(t) for t in get_enabled_ids("tools")} | {
+ str(t) for t in get_enabled_ids("mcp")
+ }
+ requested = {str(t) for t in (spec.get("tools_allowlist") or [])}
+ widened = sorted(requested - base_tools)
+ if widened:
+ raise activation_types.ActivationError(
+ "tool_escalation", f"子智能体工具集超出主体授权: {widened}"
+ )
+ if spec.get("may_delegate"):
+ raise activation_types.ActivationError("subagent_nesting", "子智能体不得再派生子智能体")
+
+ agent_id = str(spec.get("agent_id"))
+ existing = db.get(UserAgent, agent_id)
+ if existing is not None:
+ raise activation_types.ActivationError("agent_exists", f"子智能体已存在: {agent_id}")
+
+ agent = UserAgent(
+ agent_id=agent_id,
+ owner_type="user" if owner_user_id else "admin",
+ user_id=owner_user_id,
+ name=str(spec.get("name") or agent_id)[:255],
+ description=str(spec.get("responsibility") or "")[:2000],
+ system_prompt=str(spec.get("responsibility") or ""),
+ skill_ids=[str(s) for s in (spec.get("required_skills") or [])],
+ mcp_server_ids=sorted(requested),
+ kb_ids=[],
+ plugin_ids=[],
+ max_iters=12,
+ timeout=int(spec.get("timeout") or 180),
+ # Disabled until its route is approved — see the docstring.
+ is_enabled=False,
+ extra_config={
+ "origin": "evolution",
+ "candidate_id": candidate_id,
+ "route_hint": str(spec.get("route_hint") or ""),
+ "task_types": [str(t) for t in (spec.get("task_types") or [])],
+ "tools_allowlist": sorted(requested),
+ "may_delegate": False,
+ },
+ )
+ db.add(agent)
+
+ release = EvolutionRelease(
+ release_id=new_release_id(),
+ candidate_id=candidate_id,
+ target_kind="subagent",
+ target_asset_id=agent_id,
+ # Reaches nobody until the route exists, which is what shadow means.
+ stage=SHADOW,
+ traffic_percent=0,
+ scope_filter={"owner_user_id": owner_user_id} if owner_user_id else {},
+ guardrails={"proposer": candidate.proposer or "", "route_pending": True},
+ rollback_version_id="v0",
+ approved_by=approver,
+ )
+ db.add(release)
+
+ candidate.approved_by = approver
+ candidate.approved_at = datetime.now(timezone.utc)
+ db.commit()
+
+ logger.info(
+ "[activation] subagent %s created (disabled, route pending) tools=%s",
+ agent_id,
+ sorted(requested),
+ )
+ return {
+ "agent_id": agent_id,
+ "release_id": release.release_id,
+ "stage": SHADOW,
+ "is_enabled": False,
+ "tools_allowlist": sorted(requested),
+ "route_hint": str(spec.get("route_hint") or ""),
+ "next_step": "路由句需作为提示词候选单独审批后,子智能体才会被启用",
+ }
diff --git a/src/backend/core/evolution/cycle/promotion.py b/src/backend/core/evolution/cycle/promotion.py
new file mode 100644
index 00000000..2a57ef72
--- /dev/null
+++ b/src/backend/core/evolution/cycle/promotion.py
@@ -0,0 +1,317 @@
+"""Promote recurring sequences, procedures and skill combinations into proposals."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+import logging
+from core.evolution.cycle import patterns
+from core.evolution.cycle import promotion_constants
+from core.evolution.cycle import promotion_types
+
+logger = logging.getLogger(__name__)
+
+def dominant_shared_intent(
+ pattern: patterns.Pattern,
+ episodes: Optional[Sequence[Dict[str, Any]]],
+ *,
+ support_floor: int,
+):
+ """The one intent behind a sequence's supporting episodes, or ``None``.
+
+ ``None`` is a refusal, and deliberately covers the honest-uncertainty cases
+ too: no views supplied, request text too short to cluster, no clustering
+ method available. "We cannot tell whether these conversations had anything
+ in common" and "they had nothing in common" both mean no SOP claim can be
+ made — treating the first as a pass is how single-tool coincidences used to
+ become skills.
+ """
+ if not episodes:
+ return None
+ wanted = {str(e) for e in pattern.episode_ids}
+ supporting = [e for e in episodes if str(e.get("episode_id") or "") in wanted]
+ if len(supporting) < support_floor:
+ return None
+
+ from core.evolution.assessment.similarity import METHOD_NONE, cluster_intents
+
+ clusters, method = cluster_intents(supporting, min_support=2)
+ if method == METHOD_NONE or not clusters:
+ return None
+ top = clusters[0]
+ if len(top.episode_ids) < support_floor:
+ return None
+ if len(top.episode_ids) / len(supporting) < promotion_constants.INTENT_COHESION_RATIO:
+ return None
+ # 复现必须跨会话:occasions 按 chat_id 去重,同一会话里反复跑同一套
+ # 流程只算一次场合,不构成"这件事总在发生"。
+ if top.occasions < support_floor:
+ return None
+ return top
+
+
+def promote_tool_sequence_to_skill(
+ pattern: patterns.Pattern,
+ *,
+ episodes: Optional[Sequence[Dict[str, Any]]] = None,
+ min_support: Optional[int] = None,
+ skill_credit: float = 0.0,
+ workflow_credit: float = 0.0,
+ existing_skill_signatures: Optional[Dict[str, str]] = None,
+) -> Optional[promotion_types.PromotionProposal]:
+ """Compile a recurring successful tool sequence into a skill proposal.
+
+ A recurring sequence alone is *not* evidence of a reusable capability.
+ Generic tools recur across unrelated conversations by construction, and the
+ limiting case — a whole conversation whose "sequence" is one call to
+ ``view_text_file`` — says only that the task was trivial. What justifies an
+ SOP is commonality of *purpose*: the same kind of request, on separate
+ occasions, repeatedly completed by the same non-trivial procedure. So the
+ gates hold together, and each excludes a specific way of being wrong:
+
+ * the sequence is **non-trivial** (≥ :data:`MIN_TOOL_SEQUENCE_LEN` steps) —
+ wrapping one generic tool adds an asset and no knowledge;
+ * the supporting episodes **share an intent** — a dominant cluster over
+ their request text (via :mod:`core.evolution.assessment.similarity`) covers
+ ≥ :data:`INTENT_COHESION_RATIO` of them; without that the recurrence is a
+ coincidence of tooling, and no commonality means no promotion;
+ * the recurrence spans **separate conversations** — one chat re-running the
+ same procedure is one occasion, not evidence it keeps coming back;
+ * the recurrences **succeed** at ≥ :data:`SKILL_MIN_SUCCESS_RATE`.
+
+ ``episodes`` are the flattened views the pattern was discovered from; the
+ intent gate reads their request text. A caller that cannot supply them gets
+ a refusal, not the benefit of the doubt.
+
+ Named for what it reads. This was called ``promote_memory_to_skill`` and
+ carried ``from_layer="memory"``, but no memory content ever enters the
+ result — ``pattern.memory_refs`` are hashes of whatever happened to be
+ retrieved during the supporting episodes. The name promised a memory→skill
+ chain the code did not have; :func:`promote_procedural_memory_to_skill` is
+ that chain.
+
+ Refuses when orchestration is the better explanation: a stable tool ordering
+ can equally mean "the workflow is fixed" rather than "there is a reusable
+ skill here". Promoting in that case creates a skill with no independent
+ content, which then has to be maintained forever.
+ """
+ if pattern.kind != promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence:
+ return None
+ support_floor = promotion_constants.MIN_SUPPORT if min_support is None else int(min_support)
+ if len(pattern.tool_sequence) < promotion_constants.MIN_TOOL_SEQUENCE_LEN:
+ logger.info(
+ "[promotion] skipping %s: %d-step sequence is below the SOP floor",
+ pattern.signature,
+ len(pattern.tool_sequence),
+ )
+ return None
+ if pattern.support < support_floor:
+ return None
+ if pattern.success_rate < promotion_constants.SKILL_MIN_SUCCESS_RATE:
+ return None
+ if workflow_credit > skill_credit:
+ logger.info(
+ "[promotion] skipping skill promotion for %s: workflow credit higher",
+ pattern.signature,
+ )
+ return None
+
+ intent = dominant_shared_intent(pattern, episodes, support_floor=support_floor)
+ if intent is None:
+ logger.info(
+ "[promotion] skipping %s: supporting episodes share no intent — a "
+ "recurring sequence without a recurring request is not an SOP",
+ pattern.signature,
+ )
+ return None
+ covered = [e for e in pattern.episode_ids if e in set(intent.episode_ids)]
+
+ existing = (existing_skill_signatures or {}).get(pattern.signature)
+ return promotion_types.PromotionProposal(
+ from_layer="episode",
+ to_layer="skill",
+ signature=pattern.signature,
+ support=len(covered),
+ rationale=(
+ f"「{intent.representative[:40]}」类请求在 {intent.occasions} 次独立会话中"
+ f"反复出现,每次都以同一 {len(pattern.tool_sequence)} 步工具序列成功完成"
+ f"({len(covered)} 个 Episode,成功率 {pattern.success_rate:.0%})——"
+ "同类意图 + 稳定做法,具备沉淀为 SOP 的共性"
+ ),
+ # Co-retrieved memories are *context*, not sources. They are carried so
+ # a reviewer can see what was in view, and deliberately not demoted —
+ # nothing of theirs went into this skill, so down-weighting them would
+ # remove knowledge the skill does not carry.
+ source_refs=[],
+ payload={
+ "tool_sequence": pattern.tool_sequence,
+ "episode_ids": covered[:20],
+ "intent": intent.representative,
+ "occasions": intent.occasions,
+ "co_retrieved_memory_refs": list(pattern.memory_refs),
+ },
+ # An equivalent skill already exists → patch it instead of adding a twin.
+ patch_target=existing,
+ )
+
+
+def promote_procedural_memory_to_skill(
+ *,
+ memories: Sequence[Dict[str, Any]],
+ intent_cluster: Dict[str, Any],
+ occurrences: int,
+ success_rate: float,
+ plan_variety: int,
+ existing_skill_id: Optional[str] = None,
+) -> Optional[promotion_types.PromotionProposal]:
+ """Compile *how work is done here* into a skill proposal.
+
+ This is the chain the design described and the code never had. Four
+ conditions must hold together, and each excludes a specific way of being
+ wrong:
+
+ * the memories are **procedural** — a fact frozen into a document stops
+ being current, and a preference is already covered by one profile line;
+ * the intent recurs across **separate occasions** — one conversation
+ mentioning something three times is one occasion;
+ * the recurrences **succeed** — distilling a failing approach makes it the
+ house style;
+ * the plans **varied** — the same request handled identically every time is
+ already effectively a skill, and writing one down adds an asset to
+ maintain while changing nothing.
+
+ The proposal carries the memories' **content**, which is what makes this a
+ compilation rather than a rename, and their refs, which is what lets the
+ source memories be demoted when the skill goes live and restored when it is
+ rolled back.
+ """
+ procedural = [
+ m
+ for m in memories
+ if str(m.get("memory_type") or "") == "procedural" and str(m.get("content") or "").strip()
+ ]
+ if not procedural:
+ return None
+ if occurrences < promotion_constants.PROCEDURAL_MIN_OCCURRENCES:
+ return None
+ if success_rate < promotion_constants.PROCEDURAL_MIN_SUCCESS_RATE:
+ return None
+ if plan_variety < 2:
+ return None
+
+ representative = str(intent_cluster.get("representative") or "")
+ return promotion_types.PromotionProposal(
+ from_layer="memory",
+ to_layer="skill",
+ signature="proc:" + "|".join(sorted(str(m.get("ref") or "") for m in procedural)),
+ support=occurrences,
+ rationale=(
+ f"「{representative[:40]}」类请求出现 {occurrences} 次、成功率 "
+ f"{success_rate:.0%},却用了 {plan_variety} 种不同做法;"
+ f"其中 {len(procedural)} 条做法/口径已被反复检索,应编译为一次性装配的技能"
+ ),
+ source_refs=[str(m.get("ref") or "") for m in procedural if m.get("ref")],
+ payload={
+ "procedures": [
+ {
+ "rule": str(m.get("content") or ""),
+ "why": str(m.get("why") or ""),
+ "applies_to": str(m.get("applies_to") or ""),
+ }
+ for m in procedural
+ ],
+ "episode_ids": [str(e) for e in (intent_cluster.get("episode_ids") or [])][:20],
+ "intent": representative,
+ "memory_refs": [str(m.get("ref") or "") for m in procedural if m.get("ref")],
+ },
+ patch_target=existing_skill_id,
+ )
+
+
+def promote_skills_to_orchestration(
+ pattern: patterns.Pattern,
+ *,
+ combo_success_rate: float,
+ individual_success_rate: float,
+ task_types: Optional[Sequence[str]] = None,
+ skill_versions: Optional[Dict[str, str]] = None,
+) -> Optional[promotion_types.PromotionProposal]:
+ """Promote a recurring skill combination into an **agent profile** proposal.
+
+ Promoting it into a *workflow* — a fixed DAG — was the wrong target: the
+ main axis is a ReAct loop, which never executes a DAG, so a workflow asset
+ could only ever govern the separate autonomous-loop product. What a
+ recurring skill combination actually says is "for this kind of task, these
+ capabilities belong in view together", and the thing that decides what is in
+ view is the profile.
+
+ The combination must *beat its parts*: frequency alone would promote every
+ common sequence regardless of whether grouping them helps. The proposal is
+ declarative — a skill set and the task types it applies to — and carries no
+ executable content.
+ """
+ if not pattern.skill_sequence or len(pattern.skill_sequence) < 2:
+ return None
+ if pattern.support < promotion_constants.WORKFLOW_MIN_COMBO_FREQUENCY:
+ return None
+ if combo_success_rate - individual_success_rate < promotion_constants.WORKFLOW_MIN_UPLIFT:
+ return None
+
+ versions = skill_versions or {}
+ return promotion_types.PromotionProposal(
+ from_layer="skill",
+ to_layer="agent_profile",
+ signature=pattern.signature,
+ support=pattern.support,
+ rationale=(
+ f"技能组合出现 {pattern.support} 次,组合成功率 {combo_success_rate:.0%} "
+ f"显著高于零散调用 {individual_success_rate:.0%}——"
+ "应固化为该类任务的技能组合,而不是每次重新挑选"
+ ),
+ source_refs=list(pattern.skill_sequence),
+ payload={
+ "skill_ids": list(dict.fromkeys(pattern.skill_sequence)),
+ "task_types": [str(t) for t in (task_types or [])],
+ # Declared so the candidate auto-invalidates if a dependency retires
+ # — a profile pointing at a dead skill version is worse than none.
+ "depends_on": {s: versions.get(s, "") for s in pattern.skill_sequence},
+ "kind": "declarative_policy",
+ },
+ )
+
+
+def dependencies_satisfied(
+ proposal: promotion_types.PromotionProposal, live_skill_versions: Dict[str, str]
+) -> Tuple[bool, List[str]]:
+ """Whether every declared dependency still exists at the declared version.
+
+ This is the mechanism that makes cross-layer rollback work in one direction:
+ retire a skill and every profile built on it stops validating, instead of
+ quietly pointing at something that is no longer there.
+ """
+ declared = (proposal.payload or {}).get("depends_on") or {}
+ stale = [
+ skill_id
+ for skill_id, version in declared.items()
+ if version and live_skill_versions.get(skill_id) != version
+ ]
+ return (not stale), stale
+
+
+def find_overlaps(
+ signatures: Dict[str, List[str]], *, min_shared: int = 2
+) -> List[Tuple[str, str, int]]:
+ """Assets whose **declared** step sequences overlap enough to be merge candidates.
+
+ ``signatures`` must be each asset's own declared tools — the tools its
+ document names. Feeding it the tools observed in episodes where the asset
+ was merely loaded produces a merge proposal for every pair of skills that
+ happened to be enabled together, which is guaranteed rather than
+ informative: co-enabled skills share the episode's whole tool set by
+ construction.
+ """
+ overlaps: List[Tuple[str, str, int]] = []
+ asset_ids = sorted(signatures)
+ for i, a in enumerate(asset_ids):
+ for b in asset_ids[i + 1 :]:
+ shared = len(set(signatures[a]) & set(signatures[b]))
+ if shared >= min_shared:
+ overlaps.append((a, b, shared))
+ return overlaps
diff --git a/src/backend/core/evolution/cycle/promotion_constants.py b/src/backend/core/evolution/cycle/promotion_constants.py
new file mode 100644
index 00000000..992f0aa1
--- /dev/null
+++ b/src/backend/core/evolution/cycle/promotion_constants.py
@@ -0,0 +1,80 @@
+"""Pattern kinds and support thresholds for promotion."""
+from __future__ import annotations
+
+
+PATTERN_REPEATED_FAILURE = "repeated_failure"
+
+
+PATTERN_SUCCESS_SUBSEQUENCE = "success_subsequence"
+
+
+PATTERN_DRIFT = "execution_drift"
+
+
+MIN_SUPPORT = 5
+
+
+SINGLE_SHOT_CONFIDENCE = 0.85
+
+
+SKILL_MIN_SUCCESS_RATE = 0.7
+
+
+WORKFLOW_MIN_COMBO_FREQUENCY = 5
+
+
+WORKFLOW_MIN_UPLIFT = 0.05
+
+
+MIN_TOOL_SEQUENCE_LEN = 3
+
+
+INTENT_COHESION_RATIO = 0.6
+
+
+PROCEDURAL_MIN_OCCURRENCES = 3
+
+
+PROCEDURAL_MIN_SUCCESS_RATE = 0.6
+
+
+MIN_EXPOSURE_DAYS = 14
+
+
+MIN_OFFERS_BEFORE_JUDGING = 10
+
+
+MIN_OPENS_BEFORE_JUDGING = 3
+
+
+FOLLOW_THROUGH_FLOOR = 0.4
+
+
+SUCCESS_FLOOR = 0.4
+
+
+REASON_NEVER_OPENED = "never_opened"
+
+
+REASON_IGNORED_WHEN_OPENED = "ignored_when_opened"
+
+
+REASON_LOW_SUCCESS = "low_success_when_opened"
+
+
+REASON_HARMFUL = "worse_than_without"
+
+
+ACTION_LOWER_EXPOSURE = "lower_exposure"
+
+
+ACTION_REWRITE = "rewrite"
+
+
+ACTION_RETIRE = "retire"
+
+
+ACTION_ROLLBACK = "rollback"
+
+
+MIN_CONSTRAINT_CONTEXTS = 2
diff --git a/src/backend/core/evolution/cycle/promotion_types.py b/src/backend/core/evolution/cycle/promotion_types.py
new file mode 100644
index 00000000..9bde9881
--- /dev/null
+++ b/src/backend/core/evolution/cycle/promotion_types.py
@@ -0,0 +1,94 @@
+"""Promotion proposals, negative-transfer records and ordering constraints."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+@dataclass
+class PromotionProposal:
+ """A promotion the chain wants to make, before it becomes an IR candidate."""
+
+ from_layer: str
+ to_layer: str
+ signature: str
+ support: int
+ rationale: str
+ source_refs: List[str] = field(default_factory=list)
+ payload: Dict[str, Any] = field(default_factory=dict)
+ # Set when an equivalent asset already exists: promotion becomes a patch
+ # rather than a new asset, so the library does not fill with near-duplicates.
+ patch_target: Optional[str] = None
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "from_layer": self.from_layer,
+ "to_layer": self.to_layer,
+ "signature": self.signature,
+ "support": self.support,
+ "rationale": self.rationale,
+ "source_refs": list(self.source_refs),
+ "payload": self.payload,
+ "patch_target": self.patch_target,
+ }
+
+
+@dataclass
+class NegativeTransfer:
+ asset_id: str
+ task_type: str
+ before: float
+ after: float
+
+ @property
+ def delta(self) -> float:
+ return self.after - self.before
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "asset_id": self.asset_id,
+ "task_type": self.task_type,
+ "before": round(self.before, 4),
+ "after": round(self.after, 4),
+ "delta": round(self.delta, 4),
+ }
+
+
+@dataclass
+class OrderingConstraint:
+ """A pairwise ordering that held across the supporting evidence.
+
+ Carries the task types it was *validated in* and any it was *contradicted
+ in*. That distinction is what lets a genuinely context-dependent ordering be
+ used where it holds instead of being discarded everywhere.
+ """
+
+ before: str
+ after: str
+ support: int
+ contexts: int # distinct tool-set shapes it was observed in
+ validated_in: Tuple[str, ...] = ()
+ contradicted_in: Tuple[str, ...] = ()
+
+ def applies_to(self, tools: Sequence[str], task_type: Optional[str] = None) -> bool:
+ if self.before not in tools or self.after not in tools:
+ return False
+ if task_type is None:
+ # No context given: only a rule contradicted nowhere may be used.
+ return not self.contradicted_in
+ if task_type in self.contradicted_in:
+ return False
+ if task_type in self.validated_in:
+ return True
+ # An unseen context. Extending there is only safe for a rule that has
+ # never been contradicted; one known to flip somewhere is precisely the
+ # kind that should not be guessed at.
+ return not self.contradicted_in
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "before": self.before,
+ "after": self.after,
+ "support": self.support,
+ "contexts": self.contexts,
+ "validated_in": list(self.validated_in),
+ "contradicted_in": list(self.contradicted_in),
+ }
diff --git a/src/backend/core/evolution/cycle/prompt_candidates.py b/src/backend/core/evolution/cycle/prompt_candidates.py
new file mode 100644
index 00000000..629dc714
--- /dev/null
+++ b/src/backend/core/evolution/cycle/prompt_candidates.py
@@ -0,0 +1,95 @@
+"""Prompt-fragment candidates for failures attributed to the prompt."""
+from __future__ import annotations
+import hashlib
+from typing import Any, Dict, List
+from core.evolution.evidence.credit import CreditFeatures, assign_credit
+from core.evolution.ir import RISK_HIGH, EvolutionIR, InvariantViolation, Rollback, validate_ir
+import logging
+from core.evolution.cycle import candidate_records
+
+logger = logging.getLogger(__name__)
+
+def prompt_candidates(views: List[Dict[str, Any]], *, dry_run: bool) -> List[Dict[str, Any]]:
+ """Prompt fragments, for the failures attribution already blames on the prompt."""
+ from core.evolution.generators.prompt_gen import generate_prompt_candidate
+
+ payload, evidence, draft = candidate_records.run_async(
+ generate_prompt_candidate(
+ views,
+ cause="归因判定该失败源于提示词未声明相关约束",
+ existing_fragments=active_prompt_fragments(),
+ )
+ )
+ if payload is None or evidence is None:
+ return (
+ []
+ if draft.violations == ["insufficient_evidence"]
+ else [{"kind": "prompt", "written": False, "why_not": draft.to_dict()}]
+ )
+
+ decision = assign_credit(
+ CreditFeatures(
+ outcome_verdict="failed",
+ outcome_confidence=0.85,
+ prompt_missing_priority_rule=True,
+ )
+ )
+ if not decision.may_produce_candidate:
+ return []
+
+ asset_id = "auto-prompt-" + hashlib.sha256(payload["fragment"].encode("utf-8")).hexdigest()[:12]
+ ir = EvolutionIR(
+ target_kind="prompt",
+ target_asset_id=asset_id,
+ base_version="active",
+ operation="patch",
+ hypothesis=payload["cause"],
+ changes=[{"fragment": payload["fragment"], "task_types": payload["task_types"]}],
+ evidence_refs=evidence.episode_ids[:20],
+ # A prompt fragment applies to every request in scope; nothing this
+ # system produces has a wider reach.
+ risk_tier=RISK_HIGH,
+ scope={"level": "workspace"},
+ rollback=Rollback(version_id="active", triggers=["success -3pp"]),
+ evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
+ )
+ try:
+ validate_ir(ir, credit_decision=decision.to_dict())
+ except InvariantViolation as violation:
+ return [{"kind": "prompt", "written": False, "why_not": {"code": violation.code}}]
+ if dry_run:
+ return [{"dry_run": True, "kind": "prompt", "asset_id": asset_id}]
+ candidate_id = candidate_records.persist_candidate(
+ ir,
+ credit=decision.to_dict(),
+ proposer="prompt_engine",
+ evidence_refs=evidence.episode_ids[:20],
+ )
+ return (
+ [{"candidate_id": candidate_id, "kind": "prompt", "asset_id": asset_id}]
+ if candidate_id
+ else []
+ )
+
+
+def active_prompt_fragments() -> List[str]:
+ """Fragments already appended by earlier prompt candidates.
+
+ Passed to the generator so it can decline rather than restate: a prompt that
+ accumulates three phrasings of the same instruction is measurably worse than
+ one that states it once.
+ """
+ try:
+ from core.db.engine import SessionLocal as _Session
+ from core.db.models.evolution import EvolutionRelease as _Release
+
+ with _Session() as db:
+ rows = (
+ db.query(_Release)
+ .filter(_Release.target_kind == "prompt", _Release.stage == "active")
+ .all()
+ )
+ return [str(row.target_asset_id) for row in rows]
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[evolution-loop] active prompt fragments unavailable: %s", exc)
+ return []
diff --git a/src/backend/core/evolution/readiness.py b/src/backend/core/evolution/cycle/readiness.py
similarity index 93%
rename from src/backend/core/evolution/readiness.py
rename to src/backend/core/evolution/cycle/readiness.py
index 9685f4db..23daf06f 100644
--- a/src/backend/core/evolution/readiness.py
+++ b/src/backend/core/evolution/cycle/readiness.py
@@ -81,7 +81,7 @@ def _check_embedding() -> Requirement:
surface, so this makes one real call.
"""
try:
- from core.kb.kb_vector import embed_text
+ from core.models.embedding import embed_text
vector = embed_text("能力进化就绪检查", timeout=8.0)
except Exception as exc: # noqa: BLE001
@@ -162,13 +162,11 @@ def _check_generator() -> Requirement:
# run often — spends a request to re-learn something that was true 30 seconds
# ago. Short enough that fixing a misconfiguration shows up promptly.
_CACHE_TTL_S = 300.0
-_cached: Optional[Tuple[float, "Readiness"]] = None
def reset_cache() -> None:
"""Drop the cached probe, so a configuration fix is picked up immediately."""
- global _cached
- _cached = None
+ process_state().cached = None
def check_readiness(*, use_cache: bool = True) -> Readiness:
@@ -178,15 +176,26 @@ def check_readiness(*, use_cache: bool = True) -> Readiness:
are fixed by different people: an embedding endpoint is infrastructure, a
generation model is a model-configuration decision.
"""
- global _cached
now = time.monotonic()
- if use_cache and _cached is not None and now - _cached[0] < _CACHE_TTL_S:
- return _cached[1]
+ if use_cache and process_state().cached is not None and now - process_state().cached[0] < _CACHE_TTL_S:
+ return process_state().cached[1]
requirements = [_check_embedding(), _check_generator()]
readiness = Readiness(
ready=all(r.ok for r in requirements),
requirements=requirements,
)
- _cached = (now, readiness)
+ process_state().cached = (now, readiness)
return readiness
+
+
+@dataclass
+class ReadinessState:
+ cached: Optional[Tuple[float, 'Readiness']] = None
+
+
+_STATE = ReadinessState()
+
+
+def process_state() -> ReadinessState:
+ return _STATE
diff --git a/src/backend/core/evolution/release.py b/src/backend/core/evolution/cycle/release.py
similarity index 100%
rename from src/backend/core/evolution/release.py
rename to src/backend/core/evolution/cycle/release.py
diff --git a/src/backend/core/evolution/cycle/retirement.py b/src/backend/core/evolution/cycle/retirement.py
new file mode 100644
index 00000000..724d042a
--- /dev/null
+++ b/src/backend/core/evolution/cycle/retirement.py
@@ -0,0 +1,158 @@
+"""Find assets to retire and activations that caused negative transfer."""
+from __future__ import annotations
+from typing import Any, Dict, List, Sequence
+from core.evolution.cycle import promotion_constants
+from core.evolution.cycle import promotion_types
+
+def find_retirement_candidates(
+ usage: Dict[str, Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+ """Assets that are not earning their place, and what to do about each.
+
+ ``usage`` per asset:
+
+ ``offers``
+ turns where the skill was selected into the prompt (non-degraded
+ selections only).
+ ``opens``
+ turns where the model actually opened its document. This is the usage
+ signal; ``offers`` alone cannot distinguish "never chosen" from "chosen
+ and useless".
+ ``follow_through``
+ of the opens, the fraction where the subsequent tool calls stayed
+ inside what the skill declared.
+ ``success_rate``
+ of the opens, the fraction of turns that succeeded.
+ ``uplift``
+ success with the skill exposed minus success without it, where a paired
+ comparison exists. ``None`` when it does not.
+ ``exposure_days``
+ how long it has been exposed at all.
+
+ A library that only grows fills with near-duplicates and degrades selection
+ precision for everything in it — but every judgement here is a *proposal*,
+ because removing a capability produces no error, only a slow decline.
+ """
+ out: List[Dict[str, Any]] = []
+ for asset_id, stats in (usage or {}).items():
+ offers = int(stats.get("offers", 0) or 0)
+ opens = int(stats.get("opens", 0) or 0)
+ exposure_days = float(stats.get("exposure_days", 0) or 0)
+ uplift = stats.get("uplift")
+
+ if uplift is not None and float(uplift) <= -0.05:
+ out.append(
+ {
+ "asset_id": asset_id,
+ "reason": promotion_constants.REASON_HARMFUL,
+ "action": promotion_constants.ACTION_ROLLBACK,
+ "explanation": (
+ f"成对比较显示有它反而更差({float(uplift):+.0%})——"
+ "这是唯一需要立即停止曝光的情况"
+ ),
+ **stats,
+ }
+ )
+ continue
+
+ if exposure_days < promotion_constants.MIN_EXPOSURE_DAYS:
+ # Too new to judge. Saying nothing is the honest output; the old
+ # rule's answer here was "retire it", which punished every skill for
+ # having just shipped.
+ continue
+
+ if opens == 0:
+ if offers < promotion_constants.MIN_OFFERS_BEFORE_JUDGING:
+ continue
+ out.append(
+ {
+ "asset_id": asset_id,
+ "reason": promotion_constants.REASON_NEVER_OPENED,
+ "action": promotion_constants.ACTION_LOWER_EXPOSURE,
+ "explanation": (
+ f"已被挂载 {offers} 次、"
+ # A hand-authored skill has no release record and so no
+ # finite exposure window — it has been there from the
+ # start. Formatting that as a number printed "曝光 inf 天".
+ + (
+ "一直可用"
+ if exposure_days == float("inf")
+ else f"曝光 {exposure_days:.0f} 天"
+ )
+ + ",模型一次都没打开——"
+ "多半是描述没写清它解决什么问题,而不是能力本身没用"
+ ),
+ **stats,
+ }
+ )
+ continue
+
+ if opens < promotion_constants.MIN_OPENS_BEFORE_JUDGING:
+ continue
+
+ follow_through = stats.get("follow_through")
+ if follow_through is not None and float(follow_through) < promotion_constants.FOLLOW_THROUGH_FLOOR:
+ out.append(
+ {
+ "asset_id": asset_id,
+ "reason": promotion_constants.REASON_IGNORED_WHEN_OPENED,
+ "action": promotion_constants.ACTION_REWRITE,
+ "explanation": (
+ f"被打开 {opens} 次,但只有 {float(follow_through):.0%} 的情况"
+ "后续动作真的照它说的做——文档本身需要重写,不是该退役"
+ ),
+ **stats,
+ }
+ )
+ continue
+
+ success_rate = float(stats.get("success_rate", 0.0) or 0.0)
+ if success_rate < promotion_constants.SUCCESS_FLOOR:
+ out.append(
+ {
+ "asset_id": asset_id,
+ "reason": promotion_constants.REASON_LOW_SUCCESS,
+ "action": promotion_constants.ACTION_RETIRE,
+ "explanation": (
+ f"被打开 {opens} 次、照做了,但成功率只有 {success_rate:.0%}"
+ ),
+ **stats,
+ }
+ )
+ return out
+
+
+def detect_negative_transfer(
+ *,
+ asset_id: str,
+ related_task_types: Sequence[str],
+ before: Dict[str, float],
+ after: Dict[str, float],
+ threshold: float = 0.05,
+) -> List[promotion_types.NegativeTransfer]:
+ """Find task types that got worse after an activation *and are unrelated to it*.
+
+ Only unrelated tasks are examined. A dip on the task the change targeted is
+ a straightforward regression its own guardrail catches; the dangerous case
+ is collateral damage elsewhere, which no single release's guardrail is
+ watching and which accumulates silently across many individually-harmless
+ changes.
+ """
+ related = set(related_task_types)
+ findings: List[promotion_types.NegativeTransfer] = []
+ for task_type, after_value in (after or {}).items():
+ if task_type in related:
+ continue
+ before_value = (before or {}).get(task_type)
+ if before_value is None:
+ continue
+ if before_value - after_value >= threshold:
+ findings.append(
+ promotion_types.NegativeTransfer(
+ asset_id=asset_id,
+ task_type=task_type,
+ before=before_value,
+ after=after_value,
+ )
+ )
+ return findings
diff --git a/src/backend/core/evolution/runtime_binding.py b/src/backend/core/evolution/cycle/runtime_binding.py
similarity index 87%
rename from src/backend/core/evolution/runtime_binding.py
rename to src/backend/core/evolution/cycle/runtime_binding.py
index 97a953da..dcc87496 100644
--- a/src/backend/core/evolution/runtime_binding.py
+++ b/src/backend/core/evolution/cycle/runtime_binding.py
@@ -19,6 +19,7 @@
import logging
import threading
+from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
@@ -40,32 +41,29 @@
# Bundles are small and short-lived; a bounded in-process cache keeps the hot
# path free of database reads. The Episode assembler persists what it needs.
_MAX_CACHED_BUNDLES = 512
-_bundles: Dict[str, AssetBundle] = {}
-_run_to_bundle: Dict[tuple[str, str], str] = {}
-_lock = threading.Lock()
def _remember(bundle: AssetBundle, run_id: Optional[str], capability_scope: str = "") -> None:
- with _lock:
- _bundles[bundle.bundle_id] = bundle
+ with process_state().lock:
+ process_state().bundles[bundle.bundle_id] = bundle
if run_id:
- _run_to_bundle[(str(run_id), str(capability_scope))] = bundle.bundle_id
- if len(_bundles) > _MAX_CACHED_BUNDLES:
+ process_state().run_to_bundle[(str(run_id), str(capability_scope))] = bundle.bundle_id
+ if len(process_state().bundles) > _MAX_CACHED_BUNDLES:
# Cheap eviction: drop the oldest insertion. Losing a cached bundle
# is harmless — it is re-resolvable from the persisted Episode.
- for stale_key in list(_bundles.keys())[: len(_bundles) - _MAX_CACHED_BUNDLES]:
- _bundles.pop(stale_key, None)
+ for stale_key in list(process_state().bundles.keys())[: len(process_state().bundles) - _MAX_CACHED_BUNDLES]:
+ process_state().bundles.pop(stale_key, None)
def get_bundle(bundle_id: str) -> Optional[AssetBundle]:
- with _lock:
- return _bundles.get(bundle_id)
+ with process_state().lock:
+ return process_state().bundles.get(bundle_id)
def resolve_bundle_for_run(run_id: str, *, capability_scope: str = "") -> Optional[AssetBundle]:
- with _lock:
- bundle_id = _run_to_bundle.get((str(run_id), str(capability_scope)))
- return _bundles.get(bundle_id) if bundle_id else None
+ with process_state().lock:
+ bundle_id = process_state().run_to_bundle.get((str(run_id), str(capability_scope)))
+ return process_state().bundles.get(bundle_id) if bundle_id else None
def clear_run_binding(run_id: Optional[str], *, capability_scope: str = "") -> None:
@@ -77,8 +75,8 @@ def clear_run_binding(run_id: Optional[str], *, capability_scope: str = "") -> N
"""
if not run_id:
return
- with _lock:
- _run_to_bundle.pop((str(run_id), str(capability_scope)), None)
+ with process_state().lock:
+ process_state().run_to_bundle.pop((str(run_id), str(capability_scope)), None)
def _safe(resolver, label: str, degraded: List[str]) -> List[AssetRef]:
@@ -92,24 +90,17 @@ def _safe(resolver, label: str, degraded: List[str]) -> List[AssetRef]:
def _prompt_refs() -> List[AssetRef]:
- from core.services import prompt_version_service
+ from core.harness.registry.model_services import active_prompt_versions
- refs: List[AssetRef] = []
- for kind in list(prompt_version_service.KIND_SPECS):
- try:
- active = prompt_version_service.get_active_version(kind)
- except Exception:
- continue
- if active:
- refs.append(
- AssetRef(
- kind=ASSET_PROMPT,
- asset_id=kind,
- version=str(active.get("id") or "default"),
- detail={"label": active.get("label") or ""},
- )
- )
- return refs
+ return [
+ AssetRef(
+ kind=ASSET_PROMPT,
+ asset_id=kind,
+ version=str(active.get("id") or "default"),
+ detail={"label": active.get("label") or ""},
+ )
+ for kind, active in active_prompt_versions()
+ ]
def _prompt_refs_from_manifest(manifest: Dict[str, Any]) -> List[AssetRef]:
@@ -150,7 +141,7 @@ def _skill_refs(
) -> List[AssetRef]:
if not skill_ids:
return []
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
metadata: Dict[str, Any] = {}
try:
@@ -163,10 +154,10 @@ def _skill_refs(
# the same view and a later re-resolution cannot silently swap versions.
bindings: Dict[str, Dict[str, Any]] = {}
try:
- from core.capabilities.paths import capabilities_enabled
- from core.capabilities.runtime import get as get_prepared_run
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from core.harness.registry.capability_runtime import get as get_prepared_run
- if capabilities_enabled() and run_id:
+ if has_run_capabilities() and run_id:
prepared = get_prepared_run(run_id, scope_id=capability_scope)
if prepared is not None:
bindings = {
@@ -405,6 +396,20 @@ def rebind_execution_manifest(
def reset_for_tests() -> None:
- with _lock:
- _bundles.clear()
- _run_to_bundle.clear()
+ with process_state().lock:
+ process_state().bundles.clear()
+ process_state().run_to_bundle.clear()
+
+
+@dataclass
+class RuntimeBindingState:
+ bundles: Dict[str, AssetBundle] = field(default_factory=lambda: {})
+ run_to_bundle: Dict[tuple[str, str], str] = field(default_factory=lambda: {})
+ lock: object = field(default_factory=lambda: threading.Lock())
+
+
+_STATE = RuntimeBindingState()
+
+
+def process_state() -> RuntimeBindingState:
+ return _STATE
diff --git a/src/backend/core/evolution/cycle/skill_document.py b/src/backend/core/evolution/cycle/skill_document.py
new file mode 100644
index 00000000..c2ed5142
--- /dev/null
+++ b/src/backend/core/evolution/cycle/skill_document.py
@@ -0,0 +1,298 @@
+"""Render distilled skills as SKILL.md documents."""
+from __future__ import annotations
+from typing import Any, Dict, List, Sequence
+import logging
+logger = logging.getLogger(__name__)
+
+def skill_label(tools: Sequence[str], *, rules_only: bool = False) -> str:
+ """A human- and model-readable skill name.
+
+ The name and description are the *only* things the model sees before it
+ decides whether to open a skill, so they have to describe the job. The
+ candidate's ``hypothesis`` describes the evidence ("「…」类请求在 N 次独立
+ 会话中以同一序列成功完成") — true, and useless for that decision, since it
+ says nothing about when the skill applies. It belongs in the provenance
+ section, not the title.
+ """
+ head = " → ".join(tools[:4]) + ("…" if len(tools) > 4 else "")
+ return ("工具顺序规则:" if rules_only else "固定调用顺序:") + head
+
+
+def skill_description(
+ tools: Sequence[str], constraints: Sequence[Dict[str, Any]], *, rules_only: bool
+) -> str:
+ """One line, single-line by necessity — the frontmatter parser is line-based."""
+ joined = "、".join(f"`{t}`" for t in tools[:6])
+ if rules_only:
+ return (
+ f"当任务用到 {joined} 时,按已验证的先后顺序约束安排调用次序"
+ f"({len(constraints)} 条约束,来自历史成功执行)。"
+ )
+ return (
+ f"当任务需要 {joined} 且执行顺序会影响结果时使用;"
+ f"按 {' → '.join(tools)} 的顺序调用。"
+ )
+
+
+def rules_consistent_with(
+ constraints: Sequence[Dict[str, Any]], tools: Sequence[str]
+) -> List[Dict[str, Any]]:
+ """Drop rules whose direction the skill's own step list violates.
+
+ The loop attaches every rule touching a skill's tools, and scoping makes that
+ formally sound — a rule valid in ``finance`` and a sequence distilled from
+ ``trap`` genuinely can point opposite ways. But the rendered document then
+ tells the model to call A before B in its steps and B before A two sections
+ later, with nothing saying which wins. Observed on a real materialisation.
+
+ For this skill's own tool set the step list is the governing statement, so a
+ rule contradicting it is describing some other context and does not belong in
+ this document. It remains attached to the skills whose sequences agree with
+ it, and in ``ordering_constraints.json`` for any consumer that reasons about
+ scope itself.
+ """
+ if not tools:
+ return list(constraints)
+ position = {tool: index for index, tool in enumerate(tools)}
+ kept: List[Dict[str, Any]] = []
+ for rule in constraints:
+ before, after = rule.get("before"), rule.get("after")
+ if before in position and after in position and position[before] > position[after]:
+ logger.info(
+ "[activation] rule %s<%s omitted: contradicts this skill's own order",
+ before,
+ after,
+ )
+ continue
+ kept.append(rule)
+ return kept
+
+
+def ordering_rules_section(constraints: Sequence[Dict[str, Any]]) -> str:
+ """Render the learned ordering constraints into the body the model reads.
+
+ Without this the transferable half of what was learned never reaches the
+ runtime. ``ordering_constraints.json`` is written alongside, but nothing in
+ the agent path reads extra files as policy — the model reads SKILL.md, so a
+ rule that is not written here is a rule the model never receives.
+
+ Each rule carries its own scope. Stating where it was contradicted matters
+ as much as stating where it held: a rule applied in the context that
+ disproved it actively breaks that context.
+ """
+ shippable = [r for r in constraints if not (r.get("contradicted_in") or [])]
+ withheld = [r for r in constraints if (r.get("contradicted_in") or [])]
+
+ lines: List[str] = []
+ if shippable:
+ lines += ["", "## 顺序约束(可迁移)", ""]
+ lines.append("下列约束在历史成功执行中无反例,适用于本技能覆盖的工具:")
+ lines.append("")
+ for rule in shippable:
+ before, after = rule.get("before"), rule.get("after")
+ validated = [str(x) for x in (rule.get("validated_in") or [])]
+ line = f"- 先 `{before}` 再 `{after}`"
+ if validated:
+ line += f"——已在 {'、'.join(validated)} 类任务中验证"
+ line += f"(证据 {rule.get('support', 0)} 条)"
+ lines.append(line)
+
+ if withheld:
+ # Scoped rules are deliberately kept out of the model's view.
+ #
+ # Measured twice against a real model, and both framings failed:
+ #
+ # * exception as a trailing qualifier ("在 X 类任务中不适用") — the model
+ # applied the rule in X anyway, 5 of 5 times;
+ # * exception first and imperative ("若属于 X 则整条作废,那里顺序相反")
+ # — the model then applied the *reversed* order everywhere, dropping the
+ # family the rule was valid in from 50% to 17% and disturbing an
+ # unrelated control family.
+ #
+ # So a text annotation cannot make a model route on task type: it either
+ # ignores the exception or over-applies it. Handing it a rule that is
+ # wrong somewhere makes the system worse where the rule is wrong, which
+ # is a poor trade for a benefit available only where it is right.
+ #
+ # The rule is not discarded — it stays in ``ordering_constraints.json``
+ # for a runtime applier that knows the task type and can decide before
+ # injection, which is where a scope decision belongs. Until that exists,
+ # withholding is the conservative and measurably better option.
+ lines += ["", "## 已扣留的上下文相关约束", ""]
+ lines.append(
+ f"另有 {len(withheld)} 条约束在部分任务类型中被证据推翻,"
+ "因此**不在本技能中给出**——它们只在特定任务类型下成立,"
+ "由运行时按任务类型判定后注入,不交由模型自行取舍。"
+ )
+ for rule in withheld:
+ lines.append(
+ f"- (已扣留)`{rule.get('before')}` / `{rule.get('after')}`:"
+ f"在 {'、'.join(str(x) for x in (rule.get('validated_in') or []))} 成立,"
+ f"在 {'、'.join(str(x) for x in (rule.get('contradicted_in') or []))} 被推翻"
+ )
+ return "\n".join(lines)
+
+
+def skill_markdown(
+ *,
+ skill_id: str,
+ label: str,
+ description: str,
+ tools: Sequence[str],
+ evidence: int,
+ constraints: Sequence[Dict[str, Any]] = (),
+ rationale: str = "",
+ rules_only: bool = False,
+) -> str:
+ """Render the SKILL.md body for a distilled skill.
+
+ ``name`` is the skill **id** — the loader treats that field as the id and
+ rejects anything outside ``[a-z0-9_-]``. Writing the human label there made
+ every materialised skill unparseable: it still showed up in the capability
+ list (that metadata comes from table columns) while ``load_skill_full``
+ returned ``None``, so the skill was present and inert.
+
+ States its provenance in the document itself, so anyone reading the skill —
+ not just anyone reading the console — knows where it came from and on how
+ much evidence it rests.
+ """
+ # Applicability is declared for the **whole skill**, not per rule. Measured:
+ # withholding a contradicted rule from the text was not enough, because the
+ # step list generalises on its own — the model applied this skill's order to
+ # a family where the reverse is correct, 5 of 5 times, with the rule absent.
+ # So the exclusion has to bind the steps too, which means stating it once, at
+ # the top, before anything the model could copy.
+ excluded = sorted(
+ {
+ str(name)
+ for rule in constraints
+ for name in (rule.get("contradicted_in") or [])
+ if str(name)
+ }
+ )
+ covered = sorted(
+ {
+ str(name)
+ for rule in constraints
+ for name in (rule.get("validated_in") or [])
+ if str(name) and str(name) not in excluded
+ }
+ )
+ scope_block = ""
+ if excluded:
+ scope_block = (
+ "## 不适用范围(先读这一段)\n\n"
+ f"**本技能不适用于以下任务类型:{'、'.join(excluded)}。**"
+ "在这些场景中,本技能给出的顺序已被历史证据推翻——"
+ "遇到它们请完全忽略本技能,按任务本身的依赖关系自行规划。\n\n"
+ + (
+ f"本技能仅适用于:{'、'.join(covered)}。\n\n"
+ if covered
+ else ""
+ )
+ )
+
+ if rules_only:
+ body_head = (
+ scope_block
+ + "## 适用条件\n\n"
+ "当任务用到下列工具、且它们的先后顺序会影响结果时,按「顺序约束」小节排序。\n\n"
+ "## 涉及工具\n\n"
+ + "\n".join(f"- `{tool}`" for tool in tools)
+ )
+ else:
+ body_head = (
+ scope_block
+ + "## 适用条件\n\n"
+ "当任务需要按固定顺序完成下列工具调用时使用本技能。\n\n"
+ "## 步骤\n\n"
+ + "\n".join(f"{i}. `{tool}`" for i, tool in enumerate(tools, start=1))
+ )
+
+ # A rules-only skill has no governing step list, so nothing there can be
+ # contradicted; for a sequence skill the steps win and conflicting rules are
+ # about other contexts.
+ shown_rules = constraints if rules_only else rules_consistent_with(constraints, tools)
+
+ provenance = (
+ f"本技能由能力进化流水线从 {evidence} 条历史执行证据中沉淀而成,非人工撰写。"
+ )
+ if rationale:
+ provenance += f"\n\n沉淀依据:{rationale}"
+
+ return f"""---
+name: {skill_id}
+display_name: {label}
+description: {description}
+---
+
+# {label}
+
+{description}
+
+{body_head}
+{ordering_rules_section(shown_rules)}
+
+## 来源
+
+{provenance}
+"""
+
+
+def body_of(rendered: str) -> str:
+ """The body of a rendered SKILL.md, without frontmatter or provenance.
+
+ Re-rendering from the body rather than storing the generator's output
+ verbatim is what lets the id change between drafting and activation (an
+ evolution prefix, an owner suffix) without the document's ``name`` field
+ disagreeing with the row it is stored under — a mismatch that makes the
+ skill unloadable while still listing it as available.
+ """
+ text = rendered or ""
+ if text.startswith("---"):
+ end = text.find("\n---", 3)
+ if end != -1:
+ text = text[end + 4 :]
+ # Drop the rendered title/description pair and the provenance section; both
+ # are regenerated, and keeping them would double on every re-render.
+ marker = text.find("## ")
+ if marker != -1:
+ text = text[marker:]
+ source = text.find("\n## 来源")
+ if source != -1:
+ text = text[:source]
+ return text.strip()
+
+
+def tools_from_constraints(constraints: Sequence[Dict[str, Any]]) -> List[str]:
+ """The tools a set of ordering rules talks about, in an order they permit.
+
+ Not a claimed execution plan — a rules-only skill has no fixed sequence. It
+ is the tool list the skill declares, listed in an order that does not
+ contradict its own rules so the document does not read as self-refuting.
+ """
+ tools: List[str] = []
+ for rule in constraints:
+ for key in ("before", "after"):
+ name = str(rule.get(key) or "")
+ if name and name not in tools:
+ tools.append(name)
+ if not tools:
+ return []
+ from core.evolution.cycle.promotion_types import OrderingConstraint
+ from core.evolution.cycle.ordering import apply_ordering_constraints
+
+ parsed = [
+ OrderingConstraint(
+ before=str(rule.get("before") or ""),
+ after=str(rule.get("after") or ""),
+ support=int(rule.get("support") or 0),
+ contexts=int(rule.get("contexts") or 0),
+ validated_in=tuple(rule.get("validated_in") or ()),
+ # Scope is deliberately dropped for this one purpose: we are only
+ # ordering the *document's* tool list, not deciding applicability.
+ )
+ for rule in constraints
+ if rule.get("before") and rule.get("after")
+ ]
+ return apply_ordering_constraints(tools, parsed)
diff --git a/src/backend/core/evolution/cycle/skill_release.py b/src/backend/core/evolution/cycle/skill_release.py
new file mode 100644
index 00000000..1eba4793
--- /dev/null
+++ b/src/backend/core/evolution/cycle/skill_release.py
@@ -0,0 +1,355 @@
+"""Ramp, roll back and retire skill releases."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+from core.db.engine import SessionLocal
+import logging
+from core.evolution.cycle import activation_policy
+from core.evolution.cycle import activation_types
+
+logger = logging.getLogger(__name__)
+
+def advance_skill_release(
+ release_id: str,
+ *,
+ actor: str,
+ observed_metrics: Optional[Dict[str, float]] = None,
+ guardrail_thresholds: Optional[Dict[str, float]] = None,
+) -> Dict[str, Any]:
+ """Ramp a release one step: shadow → canary 5 → 25 → 100 → active.
+
+ One step per call, deliberately. A single "go live" button that walked the
+ whole ladder would reproduce the behaviour this replaces — the point of the
+ ladder is that somebody looks at the metrics between rungs.
+
+ Guardrails are checked when metrics are supplied, and a breach rolls back
+ rather than pausing: the recorded threshold already *is* the level at which
+ the release is unacceptable.
+ """
+ from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
+ from core.evolution.cycle.release import ACTIVE, CANARY, CANARY_STEPS, SHADOW, can_promote, evaluate_guardrails, next_traffic, should_auto_rollback
+
+ if not actor:
+ raise activation_types.ActivationError("actor_required", "推进发布必须署名")
+
+ with SessionLocal() as db:
+ release = db.get(EvolutionRelease, release_id)
+ if release is None:
+ raise activation_types.ActivationError("release_not_found", f"发布记录不存在: {release_id}")
+ if activation_policy.is_frozen(db):
+ raise activation_types.ActivationError("globally_frozen", "全局冻结开关已打开,禁止推进发布")
+
+ candidate = db.get(EvolutionCandidate, release.candidate_id)
+ risk = str(getattr(candidate, "risk_tier", "") or "low")
+ guardrails = dict(release.guardrails or {})
+ completed: List[str] = list(guardrails.get("completed_verification") or [])
+ proposer = str(guardrails.get("proposer") or getattr(candidate, "proposer", "") or "")
+
+ if observed_metrics:
+ thresholds = guardrail_thresholds or guardrails.get("thresholds") or {}
+ breaches = evaluate_guardrails(observed_metrics, thresholds, window="canary")
+ if should_auto_rollback(breaches):
+ detail = ", ".join(
+ f"{b.metric}={b.observed}>{b.threshold}" for b in breaches
+ )
+ db.commit()
+ rollback_skill_activation(
+ release_id,
+ actor=actor,
+ reason=f"guardrail breach: {detail}",
+ kind="guardrail",
+ )
+ return {
+ "release_id": release_id,
+ "stage": "rolled_back",
+ "guardrail_breaches": [b.to_dict() for b in breaches],
+ }
+
+ stage = str(release.stage or "")
+ traffic = int(release.traffic_percent or 0)
+
+ if stage == SHADOW:
+ target_stage, target_traffic = CANARY, CANARY_STEPS[0]
+ completed = sorted(set(completed) | {"shadow"})
+ elif stage == CANARY and traffic < CANARY_STEPS[-1]:
+ target_stage, target_traffic = CANARY, next_traffic(traffic)
+ elif stage == CANARY:
+ target_stage, target_traffic = ACTIVE, 100
+ completed = sorted(set(completed) | {"canary"})
+ else:
+ raise activation_types.ActivationError(
+ "not_rampable", f"当前阶段 {stage} 无法继续放量"
+ )
+
+ allowed, reason = can_promote(
+ current_stage=stage,
+ target_stage=target_stage,
+ risk_tier=risk,
+ # ``human_approval`` is this call: a named actor who is not the
+ # proposer. Recording it before the check keeps the requirement
+ # satisfied by the act itself rather than by a stored flag.
+ completed_verification=sorted(set(completed) | {"human_approval"})
+ if target_stage == ACTIVE
+ else completed,
+ proposer=proposer,
+ approver=actor,
+ )
+ # A same-stage traffic bump is not a transition; can_promote would reject
+ # canary→canary, which is correct for stages and wrong for ramp steps.
+ if not allowed and not (stage == target_stage == CANARY):
+ raise activation_types.ActivationError("promotion_refused", reason)
+
+ if target_stage == ACTIVE:
+ # 冲突的 L3 证据在 shadow/canary 里观测没问题,但升到全量必须先由
+ # 人裁决冲突本身——适用条件不可信的技能不配自动放大到所有人。
+ if activation_policy.has_graph_conflict(candidate):
+ raise activation_types.ActivationError(
+ "graph_conflict_requires_review",
+ "候选的 L3 图谱证据存在未裁决的冲突,禁止放量到 active",
+ )
+ activation_policy.check_pace(db, tenant_id=str((release.scope_filter or {}).get("tenant_id") or "default"))
+
+ release.stage = target_stage
+ release.traffic_percent = target_traffic
+ guardrails["completed_verification"] = completed
+ if target_stage == ACTIVE:
+ guardrails["approved_by"] = actor
+ release.guardrails = guardrails
+ release.approved_by = actor
+ if candidate is not None and target_stage == ACTIVE:
+ candidate.status = ACTIVE
+ db.commit()
+
+ logger.info(
+ "[activation] release %s → %s/%d%% by %s",
+ release_id,
+ target_stage,
+ target_traffic,
+ actor,
+ )
+ return {
+ "release_id": release_id,
+ "stage": target_stage,
+ "traffic_percent": target_traffic,
+ "completed_verification": completed,
+ "exposed_to": (
+ "all users in scope"
+ if target_stage == ACTIVE
+ else f"{target_traffic}% of users in scope"
+ ),
+ }
+
+
+def rollback_skill_activation(
+ release_id: str, *, actor: str, reason: str, kind: str = "manual"
+) -> Dict[str, Any]:
+ """Undo a materialisation, restoring the snapshot taken at activation time.
+
+ With no prior version the skill is disabled rather than deleted: history
+ referencing it must stay resolvable, and a disabled row is both reversible
+ and visible in a way a missing row is not.
+
+ ``kind`` separates "a guardrail caught it" from "a person caught it".
+ Collapsing the two would hide how well the evaluation stage is working,
+ which is the number that tells you whether the ramp is worth its cost.
+ """
+ from core.db.models import AdminSkill
+ from core.db.models.evolution import (
+ EvolutionAgentProfile,
+ EvolutionCandidate,
+ EvolutionRelease,
+ )
+ from core.evolution.cycle.release import ROLLBACK_GUARDRAIL, ROLLBACK_MANUAL, ROLLED_BACK
+
+ if not actor or not reason:
+ raise activation_types.ActivationError("audit_required", "回滚必须署名并填写理由")
+
+ with SessionLocal() as db:
+ release = db.get(EvolutionRelease, release_id)
+ if release is None:
+ raise activation_types.ActivationError("release_not_found", f"发布记录不存在: {release_id}")
+
+ previous = (release.guardrails or {}).get("previous")
+ skill_id = release.target_asset_id
+ row = (
+ db.query(AdminSkill)
+ .filter(AdminSkill.skill_id == skill_id)
+ .first()
+ )
+ invalidated: List[str] = []
+ if row is not None:
+ if previous:
+ row.skill_content = previous["skill_content"]
+ row.display_name = previous["display_name"]
+ row.description = previous["description"]
+ row.version = previous["version"]
+ row.tags = previous["tags"]
+ row.allowed_tools = previous["allowed_tools"]
+ row.extra_files = previous["extra_files"]
+ row.is_enabled = previous["is_enabled"]
+ else:
+ row.is_enabled = False
+ # No prior version means the skill just stopped being loadable,
+ # so a profile assembled on top of it is now promising a
+ # capability that fails silently — same cross-layer rule as
+ # retirement. With a restored prior version the skill stays
+ # loadable and the profiles stay valid.
+ for profile_row in (
+ db.query(EvolutionAgentProfile)
+ .filter(EvolutionAgentProfile.is_active.is_(True))
+ .all()
+ ):
+ payload = profile_row.payload or {}
+ if skill_id in (payload.get("skill_ids") or []):
+ profile_row.is_active = False
+ invalidated.append(profile_row.profile_id)
+
+ release.stage = ROLLED_BACK
+ release.rolled_back_at = datetime.now(timezone.utc)
+ release.rollback_reason = reason
+ release.rollback_kind = (
+ ROLLBACK_GUARDRAIL if kind == "guardrail" else ROLLBACK_MANUAL
+ )
+
+ candidate = db.get(EvolutionCandidate, release.candidate_id)
+ if candidate is not None:
+ candidate.status = ROLLED_BACK
+ candidate_id = release.candidate_id
+ db.commit()
+
+ # The reverse edge of the promotion chain. Materialisation down-weighted the
+ # memories this skill was compiled from; rolling the skill back must restore
+ # them in the same motion, or the knowledge stays half-priced with no
+ # carrier — retrieval skips it because a skill supposedly covers it, and the
+ # skill is gone. Same ledger the retire path already replays.
+ from core.evolution.generators.memory_apply import revert_memory_ops
+
+ restored = revert_memory_ops(candidate_id=candidate_id) if candidate_id else {}
+
+ # 血缘:这一版回滚到了哪一版。审计问「技能 v2 后来怎么了」时,答案在链上。
+ try:
+ from core.evolution.evidence import lineage
+
+ current_version = str((previous or {}).get("version") or "")
+ lineage.record_links(
+ [
+ (
+ "skill",
+ f"{skill_id}@rolled_back",
+ lineage.REL_ROLLED_BACK_TO,
+ "skill",
+ f"{skill_id}@{current_version or 'disabled'}",
+ )
+ ],
+ candidate_id=candidate_id,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[activation] rollback lineage skipped: %s", exc)
+
+ logger.info(
+ "[activation] rolled back %s (release=%s memories restored=%d profiles invalidated=%s)",
+ skill_id,
+ release_id,
+ len(restored.get("reverted", [])),
+ invalidated,
+ )
+ return {
+ "release_id": release_id,
+ "restored_previous": bool(previous),
+ "restored_memory_ops": restored.get("reverted", []),
+ "invalidated_profiles": invalidated,
+ "reason": reason,
+ }
+
+
+def retire_skill_candidate(candidate_id: str, *, approver: str) -> Dict[str, Any]:
+ """Take an approved retirement into effect.
+
+ Retirement is disabling plus a release row, never a delete. Three reasons it
+ is done this way: the evidence behind the decision stays inspectable, the
+ skill can be brought back in one operation if the decision was wrong, and
+ anything built on top of it — a profile naming it, a sub-agent requiring it
+ — can be found and invalidated rather than silently pointing at nothing.
+ """
+ from core.db.models.evolution import EvolutionCandidate, EvolutionRelease
+
+ from core.db.models import AdminSkill
+ from core.db.models.evolution import EvolutionAgentProfile
+ from core.evolution.cycle.release import RETIRED, new_release_id
+
+ if not approver:
+ raise activation_types.ActivationError("approver_required", "退役必须署名")
+
+ with SessionLocal() as db:
+ candidate = db.get(EvolutionCandidate, candidate_id)
+ if candidate is None:
+ raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}")
+ if candidate.operation != "deprecate":
+ raise activation_types.ActivationError("not_a_retirement", "该候选不是退役候选")
+ if approver == (candidate.proposer or ""):
+ raise activation_types.ActivationError("self_approval_forbidden", "生成者不能批准自己提出的退役")
+
+ skill_id = candidate.target_asset_id
+ row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
+ if row is None:
+ raise activation_types.ActivationError("skill_not_found", f"技能不存在: {skill_id}")
+
+ previous_enabled = bool(row.is_enabled)
+ row.is_enabled = False
+
+ # Cross-layer: anything assembled on top of this skill stops being
+ # applied. Leaving them active would leave a profile promising a
+ # capability that is no longer loadable, which fails silently.
+ invalidated: List[str] = []
+ for profile_row in (
+ db.query(EvolutionAgentProfile)
+ .filter(EvolutionAgentProfile.is_active.is_(True))
+ .all()
+ ):
+ payload = profile_row.payload or {}
+ if skill_id in (payload.get("skill_ids") or []):
+ profile_row.is_active = False
+ invalidated.append(profile_row.profile_id)
+
+ release = EvolutionRelease(
+ release_id=new_release_id(),
+ candidate_id=candidate_id,
+ target_kind="skill",
+ target_asset_id=skill_id,
+ stage=RETIRED,
+ traffic_percent=0,
+ scope_filter={},
+ guardrails={
+ "previous_enabled": previous_enabled,
+ "invalidated_profiles": invalidated,
+ "proposer": candidate.proposer or "",
+ },
+ rollback_version_id=row.version or "",
+ approved_by=approver,
+ )
+ db.add(release)
+
+ candidate.status = "active"
+ candidate.approved_by = approver
+ candidate.approved_at = datetime.now(timezone.utc)
+ db.commit()
+
+ # Restore the memories this skill was compiled from: the knowledge lost
+ # its cheaper carrier, so it goes back to being paid for by retrieval.
+ from core.evolution.generators.memory_apply import revert_memory_ops
+
+ source_candidate = activation_policy.source_candidate_for(db, skill_id)
+ restored = revert_memory_ops(candidate_id=source_candidate) if source_candidate else {}
+
+ logger.info(
+ "[activation] retired %s (profiles invalidated=%s memories restored=%d)",
+ skill_id,
+ invalidated,
+ len(restored.get("reverted", [])),
+ )
+ return {
+ "skill_id": skill_id,
+ "release_id": release.release_id,
+ "invalidated_profiles": invalidated,
+ "restored_memory_ops": restored.get("reverted", []),
+ }
diff --git a/src/backend/core/evolution/timescales.py b/src/backend/core/evolution/cycle/timescales.py
similarity index 99%
rename from src/backend/core/evolution/timescales.py
rename to src/backend/core/evolution/cycle/timescales.py
index bcd93632..94ef52d6 100644
--- a/src/backend/core/evolution/timescales.py
+++ b/src/backend/core/evolution/cycle/timescales.py
@@ -17,7 +17,7 @@
import logging
from dataclasses import dataclass, field
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Sequence, Tuple
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/trigger.py b/src/backend/core/evolution/cycle/trigger.py
similarity index 81%
rename from src/backend/core/evolution/trigger.py
rename to src/backend/core/evolution/cycle/trigger.py
index e90acd9c..1a5339e3 100644
--- a/src/backend/core/evolution/trigger.py
+++ b/src/backend/core/evolution/cycle/trigger.py
@@ -31,7 +31,8 @@
import logging
import os
import threading
-from datetime import datetime, timedelta, timezone
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
from typing import Any, Dict, Optional, Tuple
from core.db.engine import SessionLocal
@@ -63,12 +64,8 @@ def _env_bool(name: str, default: bool) -> bool:
# Stops a burst of conversations from queueing a cycle per turn.
DEBOUNCE_SECONDS = _env_int("EVOLUTION_TRIGGER_DEBOUNCE_S", 600)
-_state_lock = threading.Lock()
+
# Per-user debounce, so a busy person cannot crowd out everyone else's cycle.
-_personal_last_run: Dict[str, datetime] = {}
-_last_cycle_at: Optional[datetime] = None
-_last_cycle_episode_count: int = 0
-_running = False
def _now() -> datetime:
@@ -78,7 +75,7 @@ def _now() -> datetime:
def _episode_count() -> int:
"""Cheap indexed count of mineable episodes."""
from core.db.models.evolution import EvolutionEpisode
- from core.evolution.user_settings import PRIVACY_PRIVATE
+ from core.evolution.profiles.user_settings_constants import PRIVACY_PRIVATE
with SessionLocal() as db:
return (
@@ -97,10 +94,10 @@ def should_trigger(*, now: Optional[datetime] = None) -> Tuple[bool, str]:
if not AUTO_TRIGGER_ENABLED:
return False, "auto_trigger_disabled"
- with _state_lock:
- if _running:
+ with process_state().state_lock:
+ if process_state().running:
return False, "cycle_already_running"
- last_at, last_count = _last_cycle_at, _last_cycle_episode_count
+ last_at, last_count = process_state().last_cycle_at, process_state().last_cycle_episode_count
moment = now or _now()
if last_at is not None:
@@ -129,17 +126,17 @@ def _mark_started(total: int, *, now: Optional[datetime] = None) -> None:
sleeping — a debounce tested only against the real clock is a debounce
nobody tests.
"""
- global _last_cycle_at, _last_cycle_episode_count, _running
- with _state_lock:
- _running = True
- _last_cycle_at = now or _now()
- _last_cycle_episode_count = total
+ pass
+ with process_state().state_lock:
+ process_state().running = True
+ process_state().last_cycle_at = now or _now()
+ process_state().last_cycle_episode_count = total
def _mark_finished() -> None:
- global _running
- with _state_lock:
- _running = False
+ pass
+ with process_state().state_lock:
+ process_state().running = False
def run_cycle_if_due(*, limit: int = 400) -> Optional[Dict[str, Any]]:
@@ -157,11 +154,11 @@ def run_cycle_if_due(*, limit: int = 400) -> Optional[Dict[str, Any]]:
try:
total = _episode_count()
except Exception:
- total = _last_cycle_episode_count
+ total = process_state().last_cycle_episode_count
_mark_started(total)
try:
- from core.evolution.loop import run_evolution_cycle
+ from core.evolution.cycle.loop import run_evolution_cycle
report = run_evolution_cycle(limit=limit)
logger.info(
@@ -189,15 +186,15 @@ def schedule_personal(user_id: str) -> bool:
return False
moment = _now()
- with _state_lock:
- last = _personal_last_run.get(user_id)
+ with process_state().state_lock:
+ last = process_state().personal_last_run.get(user_id)
if last is not None and (moment - last).total_seconds() < DEBOUNCE_SECONDS:
return False
- _personal_last_run[user_id] = moment
+ process_state().personal_last_run[user_id] = moment
def _run() -> None:
try:
- from core.evolution.personal import run_personal_cycle
+ from core.evolution.profiles.personal import run_personal_cycle
report = run_personal_cycle(user_id)
if report.get("candidates"):
@@ -245,8 +242,8 @@ def schedule_if_due(*, limit: int = 400) -> bool:
def status() -> Dict[str, Any]:
"""Current trigger state, for the console and for answering "why not?"."""
due, reason = should_trigger()
- with _state_lock:
- last_at, last_count = _last_cycle_at, _last_cycle_episode_count
+ with process_state().state_lock:
+ last_at, last_count = process_state().last_cycle_at, process_state().last_cycle_episode_count
return {
"enabled": AUTO_TRIGGER_ENABLED,
"due": due,
@@ -259,9 +256,25 @@ def status() -> Dict[str, Any]:
def reset_for_tests() -> None:
- global _last_cycle_at, _last_cycle_episode_count, _running
- with _state_lock:
- _last_cycle_at = None
- _last_cycle_episode_count = 0
- _running = False
- _personal_last_run.clear()
+ pass
+ with process_state().state_lock:
+ process_state().last_cycle_at = None
+ process_state().last_cycle_episode_count = 0
+ process_state().running = False
+ process_state().personal_last_run.clear()
+
+
+@dataclass
+class TriggerState:
+ state_lock: object = field(default_factory=lambda: threading.Lock())
+ personal_last_run: Dict[str, datetime] = field(default_factory=lambda: {})
+ last_cycle_at: Optional[datetime] = None
+ last_cycle_episode_count: int = 0
+ running: object = False
+
+
+_STATE = TriggerState()
+
+
+def process_state() -> TriggerState:
+ return _STATE
diff --git a/src/backend/core/evolution/cycle/usage_evidence.py b/src/backend/core/evolution/cycle/usage_evidence.py
new file mode 100644
index 00000000..105693ea
--- /dev/null
+++ b/src/backend/core/evolution/cycle/usage_evidence.py
@@ -0,0 +1,283 @@
+"""Skill usage, canary uplift and exposure measured from episode evidence."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+import logging
+from core.evolution.cycle import cycle_extras_constants
+
+logger = logging.getLogger(__name__)
+
+def is_after(created_at: Any, activated_at: Any) -> bool:
+ from datetime import timezone
+
+ if created_at is None or activated_at is None:
+ return False
+ left = created_at if created_at.tzinfo else created_at.replace(tzinfo=timezone.utc)
+ right = activated_at if activated_at.tzinfo else activated_at.replace(tzinfo=timezone.utc)
+ return left >= right
+
+
+def recent_activations() -> List[Any]:
+ """``(asset_id, activated_at, related_task_types)`` for live releases.
+
+ ``related_task_types`` comes from the release's own scope: a dip on the task
+ the change targeted is an ordinary regression its guardrail already watches,
+ and treating it as negative transfer would bury the collateral damage this
+ is looking for in noise from the intended effect.
+ """
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.models.evolution import EvolutionRelease
+ from core.evolution.cycle.release import ACTIVE, CANARY
+
+ with SessionLocal() as db:
+ rows = (
+ db.query(EvolutionRelease)
+ .filter(EvolutionRelease.stage.in_([ACTIVE, CANARY]))
+ .order_by(EvolutionRelease.created_at.desc())
+ .limit(20)
+ .all()
+ )
+ return [
+ (
+ str(row.target_asset_id),
+ row.created_at,
+ [str(t) for t in ((row.scope_filter or {}).get("task_types") or [])],
+ )
+ for row in rows
+ if row.created_at is not None
+ ]
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[cycle-extras] activation ledger unavailable: %s", exc)
+ return []
+
+
+def declared_tools_for(skill_ids: Sequence[str]) -> Dict[str, List[str]]:
+ """Each skill's own declared tools, read from the skill itself."""
+ tools: Dict[str, List[str]] = {}
+ try:
+ from core.capabilities.skills.loader_factory import get_skill_loader
+
+ metadata = get_skill_loader().load_all_metadata()
+ except Exception as exc: # noqa: BLE001
+ logger.info("[cycle-extras] skill metadata unavailable, overlap skipped: %s", exc)
+ return {}
+ for skill_id in skill_ids:
+ meta = metadata.get(skill_id)
+ declared = list(getattr(meta, "allowed_tools", None) or []) if meta else []
+ if declared:
+ tools[skill_id] = [str(t) for t in declared]
+ return tools
+
+
+def skill_usage_from_views(views: Sequence[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
+ """How each skill actually fared, from evidence that distinguishes use from exposure.
+
+ ``offers`` counts turns where the skill was **available** to the model — its
+ name and description were in the prompt. Skill loading offers every enabled
+ skill, so this is a denominator, not a choice: it answers "how many chances
+ did this skill have to be picked", which is exactly what makes "never
+ opened" mean something.
+
+ ``opens`` is the choice. The gap between the two is the whole signal.
+ """
+ stats: Dict[str, Dict[str, Any]] = {}
+
+ def bucket(skill_id: str) -> Dict[str, Any]:
+ return stats.setdefault(
+ str(skill_id),
+ {
+ "offers": 0,
+ "opens": 0,
+ "opens_succeeded": 0,
+ "opens_followed_through": 0,
+ "opens_measurable": 0,
+ "exposure_days": 0.0,
+ },
+ )
+
+ declared_by_skill = declared_tools_for(
+ sorted(
+ {
+ str(opened.get("skill_id") or "")
+ for view in views
+ for opened in (view.get("skills_opened") or [])
+ if opened.get("skill_id")
+ }
+ )
+ )
+
+ for view in views:
+ for skill_id in view.get("skill_sequence") or []:
+ bucket(skill_id)["offers"] += 1
+
+ succeeded = view.get("verdict") == "success"
+ for opened in view.get("skills_opened") or []:
+ skill_id = str(opened.get("skill_id") or "")
+ if not skill_id:
+ continue
+ entry = bucket(skill_id)
+ entry["opens"] += 1
+ if succeeded:
+ entry["opens_succeeded"] += 1
+
+ # Follow-through: after reading the document, did the model do what
+ # the document is about? Approximated by whether the tools it went
+ # on to call are among the ones the skill declares.
+ #
+ # Read from the skill itself, not from the event — an event cannot
+ # carry what the skill declared *now*, and a stale copy would score
+ # a rewritten skill against its old tool list.
+ declared = {str(t) for t in (declared_by_skill.get(skill_id) or [])}
+ after = [str(t) for t in (opened.get("tools_after_open") or [])]
+ if not declared or not after:
+ # A skill that declares no tools (a rules-only skill shapes
+ # ordering, not tool choice) or a turn that called nothing after
+ # the open cannot be measured this way. Excluded from the
+ # denominator rather than counted as a pass — counting them as
+ # passes is what made follow-through 100% for everything and the
+ # "opened but ignored" verdict unreachable.
+ continue
+ entry["opens_measurable"] += 1
+ if any(tool in declared for tool in after):
+ entry["opens_followed_through"] += 1
+
+ uplift_by_skill = uplift_from_canary(views)
+ for skill_id, entry in stats.items():
+ opens = entry["opens"]
+ measurable = entry["opens_measurable"]
+ entry["success_rate"] = entry["opens_succeeded"] / opens if opens else 0.0
+ entry["follow_through"] = (
+ entry["opens_followed_through"] / measurable if measurable else None
+ )
+ entry["exposure_days"] = exposure_days(skill_id)
+ entry["uplift"] = uplift_by_skill.get(skill_id)
+ return stats
+
+
+def canary_releases() -> List[Tuple[str, str, int, Any]]:
+ """``(asset_id, release_id, traffic_percent, started_at)`` for live canaries."""
+ from core.evolution.cycle.release import CANARY
+
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.models.evolution import EvolutionRelease
+
+ with SessionLocal() as db:
+ rows = (
+ db.query(EvolutionRelease)
+ .filter(
+ EvolutionRelease.target_kind == "skill",
+ EvolutionRelease.stage == CANARY,
+ )
+ .all()
+ )
+ return [
+ (
+ str(row.target_asset_id),
+ str(row.release_id),
+ int(row.traffic_percent or 0),
+ row.created_at,
+ )
+ for row in rows
+ ]
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[cycle-extras] canary ledger unavailable: %s", exc)
+ return []
+
+
+def uplift_from_canary(
+ views: Sequence[Dict[str, Any]],
+ *,
+ ramping: Optional[Sequence[Tuple[str, str, int, Any]]] = None,
+) -> Dict[str, Optional[float]]:
+ """"Is the deployment better off with this skill?", answered by comparison.
+
+ A canary release already splits users deterministically into "sees it" and
+ "does not" (:func:`core.evolution.cycle.release.in_canary_bucket`). That split is
+ the comparison: same period, same task mix, same everything except exposure
+ to this one asset. Recomputing bucket membership here rather than storing it
+ per request means the analysis works on history too.
+
+ Returns ``None`` for a skill with no usable comparison. ``None`` and ``0.0``
+ are different answers — "we have not measured" must never read as "measured,
+ no effect", because the second one licenses a rollback decision and the
+ first one does not.
+
+ ``ramping`` is injectable so this is testable against the real bucketing
+ rather than against a re-implementation of it in a test helper.
+ """
+ from core.evolution.cycle.release import in_canary_bucket
+
+ if ramping is None:
+ ramping = canary_releases()
+
+ uplift: Dict[str, Optional[float]] = {}
+ for skill_id, release_id, traffic, started_at in ramping:
+ exposed: List[bool] = []
+ withheld: List[bool] = []
+ for view in views:
+ created_at = view.get("created_at")
+ if created_at is None or started_at is None:
+ continue
+ if not is_after(created_at, started_at):
+ continue
+ user_id = str(view.get("user_id") or "")
+ if not user_id:
+ continue
+ succeeded = view.get("verdict") == "success"
+ if in_canary_bucket(
+ release_id=release_id, subject_id=user_id, traffic_percent=traffic
+ ):
+ exposed.append(succeeded)
+ else:
+ withheld.append(succeeded)
+
+ # Both arms need enough turns for a rate to mean anything. A one-sided
+ # or tiny comparison produces a number that looks like evidence and is
+ # not, and this number can trigger an immediate rollback.
+ if len(exposed) < cycle_extras_constants.MIN_UPLIFT_SAMPLES or len(withheld) < cycle_extras_constants.MIN_UPLIFT_SAMPLES:
+ uplift[skill_id] = None
+ continue
+ uplift[skill_id] = (sum(exposed) / len(exposed)) - (
+ sum(withheld) / len(withheld)
+ )
+ return uplift
+
+
+def exposure_days(skill_id: str) -> float:
+ """How long this skill has been exposed to anyone.
+
+ Read from the release ledger. Without it, "never opened" cannot be told
+ apart from "shipped yesterday", and a retirement rule that cannot tell them
+ apart retires whatever is newest.
+ """
+ from datetime import datetime, timezone
+
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.models.evolution import EvolutionRelease
+ from core.evolution.cycle.release import ACTIVE, CANARY
+
+ with SessionLocal() as db:
+ row = (
+ db.query(EvolutionRelease)
+ .filter(
+ EvolutionRelease.target_kind == "skill",
+ EvolutionRelease.target_asset_id == skill_id,
+ EvolutionRelease.stage.in_([ACTIVE, CANARY]),
+ )
+ .order_by(EvolutionRelease.created_at.asc())
+ .first()
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[cycle-extras] exposure lookup failed for %s: %s", skill_id, exc)
+ return 0.0
+ if row is None or row.created_at is None:
+ # Not an evolved skill, or never released through the ladder. A
+ # hand-authored skill has always been exposed, so judging it on its
+ # observed usage is exactly right.
+ return float("inf")
+ created = row.created_at
+ if created.tzinfo is None:
+ created = created.replace(tzinfo=timezone.utc)
+ return (datetime.now(timezone.utc) - created).total_seconds() / 86400.0
diff --git a/src/backend/core/evolution/cycle_extras.py b/src/backend/core/evolution/cycle_extras.py
deleted file mode 100644
index 5c2b5fbf..00000000
--- a/src/backend/core/evolution/cycle_extras.py
+++ /dev/null
@@ -1,653 +0,0 @@
-"""The two engines the cycle was missing, and the direction the third lacked.
-
-A cycle used to do exactly one thing: grow the skill library from identical tool
-sequences. Memory reorganisation and capability retirement were implemented and
-unreachable, and recurrence detected from *what the user asked* did not exist at
-all.
-
-Wiring them up is not enough on its own, and the first attempt at it showed why.
-Three failures worth naming, because they are the ones this file now exists to
-not repeat:
-
-* **A finding with no exit changes nothing.** The first version returned a
- report that no code read. Everything here now produces
- :class:`~core.evolution.ir.EvolutionIR` candidates that go through the same
- security gate, review queue and release ladder as every other change. A
- proposal that cannot reach the queue is a log line.
-* **Retrieval frequency is not duplication.** Counting how often one content
- hash was retrieved and calling four hits a duplicate inverts the meaning: a
- memory retrieved in four separate conversations is the store's most useful
- entry, and merging it away is a proposal to delete what works. Duplication is
- a property of *inventory*, so it is found by scanning the store
- (:mod:`core.evolution.memory_scan`), which the event stream cannot do.
-* **Being loaded is not being used.** Retirement read a "selections" count that
- every enabled skill shared, so it ranked skills by how recently they shipped.
- Usage is ``skill.opened`` — the model deciding for itself to read the
- document — and follow-through is whether the plan afterwards matched what the
- document said.
-
-Everything here emits **candidates**. Nothing activates and nothing deletes: the
-reverse direction is where over-eager automation does damage that is hard to
-notice, because removing a capability produces no error, only a slow decline.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-logger = logging.getLogger(__name__)
-
-# An intent must recur on this many separate occasions before "we keep
-# re-planning this" is a claim about behaviour rather than about one bad
-# afternoon of rephrasing.
-MIN_INTENT_OCCASIONS = 3
-# Below this success rate a recurring intent is a *failure* pattern: the right
-# response is a fix proposal, not a distilled skill blessing the current plan.
-INTENT_MIN_SUCCESS_RATE = 0.6
-# A recurring intent handled the same way every time needs no skill — the plan
-# is already stable. The signal is variety.
-MIN_PLAN_VARIETY = 2
-
-# Injected this often that the association is not two unlucky turns.
-DEMOTE_MIN_HITS = 4
-# At or below this success rate, retrieving it is not helping.
-DEMOTE_SUCCESS_CEILING = 0.34
-
-# Turns required in *each* arm before a canary comparison is reported. Below
-# this the difference between two rates is noise, and this particular number can
-# trigger an immediate rollback — so it has the strictest sample requirement of
-# anything the decremental engine computes.
-MIN_UPLIFT_SAMPLES = 10
-
-
-def _asset_id(prefix: str, seed: str) -> str:
- return f"{prefix}-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12]
-
-
-# ── Skills: upward, from recurring intent ────────────────────────────────────
-
-
-def intent_findings(views: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
- """Recurring requests whose handling never settled.
-
- The pairing matters. Recurrence alone says the question is common;
- recurrence **with plan variety** says the system has not found a stable way
- to answer it and is paying the planning cost fresh every time. That second
- condition is what distinguishes a capability gap from a popular question.
- """
- from core.evolution.similarity import cluster_intents
-
- clusters, method = cluster_intents(views, min_support=MIN_INTENT_OCCASIONS)
- findings: List[Dict[str, Any]] = []
- for cluster in clusters:
- if cluster.occasions < MIN_INTENT_OCCASIONS:
- # Recurrence inside one conversation is a user rephrasing after a
- # bad answer, which is evidence about that answer, not about demand.
- continue
- if cluster.plan_variety < MIN_PLAN_VARIETY:
- # Same question, same plan, every time. Already effectively a skill;
- # writing one down adds an asset to maintain and changes nothing.
- continue
- if cluster.success_rate < INTENT_MIN_SUCCESS_RATE:
- # Recurring *and* failing. Distilling the current plan would encode
- # the failure. This belongs to attribution, not to promotion.
- findings.append(
- {
- "kind": "recurring_failure_intent",
- "cluster": cluster.to_dict(),
- "action": "no_skill_candidate",
- "reason": (
- f"「{cluster.representative[:30]}」类请求出现 {cluster.occasions} 次,"
- f"但成功率只有 {cluster.success_rate:.0%}——"
- "把当前做法蒸馏出来等于把失败固化"
- ),
- }
- )
- continue
- findings.append(
- {
- "kind": "recurring_intent",
- "cluster": cluster.to_dict(),
- "action": "skill_candidate",
- "asset_id": _asset_id("auto-intent", cluster.representative),
- "reason": (
- f"同一意图在 {cluster.occasions} 次独立会话中出现,却用了 "
- f"{cluster.plan_variety} 种不同的工具序列——每次都在重新规划"
- ),
- }
- )
- return {"method": method, "clusters": len(clusters), "findings": findings}
-
-
-# ── Skills: downward ─────────────────────────────────────────────────────────
-
-
-def decremental_findings(
- views: Sequence[Dict[str, Any]],
- *,
- skill_stats: Optional[Dict[str, Dict[str, Any]]] = None,
- declared_tools: Optional[Dict[str, List[str]]] = None,
-) -> Dict[str, Any]:
- """Merge, retirement and negative-transfer findings.
-
- ``skill_stats`` is derived from the episode views when not supplied.
- ``declared_tools`` maps skill id → the tools its own document names; overlap
- is computed from that and never from the tools observed in episodes where
- the skill happened to be loaded, which would pair every co-enabled skill.
- """
- from core.evolution import promotion as P
-
- stats = skill_stats if skill_stats is not None else skill_usage_from_views(views)
-
- overlaps: List[Tuple[str, str, int]] = []
- if declared_tools is None:
- declared_tools = _declared_tools_for(sorted(stats))
- if declared_tools:
- overlaps = P.find_overlaps(declared_tools)
-
- retirements = P.find_retirement_candidates(stats)
- return {
- "overlaps": [
- {"a": a, "b": b, "shared_tools": shared} for a, b, shared in overlaps
- ],
- "retirements": retirements,
- "asset_count": len(stats),
- "negative_transfer": negative_transfer_for_activations(views),
- }
-
-
-def negative_transfer_for_activations(
- views: Sequence[Dict[str, Any]],
-) -> List[Dict[str, Any]]:
- """Check each recent activation for collateral damage.
-
- The detector needs a corpus split into before and after *something*. Calling
- it on views where nothing carries that marker returns nothing on every
- corpus, which is a detector that runs and cannot find anything — the exact
- shape of defect this rework exists to remove. So the split is computed here,
- from the release ledger, once per activated asset.
- """
- activations = _recent_activations()
- if not activations:
- return []
-
- findings: List[Dict[str, Any]] = []
- for asset_id, activated_at, related in activations:
- split = [
- {**view, "after": _is_after(view.get("created_at"), activated_at)}
- for view in views
- if view.get("created_at") is not None
- ]
- if not any(v["after"] for v in split) or not any(not v["after"] for v in split):
- # One-sided evidence cannot show a change. Reporting nothing is the
- # honest output; reporting a drop from an empty baseline is not.
- continue
- findings.extend(
- negative_transfer_findings(split, asset_id=asset_id, related_task_types=related)
- )
- return findings
-
-
-def _is_after(created_at: Any, activated_at: Any) -> bool:
- from datetime import timezone
-
- if created_at is None or activated_at is None:
- return False
- left = created_at if created_at.tzinfo else created_at.replace(tzinfo=timezone.utc)
- right = activated_at if activated_at.tzinfo else activated_at.replace(tzinfo=timezone.utc)
- return left >= right
-
-
-def _recent_activations() -> List[Any]:
- """``(asset_id, activated_at, related_task_types)`` for live releases.
-
- ``related_task_types`` comes from the release's own scope: a dip on the task
- the change targeted is an ordinary regression its guardrail already watches,
- and treating it as negative transfer would bury the collateral damage this
- is looking for in noise from the intended effect.
- """
- try:
- from core.db.engine import SessionLocal
- from core.db.models.evolution import EvolutionRelease
- from core.evolution.release import ACTIVE, CANARY
-
- with SessionLocal() as db:
- rows = (
- db.query(EvolutionRelease)
- .filter(EvolutionRelease.stage.in_([ACTIVE, CANARY]))
- .order_by(EvolutionRelease.created_at.desc())
- .limit(20)
- .all()
- )
- return [
- (
- str(row.target_asset_id),
- row.created_at,
- [str(t) for t in ((row.scope_filter or {}).get("task_types") or [])],
- )
- for row in rows
- if row.created_at is not None
- ]
- except Exception as exc: # noqa: BLE001
- logger.debug("[cycle-extras] activation ledger unavailable: %s", exc)
- return []
-
-
-def _declared_tools_for(skill_ids: Sequence[str]) -> Dict[str, List[str]]:
- """Each skill's own declared tools, read from the skill itself."""
- tools: Dict[str, List[str]] = {}
- try:
- from core.agent_skills.loader import get_skill_loader
-
- metadata = get_skill_loader().load_all_metadata()
- except Exception as exc: # noqa: BLE001
- logger.info("[cycle-extras] skill metadata unavailable, overlap skipped: %s", exc)
- return {}
- for skill_id in skill_ids:
- meta = metadata.get(skill_id)
- declared = list(getattr(meta, "allowed_tools", None) or []) if meta else []
- if declared:
- tools[skill_id] = [str(t) for t in declared]
- return tools
-
-
-def skill_usage_from_views(views: Sequence[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
- """How each skill actually fared, from evidence that distinguishes use from exposure.
-
- ``offers`` counts turns where the skill was **available** to the model — its
- name and description were in the prompt. Skill loading offers every enabled
- skill, so this is a denominator, not a choice: it answers "how many chances
- did this skill have to be picked", which is exactly what makes "never
- opened" mean something.
-
- ``opens`` is the choice. The gap between the two is the whole signal.
- """
- stats: Dict[str, Dict[str, Any]] = {}
-
- def bucket(skill_id: str) -> Dict[str, Any]:
- return stats.setdefault(
- str(skill_id),
- {
- "offers": 0,
- "opens": 0,
- "opens_succeeded": 0,
- "opens_followed_through": 0,
- "opens_measurable": 0,
- "exposure_days": 0.0,
- },
- )
-
- declared_by_skill = _declared_tools_for(
- sorted(
- {
- str(opened.get("skill_id") or "")
- for view in views
- for opened in (view.get("skills_opened") or [])
- if opened.get("skill_id")
- }
- )
- )
-
- for view in views:
- for skill_id in view.get("skill_sequence") or []:
- bucket(skill_id)["offers"] += 1
-
- succeeded = view.get("verdict") == "success"
- for opened in view.get("skills_opened") or []:
- skill_id = str(opened.get("skill_id") or "")
- if not skill_id:
- continue
- entry = bucket(skill_id)
- entry["opens"] += 1
- if succeeded:
- entry["opens_succeeded"] += 1
-
- # Follow-through: after reading the document, did the model do what
- # the document is about? Approximated by whether the tools it went
- # on to call are among the ones the skill declares.
- #
- # Read from the skill itself, not from the event — an event cannot
- # carry what the skill declared *now*, and a stale copy would score
- # a rewritten skill against its old tool list.
- declared = {str(t) for t in (declared_by_skill.get(skill_id) or [])}
- after = [str(t) for t in (opened.get("tools_after_open") or [])]
- if not declared or not after:
- # A skill that declares no tools (a rules-only skill shapes
- # ordering, not tool choice) or a turn that called nothing after
- # the open cannot be measured this way. Excluded from the
- # denominator rather than counted as a pass — counting them as
- # passes is what made follow-through 100% for everything and the
- # "opened but ignored" verdict unreachable.
- continue
- entry["opens_measurable"] += 1
- if any(tool in declared for tool in after):
- entry["opens_followed_through"] += 1
-
- uplift_by_skill = _uplift_from_canary(views)
- for skill_id, entry in stats.items():
- opens = entry["opens"]
- measurable = entry["opens_measurable"]
- entry["success_rate"] = entry["opens_succeeded"] / opens if opens else 0.0
- entry["follow_through"] = (
- entry["opens_followed_through"] / measurable if measurable else None
- )
- entry["exposure_days"] = _exposure_days(skill_id)
- entry["uplift"] = uplift_by_skill.get(skill_id)
- return stats
-
-
-def _canary_releases() -> List[Tuple[str, str, int, Any]]:
- """``(asset_id, release_id, traffic_percent, started_at)`` for live canaries."""
- from core.evolution.release import CANARY
-
- try:
- from core.db.engine import SessionLocal
- from core.db.models.evolution import EvolutionRelease
-
- with SessionLocal() as db:
- rows = (
- db.query(EvolutionRelease)
- .filter(
- EvolutionRelease.target_kind == "skill",
- EvolutionRelease.stage == CANARY,
- )
- .all()
- )
- return [
- (
- str(row.target_asset_id),
- str(row.release_id),
- int(row.traffic_percent or 0),
- row.created_at,
- )
- for row in rows
- ]
- except Exception as exc: # noqa: BLE001
- logger.debug("[cycle-extras] canary ledger unavailable: %s", exc)
- return []
-
-
-def _uplift_from_canary(
- views: Sequence[Dict[str, Any]],
- *,
- ramping: Optional[Sequence[Tuple[str, str, int, Any]]] = None,
-) -> Dict[str, Optional[float]]:
- """"Is the deployment better off with this skill?", answered by comparison.
-
- A canary release already splits users deterministically into "sees it" and
- "does not" (:func:`core.evolution.release.in_canary_bucket`). That split is
- the comparison: same period, same task mix, same everything except exposure
- to this one asset. Recomputing bucket membership here rather than storing it
- per request means the analysis works on history too.
-
- Returns ``None`` for a skill with no usable comparison. ``None`` and ``0.0``
- are different answers — "we have not measured" must never read as "measured,
- no effect", because the second one licenses a rollback decision and the
- first one does not.
-
- ``ramping`` is injectable so this is testable against the real bucketing
- rather than against a re-implementation of it in a test helper.
- """
- from core.evolution.release import in_canary_bucket
-
- if ramping is None:
- ramping = _canary_releases()
-
- uplift: Dict[str, Optional[float]] = {}
- for skill_id, release_id, traffic, started_at in ramping:
- exposed: List[bool] = []
- withheld: List[bool] = []
- for view in views:
- created_at = view.get("created_at")
- if created_at is None or started_at is None:
- continue
- if not _is_after(created_at, started_at):
- continue
- user_id = str(view.get("user_id") or "")
- if not user_id:
- continue
- succeeded = view.get("verdict") == "success"
- if in_canary_bucket(
- release_id=release_id, subject_id=user_id, traffic_percent=traffic
- ):
- exposed.append(succeeded)
- else:
- withheld.append(succeeded)
-
- # Both arms need enough turns for a rate to mean anything. A one-sided
- # or tiny comparison produces a number that looks like evidence and is
- # not, and this number can trigger an immediate rollback.
- if len(exposed) < MIN_UPLIFT_SAMPLES or len(withheld) < MIN_UPLIFT_SAMPLES:
- uplift[skill_id] = None
- continue
- uplift[skill_id] = (sum(exposed) / len(exposed)) - (
- sum(withheld) / len(withheld)
- )
- return uplift
-
-
-def _exposure_days(skill_id: str) -> float:
- """How long this skill has been exposed to anyone.
-
- Read from the release ledger. Without it, "never opened" cannot be told
- apart from "shipped yesterday", and a retirement rule that cannot tell them
- apart retires whatever is newest.
- """
- from datetime import datetime, timezone
-
- try:
- from core.db.engine import SessionLocal
- from core.db.models.evolution import EvolutionRelease
- from core.evolution.release import ACTIVE, CANARY
-
- with SessionLocal() as db:
- row = (
- db.query(EvolutionRelease)
- .filter(
- EvolutionRelease.target_kind == "skill",
- EvolutionRelease.target_asset_id == skill_id,
- EvolutionRelease.stage.in_([ACTIVE, CANARY]),
- )
- .order_by(EvolutionRelease.created_at.asc())
- .first()
- )
- except Exception as exc: # noqa: BLE001
- logger.debug("[cycle-extras] exposure lookup failed for %s: %s", skill_id, exc)
- return 0.0
- if row is None or row.created_at is None:
- # Not an evolved skill, or never released through the ladder. A
- # hand-authored skill has always been exposed, so judging it on its
- # observed usage is exactly right.
- return float("inf")
- created = row.created_at
- if created.tzinfo is None:
- created = created.replace(tzinfo=timezone.utc)
- return (datetime.now(timezone.utc) - created).total_seconds() / 86400.0
-
-
-def negative_transfer_findings(
- views: Sequence[Dict[str, Any]],
- *,
- asset_id: str = "",
- related_task_types: Sequence[str] = (),
- before_key: str = "before",
- after_key: str = "after",
-) -> List[Dict[str, Any]]:
- """Task types that got worse after some activation, unrelated to it.
-
- The one failure mode of joint evolution that no single release's guardrail
- can catch: each change looks fine on the metric it was measured against, and
- the damage shows up somewhere nobody was looking.
- """
- from core.evolution import promotion as P
-
- by_type_before: Dict[str, List[bool]] = {}
- by_type_after: Dict[str, List[bool]] = {}
- for view in views:
- target = by_type_after if view.get(after_key) else by_type_before
- target.setdefault(str(view.get("task_type") or "chat"), []).append(
- view.get("verdict") == "success"
- )
-
- def rates(source: Dict[str, List[bool]]) -> Dict[str, float]:
- return {k: (sum(v) / len(v) if v else 0.0) for k, v in source.items()}
-
- findings = P.detect_negative_transfer(
- asset_id=asset_id or "unspecified",
- # Only *unrelated* task types are examined. A dip on the task the change
- # targeted is an ordinary regression its own guardrail catches; the
- # dangerous case is collateral damage where nobody is looking.
- related_task_types=list(related_task_types),
- before=rates(by_type_before),
- after=rates(by_type_after),
- )
- return [f.to_dict() for f in findings]
-
-
-# ── Memory: the half that makes it evolution rather than accumulation ────────
-
-
-def memory_findings(
- views: Sequence[Dict[str, Any]],
- *,
- scan_findings: Optional[Sequence[Any]] = None,
-) -> Dict[str, Any]:
- """Reorganisation proposals for the memory layer.
-
- Two independent sources, because they answer different questions and only
- one of them is answerable from a run's evidence:
-
- * **inventory** (``scan_findings``) — duplicates, contradictions and stale
- entries, which require reading the store;
- * **behaviour** (``views``) — memories that keep being injected into turns
- that fail, which requires reading the runs.
-
- The behavioural signal is deliberately *not* framed as duplication. A
- memory injected repeatedly is either helping or not, and the difference is
- the outcome of the turns it landed in — never the count on its own.
- """
- from core.evolution.memory_ops import (
- OP_MERGE,
- OP_REWEIGHT,
- SCOPE_USER,
- MemoryOp,
- partition_by_approval,
- )
- from core.evolution.memory_scan import (
- FINDING_CONTRADICTION,
- FINDING_DUPLICATE,
- FINDING_STALE,
- )
-
- retrieved: Dict[str, int] = {}
- helped: Dict[str, int] = {}
- for view in views:
- for ref in view.get("memory_refs") or []:
- key = str(ref)
- retrieved[key] = retrieved.get(key, 0) + 1
- if view.get("verdict") == "success":
- helped[key] = helped.get(key, 0) + 1
-
- ops: List[MemoryOp] = []
- contradictions: List[Dict[str, Any]] = []
-
- # 1. Inventory findings, from an actual scan of the store.
- for finding in scan_findings or []:
- payload = finding.to_dict() if hasattr(finding, "to_dict") else dict(finding)
- kind = payload.get("kind")
- refs = [str(r) for r in (payload.get("refs") or [])]
- if kind == FINDING_DUPLICATE and len(refs) >= 2:
- # The second entry loses. Content is not rewritten and nothing is
- # removed — the survivor already says the same thing.
- ops.append(
- MemoryOp(
- operation=OP_MERGE,
- memory_ref=refs[1],
- scope=SCOPE_USER,
- reason=payload.get("reason", ""),
- before={"injected": True, "duplicate_of": refs[0]},
- after={"injected": False, "superseded_by": refs[0]},
- )
- )
- elif kind == FINDING_STALE and refs:
- ops.append(
- MemoryOp(
- operation=OP_REWEIGHT,
- memory_ref=refs[0],
- scope=SCOPE_USER,
- reason=payload.get("reason", ""),
- before={"weight": 1.0},
- after={"weight": 0.7},
- )
- )
- elif kind == FINDING_CONTRADICTION:
- # Neither side is automatically right, and picking by recency is
- # how a corrected mistake becomes the new truth. This is reported
- # for a human, with both sides and their dates.
- contradictions.append(payload)
-
- # 2. Behavioural: injected often, associated with failure. Down-weighted,
- # never deleted — a wrong reweight is recoverable and a wrong deletion is
- # not.
- for ref, count in retrieved.items():
- if count < DEMOTE_MIN_HITS:
- continue
- rate = helped.get(ref, 0) / count
- if rate <= DEMOTE_SUCCESS_CEILING:
- ops.append(
- MemoryOp(
- operation=OP_REWEIGHT,
- memory_ref=ref,
- scope=SCOPE_USER,
- reason=(
- f"被注入 {count} 次,其中仅 {helped.get(ref, 0)} 次所在回答成功"
- "——检索到它并没有帮上忙"
- ),
- before={"weight": 1.0},
- after={"weight": 0.5},
- )
- )
-
- auto, needs_review = partition_by_approval(ops)
- return {
- "distinct_memories_retrieved": len(retrieved),
- "merge_proposals": sum(1 for op in ops if op.operation == OP_MERGE),
- "reweight_proposals": sum(1 for op in ops if op.operation == OP_REWEIGHT),
- "contradictions": contradictions,
- # Reported separately because the split *is* the governance boundary:
- # reweighting is reversible and may proceed, merging changes what gets
- # injected and may not.
- "auto_applicable": [op.to_dict() for op in auto],
- "needs_review": [op.to_dict() for op in needs_review],
- "ops": ops,
- }
-
-
-# ── One entry point for the cycle ────────────────────────────────────────────
-
-
-def run_extras(
- views: Sequence[Dict[str, Any]],
- *,
- scan_findings: Optional[Sequence[Any]] = None,
-) -> Dict[str, Any]:
- """Everything the cycle was missing, in one call.
-
- Each section is independently guarded: a failure in the decremental pass
- must not cost the intent pass its findings, because they answer different
- questions and an operator needs whichever ones succeeded.
- """
- report: Dict[str, Any] = {}
- for name, fn in (
- ("intent", lambda: intent_findings(views)),
- ("decremental", lambda: decremental_findings(views)),
- ("memory", lambda: memory_findings(views, scan_findings=scan_findings)),
- ):
- try:
- report[name] = fn()
- except Exception as exc: # noqa: BLE001
- logger.warning("[cycle-extras] %s pass failed: %s", name, exc)
- report[name] = {"error": str(exc)}
- return report
diff --git a/src/backend/core/evolution/evidence/__init__.py b/src/backend/core/evolution/evidence/__init__.py
new file mode 100644
index 00000000..09947758
--- /dev/null
+++ b/src/backend/core/evolution/evidence/__init__.py
@@ -0,0 +1 @@
+"""Evidence for evolution: traces, replay cassettes, credit assignment and lineage."""
diff --git a/src/backend/core/evolution/cassette.py b/src/backend/core/evolution/evidence/cassette.py
similarity index 94%
rename from src/backend/core/evolution/cassette.py
rename to src/backend/core/evolution/evidence/cassette.py
index 7a007f00..0b606d75 100644
--- a/src/backend/core/evolution/cassette.py
+++ b/src/backend/core/evolution/evidence/cassette.py
@@ -25,7 +25,7 @@
import json
import logging
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
logger = logging.getLogger(__name__)
@@ -180,18 +180,13 @@ def _offload_reference(result: Any) -> Optional[str]:
def _load_offloaded(result: Any) -> Optional[Any]:
- """Fetch offloaded content. Returns ``None`` when it cannot be restored."""
- reference = _offload_reference(result)
- if reference is None:
- return None
- try:
- from core.llm.offloader import load_offloaded_result # type: ignore
+ """Recorded paths alone do not carry a workspace/user authorization scope.
- restored = load_offloaded_result(reference)
- return restored if restored is not None else None
- except Exception as exc:
- logger.debug("[cassette] offload restore failed for %s: %s", reference, exc)
- return None
+ The writer has no synchronous restore API. Keep the established replay miss
+ instead of attempting an unscoped filesystem read or importing a nonexistent
+ SDK helper. Replay callers receive MISS_OFFLOADED and must use live execution.
+ """
+ return None
def build_cassette(message_ids: Sequence[str]) -> Cassette:
diff --git a/src/backend/core/evolution/counterfactual.py b/src/backend/core/evolution/evidence/counterfactual.py
similarity index 99%
rename from src/backend/core/evolution/counterfactual.py
rename to src/backend/core/evolution/evidence/counterfactual.py
index 95a2ecda..82c566eb 100644
--- a/src/backend/core/evolution/counterfactual.py
+++ b/src/backend/core/evolution/evidence/counterfactual.py
@@ -30,7 +30,7 @@
import logging
from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Callable, Dict, List, Optional, Sequence
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/credit.py b/src/backend/core/evolution/evidence/credit.py
similarity index 99%
rename from src/backend/core/evolution/credit.py
rename to src/backend/core/evolution/evidence/credit.py
index 2744e713..bdd5e752 100644
--- a/src/backend/core/evolution/credit.py
+++ b/src/backend/core/evolution/evidence/credit.py
@@ -26,7 +26,7 @@
import logging
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Tuple
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/events.py b/src/backend/core/evolution/evidence/events.py
similarity index 99%
rename from src/backend/core/evolution/events.py
rename to src/backend/core/evolution/evidence/events.py
index 217a24d6..31ac691e 100644
--- a/src/backend/core/evolution/events.py
+++ b/src/backend/core/evolution/evidence/events.py
@@ -202,7 +202,7 @@ def flush(self) -> int:
if self._flushed or not self._events:
return 0
try:
- from core.evolution.trace_store import persist_events
+ from core.evolution.evidence.trace_store import persist_events
written = persist_events(self._events)
self._flushed = True
diff --git a/src/backend/core/evolution/evidence_contract.py b/src/backend/core/evolution/evidence/evidence_contract.py
similarity index 100%
rename from src/backend/core/evolution/evidence_contract.py
rename to src/backend/core/evolution/evidence/evidence_contract.py
diff --git a/src/backend/core/evolution/evidence_resolver.py b/src/backend/core/evolution/evidence/evidence_resolver.py
similarity index 98%
rename from src/backend/core/evolution/evidence_resolver.py
rename to src/backend/core/evolution/evidence/evidence_resolver.py
index 61202484..a0946fbd 100644
--- a/src/backend/core/evolution/evidence_resolver.py
+++ b/src/backend/core/evolution/evidence/evidence_resolver.py
@@ -20,13 +20,7 @@
import logging
from typing import Any, Dict, List, Optional, Sequence
-from core.evolution.evidence_contract import (
- LAYER_GRAPH,
- LAYER_PROCEDURE,
- CapabilityEvidencePackV2,
- EvidenceRef,
- role_for_predicate,
-)
+from core.evolution.evidence.evidence_contract import LAYER_GRAPH, LAYER_PROCEDURE, CapabilityEvidencePackV2, EvidenceRef, role_for_predicate
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/exposure.py b/src/backend/core/evolution/evidence/exposure.py
similarity index 95%
rename from src/backend/core/evolution/exposure.py
rename to src/backend/core/evolution/evidence/exposure.py
index e8dc39fe..66f73d52 100644
--- a/src/backend/core/evolution/exposure.py
+++ b/src/backend/core/evolution/evidence/exposure.py
@@ -30,9 +30,9 @@
from __future__ import annotations
import logging
-from typing import Dict, Iterable, List, Optional, Sequence, Set
+from typing import Dict, Iterable, List, Sequence, Set
-from core.evolution.activation import EVOLUTION_PREFIX
+from core.evolution.cycle.activation_constants import EVOLUTION_PREFIX
logger = logging.getLogger(__name__)
@@ -90,7 +90,7 @@ def visible_evolved_skill_ids(
tenant_id: str = "default",
) -> Set[str]:
"""Of ``candidate_ids``, the evolved skill ids this subject may load."""
- from core.evolution.release import ACTIVE, CANARY, in_canary_bucket
+ from core.evolution.cycle.release import ACTIVE, CANARY, in_canary_bucket
evolved = {sid for sid in candidate_ids if is_evolved_skill_id(sid)}
if not evolved:
@@ -134,7 +134,7 @@ def _applicable_by_manifest(db, skill_ids: Set[str], *, task_type: str) -> Set[s
for it. Skills without a manifest (hand-authored, pre-manifest) pass.
"""
from core.db.models import AdminSkill
- from core.evolution.applicability import load_manifest, manifest_permits
+ from core.evolution.assessment.applicability import load_manifest, manifest_permits
rows = (
db.query(AdminSkill.skill_id, AdminSkill.extra_files)
@@ -198,7 +198,7 @@ def _resolve(session) -> Set[str]:
def exposure_state(db, skill_id: str) -> Dict[str, object]:
"""Human-readable exposure of one evolved skill, for the console."""
- from core.evolution.release import ACTIVE, CANARY, SHADOW
+ from core.evolution.cycle.release import ACTIVE, CANARY, SHADOW
latest = _latest_releases_by_asset(db, [skill_id])
release = latest.get(skill_id)
diff --git a/src/backend/core/evolution/lineage.py b/src/backend/core/evolution/evidence/lineage.py
similarity index 99%
rename from src/backend/core/evolution/lineage.py
rename to src/backend/core/evolution/evidence/lineage.py
index 98cedb8c..5612f4f2 100644
--- a/src/backend/core/evolution/lineage.py
+++ b/src/backend/core/evolution/evidence/lineage.py
@@ -16,7 +16,7 @@
import logging
from typing import Any, Dict, List, Optional, Sequence, Tuple
-from core.evolution.evidence_contract import CapabilityEvidencePackV2
+from core.evolution.evidence.evidence_contract import CapabilityEvidencePackV2
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/outcomes.py b/src/backend/core/evolution/evidence/outcomes.py
similarity index 100%
rename from src/backend/core/evolution/outcomes.py
rename to src/backend/core/evolution/evidence/outcomes.py
diff --git a/src/backend/core/evolution/replay.py b/src/backend/core/evolution/evidence/replay.py
similarity index 100%
rename from src/backend/core/evolution/replay.py
rename to src/backend/core/evolution/evidence/replay.py
diff --git a/src/backend/core/evolution/trace_assembler.py b/src/backend/core/evolution/evidence/trace_assembler.py
similarity index 98%
rename from src/backend/core/evolution/trace_assembler.py
rename to src/backend/core/evolution/evidence/trace_assembler.py
index 112c4ca6..d161ae69 100644
--- a/src/backend/core/evolution/trace_assembler.py
+++ b/src/backend/core/evolution/evidence/trace_assembler.py
@@ -24,12 +24,12 @@
import re
import uuid
from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
from core.db.engine import SessionLocal
from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent
from core.evolution.contract import AssetBundle
-from core.evolution.trace_store import attach_episode
+from core.evolution.evidence.trace_store import attach_episode
logger = logging.getLogger(__name__)
@@ -177,7 +177,7 @@ def emit_tool_events(sink, message_id: str, chat_id: str = "") -> int:
if not message_id:
return 0
try:
- from core.evolution.events import EV_SKILL_OPENED, EV_TOOL_CALLED
+ from core.evolution.evidence.events import EV_SKILL_OPENED, EV_TOOL_CALLED
with SessionLocal() as db:
rows = _tool_rows_for(db, message_id, chat_id)
diff --git a/src/backend/core/evolution/trace_store.py b/src/backend/core/evolution/evidence/trace_store.py
similarity index 97%
rename from src/backend/core/evolution/trace_store.py
rename to src/backend/core/evolution/evidence/trace_store.py
index 490e4524..944bd1ba 100644
--- a/src/backend/core/evolution/trace_store.py
+++ b/src/backend/core/evolution/evidence/trace_store.py
@@ -12,11 +12,11 @@
from __future__ import annotations
import logging
-from typing import Iterable, List, Optional, Sequence
+from typing import List, Sequence
from core.db.engine import SessionLocal
from core.db.models.evolution import EvolutionTraceEvent
-from core.evolution.events import TraceEvent
+from core.evolution.evidence.events import TraceEvent
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/generators/__init__.py b/src/backend/core/evolution/generators/__init__.py
new file mode 100644
index 00000000..6b0bafac
--- /dev/null
+++ b/src/backend/core/evolution/generators/__init__.py
@@ -0,0 +1 @@
+"""Generate assets from evolution candidates: skills and orchestration profiles."""
diff --git a/src/backend/core/evolution/graph_scan.py b/src/backend/core/evolution/generators/graph_scan.py
similarity index 100%
rename from src/backend/core/evolution/graph_scan.py
rename to src/backend/core/evolution/generators/graph_scan.py
diff --git a/src/backend/core/evolution/memory_apply.py b/src/backend/core/evolution/generators/memory_apply.py
similarity index 98%
rename from src/backend/core/evolution/memory_apply.py
rename to src/backend/core/evolution/generators/memory_apply.py
index 3ebe4a61..f879c940 100644
--- a/src/backend/core/evolution/memory_apply.py
+++ b/src/backend/core/evolution/generators/memory_apply.py
@@ -28,11 +28,7 @@
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Sequence
-from core.evolution.memory_ops import (
- OP_MERGE,
- OP_REWEIGHT,
- MemoryOp,
-)
+from core.evolution.generators.memory_ops import OP_MERGE, OP_REWEIGHT, MemoryOp
from core.memory.weights import OP_SUPERSEDE, STATUS_APPLIED, STATUS_REVERTED
logger = logging.getLogger(__name__)
@@ -243,7 +239,7 @@ def apply_memory_candidate(
"""
from core.db.engine import SessionLocal
from core.db.models.evolution import EvolutionCandidate
- from core.evolution.activation import ActivationError
+ from core.evolution.cycle.activation_types import ActivationError
with SessionLocal() as db:
candidate = db.get(EvolutionCandidate, candidate_id)
diff --git a/src/backend/core/evolution/memory_ops.py b/src/backend/core/evolution/generators/memory_ops.py
similarity index 99%
rename from src/backend/core/evolution/memory_ops.py
rename to src/backend/core/evolution/generators/memory_ops.py
index 7b749c30..ce55fd39 100644
--- a/src/backend/core/evolution/memory_ops.py
+++ b/src/backend/core/evolution/generators/memory_ops.py
@@ -25,7 +25,7 @@
import logging
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Sequence, Tuple
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/memory_scan.py b/src/backend/core/evolution/generators/memory_scan.py
similarity index 99%
rename from src/backend/core/evolution/memory_scan.py
rename to src/backend/core/evolution/generators/memory_scan.py
index fa739858..776e3725 100644
--- a/src/backend/core/evolution/memory_scan.py
+++ b/src/backend/core/evolution/generators/memory_scan.py
@@ -157,7 +157,7 @@ def _cosine(a: Sequence[float], b: Sequence[float]) -> float:
def _embed(texts: List[str]) -> Optional[List[List[float]]]:
try:
- from core.kb.kb_vector import embed_batch
+ from core.models.embedding import embed_batch
vectors = embed_batch(texts)
except Exception as exc: # noqa: BLE001
diff --git a/src/backend/core/evolution/ontology_bridge.py b/src/backend/core/evolution/generators/ontology_bridge.py
similarity index 100%
rename from src/backend/core/evolution/ontology_bridge.py
rename to src/backend/core/evolution/generators/ontology_bridge.py
diff --git a/src/backend/core/evolution/orchestration_gen.py b/src/backend/core/evolution/generators/orchestration_gen.py
similarity index 99%
rename from src/backend/core/evolution/orchestration_gen.py
rename to src/backend/core/evolution/generators/orchestration_gen.py
index b070aa04..c28f0bc1 100644
--- a/src/backend/core/evolution/orchestration_gen.py
+++ b/src/backend/core/evolution/generators/orchestration_gen.py
@@ -434,7 +434,7 @@ def profile_from_proposals(
always load these two skills" would hide the fact that the second is only
safe given the first.
"""
- from core.evolution.agent_profile import AgentProfile
+ from core.evolution.profiles.agent_profile import AgentProfile
merged = AgentProfile.from_dict(base.to_dict())
merged.profile_id = f"auto-{task_type}"
diff --git a/src/backend/core/evolution/prompt_gen.py b/src/backend/core/evolution/generators/prompt_gen.py
similarity index 100%
rename from src/backend/core/evolution/prompt_gen.py
rename to src/backend/core/evolution/generators/prompt_gen.py
diff --git a/src/backend/core/evolution/skill_candidate_engine.py b/src/backend/core/evolution/generators/skill_candidate_engine.py
similarity index 96%
rename from src/backend/core/evolution/skill_candidate_engine.py
rename to src/backend/core/evolution/generators/skill_candidate_engine.py
index 2492f929..3b234a90 100644
--- a/src/backend/core/evolution/skill_candidate_engine.py
+++ b/src/backend/core/evolution/generators/skill_candidate_engine.py
@@ -18,17 +18,14 @@
"""
from __future__ import annotations
+from core.evolution.generators import skill_gen
import logging
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional
-from core.evolution.evidence_contract import (
- CapabilityEvidencePackV2,
- build_manifest,
- validate_candidate_structure,
-)
-from core.evolution.evidence_resolver import resolve_capability_pack
+from core.evolution.evidence.evidence_contract import CapabilityEvidencePackV2, build_manifest, validate_candidate_structure
+from core.evolution.evidence.evidence_resolver import resolve_capability_pack
from core.evolution.ir import (
RISK_HIGH,
EvolutionIR,
@@ -105,7 +102,6 @@ def create_skill_candidate(
if spec.write_document:
if run_async is None:
return {"status": STATUS_REJECTED, "code": "no_async_runner"}
- import core.evolution.skill_gen as skill_gen
document, draft = run_async(
skill_gen.generate_skill_document(
@@ -240,8 +236,8 @@ def _persist(
"""
from core.db.engine import SessionLocal
from core.db.models.evolution import EvolutionCandidate
- from core.evolution import lineage
- from core.evolution.release import DRAFT
+ from core.evolution.evidence import lineage
+ from core.evolution.cycle.release import DRAFT
pack_outcome = lineage.persist_evidence_pack(pack)
decision_id = lineage.record_credit_decision(
diff --git a/src/backend/core/evolution/generators/skill_drafting.py b/src/backend/core/evolution/generators/skill_drafting.py
new file mode 100644
index 00000000..89902db3
--- /dev/null
+++ b/src/backend/core/evolution/generators/skill_drafting.py
@@ -0,0 +1,185 @@
+"""Prompt, parse and validate a model-drafted skill body."""
+from __future__ import annotations
+import re
+from typing import Any, List, Sequence, Tuple
+from core.evolution.generators import skill_evidence
+from core.evolution.generators import skill_gen_constants
+
+def format_list(items: Sequence[Any], empty: str = "(无)") -> str:
+ if not items:
+ return empty
+ lines = []
+ for item in items:
+ if isinstance(item, (list, tuple)):
+ lines.append("- " + " → ".join(str(x) for x in item))
+ elif isinstance(item, dict):
+ lines.append("- " + ";".join(f"{k}: {v}" for k, v in item.items() if v))
+ else:
+ lines.append(f"- {item}")
+ return "\n".join(lines)
+
+
+def build_prompt(pack: skill_evidence.EvidencePack, violations: Sequence[str]) -> str:
+ exclusion_hint = (
+ f"证据显示本技能在以下任务类型中被推翻,必须写进不适用范围:{'、'.join(pack.excluded_task_types)}。"
+ if pack.excluded_task_types
+ else "如果证据没有显示明确的排除场景,就写明本技能只适用于「适用条件」描述的场景,其余场景请忽略本技能。"
+ )
+ retry_hint = ""
+ if violations:
+ retry_hint = (
+ "# 上一版被打回的原因(必须修正)\n"
+ + "\n".join(f"- {v}" for v in violations)
+ + "\n"
+ )
+ return skill_gen_constants.DRAFT_PROMPT.format(
+ objective_count=len(pack.objectives),
+ objectives=format_list(pack.objectives[:10]),
+ tools=format_list([f"`{t}`" for t in pack.tools]),
+ sequences=format_list(pack.successful_sequences[:6]),
+ failures=format_list(pack.failures[:6]),
+ constraints=format_list(
+ [
+ f"先 {c.get('before')} 再 {c.get('after')}"
+ + (
+ f"(在 {'、'.join(str(x) for x in (c.get('validated_in') or []))} 中验证)"
+ if c.get("validated_in")
+ else ""
+ )
+ for c in pack.ordering_constraints
+ if not (c.get("contradicted_in") or [])
+ ]
+ ),
+ procedures=format_list(
+ [
+ p.get("rule", "") + (f"(原因:{p['why']})" if p.get("why") else "")
+ for p in pack.procedures[:10]
+ ]
+ ),
+ exclusion_hint=exclusion_hint,
+ max_chars=skill_gen_constants.MAX_BODY_CHARS,
+ retry_hint=retry_hint,
+ )
+
+
+def parse_draft(raw: str) -> Tuple[str, str, str]:
+ """``(display_name, description, body)`` from the model's reply."""
+ display_name = ""
+ description = ""
+ body = ""
+ text = (raw or "").strip()
+ if text.startswith("```"):
+ lines = text.split("\n")
+ text = "\n".join(lines[1:])
+ if text.rstrip().endswith("```"):
+ text = text.rstrip()[:-3]
+
+ body_marker = re.search(r"^BODY:\s*$", text, re.MULTILINE)
+ head = text[: body_marker.start()] if body_marker else text
+ if body_marker:
+ body = text[body_marker.end() :].strip()
+
+ for line in head.splitlines():
+ if line.upper().startswith("DISPLAY_NAME:"):
+ display_name = line.split(":", 1)[1].strip()
+ elif line.upper().startswith("DESCRIPTION:"):
+ description = line.split(":", 1)[1].strip()
+ return display_name, description, body
+
+
+def validate_draft(
+ *,
+ skill_id: str,
+ display_name: str,
+ description: str,
+ body: str,
+ allowed_tools: Sequence[str],
+) -> List[str]:
+ """Every constraint stage B was told about, checked. Returns violations."""
+ violations: List[str] = []
+
+ if not body.strip():
+ return [skill_gen_constants.REJECT_EMPTY]
+ if not display_name.strip() or not description.strip():
+ violations.append(skill_gen_constants.REJECT_NO_FRONTMATTER)
+ if not skill_gen_constants.FRONTMATTER_NAME.match(skill_id or ""):
+ # The loader treats ``name`` as the id and rejects anything outside
+ # ``[a-z0-9_-]``. A skill that fails this still appears in the capability
+ # list — that metadata comes from table columns — while loading returns
+ # nothing, so it is present and inert, which is the hardest failure of
+ # all to notice.
+ violations.append(skill_gen_constants.REJECT_BAD_NAME)
+
+ if skill_gen_constants.CODE_FENCE.search(body):
+ violations.append(skill_gen_constants.REJECT_CODE_BLOCK)
+ if skill_gen_constants.URL.search(body):
+ violations.append(skill_gen_constants.REJECT_EXTERNAL_URL)
+ if len(body) > skill_gen_constants.MAX_BODY_CHARS:
+ violations.append(skill_gen_constants.REJECT_TOO_LONG)
+ if len(body) < skill_gen_constants.MIN_BODY_CHARS:
+ violations.append(skill_gen_constants.REJECT_TOO_SHORT)
+
+ if "## 不适用范围" not in body:
+ violations.append(skill_gen_constants.REJECT_NO_SCOPE)
+ else:
+ # The exclusion has to precede anything the model could copy: measured,
+ # withholding a contradicted rule was not enough, because the step list
+ # generalises on its own.
+ scope_at = body.index("## 不适用范围")
+ earlier_section = re.search(r"^## ", body[:scope_at], re.MULTILINE)
+ if earlier_section:
+ violations.append(skill_gen_constants.REJECT_NO_SCOPE)
+ if "## 常见失败与处理" not in body and "## 步骤" not in body:
+ violations.append(skill_gen_constants.REJECT_NO_CHECKPOINTS)
+
+ allowed = {str(t) for t in allowed_tools}
+ mentioned = {m for m in skill_gen_constants.TOOL_MENTION.findall(body)}
+ # Only identifiers that look like tool names are checked: prose in backticks
+ # (a field name, a value) must not be mistaken for a tool grant, and a real
+ # tool name always appears verbatim in the evidence.
+ unknown = sorted(
+ name
+ for name in mentioned
+ if name not in allowed and "_" in name and name.islower()
+ )
+ if unknown:
+ violations.append(f"{skill_gen_constants.REJECT_UNKNOWN_TOOL}: {unknown}")
+
+ return violations
+
+
+def render_skill_markdown(
+ *,
+ skill_id: str,
+ display_name: str,
+ description: str,
+ body: str,
+ support: int,
+ rationale: str = "",
+) -> str:
+ """Wrap a validated body in the frontmatter the loader requires.
+
+ ``name`` is the skill **id**, not the label — writing the human label there
+ made every materialised skill unparseable.
+ """
+ provenance = (
+ f"本技能由能力进化流水线从 {support} 条历史执行证据中沉淀而成,经机器校验后由人工批准。"
+ )
+ if rationale:
+ provenance += f"\n\n沉淀依据:{rationale}"
+ return f"""---
+name: {skill_id}
+display_name: {display_name}
+description: {description}
+---
+
+# {display_name}
+
+{description}
+
+{body.strip()}
+
+## 来源
+
+{provenance}
+"""
diff --git a/src/backend/core/evolution/generators/skill_evidence.py b/src/backend/core/evolution/generators/skill_evidence.py
new file mode 100644
index 00000000..74a2c836
--- /dev/null
+++ b/src/backend/core/evolution/generators/skill_evidence.py
@@ -0,0 +1,124 @@
+"""The evidence pack a generated skill may be written from."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Sequence
+
+@dataclass
+class EvidencePack:
+ """Everything stage B is allowed to see, and nothing else."""
+
+ skill_id: str
+ objectives: List[str] = field(default_factory=list)
+ tools: List[str] = field(default_factory=list)
+ successful_sequences: List[List[str]] = field(default_factory=list)
+ failures: List[Dict[str, Any]] = field(default_factory=list)
+ ordering_constraints: List[Dict[str, Any]] = field(default_factory=list)
+ procedures: List[Dict[str, str]] = field(default_factory=list)
+ task_types: List[str] = field(default_factory=list)
+ excluded_task_types: List[str] = field(default_factory=list)
+ support: int = 0
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "skill_id": self.skill_id,
+ "objectives": self.objectives[:10],
+ "tools": list(self.tools),
+ "successful_sequences": self.successful_sequences[:6],
+ "failures": self.failures[:6],
+ "ordering_constraints": self.ordering_constraints,
+ "procedures": self.procedures[:10],
+ "task_types": self.task_types,
+ "excluded_task_types": self.excluded_task_types,
+ "support": self.support,
+ }
+
+
+@dataclass
+class DraftResult:
+ ok: bool
+ body: str = ""
+ display_name: str = ""
+ description: str = ""
+ violations: List[str] = field(default_factory=list)
+ attempts: int = 0
+ # ``True`` when no draft could be produced because the model was unreachable
+ # — an operational state, not a rejection, and the two must not look alike
+ # in the console.
+ generator_unavailable: bool = False
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "ok": self.ok,
+ "display_name": self.display_name,
+ "description": self.description,
+ "violations": list(self.violations),
+ "attempts": self.attempts,
+ "generator_unavailable": self.generator_unavailable,
+ "body_chars": len(self.body),
+ }
+
+
+def build_evidence_pack(
+ *,
+ skill_id: str,
+ episodes: Sequence[Dict[str, Any]],
+ ordering_constraints: Sequence[Dict[str, Any]] = (),
+ procedures: Sequence[Dict[str, str]] = (),
+) -> EvidencePack:
+ """Assemble what a skill may be written from.
+
+ Failures are included deliberately. A document distilled only from successes
+ describes a happy path and says nothing about the step where things actually
+ go wrong, which is the part a reader needs most.
+ """
+ objectives: List[str] = []
+ tools: List[str] = []
+ sequences: List[List[str]] = []
+ failures: List[Dict[str, Any]] = []
+ task_types: List[str] = []
+
+ for episode in episodes:
+ objective = str(episode.get("objective") or "").strip()
+ if objective and objective not in objectives:
+ objectives.append(objective)
+ task_type = str(episode.get("task_type") or "").strip()
+ if task_type and task_type not in task_types:
+ task_types.append(task_type)
+
+ sequence = [str(t) for t in (episode.get("tool_sequence") or [])]
+ for tool in sequence:
+ if tool not in tools:
+ tools.append(tool)
+ if episode.get("verdict") == "success":
+ if sequence and sequence not in sequences:
+ sequences.append(sequence)
+ elif episode.get("verdict") == "failed":
+ failures.append(
+ {
+ "objective": objective[:160],
+ "tool_sequence": sequence[:12],
+ "failed_at": str(episode.get("failed_tool") or ""),
+ "error": str(episode.get("error") or "")[:200],
+ }
+ )
+
+ excluded = sorted(
+ {
+ str(name)
+ for rule in ordering_constraints
+ for name in (rule.get("contradicted_in") or [])
+ if str(name)
+ }
+ )
+ return EvidencePack(
+ skill_id=skill_id,
+ objectives=objectives,
+ tools=tools,
+ successful_sequences=sequences,
+ failures=failures,
+ ordering_constraints=list(ordering_constraints),
+ procedures=[dict(p) for p in procedures],
+ task_types=[t for t in task_types if t not in excluded],
+ excluded_task_types=excluded,
+ support=len(episodes),
+ )
diff --git a/src/backend/core/evolution/generators/skill_gen.py b/src/backend/core/evolution/generators/skill_gen.py
new file mode 100644
index 00000000..0da410d6
--- /dev/null
+++ b/src/backend/core/evolution/generators/skill_gen.py
@@ -0,0 +1,95 @@
+"""Generate a skill document from evidence, retrying on validation failures."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+import logging
+from core.evolution.generators import skill_drafting
+from core.evolution.generators import skill_evidence
+from core.evolution.generators import skill_gen_constants
+
+logger = logging.getLogger(__name__)
+
+async def draft_skill(pack: skill_evidence.EvidencePack) -> skill_evidence.DraftResult:
+ """Draft and validate, retrying with the specific violations.
+
+ Returns ``ok=False`` rather than raising. A skill that cannot be written to
+ the required standard must produce no candidate at all: shipping a draft
+ that failed validation would put exactly the document the constraints exist
+ to prevent in front of the model.
+ """
+ from core.memory.extractors._base import run_llm_with_prompt
+
+ violations: List[str] = []
+ for attempt in range(1, skill_gen_constants.MAX_ATTEMPTS + 1):
+ prompt = skill_drafting.build_prompt(pack, violations)
+ raw = await run_llm_with_prompt(prompt, timeout_s=skill_gen_constants.DRAFT_TIMEOUT_S, max_tokens=2400)
+ if raw is None:
+ return skill_evidence.DraftResult(
+ ok=False,
+ violations=["generator_unavailable"],
+ attempts=attempt,
+ generator_unavailable=True,
+ )
+
+ display_name, description, body = skill_drafting.parse_draft(raw)
+ violations = skill_drafting.validate_draft(
+ skill_id=pack.skill_id,
+ display_name=display_name,
+ description=description,
+ body=body,
+ allowed_tools=pack.tools,
+ )
+ if not violations:
+ return skill_evidence.DraftResult(
+ ok=True,
+ body=body,
+ display_name=display_name[:60],
+ description=description[:200].replace("\n", " "),
+ attempts=attempt,
+ )
+ logger.info(
+ "[skill-gen] draft attempt %d rejected for %s: %s",
+ attempt, pack.skill_id, violations,
+ )
+
+ return skill_evidence.DraftResult(ok=False, violations=violations, attempts=skill_gen_constants.MAX_ATTEMPTS)
+
+
+async def generate_skill_document(
+ *,
+ skill_id: str,
+ episodes: Sequence[Dict[str, Any]],
+ ordering_constraints: Sequence[Dict[str, Any]] = (),
+ procedures: Sequence[Dict[str, str]] = (),
+ rationale: str = "",
+) -> Tuple[Optional[Dict[str, str]], skill_evidence.DraftResult]:
+ """All three stages. Returns ``(document, result)``; document is ``None`` on failure."""
+ pack = skill_evidence.build_evidence_pack(
+ skill_id=skill_id,
+ episodes=episodes,
+ ordering_constraints=ordering_constraints,
+ procedures=procedures,
+ )
+ if not pack.tools and not pack.procedures:
+ return None, skill_evidence.DraftResult(ok=False, violations=["no_evidence_to_write_from"])
+
+ result = await draft_skill(pack)
+ if not result.ok:
+ return None, result
+
+ return (
+ {
+ "skill_id": skill_id,
+ "display_name": result.display_name,
+ "description": result.description,
+ "content": skill_drafting.render_skill_markdown(
+ skill_id=skill_id,
+ display_name=result.display_name,
+ description=result.description,
+ body=result.body,
+ support=pack.support,
+ rationale=rationale,
+ ),
+ "allowed_tools": list(pack.tools),
+ },
+ result,
+ )
diff --git a/src/backend/core/evolution/generators/skill_gen_constants.py b/src/backend/core/evolution/generators/skill_gen_constants.py
new file mode 100644
index 00000000..c8bc9b4d
--- /dev/null
+++ b/src/backend/core/evolution/generators/skill_gen_constants.py
@@ -0,0 +1,109 @@
+"""Limits and patterns used to validate generated skills."""
+from __future__ import annotations
+import re
+
+MAX_ATTEMPTS = 2
+
+
+MAX_BODY_CHARS = 4000
+
+
+MIN_BODY_CHARS = 120
+
+
+DRAFT_TIMEOUT_S = 60
+
+
+CODE_FENCE = re.compile(r"```(?:\w+)?\s*\n", re.MULTILINE)
+
+
+URL = re.compile(r"https?://", re.IGNORECASE)
+
+
+FRONTMATTER_NAME = re.compile(r"^[a-z0-9_-]+$")
+
+
+TOOL_MENTION = re.compile(r"`([A-Za-z_][A-Za-z0-9_.-]*)`")
+
+
+REJECT_NO_FRONTMATTER = "missing_frontmatter"
+
+
+REJECT_BAD_NAME = "invalid_skill_id"
+
+
+REJECT_UNKNOWN_TOOL = "tool_not_in_evidence"
+
+
+REJECT_CODE_BLOCK = "contains_code_block"
+
+
+REJECT_EXTERNAL_URL = "contains_external_url"
+
+
+REJECT_TOO_LONG = "body_too_long"
+
+
+REJECT_TOO_SHORT = "body_too_short"
+
+
+REJECT_NO_SCOPE = "missing_scope_section"
+
+
+REJECT_NO_CHECKPOINTS = "missing_checkpoints"
+
+
+REJECT_EMPTY = "empty_draft"
+
+
+DRAFT_PROMPT = """你在为一个 AI 助手撰写一份「技能文档」(SKILL.md)。这份文档会被模型在遇到同类任务时读取并遵照执行。
+
+# 你拿到的证据
+
+## 用户实际提出过的请求({objective_count} 条)
+{objectives}
+
+## 成功执行时用过的工具
+{tools}
+
+## 成功的调用顺序
+{sequences}
+
+## 失败的执行(这些是最宝贵的部分,说明坑在哪)
+{failures}
+
+## 已验证的顺序约束
+{constraints}
+
+## 这些任务里沉淀下来的做法/口径
+{procedures}
+
+# 硬性要求(违反任何一条,文档会被机器校验打回)
+
+1. **只能引用上面「成功执行时用过的工具」里出现过的工具名**,一个都不能新增、不能猜。
+2. **开头第一段必须是「## 不适用范围」**,写明本技能不适用于哪些场景。{exclusion_hint}
+3. 必须有「## 适用条件」,写清楚**遇到什么样的请求才该用本技能**——要具体到能和别的任务区分开,不能写"当需要用到这些工具时"这种同义反复。
+4. 必须有「## 步骤」,每一步都要写清楚:做什么、**关键参数怎么给**、**怎么判断这一步做对了**(检查点)。
+5. 必须有「## 常见失败与处理」,基于上面的失败证据写。
+6. **禁止**出现任何代码块(``` 包裹的内容)、禁止出现外部链接(http/https)。
+7. 全文不超过 {max_chars} 字符。
+8. 用中文撰写,语气是给同事写操作手册,不要写"本技能由AI生成"之类的话。
+
+{retry_hint}
+
+# 输出格式
+
+严格按下面结构输出,不要加任何额外说明,不要用代码块包裹整篇文档:
+
+DISPLAY_NAME: <一行技能名,≤20字,说明这个技能干什么>
+DESCRIPTION: <一行描述,≤80字,说明什么时候该用它——模型只看这一行决定要不要打开本技能>
+BODY:
+## 不适用范围
+...
+## 适用条件
+...
+## 步骤
+...
+## 常见失败与处理
+...
+"""
diff --git a/src/backend/core/evolution/loop.py b/src/backend/core/evolution/loop.py
deleted file mode 100644
index 07bbaa11..00000000
--- a/src/backend/core/evolution/loop.py
+++ /dev/null
@@ -1,1576 +0,0 @@
-"""The evolution loop, wired end to end (GCE integration).
-
-Everything up to here has been a component. This module is the loop itself:
-
- Episodes → patterns → credit → promotion → IR → invariants → candidate →
- release (shadow → canary → active) → the next run binds the new version
-
-It runs off the response path, on the shared worker base, under the same cost
-budget as distillation. It cannot activate anything on its own beyond the low
-risk tier: everything else stops at a candidate awaiting review, which is the
-whole point of calling this *governed* co-evolution.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-import uuid
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
-
-from core.db.engine import SessionLocal
-from core.db.models.evolution import (
- EvolutionCandidate,
- EvolutionEpisode,
- EvolutionRelease,
- EvolutionTraceEvent,
-)
-from core.evolution import promotion as P
-from core.evolution.credit import (
- T_MEMORY,
- T_ORCHESTRATION,
- T_SKILL,
- CreditFeatures,
- aggregate_finding,
- assign_credit,
- confidence_from_evidence,
-)
-from core.evolution.ir import (
- RISK_HIGH,
- RISK_LOW,
- RISK_MEDIUM,
- EvolutionIR,
- InvariantViolation,
- Rollback,
- new_candidate_id,
- validate_ir,
-)
-from core.evolution.release import DRAFT
-from core.evolution.user_settings import PRIVACY_PRIVATE
-
-logger = logging.getLogger(__name__)
-
-
-def _view_for(episode, events) -> Dict[str, Any]:
- """One Episode plus its events, flattened into what the engines read.
-
- Three fields here did not exist and each blocked an engine outright:
-
- ``memory_refs`` now carries the **stable content ref**, not the raw content
- hash. The two look alike and are not: the ref is what the overlay keys on,
- so an op proposed against a hash could never be applied to anything.
-
- ``skills_opened`` is what the model chose to read, as opposed to
- ``skill_sequence``, which is what it was offered. Every decrement decision
- depends on the difference.
-
- ``selection_degraded`` marks turns where ranking could not run and
- everything loaded. Counting those as selections is what made every skill's
- offer count identical, and therefore useless.
- """
- from core.evolution.events import (
- EV_MEMORY_RETRIEVED,
- EV_SKILL_OPENED,
- EV_SKILL_SELECTED,
- EV_TOOL_CALLED,
- )
- from core.memory.ref_shadow import build_ref_id
- from core.memory.retrieval_types import LAYER_FACT, LAYER_GRAPH
-
- tools: List[str] = []
- skills: List[str] = []
- memory_refs: List[str] = []
- graph_refs: List[str] = []
- graph_relations: List[Dict[str, Any]] = []
- skills_opened: List[Dict[str, Any]] = []
- selection_degraded = False
- error_count = 0
-
- user_id = episode.user_id or ""
- for event in events:
- payload = event.payload or {}
- if event.event_type == EV_TOOL_CALLED:
- name = payload.get("tool_name")
- if name:
- tools.append(str(name))
- if str(payload.get("status") or "") == "error":
- error_count += 1
- elif event.event_type == EV_SKILL_SELECTED:
- if payload.get("degraded"):
- selection_degraded = True
- skills.extend(str(s) for s in (payload.get("selected") or []))
- elif event.event_type == EV_SKILL_OPENED:
- skills_opened.append(
- {
- "skill_id": str(payload.get("skill_id") or ""),
- "tools_after_open": [str(t) for t in (payload.get("tools_after_open") or [])],
- }
- )
- elif event.event_type == EV_MEMORY_RETRIEVED:
- for item in payload.get("items") or []:
- content_hash = str(item.get("content_hash") or "")
- if not content_hash:
- continue
- memory_refs.append(
- build_ref_id(
- layer=str(item.get("layer") or LAYER_FACT),
- user_id=user_id,
- workspace_id=str(item.get("workspace_id") or "default"),
- content_hash=content_hash,
- )
- )
- # L3 is deliberately tracked separately from L2. A graph relation
- # is declarative context, not a procedure, and must never enter the
- # memory→skill compiler as though it were an instruction.
- for relation in payload.get("graph") or []:
- content_hash = str(relation.get("content_hash") or "")
- if not content_hash:
- continue
- graph_refs.append(
- build_ref_id(
- layer=LAYER_GRAPH,
- user_id=user_id,
- workspace_id=str(relation.get("workspace_id") or "default"),
- content_hash=content_hash,
- )
- )
- # The full reference, kept for the evidence resolver: a pack's
- # L3 entry must carry the store's stable relation_id so an
- # activated skill's graph evidence can be resolved back to the
- # exact Neo4j edge — a bare ref id cannot do that.
- graph_relations.append(
- {
- "relation_id": str(relation.get("relation_id") or ""),
- "content_hash": content_hash,
- "predicate": str(relation.get("predicate") or ""),
- "confidence": relation.get("confidence") or 0.0,
- "workspace_id": str(relation.get("workspace_id") or "default"),
- }
- )
-
- outcome = episode.outcome or {}
- return {
- "episode_id": episode.episode_id,
- "chat_id": episode.chat_id or "",
- "user_id": user_id,
- "tenant_id": episode.tenant_id or "default",
- "verdict": outcome.get("verdict")
- or ("success" if (episode.quality_score or 0) >= 0.65 else "unknown"),
- "tool_sequence": tools,
- "skill_sequence": skills,
- "skills_opened": skills_opened,
- "selection_degraded": selection_degraded,
- "memory_refs": memory_refs,
- "graph_refs": graph_refs,
- "graph_relations": graph_relations,
- "task_type": episode.task_type or "chat",
- "backfilled": bool(episode.backfilled),
- # Repeated tool errors are the observable form of "this run was stuck",
- # which sub-agent derivation needs and nothing else recorded.
- "stalled": error_count >= 3,
- # What the user actually asked. Clustering only ever saw tool sequences
- # before this, so "the same question keeps coming back" was not a
- # detectable event. Sanitised at write time, so this is the preview
- # rather than the raw text.
- "objective": episode.objective_preview or "",
- "objective_hash": episode.objective_hash or "",
- "attributed_to": str(outcome.get("attributed_to") or ""),
- # Needed to place an episode before or after an activation. Without it
- # negative-transfer detection has no way to split the corpus and returns
- # nothing on every corpus — wired, running, and structurally incapable
- # of finding anything.
- "created_at": episode.created_at,
- }
-
-
-def load_episode_views(*, limit: int = 500, since_days: int = 30) -> List[Dict[str, Any]]:
- """Flatten Episodes plus their trace events into the shape pattern mining wants.
-
- Backfilled Episodes are included on purpose: they cannot be replayed, but
- they are perfectly good for discovering what recurs — which is exactly why
- the backfill was worth doing.
- """
- views: List[Dict[str, Any]] = []
- try:
- with SessionLocal() as db:
- # The single enforcement point for per-user opt-out. Episodes from
- # users who declined to contribute are still recorded (they power
- # that user's own card and history) but never feed cross-user
- # pattern mining. Filtering here rather than at each call site is
- # what makes the promise checkable.
- episodes = (
- db.query(EvolutionEpisode)
- .filter(EvolutionEpisode.privacy_class != PRIVACY_PRIVATE)
- .order_by(EvolutionEpisode.created_at.desc())
- .limit(limit)
- .all()
- )
- if not episodes:
- return []
-
- ids = [e.episode_id for e in episodes]
- events = (
- db.query(EvolutionTraceEvent)
- .filter(EvolutionTraceEvent.episode_id.in_(ids))
- .order_by(EvolutionTraceEvent.seq.asc())
- .all()
- )
- by_episode: Dict[str, List[EvolutionTraceEvent]] = {}
- for event in events:
- by_episode.setdefault(event.episode_id, []).append(event)
-
- for episode in episodes:
- views.append(_view_for(episode, by_episode.get(episode.episode_id, [])))
- except Exception as exc:
- logger.warning("[evolution-loop] loading episodes failed: %s", exc)
- return []
- return views
-
-
-def _features_for(pattern: P.Pattern) -> CreditFeatures:
- """Translate a discovered pattern into attribution features.
-
- Success-subsequence tool patterns no longer come through here: they are an
- aggregate observation over successful runs, and inventing a ``failed``
- verdict to route them through the failure attributor is the exact move
- :func:`core.evolution.credit.aggregate_finding` exists to replace.
- """
- if pattern.kind == P.PATTERN_REPEATED_FAILURE:
- return CreditFeatures(
- outcome_verdict="failed",
- outcome_confidence=0.8,
- loop_stagnation=True,
- no_diff_iterations=2,
- )
- return CreditFeatures(outcome_verdict="unknown", outcome_confidence=0.3)
-
-
-def _features_for_ordering_rules(constraints: List[Any]) -> CreditFeatures:
- """Attribution features for "a stable ordering keeps being re-derived".
-
- The mapping, stated so it can be argued with:
-
- * ``repeated_tool_subsequence`` — a pairwise ordering recurring across
- different tool sets is a repeated successful subsequence that is
- re-planned on every occurrence, which is what the feature means.
- * ``outcome_verdict="failed"`` — the runs this would fix are the ones that
- ordered the calls wrongly; the successful ones are the evidence, not the
- target.
-
- ``step_order_suspect`` is deliberately *not* set even though ordering is the
- subject. That feature says "this pipeline's step order looks wrong", a
- workflow-level claim, and asserting it ties workflow with skill at 0.65 —
- the assigner then correctly reports ``unidentifiable`` and nothing is
- produced. The evidence here does separate them: an ordering that holds
- across several different tool sets and task types is reusable knowledge, not
- one fixed pipeline, and "the workflow is fixed" is the reading it rules out.
-
- Confidence sits just above the ``MIN_CONFIDENCE`` floor on purpose: with
- support and context breadth at their minimum it lands below the floor and
- resolves to No-Update, so thin evidence cannot mint a permanent asset.
- """
- support = max((int(getattr(c, "support", 0) or 0) for c in constraints), default=0)
- contexts = max((int(getattr(c, "contexts", 0) or 0) for c in constraints), default=0)
- return CreditFeatures(
- outcome_verdict="failed",
- outcome_confidence=min(0.9, 0.6 + 0.05 * support + 0.05 * contexts),
- repeated_tool_subsequence=True,
- )
-
-
-def _uncovered_constraints(constraints: List[Any], covered_tool_sets: List[Any]) -> List[Any]:
- """Rules that no sequence candidate would carry into the runtime.
-
- A rule attached to a materialised sequence already reaches the agent through
- that skill. The ones worth a candidate of their own are the rest — and on
- real traffic that is most of them, because a rule earns its scope by
- appearing in *several* tool-set shapes and no single sequence covers them
- all.
- """
- out: List[Any] = []
- for constraint in constraints:
- pair = {constraint.before, constraint.after}
- if any(pair <= set(tools) for tools in covered_tool_sets):
- continue
- out.append(constraint)
- return out
-
-
-def _persist_candidate(
- ir: EvolutionIR,
- *,
- credit: Dict[str, Any],
- proposer: str,
- evidence_refs: List[str],
-) -> Optional[str]:
- """Store a candidate, collapsing duplicates rather than failing on them."""
- from core.evolution import lineage
-
- # Attribution outlives the candidate that happened to carry it: every
- # decision that reaches persistence gets its own ledger row, and the stored
- # copy inside the candidate becomes a cache keyed by ``decision_id``.
- decision_id = lineage.record_credit_decision(
- credit, episode_id=(evidence_refs[0] if evidence_refs else "")
- )
- credit = {**credit, "decision_id": decision_id}
-
- checksum = ir.change_checksum()
- try:
- with SessionLocal() as db:
- existing = (
- db.query(EvolutionCandidate)
- .filter(
- EvolutionCandidate.target_kind == ir.target_kind,
- EvolutionCandidate.target_asset_id == ir.target_asset_id,
- EvolutionCandidate.base_version_id == (ir.base_version or None),
- EvolutionCandidate.change_checksum == checksum,
- )
- .first()
- )
- if existing is not None:
- # Same edit proposed again from fresh evidence: one review item,
- # not two.
- return existing.candidate_id
-
- candidate_id = new_candidate_id()
- db.add(
- EvolutionCandidate(
- candidate_id=candidate_id,
- target_kind=ir.target_kind,
- target_asset_id=ir.target_asset_id,
- base_version_id=ir.base_version or None,
- operation=ir.operation,
- ir=ir.to_dict(),
- hypothesis=ir.hypothesis,
- change_checksum=checksum,
- evidence_refs=evidence_refs,
- credit_decision=credit,
- risk_tier=ir.risk_tier,
- scope=ir.scope,
- status=DRAFT,
- proposer=proposer,
- )
- )
- db.commit()
- except Exception as exc:
- logger.warning("[evolution-loop] persisting candidate failed: %s", exc)
- return None
-
- # No backfill onto the turn cards. A skill candidate is a batch product
- # reviewed in the console; pushing it back into a conversation from days ago
- # put capability claims on a card whose only subject is what that turn
- # remembered. The candidate's own evidence_refs still record which episodes
- # produced it, which is where that link belongs.
- return candidate_id
-
-
-def cross_user_mining_available() -> bool:
- """Whether fleet-wide mining may run in this edition.
-
- Personal evolution runs everywhere — it needs no aggregation across people
- and its output reaches only its owner. Mining *across* users and releasing
- to everyone is a governance surface, so it is enterprise-gated. The
- community edition keeps a complete personal evolution path rather than a
- degraded one.
- """
- try:
- # Resolved by name rather than a literal enterprise-tree import, so the
- # community build's boundary scan does not read this file as reaching
- # across the edition boundary. The behaviour is unchanged: the module is
- # physically absent in CE and the resulting ImportError still lands in
- # the handler below.
- from importlib import import_module
-
- features = import_module("edition_ee.licensing.features")
- manager = import_module("edition_ee.licensing.manager")
-
- return bool(manager.license_manager.has(features.Feature.EVOLUTION_CONTROL))
- except Exception:
- # The community tree physically excludes edition_ee, so an import error
- # here is the expected CE signal rather than a fault: personal only.
- return False
-
-
-def _run_async(coro) -> Any:
- """Run one coroutine from the cycle's synchronous body.
-
- The cycle is a background job and must run in a worker thread. Calling it
- from inside a running loop is a programming error and is raised as one:
- degrading to "skip the generators" would produce a cycle that silently
- stopped writing skills, which is the failure mode this whole rework exists
- to eliminate.
- """
- import asyncio
-
- try:
- asyncio.get_running_loop()
- except RuntimeError:
- return asyncio.run(coro)
- coro.close()
- raise RuntimeError(
- "run_evolution_cycle must not be called from a running event loop; "
- "dispatch it with run_in_executor"
- )
-
-
-def _memory_scan_findings(views: List[Dict[str, Any]]) -> List[Any]:
- """Scan every contributing user's store.
-
- Per user, because the store is per user: near-duplicates and contradictions
- only mean anything within one person's memories, and a cross-user scan would
- both mean nothing and read what it has no business reading.
- """
- from core.evolution.memory_scan import scan_user_memories
-
- retrieved: Dict[str, Dict[str, int]] = {}
- for view in views:
- user_id = str(view.get("user_id") or "")
- if not user_id:
- continue
- bucket = retrieved.setdefault(user_id, {})
- for ref in view.get("memory_refs") or []:
- bucket[str(ref)] = bucket.get(str(ref), 0) + 1
-
- findings: List[Any] = []
- for user_id, refs in retrieved.items():
- try:
- _, user_findings = _run_async(scan_user_memories(user_id, retrieved_refs=refs))
- except Exception as exc: # noqa: BLE001
- logger.warning("[evolution-loop] memory scan failed for %s: %s", user_id, exc)
- continue
- findings.extend(user_findings)
- return findings
-
-
-def _memory_candidates(
- extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool
-) -> List[Dict[str, Any]]:
- """One memory candidate per user, carrying that user's operations.
-
- Grouped per user rather than per operation because the approval question is
- "may the system reorganise *my* memories", and answering it forty times for
- forty entries is not review, it is attrition.
- """
- memory = extras.get("memory") or {}
- ops = memory.get("ops") or []
- if not ops:
- return []
-
- owner_by_ref: Dict[str, str] = {}
- for view in views:
- user_id = str(view.get("user_id") or "")
- for ref in view.get("memory_refs") or []:
- owner_by_ref.setdefault(str(ref), user_id)
-
- by_user: Dict[str, List[Any]] = {}
- for op in ops:
- owner = owner_by_ref.get(op.memory_ref, "")
- if not owner:
- # An operation whose subject cannot be identified must not be
- # proposed: applying it would need a user id, and guessing one is
- # how a change lands on the wrong person's memories.
- continue
- by_user.setdefault(owner, []).append(op)
-
- created: List[Dict[str, Any]] = []
- for user_id, user_ops in by_user.items():
- # Duplicates, contradictions and stale entries are properties of the
- # store, found by scanning it — not a conflict with any one request.
- decision = aggregate_finding(
- target=T_MEMORY,
- explanation=";".join(op.reason for op in user_ops[:3]),
- confidence=confidence_from_evidence(len(user_ops), saturates_at=8),
- evidence_count=len(user_ops),
- )
- if not decision.may_produce_candidate:
- continue
-
- asset_id = "mem-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12]
- ir = EvolutionIR(
- target_kind="memory",
- target_asset_id=asset_id,
- base_version="current",
- operation="merge" if any(op.operation == "merge" for op in user_ops) else "reweight",
- hypothesis=(
- f"记忆库有 {len(user_ops)} 处需要重组:"
- + ";".join(op.reason for op in user_ops[:3])
- ),
- changes=[
- {
- "user_id": user_id,
- "operations": [op.to_dict() for op in user_ops],
- }
- ],
- evidence_refs=[op.memory_ref for op in user_ops][:20],
- # Changing what a person's agent remembers is never low risk, and
- # the scope is that person alone.
- risk_tier=RISK_MEDIUM,
- scope={"level": "user", "user_id": user_id},
- rollback=Rollback(version_id="current", triggers=["user_disable"]),
- evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"},
- )
- try:
- validate_ir(
- ir, credit_decision=decision.to_dict(), allowed_scopes=["user", "workspace"]
- )
- except InvariantViolation as violation:
- logger.info("[evolution-loop] memory candidate rejected: %s", violation.code)
- continue
- if dry_run:
- created.append({"dry_run": True, "kind": "memory", "user_id": user_id})
- continue
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="memory_engine",
- evidence_refs=[op.memory_ref for op in user_ops][:20],
- )
- if candidate_id:
- created.append(
- {
- "candidate_id": candidate_id,
- "kind": "memory",
- "user_id": user_id,
- "operations": len(user_ops),
- }
- )
- return created
-
-
-def _intent_skill_candidates(
- extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool
-) -> List[Dict[str, Any]]:
- """Recurring intents, compiled through the unified skill candidate engine.
-
- The intent finding says a capability is missing; it does not say what the
- capability is. The engine resolves the cluster's evidence into an immutable
- pack, derives the deterministic structure (tools, boundaries, dependencies),
- has the model write only the document, and validates the layered contract —
- the same chain the tool-sequence path uses, so identical evidence can no
- longer produce structurally different skills depending on which engine
- noticed it first.
- """
- from core.evolution.skill_candidate_engine import (
- STATUS_CREATED,
- STATUS_DRY_RUN,
- STATUS_NOT_WRITTEN,
- STATUS_REJECTED,
- SkillCandidateSpec,
- create_skill_candidate,
- )
-
- findings = [
- f
- for f in (extras.get("intent") or {}).get("findings", [])
- if f.get("action") == "skill_candidate"
- ]
- if not findings:
- return []
-
- by_episode = {str(v.get("episode_id") or ""): v for v in views}
- created: List[Dict[str, Any]] = []
-
- for finding in findings:
- cluster = finding.get("cluster") or {}
- episode_ids = [str(e) for e in (cluster.get("episode_ids") or [])]
- episodes = [by_episode[e] for e in episode_ids if e in by_episode]
- if not episodes:
- continue
-
- # The cluster mostly *succeeded* — that is a precondition for
- # distilling it. Declaring the turns failed to satisfy the failure
- # attributor would contradict the very check that let this through.
- decision = aggregate_finding(
- target=T_SKILL,
- explanation=str(finding.get("reason") or ""),
- confidence=confidence_from_evidence(len(episode_ids), saturates_at=10),
- evidence_count=len(episode_ids),
- )
- if not decision.may_produce_candidate:
- continue
-
- asset_id = str(finding.get("asset_id") or "")
- spec = SkillCandidateSpec(
- asset_id=asset_id,
- proposer="intent_engine",
- hypothesis=str(finding.get("reason") or ""),
- views=episodes,
- intent_cluster=cluster,
- procedures=_procedures_for_cluster(cluster, views),
- operation="new",
- base_version="v0",
- risk_tier=RISK_HIGH,
- scope={"level": "workspace"},
- tenant_id=str(episodes[0].get("tenant_id") or "default"),
- write_document=True,
- )
- outcome = create_skill_candidate(
- spec, credit=decision.to_dict(), dry_run=dry_run, run_async=_run_async
- )
- status = outcome.get("status")
- if status == STATUS_NOT_WRITTEN:
- created.append(
- {
- "kind": "recurring_intent",
- "asset_id": asset_id,
- "written": False,
- "why_not": outcome.get("why_not") or {},
- }
- )
- elif status == STATUS_REJECTED:
- logger.info(
- "[evolution-loop] intent candidate rejected: %s %s",
- outcome.get("code"),
- outcome.get("violations") or "",
- )
- elif status == STATUS_DRY_RUN:
- created.append({"dry_run": True, "kind": "recurring_intent", "asset_id": asset_id})
- elif status == STATUS_CREATED:
- created.append(
- {
- "candidate_id": outcome["candidate_id"],
- "kind": "recurring_intent",
- "asset_id": asset_id,
- "written": True,
- "attempts": outcome.get("attempts", 0),
- "evidence_pack_id": outcome.get("pack_id", ""),
- }
- )
- return created
-
-
-def _procedures_for_cluster(
- cluster: Dict[str, Any], views: List[Dict[str, Any]]
-) -> List[Dict[str, str]]:
- """The procedural memories retrieved during this intent's episodes.
-
- These are what makes the resulting document a *compilation* of memory rather
- than a restatement of a tool list — and their refs are what gets demoted
- once the skill is live.
- """
- from core.evolution.memory_scan import procedural_records, to_records
- from core.memory.ref_shadow import build_ref_id
- from core.memory.retrieval_types import LAYER_FACT
-
- wanted = {str(ref) for ref in (cluster.get("memory_refs") or [])}
- if not wanted:
- return []
-
- episode_ids = {str(e) for e in (cluster.get("episode_ids") or [])}
- users = {
- str(v.get("user_id") or "")
- for v in views
- if str(v.get("episode_id") or "") in episode_ids and v.get("user_id")
- }
-
- procedures: List[Dict[str, str]] = []
- for user_id in users:
- try:
- from core.memory.service import get_all_memories
-
- raw = _run_async(get_all_memories(user_id, top_k=500))
- except Exception as exc: # noqa: BLE001
- logger.debug("[evolution-loop] procedure lookup failed for %s: %s", user_id, exc)
- continue
- for record in procedural_records(to_records(raw)):
- ref = build_ref_id(
- layer=LAYER_FACT,
- user_id=user_id,
- workspace_id=record.workspace_id,
- content_hash=record.content_hash,
- )
- if ref in wanted:
- procedures.append(
- {
- "ref": ref,
- # Carried because a fleet-wide skill can be compiled from
- # several people's memories, and demotion is per-store: a
- # bare list of refs has no owner to apply them against, so
- # the reverse edge would silently do nothing for exactly
- # the candidates that need it most.
- "user_id": user_id,
- "rule": record.content,
- "why": record.why,
- "applies_to": record.applies_to,
- }
- )
- return procedures
-
-
-def _decremental_candidates(extras: Dict[str, Any], *, dry_run: bool) -> List[Dict[str, Any]]:
- """Retirement and merge findings, as candidates rather than as a list.
-
- Retiring goes through the same approval path as creating. Removing a
- capability produces no error — only a slow decline — so it is precisely the
- direction that must not be automatic.
- """
- from core.evolution.promotion import ACTION_REWRITE
-
- decremental = extras.get("decremental") or {}
- created: List[Dict[str, Any]] = []
-
- for item in decremental.get("retirements") or []:
- asset_id = str(item.get("asset_id") or "")
- if not asset_id:
- continue
- action = str(item.get("action") or "")
- if action == ACTION_REWRITE:
- # "Opened and ignored" is a content problem. Proposing retirement
- # would remove a capability the model wanted; the fix is a rewrite,
- # and that is a different candidate written from failure evidence.
- created.append({"kind": "needs_rewrite", "asset_id": asset_id, **item})
- continue
-
- decision = aggregate_finding(
- target=T_SKILL,
- explanation=str(item.get("explanation") or item.get("reason") or ""),
- confidence=confidence_from_evidence(int(item.get("offers", 0) or 0), saturates_at=20),
- evidence_count=int(item.get("offers", 0) or 0),
- )
- if not decision.may_produce_candidate:
- continue
-
- ir = EvolutionIR(
- target_kind="skill",
- target_asset_id=asset_id,
- base_version="current",
- operation="deprecate",
- hypothesis=str(item.get("explanation") or item.get("reason") or ""),
- changes=[
- {
- "retire_reason": item.get("reason"),
- "stats": {
- k: v for k, v in item.items() if k not in ("asset_id", "explanation")
- },
- }
- ],
- evidence_refs=[asset_id],
- risk_tier=RISK_MEDIUM,
- scope={"level": "workspace"},
- rollback=Rollback(version_id="current", triggers=["manual"]),
- evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
- )
- try:
- validate_ir(ir, credit_decision=decision.to_dict())
- except InvariantViolation as violation:
- logger.info("[evolution-loop] retirement rejected: %s", violation.code)
- continue
- if dry_run:
- created.append({"dry_run": True, "kind": "retirement", "asset_id": asset_id})
- continue
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="decremental_engine",
- evidence_refs=[asset_id],
- )
- if candidate_id:
- created.append(
- {
- "candidate_id": candidate_id,
- "kind": "retirement",
- "asset_id": asset_id,
- "reason": item.get("reason"),
- }
- )
- return created
-
-
-def _orchestration_candidates(
- views: List[Dict[str, Any]], *, dry_run: bool
-) -> List[Dict[str, Any]]:
- """Profile and sub-agent candidates.
-
- The tool authority every candidate here is measured against is the *current*
- grant, read once. A profile may only ever narrow it, and a derived sub-agent
- may only ever receive a subset — both enforced by the IR gate rather than by
- the generator, so the check holds even for a candidate that arrived by some
- other route.
- """
- from core.config.catalog import get_enabled_ids
- from core.evolution.agent_profile import TARGET_KIND, builtin_profile, validate_profile
- from core.evolution.orchestration_gen import (
- generate_orchestration_proposals,
- profile_from_proposals,
- )
-
- base_tools = [str(t) for t in get_enabled_ids("tools")] + [
- str(t) for t in get_enabled_ids("mcp")
- ]
- offered_tools = {str(view.get("task_type") or "chat"): base_tools for view in views}
- proposals, subagents = generate_orchestration_proposals(
- views, offered_tools=offered_tools, parent_tools=base_tools
- )
- created: List[Dict[str, Any]] = []
-
- task_types = sorted({p.task_type for p in proposals})
- for task_type in task_types:
- task_proposals = [p for p in proposals if p.task_type == task_type]
- profile = profile_from_proposals(
- task_proposals, task_type=task_type, base=builtin_profile()
- )
- ok, problems = validate_profile(profile, base_tools=base_tools)
- if not ok:
- logger.info("[evolution-loop] profile %s rejected: %s", profile.profile_id, problems)
- continue
-
- evidence = [ref for p in task_proposals for ref in p.evidence_refs][:20]
- # An observation across the corpus, not a failure attributed to one run.
- # Inventing failure features to make the attributor return "workflow"
- # is what put "过程停滞:重复动作、无产出增量" in front of reviewers for a
- # candidate whose actual reason was "eight tools were never used".
- decision = aggregate_finding(
- target=T_ORCHESTRATION,
- explanation=";".join(p.rationale for p in task_proposals),
- confidence=confidence_from_evidence(sum(p.support for p in task_proposals)),
- evidence_count=len(evidence),
- )
- if not decision.may_produce_candidate:
- continue
- ir = EvolutionIR(
- target_kind=TARGET_KIND,
- target_asset_id=profile.profile_id,
- base_version="builtin",
- operation="new",
- hypothesis=";".join(p.rationale for p in task_proposals),
- changes=[
- {
- "profile": profile.to_dict(),
- "tool_allowlist": profile.tool_allowlist or [],
- }
- ],
- evidence_refs=evidence,
- requested_tools=list(profile.tool_allowlist or []),
- risk_tier=RISK_MEDIUM,
- scope={"level": "workspace"},
- rollback=Rollback(version_id="builtin", triggers=["success -3pp"]),
- evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
- )
- try:
- validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools)
- except InvariantViolation as violation:
- logger.info("[evolution-loop] profile candidate rejected: %s", violation.code)
- continue
- if dry_run:
- created.append({"dry_run": True, "kind": "agent_profile", "task_type": task_type})
- continue
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="orchestration_engine",
- evidence_refs=evidence,
- )
- if candidate_id:
- created.append(
- {"candidate_id": candidate_id, "kind": "agent_profile", "task_type": task_type}
- )
-
- for subagent in subagents:
- decision = aggregate_finding(
- target=T_ORCHESTRATION,
- explanation=subagent.rationale,
- confidence=confidence_from_evidence(subagent.support),
- evidence_count=len(subagent.evidence_refs),
- )
- if not decision.may_produce_candidate:
- continue
- payload = subagent.to_dict()
- ir = EvolutionIR(
- target_kind="subagent",
- target_asset_id=subagent.agent_id,
- base_version="v0",
- operation="new",
- hypothesis=subagent.rationale,
- changes=[payload],
- evidence_refs=subagent.evidence_refs,
- requested_tools=list(subagent.tools_allowlist),
- # Creating an agent is the widest-reaching change this system can
- # propose, and the route into it changes the parent's prompt.
- risk_tier=RISK_HIGH,
- scope={"level": "workspace"},
- rollback=Rollback(version_id="v0", triggers=["manual"]),
- evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
- )
- try:
- validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools)
- except InvariantViolation as violation:
- logger.info("[evolution-loop] subagent candidate rejected: %s", violation.code)
- continue
- if dry_run:
- created.append({"dry_run": True, "kind": "subagent", "name": subagent.name})
- continue
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="orchestration_engine",
- evidence_refs=subagent.evidence_refs,
- )
- if candidate_id:
- created.append(
- {"candidate_id": candidate_id, "kind": "subagent", "name": subagent.name}
- )
- return created
-
-
-def _prompt_candidates(views: List[Dict[str, Any]], *, dry_run: bool) -> List[Dict[str, Any]]:
- """Prompt fragments, for the failures attribution already blames on the prompt."""
- from core.evolution.prompt_gen import generate_prompt_candidate
-
- payload, evidence, draft = _run_async(
- generate_prompt_candidate(
- views,
- cause="归因判定该失败源于提示词未声明相关约束",
- existing_fragments=_active_prompt_fragments(),
- )
- )
- if payload is None or evidence is None:
- return (
- []
- if draft.violations == ["insufficient_evidence"]
- else [{"kind": "prompt", "written": False, "why_not": draft.to_dict()}]
- )
-
- decision = assign_credit(
- CreditFeatures(
- outcome_verdict="failed",
- outcome_confidence=0.85,
- prompt_missing_priority_rule=True,
- )
- )
- if not decision.may_produce_candidate:
- return []
-
- asset_id = "auto-prompt-" + hashlib.sha256(payload["fragment"].encode("utf-8")).hexdigest()[:12]
- ir = EvolutionIR(
- target_kind="prompt",
- target_asset_id=asset_id,
- base_version="active",
- operation="patch",
- hypothesis=payload["cause"],
- changes=[{"fragment": payload["fragment"], "task_types": payload["task_types"]}],
- evidence_refs=evidence.episode_ids[:20],
- # A prompt fragment applies to every request in scope; nothing this
- # system produces has a wider reach.
- risk_tier=RISK_HIGH,
- scope={"level": "workspace"},
- rollback=Rollback(version_id="active", triggers=["success -3pp"]),
- evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"},
- )
- try:
- validate_ir(ir, credit_decision=decision.to_dict())
- except InvariantViolation as violation:
- return [{"kind": "prompt", "written": False, "why_not": {"code": violation.code}}]
- if dry_run:
- return [{"dry_run": True, "kind": "prompt", "asset_id": asset_id}]
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="prompt_engine",
- evidence_refs=evidence.episode_ids[:20],
- )
- return (
- [{"candidate_id": candidate_id, "kind": "prompt", "asset_id": asset_id}]
- if candidate_id
- else []
- )
-
-
-def _active_prompt_fragments() -> List[str]:
- """Fragments already appended by earlier prompt candidates.
-
- Passed to the generator so it can decline rather than restate: a prompt that
- accumulates three phrasings of the same instruction is measurably worse than
- one that states it once.
- """
- try:
- from core.db.engine import SessionLocal as _Session
- from core.db.models.evolution import EvolutionRelease as _Release
-
- with _Session() as db:
- rows = (
- db.query(_Release)
- .filter(_Release.target_kind == "prompt", _Release.stage == "active")
- .all()
- )
- return [str(row.target_asset_id) for row in rows]
- except Exception as exc: # noqa: BLE001
- logger.debug("[evolution-loop] active prompt fragments unavailable: %s", exc)
- return []
-
-
-def _graph_evolution_candidates(
- views: List[Dict[str, Any]], *, dry_run: bool
-) -> List[Dict[str, Any]]:
- """L3 的演进:强化/降权自动执行,语义变更走人工候选。
-
- Per user, because the graph is per user scope: a contradiction between two
- people's graphs is not a contradiction at all.
- """
- from core.config.settings import settings
-
- if not (settings.memory.enabled and settings.memory.graph_enabled):
- return []
-
- from core.evolution.graph_scan import apply_graph_ops, scan_graph_relations
-
- observed_by_user: Dict[str, Dict[str, int]] = {}
- for view in views:
- user_id = str(view.get("user_id") or "")
- if not user_id:
- continue
- bucket = observed_by_user.setdefault(user_id, {})
- for relation in view.get("graph_relations") or []:
- relation_id = str(relation.get("relation_id") or "")
- if relation_id:
- bucket[relation_id] = bucket.get(relation_id, 0) + 1
-
- created: List[Dict[str, Any]] = []
- for user_id, observed in observed_by_user.items():
- try:
- from core.memory.graph import list_graph_relations
-
- relations = _run_async(list_graph_relations(user_id, limit=200))
- except Exception as exc: # noqa: BLE001
- logger.debug("[evolution-loop] graph listing failed for %s: %s", user_id, exc)
- continue
- if not relations:
- continue
-
- ops = scan_graph_relations(relations, observed_relation_ids=observed)
- if not ops:
- continue
- if dry_run:
- created.append(
- {"dry_run": True, "kind": "graph", "user_id": user_id, "ops": len(ops)}
- )
- continue
-
- outcome = apply_graph_ops(ops, scope_user_id=user_id)
- manual = outcome.get("manual") or []
- entry: Dict[str, Any] = {
- "kind": "graph",
- "user_id": user_id,
- "auto_applied": len(outcome.get("applied") or []),
- "manual_proposed": len(manual),
- }
- if manual:
- decision = aggregate_finding(
- target=T_MEMORY,
- explanation=";".join(str(op.get("reason") or "") for op in manual[:3]),
- confidence=confidence_from_evidence(len(manual), saturates_at=6),
- evidence_count=len(manual),
- )
- if decision.may_produce_candidate:
- asset_id = "graph-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12]
- ir = EvolutionIR(
- target_kind="memory",
- target_asset_id=asset_id,
- base_version="current",
- operation="update",
- hypothesis=(
- f"图谱有 {len(manual)} 处语义变更需要人工裁决:"
- + ";".join(str(op.get("reason") or "") for op in manual[:3])
- ),
- changes=[{"user_id": user_id, "graph_ops": manual}],
- evidence_refs=[
- str(op.get("relation_id") or "") for op in manual if op.get("relation_id")
- ][:20],
- risk_tier=RISK_MEDIUM,
- scope={"level": "user", "user_id": user_id},
- rollback=Rollback(version_id="current", triggers=["manual"]),
- evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"},
- )
- try:
- validate_ir(
- ir,
- credit_decision=decision.to_dict(),
- allowed_scopes=["user", "workspace"],
- )
- except InvariantViolation as violation:
- logger.info(
- "[evolution-loop] graph candidate rejected: %s", violation.code
- )
- else:
- candidate_id = _persist_candidate(
- ir,
- credit=decision.to_dict(),
- proposer="graph_engine",
- evidence_refs=list(ir.evidence_refs),
- )
- if candidate_id:
- entry["candidate_id"] = candidate_id
- created.append(entry)
- return created
-
-
-def _run_other_engines(views: List[Dict[str, Any]], *, dry_run: bool) -> Dict[str, Any]:
- """Memory, decremental, intent, orchestration, prompt and graph — each with an exit.
-
- Independently guarded: a failure in one engine must not cost the others
- their findings, because they answer different questions and an operator
- needs whichever ones succeeded.
- """
- from core.evolution.cycle_extras import run_extras
-
- scan_findings = _memory_scan_findings(views)
- extras = run_extras(views, scan_findings=scan_findings)
-
- produced: List[Dict[str, Any]] = []
- for name, fn in (
- ("memory", lambda: _memory_candidates(extras, views, dry_run=dry_run)),
- ("intent", lambda: _intent_skill_candidates(extras, views, dry_run=dry_run)),
- ("decremental", lambda: _decremental_candidates(extras, dry_run=dry_run)),
- ("orchestration", lambda: _orchestration_candidates(views, dry_run=dry_run)),
- ("prompt", lambda: _prompt_candidates(views, dry_run=dry_run)),
- ("graph", lambda: _graph_evolution_candidates(views, dry_run=dry_run)),
- ):
- try:
- produced.extend(fn())
- except Exception as exc: # noqa: BLE001
- logger.warning("[evolution-loop] %s engine failed: %s", name, exc)
- produced.append({"kind": name, "error": str(exc)})
-
- # ``ops`` holds MemoryOp objects for the candidate builder; the report is
- # serialised into the console, so it must not carry them.
- memory_report = dict(extras.get("memory") or {})
- memory_report.pop("ops", None)
- extras["memory"] = memory_report
- extras["memory"]["scan_findings"] = [
- f.to_dict() if hasattr(f, "to_dict") else f for f in scan_findings
- ][:50]
-
- return {"extras": extras, "engine_candidates": produced}
-
-
-def run_evolution_cycle(*, limit: int = 500, dry_run: bool = False) -> Dict[str, Any]:
- """One fleet-wide pass: discover → attribute → promote → validate → persist.
-
- Returns a report rather than raising: this runs as a background job and a
- bad cycle must leave the product untouched.
- """
- if not cross_user_mining_available():
- logger.info("[evolution-loop] cross-user mining unavailable; personal only")
- return {
- "episodes": 0,
- "patterns": 0,
- "candidates": [],
- "rejected": [],
- "no_update": 0,
- "skipped": "cross_user_mining_requires_enterprise",
- }
- # Refuse rather than degrade. Without an embedding service two of the three
- # engines lose their primary path and the cycle reports success having found
- # nothing — which is indistinguishable from "there was nothing to find", and
- # is the state nobody investigates.
- from core.evolution.readiness import check_readiness
-
- readiness = check_readiness()
- if not readiness.ready:
- logger.warning("[evolution-loop] not ready, refusing to run: %s", readiness.blocking)
- return {
- "episodes": 0,
- "patterns": 0,
- "candidates": [],
- "rejected": [],
- "no_update": 0,
- "skipped": "not_ready",
- "readiness": readiness.to_dict(),
- }
-
- report: Dict[str, Any] = {
- "episodes": 0,
- "patterns": 0,
- "proposals": 0,
- "candidates": [],
- "rejected": [],
- "no_update": 0,
- "readiness": readiness.to_dict(),
- }
-
- views = load_episode_views(limit=limit)
- report["episodes"] = len(views)
- if not views:
- return report
-
- patterns = P.discover_patterns(views)
- report["patterns"] = len(patterns)
-
- # Transferable ordering rules, extracted once across the whole corpus rather
- # than per pattern. A rigid sequence only helps tasks whose tool set it
- # happens to cover; the ordering rule behind it helps every task making the
- # same mistake, which is the difference between memorising and learning.
- constraints = P.extract_ordering_constraints(views)
- report["ordering_constraints"] = [c.to_dict() for c in constraints]
-
- from core.evolution.skill_candidate_engine import (
- STATUS_CREATED,
- STATUS_DRY_RUN,
- STATUS_REJECTED,
- SkillCandidateSpec,
- create_skill_candidate,
- )
-
- existing_signatures: Dict[str, str] = {}
- covered_tool_sets: List[List[str]] = []
- by_episode_id = {str(v.get("episode_id") or ""): v for v in views}
-
- for pattern in patterns:
- if pattern.kind == P.PATTERN_SUCCESS_SUBSEQUENCE and pattern.tool_sequence:
- # An observation across many *successful* runs — aggregate decision,
- # not failure attribution with invented features.
- decision = aggregate_finding(
- target=T_SKILL,
- explanation=(
- f"{pattern.support} 个成功 Episode 复现同一 "
- f"{len(pattern.tool_sequence)} 步工具序列"
- f"(成功率 {pattern.success_rate:.0%})"
- ),
- confidence=confidence_from_evidence(pattern.support, saturates_at=10),
- evidence_count=pattern.support,
- )
- else:
- decision = assign_credit(_features_for(pattern))
- if not decision.may_produce_candidate:
- # No-update / unidentifiable / a non-asset cause. Refusing here is
- # the mechanism that stops a KB gap being "fixed" by editing a skill.
- report["no_update"] += 1
- continue
-
- proposal = P.promote_tool_sequence_to_skill(
- pattern,
- episodes=views,
- skill_credit=decision.scores.get("skill", 0.0),
- workflow_credit=decision.scores.get("workflow", 0.0),
- existing_skill_signatures=existing_signatures,
- )
- if proposal is None:
- continue
- report["proposals"] += 1
-
- # Stable, content-derived id. Python's built-in hash() is randomised per
- # process, so using it here minted a fresh asset id on every cycle and the
- # de-duplication constraint never matched — one candidate per run, forever.
- asset_id = proposal.patch_target or (
- "auto-" + hashlib.sha256(proposal.signature.encode("utf-8")).hexdigest()[:12]
- )
- sequence = [str(t) for t in proposal.payload.get("tool_sequence", [])]
- supporting = [
- by_episode_id[e]
- for e in (proposal.payload.get("episode_ids") or [])
- if e in by_episode_id
- ]
-
- # 规则门测统计,测不了语义——意图簇可能是巧合归并,序列可能是对任何
- # 任务都成立的通用组合。规则幸存者再过一道 LLM 裁决,拒绝即不产候选。
- from core.evolution.sop_judge import adjudicate_sequence_sop
-
- verdict = _run_async(
- adjudicate_sequence_sop(
- representative=str(proposal.payload.get("intent") or ""),
- objectives=[str(v.get("objective") or "") for v in supporting],
- tool_sequence=sequence,
- occasions=int(proposal.payload.get("occasions") or 0),
- support=proposal.support,
- success_rate=pattern.success_rate,
- )
- )
- if not verdict.worthy:
- report["rejected"].append(
- {
- "signature": proposal.signature,
- "code": (
- "sop_judge_unavailable"
- if verdict.judge_unavailable
- else "sop_judge_refused"
- ),
- "reason": verdict.reason,
- }
- )
- continue
- spec = SkillCandidateSpec(
- asset_id=asset_id,
- proposer="promotion_chain",
- hypothesis=proposal.rationale,
- views=supporting,
- tool_sequence=sequence,
- # Attach every rule whose tools this skill touches, and carry its
- # scope along with it. Attaching must not apply the usability
- # policy: doing so dropped exactly the context-dependent rules the
- # scoping exists to preserve — a rule contradicted in one task type
- # is still valid in the ones where it held, and that decision
- # belongs at use time, where the context is known.
- ordering_constraints=[
- c.to_dict()
- for c in constraints
- if c.before in sequence and c.after in sequence
- ],
- operation="patch" if proposal.patch_target else "new",
- base_version="current" if proposal.patch_target else "v0",
- # A brand-new capability is riskier than tweaking an existing one.
- risk_tier=RISK_MEDIUM if proposal.patch_target else RISK_HIGH,
- scope={"level": "workspace"},
- tenant_id=str(
- (supporting[0].get("tenant_id") if supporting else "") or "default"
- ),
- write_document=False,
- )
- # 裁决结论随候选进 credit_decision 落库,评审员能看到机器放行的理由。
- outcome = create_skill_candidate(
- spec,
- credit={**decision.to_dict(), "sop_judge": verdict.to_dict()},
- dry_run=dry_run,
- )
- status = outcome.get("status")
- if status == STATUS_REJECTED:
- report["rejected"].append(
- {"signature": proposal.signature, "code": outcome.get("code")}
- )
- elif status == STATUS_DRY_RUN:
- report["candidates"].append({"dry_run": True, "signature": proposal.signature})
- elif status == STATUS_CREATED:
- covered_tool_sets.append(sequence)
- report["candidates"].append(
- {
- "candidate_id": outcome["candidate_id"],
- "signature": proposal.signature,
- "evidence_pack_id": outcome.get("pack_id", ""),
- }
- )
-
- # ── Ordering rules that no sequence candidate carries ────────────────────
- #
- # Without this the only shape of learning that can become an asset is "five
- # runs repeated one identical tool sequence", which on real traffic is close
- # to unobservable — measured on this deployment's history, every episode's
- # tool sequence was unique. The rules are the part that generalises, so they
- # get to stand on their own rather than only riding along on a sequence that
- # happens to qualify.
- orphan_rules = _uncovered_constraints(constraints, covered_tool_sets)
- if orphan_rules:
- rule_decision = assign_credit(_features_for_ordering_rules(orphan_rules))
- if not rule_decision.may_produce_candidate:
- report["no_update"] += 1
- else:
- rule_payload = [c.to_dict() for c in orphan_rules]
- signature = "|".join(sorted(f"{c.before}<{c.after}" for c in orphan_rules))
- asset_id = "auto-rules-" + hashlib.sha256(signature.encode("utf-8")).hexdigest()[:12]
- supporting_views = [
- view
- for view in views
- if view.get("verdict") == "success"
- and any(
- c.before in (view.get("tool_sequence") or [])
- and c.after in (view.get("tool_sequence") or [])
- for c in orphan_rules
- )
- ]
- rule_spec = SkillCandidateSpec(
- asset_id=asset_id,
- proposer="promotion_chain",
- hypothesis=(
- f"{len(orphan_rules)} 条工具顺序约束在 {len(supporting_views)} 条成功执行中"
- "稳定成立且跨多种工具组合复现,但规划阶段每次仍可能排错"
- ),
- views=supporting_views,
- # Deliberately no tool sequence: this asset claims no fixed
- # order of its own, only constraints. Materialisation renders
- # it as rules.
- tool_sequence=[],
- ordering_constraints=rule_payload,
- operation="new",
- base_version="v0",
- risk_tier=RISK_HIGH,
- scope={"level": "workspace"},
- tenant_id=str(
- (supporting_views[0].get("tenant_id") if supporting_views else "")
- or "default"
- ),
- write_document=False,
- )
- outcome = create_skill_candidate(
- rule_spec, credit=rule_decision.to_dict(), dry_run=dry_run
- )
- status = outcome.get("status")
- if status == STATUS_REJECTED:
- report["rejected"].append({"signature": signature, "code": outcome.get("code")})
- else:
- report["proposals"] += 1
- if status == STATUS_DRY_RUN:
- report["candidates"].append(
- {"dry_run": True, "signature": signature, "kind": "ordering_rules"}
- )
- elif status == STATUS_CREATED:
- report["candidates"].append(
- {
- "candidate_id": outcome["candidate_id"],
- "signature": signature,
- "kind": "ordering_rules",
- "evidence_pack_id": outcome.get("pack_id", ""),
- }
- )
-
- # The other two engines, plus orchestration. Every finding below becomes a
- # candidate in the same table, behind the same security gate and the same
- # review queue as the skill candidates above — a finding that stops at a
- # report is a log line, not an engine.
- report.update(_run_other_engines(views, dry_run=dry_run))
-
- logger.info(
- "[evolution-loop] cycle done: episodes=%d patterns=%d candidates=%d no_update=%d",
- report["episodes"],
- report["patterns"],
- len(report["candidates"]),
- report["no_update"],
- )
- return report
-
-
-# ── Advancing candidates through the release pipeline ────────────────────────
-
-
-def _record_evaluation(candidate_id: str, report: Any, eval_type: str) -> None:
- """Persist an evaluation result against its candidate."""
- from core.db.models.evolution import EvolutionEvaluation
-
- try:
- payload = report.to_dict()
- with SessionLocal() as db:
- db.add(
- EvolutionEvaluation(
- evaluation_id=payload["evaluation_id"],
- candidate_id=candidate_id,
- eval_type=eval_type,
- # Frozen: a dataset edited after the fact invalidates every
- # conclusion computed from it.
- dataset_snapshot=payload["dataset_snapshot"],
- baseline_metrics=payload["baseline"],
- candidate_metrics=payload["candidate"],
- guardrails={"breaches": payload["guardrail_breaches"]},
- verdict=payload["verdict"],
- effect_size=payload["effect_size"],
- p_value=payload["p_value"],
- sample_size=payload["sample_size"],
- )
- )
- db.commit()
- except Exception as exc:
- logger.warning("[evolution-loop] recording evaluation failed: %s", exc)
-
-
-def _replay_coverage_for(candidate_id: str, task_ids: List[str]) -> Optional[Dict[str, Any]]:
- """Coverage assessment for candidates that carry an evidence pack.
-
- Returns ``None`` for pre-pack candidates — the gate governs only what the
- unified engine produced, so legacy candidates keep their old semantics.
- """
- try:
- from core.evolution import lineage
- from core.evolution.applicability import assess_replay_coverage
-
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- pack_id = str(getattr(candidate, "evidence_pack_id", "") or "")
- if not pack_id:
- return None
- pack = lineage.load_evidence_pack(pack_id)
- if pack is None:
- return None
- return assess_replay_coverage(pack, task_ids)
- except Exception as exc: # noqa: BLE001
- logger.debug("[evolution-loop] replay coverage check skipped: %s", exc)
- return None
-
-
-def evaluate_candidate(
- candidate_id: str,
- run_arm: Any,
- task_ids: List[str],
-) -> Optional[Dict[str, Any]]:
- """Run tiered replay for a candidate and advance it if it earns the step.
-
- ``run_arm(task_id, is_candidate)`` is injected: the caller owns isolation
- (frozen bundle, cassette, sandbox snapshot). Only ``replay_passed`` is
- reachable from here — shadow and canary need real traffic, and activation
- needs a human, so this can never quietly ship anything.
- """
- from core.evolution.release import DRAFT, REJECTED, REPLAY_PASSED, is_legal_transition
- from core.evolution.replay import VERDICT_IMPROVED, VERDICT_REGRESSED, run_tiered_replay
-
- tiered = run_tiered_replay(task_ids, run_arm)
- report = tiered.report or tiered.sentinel_report
- if report is None:
- return None
-
- _record_evaluation(candidate_id, report, "replay" if tiered.passed_sentinel else "sentinel")
-
- # Replay 覆盖闸门:带证据 pack 的候选,其回放集必须覆盖 pack 声称的行为面
- # (成功 / 失败恢复 / 负样本 / 图谱依赖两类场景)。只用成功案例喂出来的
- # "improved" 是单面证据,不允许换来 replay_passed;回归照常生效——安全
- # 方向的结论不需要覆盖完整也成立。
- coverage: Optional[Dict[str, Any]] = None
- if report.verdict == VERDICT_IMPROVED:
- coverage = _replay_coverage_for(candidate_id, task_ids)
- if coverage is not None and not coverage.get("covered", True):
- return {
- "candidate_id": candidate_id,
- "verdict": report.verdict,
- "status": DRAFT,
- "transition": "held_insufficient_coverage",
- "coverage": coverage,
- "tasks_run": tiered.tasks_run,
- "effect_size": report.effect_size,
- "p_value": report.p_value,
- }
-
- try:
- with SessionLocal() as db:
- candidate = db.get(EvolutionCandidate, candidate_id)
- if candidate is None:
- return None
- current = candidate.status
-
- # Re-evaluating a candidate that has already moved on is a no-op,
- # not a failure. Collapsing the two into None left callers unable to
- # tell "already advanced" from "something broke".
- target = (
- REJECTED
- if report.verdict == VERDICT_REGRESSED
- else (REPLAY_PASSED if report.verdict == VERDICT_IMPROVED else current)
- )
- if target != current and not is_legal_transition(current, target):
- return {
- "candidate_id": candidate_id,
- "verdict": report.verdict,
- "status": current,
- "transition": "skipped_already_advanced",
- "tasks_run": tiered.tasks_run,
- "effect_size": report.effect_size,
- "p_value": report.p_value,
- }
-
- if report.verdict == VERDICT_REGRESSED:
- candidate.status = REJECTED
- candidate.reject_reason = (
- f"replay regression: effect={report.effect_size:.3f} "
- f"p={report.p_value:.4f} {report.guardrail_breaches}"
- )
- elif report.verdict == VERDICT_IMPROVED:
- candidate.status = REPLAY_PASSED
- else:
- # Neutral or insufficient: stays a draft. Not advancing is the
- # correct outcome — "we could not show it helps" is not "it helps".
- pass
- db.commit()
- return {
- "candidate_id": candidate_id,
- "verdict": report.verdict,
- "status": candidate.status,
- "transition": "applied" if candidate.status != current else "held",
- "tasks_run": tiered.tasks_run,
- "effect_size": report.effect_size,
- "p_value": report.p_value,
- }
- except Exception as exc:
- logger.warning("[evolution-loop] advancing candidate failed: %s", exc)
- return None
diff --git a/src/backend/core/evolution/profiles/__init__.py b/src/backend/core/evolution/profiles/__init__.py
new file mode 100644
index 00000000..b8789016
--- /dev/null
+++ b/src/backend/core/evolution/profiles/__init__.py
@@ -0,0 +1 @@
+"""Agent profiles and per-user evolution settings and reviews."""
diff --git a/src/backend/core/evolution/profiles/agent_profile.py b/src/backend/core/evolution/profiles/agent_profile.py
new file mode 100644
index 00000000..80589412
--- /dev/null
+++ b/src/backend/core/evolution/profiles/agent_profile.py
@@ -0,0 +1,259 @@
+"""Agent profiles: one ReAct configuration expressed as data, with validation."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+from core.evolution.profiles import agent_profile_constants
+
+@dataclass
+class InterventionRule:
+ """signal → threshold → action."""
+
+ signal: str
+ threshold: int
+ action: str
+ note: str = ""
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "signal": self.signal,
+ "threshold": self.threshold,
+ "action": self.action,
+ "note": self.note,
+ }
+
+ @classmethod
+ def from_dict(cls, raw: Dict[str, Any]) -> "InterventionRule":
+ return cls(
+ signal=str(raw.get("signal") or ""),
+ threshold=int(raw.get("threshold") or 0),
+ action=str(raw.get("action") or ""),
+ note=str(raw.get("note") or ""),
+ )
+
+
+@dataclass
+class SubagentRoute:
+ """Hand this shape of sub-task to a separate context.
+
+ ``agent_id`` points at a real sub-agent; the route is only the *decision to
+ delegate*. Creating the sub-agent is a separate, separately-approved change
+ (see :mod:`core.evolution.generators.orchestration_gen`), because a route to an agent
+ that does not exist is a dead end and an agent nobody routes to is dead
+ weight — they have to be reviewed as one change but recorded as two.
+ """
+
+ agent_id: str
+ when: str = ""
+ task_types: List[str] = field(default_factory=list)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {"agent_id": self.agent_id, "when": self.when, "task_types": list(self.task_types)}
+
+ @classmethod
+ def from_dict(cls, raw: Dict[str, Any]) -> "SubagentRoute":
+ return cls(
+ agent_id=str(raw.get("agent_id") or ""),
+ when=str(raw.get("when") or ""),
+ task_types=[str(t) for t in (raw.get("task_types") or [])],
+ )
+
+
+@dataclass
+class AgentProfile:
+ """One assembled ReAct configuration, as data."""
+
+ profile_id: str = "builtin"
+ version: str = "v0"
+ # Which tasks this profile governs. Empty means "every task in scope" — the
+ # deployment default. A profile with task types is only consulted for those,
+ # which is how a capability learned in one family stops leaking into others.
+ task_types: List[str] = field(default_factory=list)
+ scope: Dict[str, str] = field(default_factory=dict)
+
+ prompt_fragments: List[str] = field(default_factory=list)
+ # ``None`` means "no restriction beyond the base grant". An empty list means
+ # "no tools", which is a legitimate profile for a pure-reasoning task type
+ # and must not be confused with the former.
+ tool_allowlist: Optional[List[str]] = None
+ # Which skills this task type may load at all. Empty means "no restriction"
+ # — skill loading keeps its own logic and this only ever narrows it, and
+ # only for task types an approved profile actually covers.
+ skill_ids: List[str] = field(default_factory=list)
+ subagent_routes: List[SubagentRoute] = field(default_factory=list)
+
+ memory_budget_ms: int = 600
+ # 0 = 不限(默认)。See UNBOUNDED_REACT_TURNS: the main agent's round ceiling
+ # was removed, so a turn budget here is an opt-in narrowing a published
+ # profile performs deliberately, not a floor every profile inherits.
+ max_react_turns: int = agent_profile_constants.UNBOUNDED_REACT_TURNS
+ budget_multiplier: float = 1.0
+ reviewer_checkpoints: List[str] = field(default_factory=list)
+ intervention_rules: List[InterventionRule] = field(default_factory=list)
+
+ # The autonomous loop's own caps. Held as their own fields rather than
+ # derived from ``max_react_turns``: the two loops bound different things —
+ # turns inside one answer versus attempts at one requirement across many
+ # answers — and a ratio between them would be an invented relationship that
+ # quietly changes one whenever the other is tuned.
+ loop_max_attempts_per_requirement: int = 6
+ loop_strategy_change_after: int = 2
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "profile_id": self.profile_id,
+ "version": self.version,
+ "task_types": list(self.task_types),
+ "scope": dict(self.scope),
+ "prompt_fragments": list(self.prompt_fragments),
+ "tool_allowlist": None if self.tool_allowlist is None else list(self.tool_allowlist),
+ "skill_ids": list(self.skill_ids),
+ "subagent_routes": [r.to_dict() for r in self.subagent_routes],
+ "memory_budget_ms": self.memory_budget_ms,
+ "max_react_turns": self.max_react_turns,
+ "budget_multiplier": self.budget_multiplier,
+ "reviewer_checkpoints": list(self.reviewer_checkpoints),
+ "intervention_rules": [r.to_dict() for r in self.intervention_rules],
+ "loop_max_attempts_per_requirement": self.loop_max_attempts_per_requirement,
+ "loop_strategy_change_after": self.loop_strategy_change_after,
+ }
+
+ @classmethod
+ def from_dict(cls, raw: Dict[str, Any]) -> "AgentProfile":
+ allowlist = raw.get("tool_allowlist", None)
+ return cls(
+ profile_id=str(raw.get("profile_id") or "builtin"),
+ version=str(raw.get("version") or "v0"),
+ task_types=[str(t) for t in (raw.get("task_types") or [])],
+ scope=dict(raw.get("scope") or {}),
+ prompt_fragments=[str(f) for f in (raw.get("prompt_fragments") or [])],
+ tool_allowlist=None if allowlist is None else [str(t) for t in allowlist],
+ skill_ids=[str(s) for s in (raw.get("skill_ids") or [])],
+ subagent_routes=[
+ SubagentRoute.from_dict(r) for r in (raw.get("subagent_routes") or [])
+ ],
+ memory_budget_ms=int(raw.get("memory_budget_ms") or 600),
+ # Missing key → unbounded, not an invented ceiling: a profile
+ # published for unrelated reasons (tool allowlist, memory budget)
+ # must not silently re-fence the main loop.
+ max_react_turns=int(raw.get("max_react_turns") or agent_profile_constants.UNBOUNDED_REACT_TURNS),
+ budget_multiplier=float(raw.get("budget_multiplier") or 1.0),
+ reviewer_checkpoints=[str(c) for c in (raw.get("reviewer_checkpoints") or [])],
+ intervention_rules=[
+ InterventionRule.from_dict(r) for r in (raw.get("intervention_rules") or [])
+ ],
+ loop_max_attempts_per_requirement=int(
+ raw.get("loop_max_attempts_per_requirement") or 6
+ ),
+ loop_strategy_change_after=int(raw.get("loop_strategy_change_after") or 2),
+ )
+
+ def applies_to(self, task_type: str) -> bool:
+ return not self.task_types or str(task_type or "chat") in self.task_types
+
+
+def builtin_profile() -> AgentProfile:
+ """The deployment default.
+
+ Reproduces today's behaviour exactly — no tool restriction, no forced skill
+ set, the retrieval budget and turn cap the runtime already uses — so
+ installing profile resolution changes nothing until a profile is published.
+ """
+ return AgentProfile(
+ profile_id="builtin",
+ version="v0",
+ tool_allowlist=None,
+ memory_budget_ms=600,
+ # The main agent's runtime default, reproduced exactly — which is now
+ # "no cap". This field only bounds the loop when a published profile
+ # sets it on purpose.
+ max_react_turns=agent_profile_constants.UNBOUNDED_REACT_TURNS,
+ budget_multiplier=1.0,
+ loop_max_attempts_per_requirement=6,
+ loop_strategy_change_after=2,
+ intervention_rules=[
+ InterventionRule(agent_profile_constants.SIGNAL_NO_PROGRESS, 2, agent_profile_constants.ACTION_CHANGE_STRATEGY),
+ InterventionRule(agent_profile_constants.SIGNAL_REPEATED_ACTIONS, 3, agent_profile_constants.ACTION_NARROW_SCOPE),
+ InterventionRule(agent_profile_constants.SIGNAL_TOOL_ERROR_STREAK, 3, agent_profile_constants.ACTION_CHANGE_STRATEGY),
+ InterventionRule(agent_profile_constants.SIGNAL_BUDGET_LOW, 1, agent_profile_constants.ACTION_ESCALATE),
+ ],
+ )
+
+
+def validate_profile(
+ profile: AgentProfile,
+ *,
+ base_tools: Optional[Sequence[str]] = None,
+ known_skill_ids: Optional[Sequence[str]] = None,
+ known_agent_ids: Optional[Sequence[str]] = None,
+) -> Tuple[bool, List[str]]:
+ """Structural and security validation.
+
+ ``base_tools`` is the authority this profile is allowed to draw from. A
+ profile naming a tool outside it is refused — orchestration evolution may
+ narrow authority and may never widen it, and this is the check that makes
+ "narrow only" a property of the system rather than an intention.
+ """
+ problems: List[str] = []
+
+ if not profile.profile_id:
+ problems.append("profile_id 不能为空")
+
+ if profile.tool_allowlist is not None and base_tools is not None:
+ widened = sorted(set(profile.tool_allowlist) - set(base_tools))
+ if widened:
+ problems.append(f"工具授权越界(只允许收敛,不允许扩权): {widened}")
+
+ if known_skill_ids is not None:
+ unknown = sorted(set(profile.skill_ids) - set(known_skill_ids))
+ if unknown:
+ problems.append(f"引用了不存在的技能: {unknown}")
+
+ seen_agents = set()
+ for route in profile.subagent_routes:
+ if not route.agent_id:
+ problems.append("子智能体路由缺少 agent_id")
+ continue
+ if route.agent_id in seen_agents:
+ problems.append(f"重复的子智能体路由: {route.agent_id}")
+ seen_agents.add(route.agent_id)
+ if known_agent_ids is not None and route.agent_id not in set(known_agent_ids):
+ problems.append(f"路由指向不存在的子智能体: {route.agent_id}")
+
+ for rule in profile.intervention_rules:
+ if rule.signal not in agent_profile_constants.ALLOWED_SIGNALS:
+ problems.append(f"未知信号 {rule.signal}")
+ if rule.action not in agent_profile_constants.ALLOWED_ACTIONS:
+ problems.append(f"未知动作 {rule.action}")
+ if rule.threshold < 1:
+ problems.append(f"{rule.signal} 阈值必须 ≥1(0 会让干预立刻触发)")
+
+ if not (agent_profile_constants.MIN_BUDGET_MULTIPLIER <= profile.budget_multiplier <= agent_profile_constants.MAX_BUDGET_MULTIPLIER):
+ problems.append(
+ f"预算倍数 {profile.budget_multiplier} 超出 "
+ f"[{agent_profile_constants.MIN_BUDGET_MULTIPLIER}, {agent_profile_constants.MAX_BUDGET_MULTIPLIER}]"
+ )
+ if profile.max_react_turns != agent_profile_constants.UNBOUNDED_REACT_TURNS and not (
+ agent_profile_constants.MIN_REACT_TURNS <= profile.max_react_turns <= agent_profile_constants.MAX_REACT_TURNS
+ ):
+ problems.append(
+ f"最大轮数 {profile.max_react_turns} 超出 "
+ f"[{agent_profile_constants.MIN_REACT_TURNS}, {agent_profile_constants.MAX_REACT_TURNS}]({agent_profile_constants.UNBOUNDED_REACT_TURNS} 表示不限)"
+ )
+ if not (agent_profile_constants.MIN_MEMORY_BUDGET_MS <= profile.memory_budget_ms <= agent_profile_constants.MAX_MEMORY_BUDGET_MS):
+ problems.append(
+ f"记忆检索预算 {profile.memory_budget_ms}ms 超出 "
+ f"[{agent_profile_constants.MIN_MEMORY_BUDGET_MS}, {agent_profile_constants.MAX_MEMORY_BUDGET_MS}]"
+ )
+ if not (
+ agent_profile_constants.MIN_LOOP_ATTEMPTS
+ <= profile.loop_max_attempts_per_requirement
+ <= agent_profile_constants.MAX_LOOP_ATTEMPTS
+ ):
+ problems.append(
+ f"单需求最大尝试次数 {profile.loop_max_attempts_per_requirement} 超出 "
+ f"[{agent_profile_constants.MIN_LOOP_ATTEMPTS}, {agent_profile_constants.MAX_LOOP_ATTEMPTS}]"
+ )
+ if profile.loop_strategy_change_after < 1:
+ problems.append("换策略阈值必须 ≥1(0 会让每一轮都换策略)")
+
+ return (not problems), problems
diff --git a/src/backend/core/evolution/profiles/agent_profile_constants.py b/src/backend/core/evolution/profiles/agent_profile_constants.py
new file mode 100644
index 00000000..8f937ff1
--- /dev/null
+++ b/src/backend/core/evolution/profiles/agent_profile_constants.py
@@ -0,0 +1,102 @@
+"""Signals, actions and limits allowed in agent profiles."""
+from __future__ import annotations
+
+
+SIGNAL_NO_PROGRESS = "no_progress"
+
+
+SIGNAL_REPEATED_ACTIONS = "repeated_actions"
+
+
+SIGNAL_NO_DIFF = "no_diff"
+
+
+SIGNAL_REVIEWER_FLAT = "reviewer_score_flat"
+
+
+SIGNAL_BUDGET_LOW = "budget_low"
+
+
+SIGNAL_TOOL_ERROR_STREAK = "tool_error_streak"
+
+
+ALLOWED_SIGNALS = (
+ SIGNAL_NO_PROGRESS,
+ SIGNAL_REPEATED_ACTIONS,
+ SIGNAL_NO_DIFF,
+ SIGNAL_REVIEWER_FLAT,
+ SIGNAL_BUDGET_LOW,
+ SIGNAL_TOOL_ERROR_STREAK,
+)
+
+
+REACT_OBSERVABLE_SIGNALS = frozenset(
+ {SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_TOOL_ERROR_STREAK}
+)
+
+
+LOOP_ONLY_SIGNALS = frozenset(
+ {SIGNAL_NO_DIFF, SIGNAL_REVIEWER_FLAT, SIGNAL_BUDGET_LOW}
+)
+
+
+ACTION_RETRY = "retry"
+
+
+ACTION_CHANGE_STRATEGY = "change_strategy"
+
+
+ACTION_ROLLBACK_AND_FORK = "rollback_and_fork"
+
+
+ACTION_NARROW_SCOPE = "narrow_scope"
+
+
+ACTION_DELEGATE = "delegate"
+
+
+ACTION_ESCALATE = "escalate"
+
+
+ACTION_STOP = "stop"
+
+
+ALLOWED_ACTIONS = (
+ ACTION_RETRY,
+ ACTION_CHANGE_STRATEGY,
+ ACTION_ROLLBACK_AND_FORK,
+ ACTION_NARROW_SCOPE,
+ ACTION_DELEGATE,
+ ACTION_ESCALATE,
+ ACTION_STOP,
+)
+
+
+MIN_BUDGET_MULTIPLIER = 0.25
+
+
+MAX_BUDGET_MULTIPLIER = 3.0
+
+
+MIN_REACT_TURNS = 3
+
+
+MAX_REACT_TURNS = 80
+
+
+UNBOUNDED_REACT_TURNS = 0
+
+
+MIN_MEMORY_BUDGET_MS = 100
+
+
+MAX_MEMORY_BUDGET_MS = 5000
+
+
+MIN_LOOP_ATTEMPTS = 2
+
+
+MAX_LOOP_ATTEMPTS = 20
+
+
+TARGET_KIND = "agent_profile"
diff --git a/src/backend/core/evolution/profiles/change_preview.py b/src/backend/core/evolution/profiles/change_preview.py
new file mode 100644
index 00000000..b5d1ae64
--- /dev/null
+++ b/src/backend/core/evolution/profiles/change_preview.py
@@ -0,0 +1,85 @@
+"""Show the concrete content of a proposed evolution change."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+
+def change_preview(candidate) -> Dict[str, Any]:
+ """The concrete content of the proposed change.
+
+ A one-line hypothesis explains *why* something is proposed and says nothing
+ about *what* it is. Approving a distilled skill without being able to read
+ its body is not a decision, it is a guess — the text was already in the IR,
+ it was simply never sent to the page that asks the user to accept it.
+ """
+ ir = candidate.ir or {}
+ kind = str(candidate.target_kind or "")
+ changes = ir.get("changes") or []
+ for change in changes:
+ if kind == "skill":
+ document = change.get("document")
+ if isinstance(document, dict) and document.get("content"):
+ return {
+ "type": "skill_document",
+ "display_name": str(document.get("display_name") or ""),
+ "description": str(document.get("description") or ""),
+ "allowed_tools": [str(t) for t in (document.get("allowed_tools") or [])],
+ "content": str(document.get("content") or ""),
+ }
+ if kind == "memory":
+ operations = change.get("operations") or []
+ if operations:
+ return {
+ "type": "memory_ops",
+ "operations": [
+ {
+ "operation": str(op.get("operation") or ""),
+ "text": str(op.get("text") or op.get("memory_ref") or ""),
+ "reason": str(op.get("reason") or ""),
+ "before": op.get("before"),
+ "after": op.get("after"),
+ }
+ for op in operations
+ ],
+ }
+ if kind == "skill":
+ # Sequence-only candidates are the common output of the personal
+ # evolution path. They do not carry an authored SKILL.md, but approval
+ # still materialises a concrete name, description and ordered set of
+ # steps. Surface that exact structure instead of returning {}, which
+ # made the frontend silently remove the detail button.
+ tools: List[str] = []
+ constraints: List[Dict[str, Any]] = []
+ for change in changes:
+ sequence = change.get("tool_sequence")
+ if isinstance(sequence, list) and sequence and not tools:
+ tools = [str(tool) for tool in sequence]
+ rules = change.get("ordering_constraints")
+ if isinstance(rules, list) and rules:
+ constraints = [dict(rule) for rule in rules if isinstance(rule, dict)]
+
+ rules_only = not tools and bool(constraints)
+ if rules_only:
+ from core.evolution.cycle.skill_document import tools_from_constraints as _tools_from_constraints
+
+ tools = _tools_from_constraints(constraints)
+ if tools:
+ # Reuse the materialiser's wording so the preview and the installed
+ # skill never disagree about what the user accepted.
+ from core.evolution.cycle.skill_document import skill_description as _skill_description, skill_label as _skill_label
+
+ return {
+ "type": "skill_sequence",
+ "display_name": _skill_label(tools, rules_only=rules_only),
+ "description": _skill_description(tools, constraints, rules_only=rules_only),
+ "allowed_tools": tools,
+ "steps": tools,
+ "ordering_constraints": constraints,
+ }
+ return {}
+
+
+def tool_sequence_of_ir(ir: Optional[Dict[str, Any]]) -> List[str]:
+ for change in (ir or {}).get("changes") or []:
+ sequence = change.get("tool_sequence")
+ if isinstance(sequence, list) and sequence:
+ return [str(t) for t in sequence]
+ return []
diff --git a/src/backend/core/evolution/console_view.py b/src/backend/core/evolution/profiles/console_view.py
similarity index 99%
rename from src/backend/core/evolution/console_view.py
rename to src/backend/core/evolution/profiles/console_view.py
index ff40ed23..c5c08580 100644
--- a/src/backend/core/evolution/console_view.py
+++ b/src/backend/core/evolution/profiles/console_view.py
@@ -105,5 +105,3 @@ def release_to_dict(row: EvolutionRelease) -> Dict[str, Any]:
"approved_by": row.approved_by,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
-
-
diff --git a/src/backend/core/evolution/federation.py b/src/backend/core/evolution/profiles/federation.py
similarity index 99%
rename from src/backend/core/evolution/federation.py
rename to src/backend/core/evolution/profiles/federation.py
index f42b10f7..e42a1528 100644
--- a/src/backend/core/evolution/federation.py
+++ b/src/backend/core/evolution/profiles/federation.py
@@ -26,7 +26,7 @@
import logging
import re
from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Sequence
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/personal.py b/src/backend/core/evolution/profiles/personal.py
similarity index 88%
rename from src/backend/core/evolution/personal.py
rename to src/backend/core/evolution/profiles/personal.py
index 2d63db08..039f8240 100644
--- a/src/backend/core/evolution/personal.py
+++ b/src/backend/core/evolution/profiles/personal.py
@@ -22,17 +22,19 @@
"""
from __future__ import annotations
+from core.evolution.cycle import ordering
+from core.evolution.cycle import patterns
+from core.evolution.cycle import promotion
+from core.evolution.cycle import promotion_constants
import hashlib
import logging
from typing import Any, Dict, List, Optional
from core.db.engine import SessionLocal
-from core.evolution import promotion as P
-from core.evolution.credit import T_SKILL, aggregate_finding, confidence_from_evidence
+from core.evolution.evidence.credit import T_SKILL, aggregate_finding, confidence_from_evidence
from core.evolution.ir import (
RISK_LOW,
- RISK_MEDIUM,
EvolutionIR,
InvariantViolation,
Rollback,
@@ -86,7 +88,7 @@ def load_personal_episodes(user_id: str, *, limit: int = 300) -> List[Dict[str,
# this drifted apart once already: the personal path kept reading
# raw content hashes as memory refs, which the overlay cannot key
# on, so a personal memory candidate could never have been applied.
- from core.evolution.loop import _view_for
+ from core.evolution.cycle.episode_views import view_for as _view_for
for episode in episodes:
views.append(_view_for(episode, by_episode.get(episode.episode_id, [])))
@@ -96,14 +98,14 @@ def load_personal_episodes(user_id: str, *, limit: int = 300) -> List[Dict[str,
return views
-def _discover(views: List[Dict[str, Any]]) -> List[P.Pattern]:
+def _discover(views: List[Dict[str, Any]]) -> List[patterns.Pattern]:
"""Pattern discovery at the personal support threshold."""
- original = P.MIN_SUPPORT
+ original = promotion_constants.MIN_SUPPORT
try:
- P.MIN_SUPPORT = PERSONAL_MIN_SUPPORT
- return P.discover_patterns(views)
+ promotion_constants.MIN_SUPPORT = PERSONAL_MIN_SUPPORT
+ return patterns.discover_patterns(views)
finally:
- P.MIN_SUPPORT = original
+ promotion_constants.MIN_SUPPORT = original
def run_personal_cycle(
@@ -117,8 +119,8 @@ def run_personal_cycle(
Returns a report rather than raising — this runs in the background and a bad
cycle must leave the product untouched.
"""
- from core.evolution.prefs import resolve_prefs
- from core.evolution.loop import _persist_candidate
+ from core.evolution.profiles.prefs import resolve_prefs
+ from core.evolution.cycle.candidate_records import persist_candidate as _persist_candidate
report: Dict[str, Any] = {
"user_id": user_id,
@@ -130,7 +132,7 @@ def run_personal_cycle(
"scope": "personal",
}
- from core.evolution.readiness import check_readiness
+ from core.evolution.cycle.readiness import check_readiness
readiness = check_readiness()
report["readiness"] = readiness.to_dict()
@@ -151,17 +153,17 @@ def run_personal_cycle(
if not views:
return report
- patterns = _discover(views)
- report["patterns"] = len(patterns)
- constraints = P.extract_ordering_constraints(
+ discovered = _discover(views)
+ report["patterns"] = len(discovered)
+ constraints = ordering.extract_ordering_constraints(
views, min_support=PERSONAL_MIN_SUPPORT, min_contexts=1
)
report["ordering_constraints"] = [c.to_dict() for c in constraints]
known_concepts = _known_concepts(prefs)
- for pattern in patterns:
- if pattern.kind != P.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence:
+ for pattern in discovered:
+ if pattern.kind != promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence:
continue
# 这是"多次成功里观察到的共性",不是某次失败的归因,所以走聚合决策。
# 旧实现在这里伪造 outcome_verdict="failed" 逼归因器选中技能引擎——
@@ -183,7 +185,7 @@ def run_personal_cycle(
# min_support 必须显式传入:发现阶段用的个人门槛(3)在 _discover 返回
# 后已恢复为全局值(5),不传的话支持度 3–4 的个人模式会在这里被静默拒绝。
- proposal = P.promote_tool_sequence_to_skill(
+ proposal = promotion.promote_tool_sequence_to_skill(
pattern,
episodes=views,
min_support=PERSONAL_MIN_SUPPORT,
@@ -196,8 +198,8 @@ def run_personal_cycle(
# 规则门之后的 LLM 语义裁决:意图簇是否巧合归并、序列是否只是对任何
# 任务都成立的通用组合。拒绝即不产候选;模型不可用同样拒绝,但在
# 报告里可区分。
- from core.evolution.loop import _run_async
- from core.evolution.sop_judge import adjudicate_sequence_sop
+ from core.evolution.cycle.candidate_records import run_async as _run_async
+ from core.evolution.assessment.sop_judge import adjudicate_sequence_sop
covered = {str(e) for e in (proposal.payload.get("episode_ids") or [])}
verdict = _run_async(
@@ -313,13 +315,9 @@ def _personal_other_engines(
*every* request in scope is assembled from, so a single user's evidence is
not a basis for them — this is the scope boundary, not a missing feature.
"""
- from core.evolution.cycle_extras import run_extras
- from core.evolution.loop import (
- _decremental_candidates,
- _intent_skill_candidates,
- _memory_candidates,
- _memory_scan_findings,
- )
+ from core.evolution.cycle.cycle_extras import run_extras
+ from core.evolution.cycle.composition_candidates import decremental_candidates as _decremental_candidates
+ from core.evolution.cycle.memory_candidates import intent_skill_candidates as _intent_skill_candidates, memory_candidates as _memory_candidates, memory_scan_findings as _memory_scan_findings
scan_findings = _memory_scan_findings(views)
extras = run_extras(views, scan_findings=scan_findings)
diff --git a/src/backend/core/evolution/user_settings.py b/src/backend/core/evolution/profiles/personal_review.py
similarity index 60%
rename from src/backend/core/evolution/user_settings.py
rename to src/backend/core/evolution/profiles/personal_review.py
index 0a4ff203..2727e425 100644
--- a/src/backend/core/evolution/user_settings.py
+++ b/src/backend/core/evolution/profiles/personal_review.py
@@ -1,109 +1,13 @@
-"""Per-user evolution participation, derived from the single evolution switch.
-
-There used to be two user-facing switches: "start evolution" (does the loop
-distil for me) and "contribute to evolution" (do my conversations feed
-cross-user learning). The distinction was real but nobody could tell the two
-apart in the settings panel, so the product now has exactly one switch —
-``evolution_prefs.enabled`` — and contribution follows it: evolution on means
-the loop may distil for this user *and* their conversations may serve as
-evidence within the tenant; off means neither.
-
-Enforcement still lives at one point rather than being scattered: the
-Episode's ``privacy_class``. With the switch off, episodes are marked
-``private`` and pattern mining filters those out — so the user still gets
-their own in-chat card and their own history, while nothing of theirs feeds
-cross-user learning. Anything weaker would make the promise unverifiable.
-"""
-
+"""Let users review, approve and activate candidates for themselves."""
from __future__ import annotations
-
-import logging
from typing import Any, Dict, List, Optional
-
from core.db.engine import SessionLocal
+import logging
+from core.evolution.profiles import change_preview
+from core.evolution.profiles import user_settings_constants
logger = logging.getLogger(__name__)
-# Response-field name kept from the era of a separate contribution switch, so
-# older bundled frontends that read /evolution/settings keep working.
-SETTING_KEY = "evolution_contribution_enabled"
-
-# Episodes from an opted-out user. Recorded for that user's own benefit (their
-# card, their history) but excluded from anything cross-user.
-PRIVACY_PRIVATE = "private"
-PRIVACY_TENANT = "tenant"
-
-
-def is_contribution_enabled(metadata: Optional[Dict[str, Any]]) -> bool:
- """Whether this user's episodes may feed the evolution loop.
-
- Reads the single evolution switch (``evolution_prefs.enabled``), so one
- stored value governs both distilling *for* the user and learning *from*
- them. Off by default: consent is collected by the first-run wizard, and a
- user who never enabled evolution contributes nothing. "On" only ever means
- *within the tenant*: cross-tenant sharing is a separate, independently-
- gated decision (see ``federation``).
- """
- from core.evolution.prefs import PREFS_KEY, normalise
-
- if not metadata:
- return False
- return bool(normalise(metadata.get(PREFS_KEY)).get("enabled", False))
-
-
-def privacy_class_for(metadata: Optional[Dict[str, Any]]) -> str:
- """The Episode privacy class implied by the user's setting."""
- return PRIVACY_TENANT if is_contribution_enabled(metadata) else PRIVACY_PRIVATE
-
-
-def resolve_for_user(user_id: str) -> Dict[str, Any]:
- """Read the setting for one user. Degrades to the default, never raises."""
- if not user_id:
- return {SETTING_KEY: False, "privacy_class": PRIVACY_PRIVATE}
- try:
- from core.db.models import UserShadow
-
- with SessionLocal() as db:
- row = db.query(UserShadow).filter(UserShadow.user_id == user_id).first()
- metadata = (getattr(row, "extra_data", None) or {}) if row else {}
- except Exception as exc:
- logger.debug("[evo-settings] read failed for %s: %s", user_id, exc)
- metadata = {}
-
- enabled = is_contribution_enabled(metadata)
- return {SETTING_KEY: enabled, "privacy_class": privacy_class_for(metadata)}
-
-
-def update_for_user(user_id: str, enabled: bool) -> Dict[str, Any]:
- """Persist the setting by writing the single evolution switch.
-
- Kept as an alias so older bundled frontends PATCHing /evolution/settings
- still work; it writes the same ``evolution_prefs.enabled`` the preferences
- panel writes, never a separate value that could drift from it.
-
- Turning it off does **not** retroactively delete already-contributed
- evidence — that would silently rewrite history other candidates were derived
- from. It stops future episodes from contributing; withdrawing past evidence
- is a separate, explicit action so its consequences stay visible.
- """
- from core.evolution.prefs import update_prefs
-
- merged = update_prefs(user_id, {"enabled": bool(enabled)})
- on = bool(merged.get("enabled", False))
- return {
- SETTING_KEY: on,
- "privacy_class": PRIVACY_TENANT if on else PRIVACY_PRIVATE,
- "note": "仅影响此后新产生的证据;已贡献的证据需单独撤回",
- }
-
-
-# A candidate is the user's to approve when their own episodes account for at
-# least this share of its evidence. Below it the capability was learned mostly
-# from other people's work, and one person approving it for everyone is a
-# different decision that belongs with an administrator.
-OWN_EVIDENCE_THRESHOLD = 0.6
-
-
def personal_action(candidate) -> Optional[Dict[str, str]]:
"""What this user can actually do with a candidate, or ``None``.
@@ -152,7 +56,7 @@ def personal_action(candidate) -> Optional[Dict[str, str]]:
# user — but only where the candidate has already passed replay, or its
# tier owes nothing beyond replay. Listing a medium-risk draft here put
# a button on screen that answered "force 只允许用于 low 风险候选".
- from core.evolution.release import ACTIVE, REPLAY_PASSED
+ from core.evolution.cycle.release import ACTIVE, REPLAY_PASSED
replayed = str(candidate.status or "") in (REPLAY_PASSED, ACTIVE)
if not replayed and risk != "low":
@@ -167,7 +71,7 @@ def personal_action(candidate) -> Optional[Dict[str, str]]:
return None
-def _personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set:
+def personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set:
"""Candidates this user has already turned into a live personal release.
A personal activation deliberately leaves the candidate's status untouched
@@ -182,7 +86,7 @@ def _personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set
out = set()
try:
from core.db.models.evolution import EvolutionRelease
- from core.evolution.release import TERMINAL_STAGES
+ from core.evolution.cycle.release import TERMINAL_STAGES
for release in (
db.query(EvolutionRelease)
@@ -240,7 +144,7 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]:
# by design (its life continues for others), so without this it would
# reappear on every refresh — an "enable" button for something already
# enabled, whose second press just stamps out a duplicate release.
- already_mine = _personally_activated_ids(
+ already_mine = personally_activated_ids(
db, user_id, [c.candidate_id for c in candidates]
)
@@ -253,12 +157,12 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]:
continue
mine = refs & own_ids
share = len(mine) / len(refs)
- if share < OWN_EVIDENCE_THRESHOLD:
+ if share < user_settings_constants.OWN_EVIDENCE_THRESHOLD:
continue
action = personal_action(candidate)
if action is None:
continue
- change_preview = _change_preview(candidate)
+ preview = change_preview.change_preview(candidate)
if str(candidate.target_kind or "") == "skill":
# Legacy personal cycles could promote a single repeated
# tool call as a "sequence". Such a candidate has neither
@@ -266,12 +170,12 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]:
# approving it would install a tautological skill. Keep
# the evidence intact for audit, but do not present noise as
# a decision the user ought to make.
- if change_preview.get("type") == "skill_sequence":
- steps = change_preview.get("steps") or []
- rules = change_preview.get("ordering_constraints") or []
+ if preview.get("type") == "skill_sequence":
+ steps = preview.get("steps") or []
+ rules = preview.get("ordering_constraints") or []
if len(steps) < 2 and not rules:
continue
- elif change_preview.get("type") != "skill_document":
+ elif preview.get("type") != "skill_document":
# A skill with no inspectable content is not informed
# consent and would also fail materialisation later.
continue
@@ -286,14 +190,14 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]:
"your_episodes": len(mine),
"total_evidence": len(refs),
"own_share": round(share, 3),
- "tool_sequence": _tool_sequence_of_ir(candidate.ir),
+ "tool_sequence": change_preview.tool_sequence_of_ir(candidate.ir),
# What the button does, and what it will change. Without
# these the row states a diagnosis and offers a verb that
# does not follow from it.
"action": action["action"],
"action_label": action["label"],
"action_effect": action["effect"],
- "change": change_preview,
+ "change": preview,
"created_at": candidate.created_at.isoformat()
if candidate.created_at
else None,
@@ -307,89 +211,6 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]:
return []
-def _change_preview(candidate) -> Dict[str, Any]:
- """The concrete content of the proposed change.
-
- A one-line hypothesis explains *why* something is proposed and says nothing
- about *what* it is. Approving a distilled skill without being able to read
- its body is not a decision, it is a guess — the text was already in the IR,
- it was simply never sent to the page that asks the user to accept it.
- """
- ir = candidate.ir or {}
- kind = str(candidate.target_kind or "")
- changes = ir.get("changes") or []
- for change in changes:
- if kind == "skill":
- document = change.get("document")
- if isinstance(document, dict) and document.get("content"):
- return {
- "type": "skill_document",
- "display_name": str(document.get("display_name") or ""),
- "description": str(document.get("description") or ""),
- "allowed_tools": [str(t) for t in (document.get("allowed_tools") or [])],
- "content": str(document.get("content") or ""),
- }
- if kind == "memory":
- operations = change.get("operations") or []
- if operations:
- return {
- "type": "memory_ops",
- "operations": [
- {
- "operation": str(op.get("operation") or ""),
- "text": str(op.get("text") or op.get("memory_ref") or ""),
- "reason": str(op.get("reason") or ""),
- "before": op.get("before"),
- "after": op.get("after"),
- }
- for op in operations
- ],
- }
- if kind == "skill":
- # Sequence-only candidates are the common output of the personal
- # evolution path. They do not carry an authored SKILL.md, but approval
- # still materialises a concrete name, description and ordered set of
- # steps. Surface that exact structure instead of returning {}, which
- # made the frontend silently remove the detail button.
- tools: List[str] = []
- constraints: List[Dict[str, Any]] = []
- for change in changes:
- sequence = change.get("tool_sequence")
- if isinstance(sequence, list) and sequence and not tools:
- tools = [str(tool) for tool in sequence]
- rules = change.get("ordering_constraints")
- if isinstance(rules, list) and rules:
- constraints = [dict(rule) for rule in rules if isinstance(rule, dict)]
-
- rules_only = not tools and bool(constraints)
- if rules_only:
- from core.evolution.activation import _tools_from_constraints
-
- tools = _tools_from_constraints(constraints)
- if tools:
- # Reuse the materialiser's wording so the preview and the installed
- # skill never disagree about what the user accepted.
- from core.evolution.activation import _skill_description, _skill_label
-
- return {
- "type": "skill_sequence",
- "display_name": _skill_label(tools, rules_only=rules_only),
- "description": _skill_description(tools, constraints, rules_only=rules_only),
- "allowed_tools": tools,
- "steps": tools,
- "ordering_constraints": constraints,
- }
- return {}
-
-
-def _tool_sequence_of_ir(ir: Optional[Dict[str, Any]]) -> List[str]:
- for change in (ir or {}).get("changes") or []:
- sequence = change.get("tool_sequence")
- if isinstance(sequence, list) and sequence:
- return [str(t) for t in sequence]
- return []
-
-
def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]:
"""Approve a candidate and activate it **for this user only**.
@@ -397,7 +218,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]:
anyone else's agent does. That is what makes personal approval safe enough
to sit in a settings panel rather than behind an administrator.
"""
- from core.evolution.activation import materialise_skill_candidate
+ from core.evolution.cycle.activation import materialise_skill_candidate
allowed = {c["candidate_id"]: c for c in pending_for_user(user_id, limit=200)}
row = allowed.get(candidate_id)
@@ -406,7 +227,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]:
# or double click lands here, and telling that user they lack permission
# over a skill they just installed reads as a failure.
with SessionLocal() as db:
- if candidate_id in _personally_activated_ids(db, user_id, [candidate_id]):
+ if candidate_id in personally_activated_ids(db, user_id, [candidate_id]):
raise PermissionError("该候选你已启用过,对应能力已在你的技能列表中,无需重复启用")
raise PermissionError(
"该候选无法由你单独批准:可能主要来自他人的对话证据,或属于影响全体成员的编排/退役变更,需管理员审批"
@@ -416,7 +237,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]:
# skill materialiser is what made memory candidates fail with
# "暂不支持物化 memory 候选" on every click.
if row["action"] == "apply_memory":
- from core.evolution.memory_apply import apply_memory_candidate
+ from core.evolution.generators.memory_apply import apply_memory_candidate
result = apply_memory_candidate(
candidate_id, approver=f"user:{user_id}", require_user_id=user_id
@@ -468,7 +289,7 @@ def summarise_contributions(user_id: str) -> Dict[str, Any]:
return empty
episode_ids = {e.episode_id for e in episodes}
- contributed = [e for e in episodes if e.privacy_class != PRIVACY_PRIVATE]
+ contributed = [e for e in episodes if e.privacy_class != user_settings_constants.PRIVACY_PRIVATE]
memory_written = 0
for event in (
diff --git a/src/backend/core/evolution/policies.py b/src/backend/core/evolution/profiles/policies.py
similarity index 88%
rename from src/backend/core/evolution/policies.py
rename to src/backend/core/evolution/profiles/policies.py
index e9e1dda5..abfe1e10 100644
--- a/src/backend/core/evolution/policies.py
+++ b/src/backend/core/evolution/profiles/policies.py
@@ -17,7 +17,7 @@
fix for that shipped as a ``tenant_id`` parameter no caller passed.
All three are gone. There is one orchestration asset —
-:class:`~core.evolution.agent_profile.AgentProfile` — and this module is the
+:class:`~core.evolution.profiles.agent_profile.AgentProfile` — and this module is the
**projection** of it that the autonomous loop consumes. A projection rather than
a copy: there is no second store to publish to, no second thing to keep in sync,
and no way for the loop's numbers to disagree with the profile they came from.
@@ -29,23 +29,9 @@
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
-from core.evolution.agent_profile import (
- ACTION_CHANGE_STRATEGY,
- ACTION_ESCALATE,
- ACTION_NARROW_SCOPE,
- ACTION_RETRY,
- ACTION_ROLLBACK_AND_FORK,
- ACTION_STOP,
- ALLOWED_ACTIONS,
- SIGNAL_BUDGET_LOW,
- SIGNAL_NO_DIFF,
- SIGNAL_NO_PROGRESS,
- SIGNAL_REPEATED_ACTIONS,
- SIGNAL_REVIEWER_FLAT,
- AgentProfile,
- InterventionRule,
- load_active_profile,
-)
+from core.evolution.profiles.agent_profile_constants import ACTION_CHANGE_STRATEGY, ACTION_ESCALATE, ACTION_NARROW_SCOPE, ACTION_RETRY, ACTION_ROLLBACK_AND_FORK, ACTION_STOP, ALLOWED_ACTIONS, SIGNAL_BUDGET_LOW, SIGNAL_NO_DIFF, SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_REVIEWER_FLAT
+from core.evolution.profiles.agent_profile import AgentProfile, InterventionRule
+from core.evolution.profiles.profile_selection import load_active_profile
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/prefs.py b/src/backend/core/evolution/profiles/prefs.py
similarity index 100%
rename from src/backend/core/evolution/prefs.py
rename to src/backend/core/evolution/profiles/prefs.py
diff --git a/src/backend/core/evolution/profiles/profile_selection.py b/src/backend/core/evolution/profiles/profile_selection.py
new file mode 100644
index 00000000..6d0fef54
--- /dev/null
+++ b/src/backend/core/evolution/profiles/profile_selection.py
@@ -0,0 +1,147 @@
+"""Pick the agent profile that governs a run."""
+from __future__ import annotations
+from core.evolution.profiles import agent_profile
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+import logging
+logger = logging.getLogger(__name__)
+
+def scope_rank(scope: Dict[str, str], *, user_id: str, tenant_id: str) -> Optional[int]:
+ """How specific this profile is to the subject, or ``None`` if it is not.
+
+ Lower is more specific. Returning ``None`` for a profile owned by another
+ subject is the whole point: it must not act as a fallback.
+ """
+ owner = str((scope or {}).get("user_id") or "")
+ tenant = str((scope or {}).get("tenant_id") or "")
+ if owner:
+ return 0 if owner == user_id else None
+ if tenant:
+ return 1 if tenant == (tenant_id or "default") else None
+ return 2
+
+
+def pick_profile(
+ profiles: Sequence[agent_profile.AgentProfile],
+ *,
+ task_type: str,
+ user_id: str = "",
+ tenant_id: str = "default",
+) -> Optional[agent_profile.AgentProfile]:
+ """The one profile that governs this run, or ``None``.
+
+ A task-type-specific profile beats a general one at the same scope level:
+ the narrower statement is the more deliberate one.
+ """
+ ranked: List[Tuple[int, int, agent_profile.AgentProfile]] = []
+ for profile in profiles:
+ if not profile.applies_to(task_type):
+ continue
+ rank = scope_rank(profile.scope, user_id=user_id, tenant_id=tenant_id)
+ if rank is None:
+ continue
+ ranked.append((rank, 0 if profile.task_types else 1, profile))
+ if not ranked:
+ return None
+ ranked.sort(key=lambda item: (item[0], item[1]))
+ return ranked[0][2]
+
+
+def profile_from_row(row: Any) -> agent_profile.AgentProfile:
+ """The profile a stored row describes, with the columns believed over payload.
+
+ ``profile_id`` / ``version`` / ``task_types`` / ``scope`` exist as real
+ columns *and* inside ``payload`` (:meth:`AgentProfile.to_dict` writes them),
+ so for anything published through :mod:`core.evolution.cycle.activation` the two
+ agree and this is a no-op. They stop agreeing for a row written by hand or
+ by a seeding script, and reading only ``payload`` then fails open in the
+ worst possible direction: a missing ``task_types`` parses as ``[]``, which
+ :meth:`AgentProfile.applies_to` treats as *every* task type, and a missing
+ ``scope`` parses as ``{}``, which :func:`_scope_rank` treats as *every*
+ subject. A profile published for one task type silently governs every run —
+ reported under the id ``builtin``, because that is what
+ :meth:`AgentProfile.from_dict` falls back to, so the log line naming the
+ culprit names the deployment default instead.
+
+ The two narrowing fields resolve to whichever side is non-empty rather than
+ to the column unconditionally: an empty column against a populated payload
+ is the same widening failure with the sides swapped. When both are populated
+ and disagree the column wins — it is what the activation uniqueness check and
+ the console read — and the disagreement is logged, because it means something
+ wrote this row outside the activation path.
+ """
+ raw = dict(row.payload or {})
+
+ # Identity: the column is the primary key and NOT NULL, so it is always the
+ # more trustworthy of the two, and a wrong id here is what makes a bad
+ # profile untraceable in the logs.
+ raw["profile_id"] = row.profile_id
+ raw["version"] = row.version
+
+ col_types = [str(t) for t in (row.task_types or [])]
+ payload_types = [str(t) for t in (raw.get("task_types") or [])]
+ if col_types and payload_types and set(col_types) != set(payload_types):
+ logger.warning(
+ "[agent-profile] %s: task_types column %s != payload %s, using column",
+ row.profile_id, col_types, payload_types,
+ )
+ raw["task_types"] = col_types or payload_types
+
+ col_scope = dict(row.scope or {})
+ payload_scope = dict(raw.get("scope") or {})
+ if col_scope and payload_scope and col_scope != payload_scope:
+ logger.warning(
+ "[agent-profile] %s: scope column %s != payload %s, using column",
+ row.profile_id, col_scope, payload_scope,
+ )
+ raw["scope"] = col_scope or payload_scope
+
+ return agent_profile.AgentProfile.from_dict(raw)
+
+
+def load_active_profile(
+ *,
+ task_type: str = "chat",
+ user_id: str = "",
+ tenant_id: str = "default",
+) -> agent_profile.AgentProfile:
+ """The profile in force for this run.
+
+ Read fresh on every call. Resolving once at import — which is what the old
+ orchestration policy did — means a published change needs a process restart
+ to take effect and different replicas disagree in the meantime, neither of
+ which is acceptable for an asset whose entire purpose is to change behaviour
+ between runs.
+ """
+ from core.db.engine import SessionLocal
+ from core.db.models.evolution import EvolutionAgentProfile
+
+ try:
+ with SessionLocal() as db:
+ rows = (
+ db.query(EvolutionAgentProfile)
+ .filter(EvolutionAgentProfile.is_active.is_(True))
+ .order_by(EvolutionAgentProfile.updated_at.desc())
+ .all()
+ )
+ profiles = [profile_from_row(row) for row in rows]
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[agent-profile] load failed, using built-in: %s", exc)
+ return agent_profile.builtin_profile()
+
+ chosen = pick_profile(
+ profiles, task_type=task_type, user_id=user_id, tenant_id=tenant_id
+ )
+ if chosen is None:
+ return agent_profile.builtin_profile()
+ ok, problems = agent_profile.validate_profile(chosen)
+ if not ok:
+ # A stored profile that no longer validates is not applied. The
+ # alternative — applying it partially — produces a configuration nobody
+ # reviewed and nobody can reproduce.
+ logger.warning(
+ "[agent-profile] stored profile %s rejected at load: %s",
+ chosen.profile_id,
+ problems,
+ )
+ return agent_profile.builtin_profile()
+ return chosen
diff --git a/src/backend/core/evolution/profiles/stall_hook.py b/src/backend/core/evolution/profiles/stall_hook.py
new file mode 100644
index 00000000..97b22316
--- /dev/null
+++ b/src/backend/core/evolution/profiles/stall_hook.py
@@ -0,0 +1,103 @@
+"""Stall decisions from neutral tool attempts, without framework objects."""
+import json
+import logging
+from core.harness.context_items import reminder_item
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage
+
+logger = logging.getLogger(__name__)
+
+
+class StallHook:
+ def __init__(self, rules):
+ self._rules = list(rules or [])
+ self._signals = {}
+ self._last_call = None
+ self._fired = set()
+
+ def __call__(self, invocation):
+ if not self._rules:
+ return Decision()
+ if invocation.stage == HookStage.BEFORE_TOOL:
+ signature = (invocation.operation_name,
+ json.dumps(invocation.data.get("raw_input"), sort_keys=True, default=str))
+ self._signals["repeated_actions"] = (self._signals.get("repeated_actions", 0) + 1
+ if signature == self._last_call else 0)
+ self._last_call = signature
+ return Decision()
+ response = invocation.data.get("tool_result") or {}
+ failed = response.get("state") in {"error", "denied", "interrupted"}
+ for signal in ("tool_error_streak", "no_progress"):
+ self._signals[signal] = self._signals.get(signal, 0) + 1 if failed else 0
+ return self.intervene(invocation)
+
+ def intervene(self, invocation):
+ from core.evolution.profiles.agent_profile_constants import ACTION_CHANGE_STRATEGY, ACTION_DELEGATE, ACTION_ESCALATE, ACTION_NARROW_SCOPE, ACTION_RETRY, ACTION_ROLLBACK_AND_FORK, ACTION_STOP
+
+ applicable = [
+ rule
+ for rule in self._rules
+ if self._signals.get(getattr(rule, "signal", ""), 0)
+ >= int(getattr(rule, "threshold", 0) or 0)
+ and int(getattr(rule, "threshold", 0) or 0) > 0
+ ]
+ if not applicable:
+ return
+ # Most-specific first: a long stall escalates rather than repeating the
+ # mild response that already failed to help.
+ rule = max(applicable, key=lambda r: int(getattr(r, "threshold", 0) or 0))
+ action = str(getattr(rule, "action", ""))
+ signal = str(getattr(rule, "signal", ""))
+
+ # Fire once per (signal, action). Repeating the same reminder every
+ # iteration turns it into noise the model learns to skip.
+ key = f"{signal}:{action}"
+ if key in self._fired:
+ return
+ self._fired.add(key)
+
+ observed = {
+ "repeated_actions": "你已经用完全相同的参数重复调用同一个工具",
+ "tool_error_streak": "连续多次工具调用失败",
+ "no_progress": "连续多轮没有取得任何可用结果",
+ }.get(signal, "当前执行已停滞")
+
+ guidance = {
+ ACTION_CHANGE_STRATEGY: "换一条思路:不要重试同样的调用,改用别的工具或别的切入角度。",
+ ACTION_NARROW_SCOPE: "缩小范围:先只解决其中最小的一个子问题,拿到确定结果再往下走。",
+ ACTION_RETRY: "可以再试一次,但必须改变参数或前置条件,原样重试不会有不同结果。",
+ ACTION_DELEGATE: "把这段工作交给合适的子智能体处理,只取回结论。",
+ ACTION_ESCALATE: "停止尝试,向用户说明卡在哪里、还缺什么信息。",
+ ACTION_STOP: "停止继续尝试,如实汇报已完成的部分和未完成的原因。",
+ # Loop-only: a single answer has nothing to roll back to.
+ ACTION_ROLLBACK_AND_FORK: "放弃当前路径,回到上一个可用状态重新规划。",
+ }.get(action, "")
+ if not guidance:
+ return
+
+ logger.info(
+ "[stall-intervention] %s >= %s -> %s",
+ signal,
+ getattr(rule, "threshold", 0),
+ action,
+ )
+ message = (
+ f"{observed}。{guidance}\n"
+ f"(该干预来自当前生效的编排配置:{signal} ≥ "
+ f"{getattr(rule, 'threshold', 0)} → {action})"
+ )
+ return Decision.modify({
+ "context_items": [
+ *invocation.data.get("context_items", ()),
+ reminder_item(message, "harness:stall_intervention"),
+ ],
+ })
+
+
+
+
+def stall_hooks(rules):
+ policy = StallHook(rules)
+ return (HookSpec("stall_attempt", HookStage.BEFORE_TOOL, policy, priority=400,
+ failure_mode=FailureMode.OPEN),
+ HookSpec("stall_result", HookStage.AFTER_TOOL, policy, priority=400,
+ failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"})))
diff --git a/src/backend/core/evolution/profiles/user_settings.py b/src/backend/core/evolution/profiles/user_settings.py
new file mode 100644
index 00000000..bb62a77e
--- /dev/null
+++ b/src/backend/core/evolution/profiles/user_settings.py
@@ -0,0 +1,70 @@
+"""Read and update whether a user's episodes feed the evolution loop."""
+from __future__ import annotations
+from typing import Any, Dict, Optional
+from core.db.engine import SessionLocal
+import logging
+from core.evolution.profiles import user_settings_constants
+
+logger = logging.getLogger(__name__)
+
+def is_contribution_enabled(metadata: Optional[Dict[str, Any]]) -> bool:
+ """Whether this user's episodes may feed the evolution loop.
+
+ Reads the single evolution switch (``evolution_prefs.enabled``), so one
+ stored value governs both distilling *for* the user and learning *from*
+ them. Off by default: consent is collected by the first-run wizard, and a
+ user who never enabled evolution contributes nothing. "On" only ever means
+ *within the tenant*: cross-tenant sharing is a separate, independently-
+ gated decision (see ``federation``).
+ """
+ from core.evolution.profiles.prefs import PREFS_KEY, normalise
+
+ if not metadata:
+ return False
+ return bool(normalise(metadata.get(PREFS_KEY)).get("enabled", False))
+
+
+def privacy_class_for(metadata: Optional[Dict[str, Any]]) -> str:
+ """The Episode privacy class implied by the user's setting."""
+ return user_settings_constants.PRIVACY_TENANT if is_contribution_enabled(metadata) else user_settings_constants.PRIVACY_PRIVATE
+
+
+def resolve_for_user(user_id: str) -> Dict[str, Any]:
+ """Read the setting for one user. Degrades to the default, never raises."""
+ if not user_id:
+ return {user_settings_constants.SETTING_KEY: False, "privacy_class": user_settings_constants.PRIVACY_PRIVATE}
+ try:
+ from core.db.models import UserShadow
+
+ with SessionLocal() as db:
+ row = db.query(UserShadow).filter(UserShadow.user_id == user_id).first()
+ metadata = (getattr(row, "extra_data", None) or {}) if row else {}
+ except Exception as exc:
+ logger.debug("[evo-settings] read failed for %s: %s", user_id, exc)
+ metadata = {}
+
+ enabled = is_contribution_enabled(metadata)
+ return {user_settings_constants.SETTING_KEY: enabled, "privacy_class": privacy_class_for(metadata)}
+
+
+def update_for_user(user_id: str, enabled: bool) -> Dict[str, Any]:
+ """Persist the setting by writing the single evolution switch.
+
+ Kept as an alias so older bundled frontends PATCHing /evolution/settings
+ still work; it writes the same ``evolution_prefs.enabled`` the preferences
+ panel writes, never a separate value that could drift from it.
+
+ Turning it off does **not** retroactively delete already-contributed
+ evidence — that would silently rewrite history other candidates were derived
+ from. It stops future episodes from contributing; withdrawing past evidence
+ is a separate, explicit action so its consequences stay visible.
+ """
+ from core.evolution.profiles.prefs import update_prefs
+
+ merged = update_prefs(user_id, {"enabled": bool(enabled)})
+ on = bool(merged.get("enabled", False))
+ return {
+ user_settings_constants.SETTING_KEY: on,
+ "privacy_class": user_settings_constants.PRIVACY_TENANT if on else user_settings_constants.PRIVACY_PRIVATE,
+ "note": "仅影响此后新产生的证据;已贡献的证据需单独撤回",
+ }
diff --git a/src/backend/core/evolution/profiles/user_settings_constants.py b/src/backend/core/evolution/profiles/user_settings_constants.py
new file mode 100644
index 00000000..7cceead8
--- /dev/null
+++ b/src/backend/core/evolution/profiles/user_settings_constants.py
@@ -0,0 +1,14 @@
+"""Setting key and privacy classes for evolution contributions."""
+from __future__ import annotations
+
+
+SETTING_KEY = "evolution_contribution_enabled"
+
+
+PRIVACY_PRIVATE = "private"
+
+
+PRIVACY_TENANT = "tenant"
+
+
+OWN_EVIDENCE_THRESHOLD = 0.6
diff --git a/src/backend/core/evolution/promotion.py b/src/backend/core/evolution/promotion.py
deleted file mode 100644
index f71a631c..00000000
--- a/src/backend/core/evolution/promotion.py
+++ /dev/null
@@ -1,890 +0,0 @@
-"""Cross-engine capability promotion (GCE tickets 32 / 33 / 34 / 36).
-
-This is the part that makes the system *learn* rather than merely record.
-Capability flows up a chain:
-
- raw traces → episodic memory → skill candidate → workflow candidate → domain
- constraint → released asset
-
-The idea is that experience should not stay in whichever form it first arrived
-in. A pattern that keeps recurring as scattered memories is re-retrieved and
-re-planned on every occurrence; compiled into a skill, it is assembled once. A
-combination of skills that keeps appearing in the same order is re-derived every
-time; promoted to a workflow, it becomes reusable — which is the direct answer
-to "orchestration never accumulates", the largest gap in the current system.
-
-Two design commitments:
-
-* **Promotion, not duplication.** A promoted memory is down-weighted rather
- than deleted, and the link is bidirectional, so "why does this skill exist?"
- has an answer and a rollback can restore the original weighting.
-* **Aggregate evidence by default.** A single high-confidence failure may
- trigger a candidate, but the normal entry point is a recurring cluster.
- Otherwise the system permanently encodes noise.
-
-Flow also runs downhill: overlapping assets merge, unused ones retire, and
-**negative transfer** — a change that quietly degrades tasks it has nothing to do
-with — is detected. That last one is the most dangerous failure mode of joint
-evolution and the easiest to miss, because no single release's guardrail ever
-trips.
-"""
-
-from __future__ import annotations
-
-import logging
-from collections import Counter, defaultdict
-from dataclasses import dataclass, field
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
-
-logger = logging.getLogger(__name__)
-
-# Pattern families discovered from episode clusters.
-PATTERN_REPEATED_FAILURE = "repeated_failure"
-PATTERN_SUCCESS_SUBSEQUENCE = "success_subsequence"
-PATTERN_DRIFT = "execution_drift"
-
-# Minimum cluster size before anything downstream is produced. A lower bar would
-# let one-off noise become a permanent asset.
-MIN_SUPPORT = 5
-# A single failure may still trigger, but is flagged so reviewers can see that
-# it rests on one observation.
-SINGLE_SHOT_CONFIDENCE = 0.85
-
-# Promotion thresholds.
-SKILL_MIN_SUCCESS_RATE = 0.7
-WORKFLOW_MIN_COMBO_FREQUENCY = 5
-# A combination must beat its parts, not merely be frequent — otherwise every
-# common sequence becomes a workflow whether or not the grouping helps.
-WORKFLOW_MIN_UPLIFT = 0.05
-
-# 一段 SOP 至少要有的步骤数。长度 1–2 的"序列"几乎总是通用工具的巧合复现——
-# 整轮对话只调一次 view_text_file 也会形成"相同序列"——包装它们不携带任何
-# 过程知识,只往评审队列里添噪声。
-MIN_TOOL_SEQUENCE_LEN = 3
-# 支撑 Episode 中归入同一意图簇的最低占比。序列相同只说明工具巧合,
-# 意图相同才说明存在可复用的做事方法——这是把一段序列当 SOP 的前提。
-INTENT_COHESION_RATIO = 0.6
-
-
-@dataclass
-class Pattern:
- """A recurring shape found across episodes."""
-
- kind: str
- signature: str
- support: int
- episode_ids: List[str] = field(default_factory=list)
- success_rate: float = 0.0
- tool_sequence: List[str] = field(default_factory=list)
- skill_sequence: List[str] = field(default_factory=list)
- memory_refs: List[str] = field(default_factory=list)
- single_shot: bool = False
-
- @property
- def meets_support(self) -> bool:
- return self.support >= MIN_SUPPORT or self.single_shot
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "kind": self.kind,
- "signature": self.signature,
- "support": self.support,
- "success_rate": round(self.success_rate, 3),
- "tool_sequence": self.tool_sequence,
- "skill_sequence": self.skill_sequence,
- "memory_refs": self.memory_refs,
- "single_shot": self.single_shot,
- "episode_ids": self.episode_ids[:20],
- }
-
-
-def _signature(items: Sequence[str]) -> str:
- return "→".join(items)
-
-
-def discover_patterns(episodes: List[Dict[str, Any]]) -> List[Pattern]:
- """Cluster episodes into recurring shapes.
-
- ``episodes`` are plain dicts so this stays testable without a database and
- usable over backfilled history (which has no asset bundle and therefore
- cannot be replayed, but is perfectly good for finding patterns).
- """
- by_tool_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
- by_skill_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
-
- for episode in episodes:
- tools = [str(t) for t in (episode.get("tool_sequence") or [])]
- skills = [str(s) for s in (episode.get("skill_sequence") or [])]
- if tools:
- by_tool_seq[_signature(tools)].append(episode)
- if len(skills) >= 2:
- by_skill_seq[_signature(skills)].append(episode)
-
- patterns: List[Pattern] = []
-
- for signature, group in by_tool_seq.items():
- successes = [e for e in group if e.get("verdict") == "success"]
- failures = [e for e in group if e.get("verdict") == "failed"]
- rate = len(successes) / len(group) if group else 0.0
-
- if len(successes) >= MIN_SUPPORT and rate >= SKILL_MIN_SUCCESS_RATE:
- patterns.append(
- Pattern(
- kind=PATTERN_SUCCESS_SUBSEQUENCE,
- signature=signature,
- support=len(successes),
- episode_ids=[e.get("episode_id", "") for e in successes],
- success_rate=rate,
- tool_sequence=signature.split("→"),
- memory_refs=sorted(
- {
- ref
- for e in successes
- for ref in (e.get("memory_refs") or [])
- }
- ),
- )
- )
- if len(failures) >= MIN_SUPPORT:
- patterns.append(
- Pattern(
- kind=PATTERN_REPEATED_FAILURE,
- signature=signature,
- support=len(failures),
- episode_ids=[e.get("episode_id", "") for e in failures],
- success_rate=rate,
- tool_sequence=signature.split("→"),
- )
- )
-
- for signature, group in by_skill_seq.items():
- if len(group) >= WORKFLOW_MIN_COMBO_FREQUENCY:
- successes = [e for e in group if e.get("verdict") == "success"]
- patterns.append(
- Pattern(
- kind=PATTERN_SUCCESS_SUBSEQUENCE,
- signature=signature,
- support=len(group),
- episode_ids=[e.get("episode_id", "") for e in group],
- success_rate=len(successes) / len(group),
- skill_sequence=signature.split("→"),
- )
- )
-
- return [p for p in patterns if p.meets_support]
-
-
-# ── Memory → Skill ───────────────────────────────────────────────────────────
-
-
-@dataclass
-class PromotionProposal:
- """A promotion the chain wants to make, before it becomes an IR candidate."""
-
- from_layer: str
- to_layer: str
- signature: str
- support: int
- rationale: str
- source_refs: List[str] = field(default_factory=list)
- payload: Dict[str, Any] = field(default_factory=dict)
- # Set when an equivalent asset already exists: promotion becomes a patch
- # rather than a new asset, so the library does not fill with near-duplicates.
- patch_target: Optional[str] = None
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "from_layer": self.from_layer,
- "to_layer": self.to_layer,
- "signature": self.signature,
- "support": self.support,
- "rationale": self.rationale,
- "source_refs": list(self.source_refs),
- "payload": self.payload,
- "patch_target": self.patch_target,
- }
-
-
-def _dominant_shared_intent(
- pattern: Pattern,
- episodes: Optional[Sequence[Dict[str, Any]]],
- *,
- support_floor: int,
-):
- """The one intent behind a sequence's supporting episodes, or ``None``.
-
- ``None`` is a refusal, and deliberately covers the honest-uncertainty cases
- too: no views supplied, request text too short to cluster, no clustering
- method available. "We cannot tell whether these conversations had anything
- in common" and "they had nothing in common" both mean no SOP claim can be
- made — treating the first as a pass is how single-tool coincidences used to
- become skills.
- """
- if not episodes:
- return None
- wanted = {str(e) for e in pattern.episode_ids}
- supporting = [e for e in episodes if str(e.get("episode_id") or "") in wanted]
- if len(supporting) < support_floor:
- return None
-
- from core.evolution.similarity import METHOD_NONE, cluster_intents
-
- clusters, method = cluster_intents(supporting, min_support=2)
- if method == METHOD_NONE or not clusters:
- return None
- top = clusters[0]
- if len(top.episode_ids) < support_floor:
- return None
- if len(top.episode_ids) / len(supporting) < INTENT_COHESION_RATIO:
- return None
- # 复现必须跨会话:occasions 按 chat_id 去重,同一会话里反复跑同一套
- # 流程只算一次场合,不构成"这件事总在发生"。
- if top.occasions < support_floor:
- return None
- return top
-
-
-def promote_tool_sequence_to_skill(
- pattern: Pattern,
- *,
- episodes: Optional[Sequence[Dict[str, Any]]] = None,
- min_support: Optional[int] = None,
- skill_credit: float = 0.0,
- workflow_credit: float = 0.0,
- existing_skill_signatures: Optional[Dict[str, str]] = None,
-) -> Optional[PromotionProposal]:
- """Compile a recurring successful tool sequence into a skill proposal.
-
- A recurring sequence alone is *not* evidence of a reusable capability.
- Generic tools recur across unrelated conversations by construction, and the
- limiting case — a whole conversation whose "sequence" is one call to
- ``view_text_file`` — says only that the task was trivial. What justifies an
- SOP is commonality of *purpose*: the same kind of request, on separate
- occasions, repeatedly completed by the same non-trivial procedure. So the
- gates hold together, and each excludes a specific way of being wrong:
-
- * the sequence is **non-trivial** (≥ :data:`MIN_TOOL_SEQUENCE_LEN` steps) —
- wrapping one generic tool adds an asset and no knowledge;
- * the supporting episodes **share an intent** — a dominant cluster over
- their request text (via :mod:`core.evolution.similarity`) covers
- ≥ :data:`INTENT_COHESION_RATIO` of them; without that the recurrence is a
- coincidence of tooling, and no commonality means no promotion;
- * the recurrence spans **separate conversations** — one chat re-running the
- same procedure is one occasion, not evidence it keeps coming back;
- * the recurrences **succeed** at ≥ :data:`SKILL_MIN_SUCCESS_RATE`.
-
- ``episodes`` are the flattened views the pattern was discovered from; the
- intent gate reads their request text. A caller that cannot supply them gets
- a refusal, not the benefit of the doubt.
-
- Named for what it reads. This was called ``promote_memory_to_skill`` and
- carried ``from_layer="memory"``, but no memory content ever enters the
- result — ``pattern.memory_refs`` are hashes of whatever happened to be
- retrieved during the supporting episodes. The name promised a memory→skill
- chain the code did not have; :func:`promote_procedural_memory_to_skill` is
- that chain.
-
- Refuses when orchestration is the better explanation: a stable tool ordering
- can equally mean "the workflow is fixed" rather than "there is a reusable
- skill here". Promoting in that case creates a skill with no independent
- content, which then has to be maintained forever.
- """
- if pattern.kind != PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence:
- return None
- support_floor = MIN_SUPPORT if min_support is None else int(min_support)
- if len(pattern.tool_sequence) < MIN_TOOL_SEQUENCE_LEN:
- logger.info(
- "[promotion] skipping %s: %d-step sequence is below the SOP floor",
- pattern.signature,
- len(pattern.tool_sequence),
- )
- return None
- if pattern.support < support_floor:
- return None
- if pattern.success_rate < SKILL_MIN_SUCCESS_RATE:
- return None
- if workflow_credit > skill_credit:
- logger.info(
- "[promotion] skipping skill promotion for %s: workflow credit higher",
- pattern.signature,
- )
- return None
-
- intent = _dominant_shared_intent(pattern, episodes, support_floor=support_floor)
- if intent is None:
- logger.info(
- "[promotion] skipping %s: supporting episodes share no intent — a "
- "recurring sequence without a recurring request is not an SOP",
- pattern.signature,
- )
- return None
- covered = [e for e in pattern.episode_ids if e in set(intent.episode_ids)]
-
- existing = (existing_skill_signatures or {}).get(pattern.signature)
- return PromotionProposal(
- from_layer="episode",
- to_layer="skill",
- signature=pattern.signature,
- support=len(covered),
- rationale=(
- f"「{intent.representative[:40]}」类请求在 {intent.occasions} 次独立会话中"
- f"反复出现,每次都以同一 {len(pattern.tool_sequence)} 步工具序列成功完成"
- f"({len(covered)} 个 Episode,成功率 {pattern.success_rate:.0%})——"
- "同类意图 + 稳定做法,具备沉淀为 SOP 的共性"
- ),
- # Co-retrieved memories are *context*, not sources. They are carried so
- # a reviewer can see what was in view, and deliberately not demoted —
- # nothing of theirs went into this skill, so down-weighting them would
- # remove knowledge the skill does not carry.
- source_refs=[],
- payload={
- "tool_sequence": pattern.tool_sequence,
- "episode_ids": covered[:20],
- "intent": intent.representative,
- "occasions": intent.occasions,
- "co_retrieved_memory_refs": list(pattern.memory_refs),
- },
- # An equivalent skill already exists → patch it instead of adding a twin.
- patch_target=existing,
- )
-
-
-# Enough separate occasions that "this keeps coming up" is a claim about
-# behaviour rather than about one busy afternoon.
-PROCEDURAL_MIN_OCCURRENCES = 3
-# Below this the procedure is being followed into failures, and compiling it
-# would encode the failure as the house style.
-PROCEDURAL_MIN_SUCCESS_RATE = 0.6
-
-
-def promote_procedural_memory_to_skill(
- *,
- memories: Sequence[Dict[str, Any]],
- intent_cluster: Dict[str, Any],
- occurrences: int,
- success_rate: float,
- plan_variety: int,
- existing_skill_id: Optional[str] = None,
-) -> Optional[PromotionProposal]:
- """Compile *how work is done here* into a skill proposal.
-
- This is the chain the design described and the code never had. Four
- conditions must hold together, and each excludes a specific way of being
- wrong:
-
- * the memories are **procedural** — a fact frozen into a document stops
- being current, and a preference is already covered by one profile line;
- * the intent recurs across **separate occasions** — one conversation
- mentioning something three times is one occasion;
- * the recurrences **succeed** — distilling a failing approach makes it the
- house style;
- * the plans **varied** — the same request handled identically every time is
- already effectively a skill, and writing one down adds an asset to
- maintain while changing nothing.
-
- The proposal carries the memories' **content**, which is what makes this a
- compilation rather than a rename, and their refs, which is what lets the
- source memories be demoted when the skill goes live and restored when it is
- rolled back.
- """
- procedural = [
- m
- for m in memories
- if str(m.get("memory_type") or "") == "procedural" and str(m.get("content") or "").strip()
- ]
- if not procedural:
- return None
- if occurrences < PROCEDURAL_MIN_OCCURRENCES:
- return None
- if success_rate < PROCEDURAL_MIN_SUCCESS_RATE:
- return None
- if plan_variety < 2:
- return None
-
- representative = str(intent_cluster.get("representative") or "")
- return PromotionProposal(
- from_layer="memory",
- to_layer="skill",
- signature="proc:" + "|".join(sorted(str(m.get("ref") or "") for m in procedural)),
- support=occurrences,
- rationale=(
- f"「{representative[:40]}」类请求出现 {occurrences} 次、成功率 "
- f"{success_rate:.0%},却用了 {plan_variety} 种不同做法;"
- f"其中 {len(procedural)} 条做法/口径已被反复检索,应编译为一次性装配的技能"
- ),
- source_refs=[str(m.get("ref") or "") for m in procedural if m.get("ref")],
- payload={
- "procedures": [
- {
- "rule": str(m.get("content") or ""),
- "why": str(m.get("why") or ""),
- "applies_to": str(m.get("applies_to") or ""),
- }
- for m in procedural
- ],
- "episode_ids": [str(e) for e in (intent_cluster.get("episode_ids") or [])][:20],
- "intent": representative,
- "memory_refs": [str(m.get("ref") or "") for m in procedural if m.get("ref")],
- },
- patch_target=existing_skill_id,
- )
-
-
-# Down-weighting the memories a skill was compiled from lives in
-# :mod:`core.evolution.memory_apply`, because it is an applied change with an
-# undo rather than a list of dictionaries. The version that used to sit here
-# returned a plan nothing executed, and its ``restore_on_rollback: True`` was a
-# promise no code kept.
-
-
-# ── Skill → Orchestration ────────────────────────────────────────────────────
-
-
-def promote_skills_to_orchestration(
- pattern: Pattern,
- *,
- combo_success_rate: float,
- individual_success_rate: float,
- task_types: Optional[Sequence[str]] = None,
- skill_versions: Optional[Dict[str, str]] = None,
-) -> Optional[PromotionProposal]:
- """Promote a recurring skill combination into an **agent profile** proposal.
-
- Promoting it into a *workflow* — a fixed DAG — was the wrong target: the
- main axis is a ReAct loop, which never executes a DAG, so a workflow asset
- could only ever govern the separate autonomous-loop product. What a
- recurring skill combination actually says is "for this kind of task, these
- capabilities belong in view together", and the thing that decides what is in
- view is the profile.
-
- The combination must *beat its parts*: frequency alone would promote every
- common sequence regardless of whether grouping them helps. The proposal is
- declarative — a skill set and the task types it applies to — and carries no
- executable content.
- """
- if not pattern.skill_sequence or len(pattern.skill_sequence) < 2:
- return None
- if pattern.support < WORKFLOW_MIN_COMBO_FREQUENCY:
- return None
- if combo_success_rate - individual_success_rate < WORKFLOW_MIN_UPLIFT:
- return None
-
- versions = skill_versions or {}
- return PromotionProposal(
- from_layer="skill",
- to_layer="agent_profile",
- signature=pattern.signature,
- support=pattern.support,
- rationale=(
- f"技能组合出现 {pattern.support} 次,组合成功率 {combo_success_rate:.0%} "
- f"显著高于零散调用 {individual_success_rate:.0%}——"
- "应固化为该类任务的技能组合,而不是每次重新挑选"
- ),
- source_refs=list(pattern.skill_sequence),
- payload={
- "skill_ids": list(dict.fromkeys(pattern.skill_sequence)),
- "task_types": [str(t) for t in (task_types or [])],
- # Declared so the candidate auto-invalidates if a dependency retires
- # — a profile pointing at a dead skill version is worse than none.
- "depends_on": {s: versions.get(s, "") for s in pattern.skill_sequence},
- "kind": "declarative_policy",
- },
- )
-
-
-def dependencies_satisfied(
- proposal: PromotionProposal, live_skill_versions: Dict[str, str]
-) -> Tuple[bool, List[str]]:
- """Whether every declared dependency still exists at the declared version.
-
- This is the mechanism that makes cross-layer rollback work in one direction:
- retire a skill and every profile built on it stops validating, instead of
- quietly pointing at something that is no longer there.
- """
- declared = (proposal.payload or {}).get("depends_on") or {}
- stale = [
- skill_id
- for skill_id, version in declared.items()
- if version and live_skill_versions.get(skill_id) != version
- ]
- return (not stale), stale
-
-
-# ── Downhill flow: overlap, retirement, negative transfer ────────────────────
-
-
-def find_overlaps(
- signatures: Dict[str, List[str]], *, min_shared: int = 2
-) -> List[Tuple[str, str, int]]:
- """Assets whose **declared** step sequences overlap enough to be merge candidates.
-
- ``signatures`` must be each asset's own declared tools — the tools its
- document names. Feeding it the tools observed in episodes where the asset
- was merely loaded produces a merge proposal for every pair of skills that
- happened to be enabled together, which is guaranteed rather than
- informative: co-enabled skills share the episode's whole tool set by
- construction.
- """
- overlaps: List[Tuple[str, str, int]] = []
- asset_ids = sorted(signatures)
- for i, a in enumerate(asset_ids):
- for b in asset_ids[i + 1 :]:
- shared = len(set(signatures[a]) & set(signatures[b]))
- if shared >= min_shared:
- overlaps.append((a, b, shared))
- return overlaps
-
-
-# How long a skill must have been exposed before "never opened" means anything.
-# Retiring on a short window measures release recency, not usefulness — which
-# is exactly what the old selection-count rule did, since every enabled skill
-# recorded the same count and the only real variable was how recently a skill
-# had shipped.
-MIN_EXPOSURE_DAYS = 14
-# Offered this many times without ever being opened → the model consistently
-# decides it is not relevant.
-MIN_OFFERS_BEFORE_JUDGING = 10
-# Opened this many times before follow-through is worth judging.
-MIN_OPENS_BEFORE_JUDGING = 3
-# Opened, but the plan afterwards ignored what it said.
-FOLLOW_THROUGH_FLOOR = 0.4
-SUCCESS_FLOOR = 0.4
-
-REASON_NEVER_OPENED = "never_opened"
-REASON_IGNORED_WHEN_OPENED = "ignored_when_opened"
-REASON_LOW_SUCCESS = "low_success_when_opened"
-REASON_HARMFUL = "worse_than_without"
-
-# What to do about it. Distinct because the fixes are distinct: a skill nobody
-# opens has a discovery problem (its description does not match the requests it
-# serves), a skill that gets opened and ignored has a *content* problem, and a
-# skill that makes things worse has to stop being exposed now.
-ACTION_LOWER_EXPOSURE = "lower_exposure"
-ACTION_REWRITE = "rewrite"
-ACTION_RETIRE = "retire"
-ACTION_ROLLBACK = "rollback"
-
-
-def find_retirement_candidates(
- usage: Dict[str, Dict[str, Any]],
-) -> List[Dict[str, Any]]:
- """Assets that are not earning their place, and what to do about each.
-
- ``usage`` per asset:
-
- ``offers``
- turns where the skill was selected into the prompt (non-degraded
- selections only).
- ``opens``
- turns where the model actually opened its document. This is the usage
- signal; ``offers`` alone cannot distinguish "never chosen" from "chosen
- and useless".
- ``follow_through``
- of the opens, the fraction where the subsequent tool calls stayed
- inside what the skill declared.
- ``success_rate``
- of the opens, the fraction of turns that succeeded.
- ``uplift``
- success with the skill exposed minus success without it, where a paired
- comparison exists. ``None`` when it does not.
- ``exposure_days``
- how long it has been exposed at all.
-
- A library that only grows fills with near-duplicates and degrades selection
- precision for everything in it — but every judgement here is a *proposal*,
- because removing a capability produces no error, only a slow decline.
- """
- out: List[Dict[str, Any]] = []
- for asset_id, stats in (usage or {}).items():
- offers = int(stats.get("offers", 0) or 0)
- opens = int(stats.get("opens", 0) or 0)
- exposure_days = float(stats.get("exposure_days", 0) or 0)
- uplift = stats.get("uplift")
-
- if uplift is not None and float(uplift) <= -0.05:
- out.append(
- {
- "asset_id": asset_id,
- "reason": REASON_HARMFUL,
- "action": ACTION_ROLLBACK,
- "explanation": (
- f"成对比较显示有它反而更差({float(uplift):+.0%})——"
- "这是唯一需要立即停止曝光的情况"
- ),
- **stats,
- }
- )
- continue
-
- if exposure_days < MIN_EXPOSURE_DAYS:
- # Too new to judge. Saying nothing is the honest output; the old
- # rule's answer here was "retire it", which punished every skill for
- # having just shipped.
- continue
-
- if opens == 0:
- if offers < MIN_OFFERS_BEFORE_JUDGING:
- continue
- out.append(
- {
- "asset_id": asset_id,
- "reason": REASON_NEVER_OPENED,
- "action": ACTION_LOWER_EXPOSURE,
- "explanation": (
- f"已被挂载 {offers} 次、"
- # A hand-authored skill has no release record and so no
- # finite exposure window — it has been there from the
- # start. Formatting that as a number printed "曝光 inf 天".
- + (
- "一直可用"
- if exposure_days == float("inf")
- else f"曝光 {exposure_days:.0f} 天"
- )
- + ",模型一次都没打开——"
- "多半是描述没写清它解决什么问题,而不是能力本身没用"
- ),
- **stats,
- }
- )
- continue
-
- if opens < MIN_OPENS_BEFORE_JUDGING:
- continue
-
- follow_through = stats.get("follow_through")
- if follow_through is not None and float(follow_through) < FOLLOW_THROUGH_FLOOR:
- out.append(
- {
- "asset_id": asset_id,
- "reason": REASON_IGNORED_WHEN_OPENED,
- "action": ACTION_REWRITE,
- "explanation": (
- f"被打开 {opens} 次,但只有 {float(follow_through):.0%} 的情况"
- "后续动作真的照它说的做——文档本身需要重写,不是该退役"
- ),
- **stats,
- }
- )
- continue
-
- success_rate = float(stats.get("success_rate", 0.0) or 0.0)
- if success_rate < SUCCESS_FLOOR:
- out.append(
- {
- "asset_id": asset_id,
- "reason": REASON_LOW_SUCCESS,
- "action": ACTION_RETIRE,
- "explanation": (
- f"被打开 {opens} 次、照做了,但成功率只有 {success_rate:.0%}"
- ),
- **stats,
- }
- )
- return out
-
-
-@dataclass
-class NegativeTransfer:
- asset_id: str
- task_type: str
- before: float
- after: float
-
- @property
- def delta(self) -> float:
- return self.after - self.before
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "asset_id": self.asset_id,
- "task_type": self.task_type,
- "before": round(self.before, 4),
- "after": round(self.after, 4),
- "delta": round(self.delta, 4),
- }
-
-
-def detect_negative_transfer(
- *,
- asset_id: str,
- related_task_types: Sequence[str],
- before: Dict[str, float],
- after: Dict[str, float],
- threshold: float = 0.05,
-) -> List[NegativeTransfer]:
- """Find task types that got worse after an activation *and are unrelated to it*.
-
- Only unrelated tasks are examined. A dip on the task the change targeted is
- a straightforward regression its own guardrail catches; the dangerous case
- is collateral damage elsewhere, which no single release's guardrail is
- watching and which accumulates silently across many individually-harmless
- changes.
- """
- related = set(related_task_types)
- findings: List[NegativeTransfer] = []
- for task_type, after_value in (after or {}).items():
- if task_type in related:
- continue
- before_value = (before or {}).get(task_type)
- if before_value is None:
- continue
- if before_value - after_value >= threshold:
- findings.append(
- NegativeTransfer(
- asset_id=asset_id,
- task_type=task_type,
- before=before_value,
- after=after_value,
- )
- )
- return findings
-
-
-# ── Transferable ordering constraints ────────────────────────────────────────
-
-
-@dataclass
-class OrderingConstraint:
- """A pairwise ordering that held across the supporting evidence.
-
- Carries the task types it was *validated in* and any it was *contradicted
- in*. That distinction is what lets a genuinely context-dependent ordering be
- used where it holds instead of being discarded everywhere.
- """
-
- before: str
- after: str
- support: int
- contexts: int # distinct tool-set shapes it was observed in
- validated_in: Tuple[str, ...] = ()
- contradicted_in: Tuple[str, ...] = ()
-
- def applies_to(self, tools: Sequence[str], task_type: Optional[str] = None) -> bool:
- if self.before not in tools or self.after not in tools:
- return False
- if task_type is None:
- # No context given: only a rule contradicted nowhere may be used.
- return not self.contradicted_in
- if task_type in self.contradicted_in:
- return False
- if task_type in self.validated_in:
- return True
- # An unseen context. Extending there is only safe for a rule that has
- # never been contradicted; one known to flip somewhere is precisely the
- # kind that should not be guessed at.
- return not self.contradicted_in
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "before": self.before,
- "after": self.after,
- "support": self.support,
- "contexts": self.contexts,
- "validated_in": list(self.validated_in),
- "contradicted_in": list(self.contradicted_in),
- }
-
-
-# A constraint seen in only one tool-set shape is indistinguishable from "that
-# is just how this one sequence goes". Requiring two distinct shapes is what
-# separates a transferable rule from a memorised sequence — and it is the whole
-# difference between a skill that helps one task family and one that helps
-# every family making the same mistake.
-MIN_CONSTRAINT_CONTEXTS = 2
-
-
-def extract_ordering_constraints(
- episodes: Sequence[Dict[str, Any]],
- *,
- min_support: int = MIN_SUPPORT,
- min_contexts: int = MIN_CONSTRAINT_CONTEXTS,
-) -> List[OrderingConstraint]:
- """Find orderings that hold *invariantly* across successful episodes.
-
- Conflicting evidence vetoes a constraint outright rather than letting the
- majority win. A pair that appears both ways round is not a rule — it is a
- signal that ordering depends on context the system cannot yet see, and
- promoting the popular direction would actively break the minority case.
- """
- forward: Dict[Tuple[str, str], Set[str]] = defaultdict(set)
- by_task_type: Dict[Tuple[str, str], Set[str]] = defaultdict(set)
-
- for episode in episodes:
- if episode.get("verdict") != "success":
- continue
- tools = [str(t) for t in (episode.get("tool_sequence") or [])]
- if len(tools) < 2:
- continue
- shape = _signature(sorted(set(tools)))
- task_type = str(episode.get("task_type") or "unknown")
- for i, earlier in enumerate(tools):
- for later in tools[i + 1 :]:
- if earlier == later:
- continue
- forward[(earlier, later)].add(shape)
- by_task_type[(earlier, later)].add(task_type)
-
- constraints: List[OrderingConstraint] = []
- for (a, b), shapes in forward.items():
- validated = by_task_type[(a, b)]
- contradicted = by_task_type.get((b, a), set())
-
- # A pair that flips *within the same task type* is genuinely ambiguous
- # there — no amount of context resolves it, so the rule is dropped for
- # that type rather than resolved by majority.
- ambiguous = validated & contradicted
- validated = validated - ambiguous
- if not validated:
- continue
-
- support = sum(
- 1
- for e in episodes
- if e.get("verdict") == "success"
- and a in (e.get("tool_sequence") or [])
- and b in (e.get("tool_sequence") or [])
- and str(e.get("task_type") or "unknown") in validated
- )
- if support < min_support or len(shapes) < min_contexts:
- continue
- constraints.append(
- OrderingConstraint(
- before=a,
- after=b,
- support=support,
- contexts=len(shapes),
- validated_in=tuple(sorted(validated)),
- contradicted_in=tuple(sorted(contradicted | ambiguous)),
- )
- )
-
- constraints.sort(key=lambda c: (c.contexts, c.support), reverse=True)
- return constraints
-
-
-def apply_ordering_constraints(
- tools: Sequence[str],
- constraints: Sequence[OrderingConstraint],
- task_type: Optional[str] = None,
-) -> List[str]:
- """Reorder a task's tools to satisfy every applicable constraint.
-
- A stable topological sort: tools with no constraint between them keep their
- original relative order, so applying a rule about one pair never silently
- reshuffles everything else.
- """
- items = list(tools)
- applicable = [c for c in constraints if c.applies_to(items, task_type)]
- if not applicable:
- return items
-
- for _ in range(len(items)):
- changed = False
- for constraint in applicable:
- i, j = items.index(constraint.before), items.index(constraint.after)
- if i > j:
- items.insert(j, items.pop(i))
- changed = True
- if not changed:
- break
- return items
diff --git a/src/backend/core/evolution/settlement/__init__.py b/src/backend/core/evolution/settlement/__init__.py
new file mode 100644
index 00000000..fc4d4e7c
--- /dev/null
+++ b/src/backend/core/evolution/settlement/__init__.py
@@ -0,0 +1 @@
+"""Settle evolution outcomes back into memory."""
diff --git a/src/backend/core/evolution/settlement/memory_service.py b/src/backend/core/evolution/settlement/memory_service.py
new file mode 100644
index 00000000..47bd8198
--- /dev/null
+++ b/src/backend/core/evolution/settlement/memory_service.py
@@ -0,0 +1,39 @@
+"""Evolution implementation of the memory settlement boundary."""
+from core.db.models import ChatMessage, EvolutionEpisode
+from core.evolution.settlement import settlement, settlement_runner, settlement_store
+from core.memory.settlement_port import register_memory_settlement
+
+
+class EvolutionMemorySettlement:
+ def report_writes(self, message_id, *, items, failed):
+ settlement_runner.report_memory_writes(message_id, items=items, failed=failed)
+
+ def update_entry_text(self, message_id, handle, text):
+ return settlement_store.update_entry_text(message_id, handle, text)
+
+ def build_summary(self, message_id, *, session_factory, items, failed):
+ settlement_runner.acknowledge_durable_settlement(message_id)
+ with session_factory() as db:
+ episode = db.query(EvolutionEpisode.episode_id).filter(
+ EvolutionEpisode.message_id == message_id
+ ).first()
+ return settlement.settle_turn(
+ message_id=message_id,
+ episode_id=str(episode[0]) if episode is not None else "",
+ memory_entries=items, memory_failed=failed, memory_enabled=True,
+ ).to_dict()
+
+ def write_summary(self, db, message_id, summary):
+ # Called inside the outbox acknowledgement transaction. Never commit here.
+ message = db.query(ChatMessage).filter(ChatMessage.message_id == message_id).first()
+ if message is None:
+ raise RuntimeError("assistant message is not durable yet; settlement will retry")
+ if not isinstance(summary, dict):
+ raise RuntimeError("settlement result is missing its durable summary")
+ extra = dict(message.extra_data or {})
+ extra[settlement_store.METADATA_KEY] = summary
+ message.extra_data = extra
+
+
+def register() -> None:
+ register_memory_settlement(EvolutionMemorySettlement())
diff --git a/src/backend/core/evolution/settlement.py b/src/backend/core/evolution/settlement/settlement.py
similarity index 100%
rename from src/backend/core/evolution/settlement.py
rename to src/backend/core/evolution/settlement/settlement.py
diff --git a/src/backend/core/evolution/settlement_runner.py b/src/backend/core/evolution/settlement/settlement_runner.py
similarity index 88%
rename from src/backend/core/evolution/settlement_runner.py
rename to src/backend/core/evolution/settlement/settlement_runner.py
index 5dbaec71..fb990a70 100644
--- a/src/backend/core/evolution/settlement_runner.py
+++ b/src/backend/core/evolution/settlement/settlement_runner.py
@@ -36,8 +36,6 @@
# early would report zero writes for a turn that was about to write.
MEMORY_WAIT_SECONDS = 45.0
-_lock = threading.Lock()
-
@dataclass
class _PendingTurn:
@@ -53,8 +51,6 @@ class _PendingTurn:
timer: Optional[threading.Timer] = None
-_pending: Dict[str, _PendingTurn] = {}
-
# Turns already settled. A late duplicate — the watchdog fired and the pipeline
# reported afterwards — must not settle a second time: the second pass would
# start from an empty entry and overwrite a good summary with an empty one.
@@ -70,10 +66,10 @@ def _remember_settled(message_id: str) -> None:
def _entry(message_id: str) -> _PendingTurn:
- entry = _pending.get(message_id)
+ entry = process_state().pending.get(message_id)
if entry is None:
entry = _PendingTurn(message_id=message_id)
- _pending[message_id] = entry
+ process_state().pending[message_id] = entry
return entry
@@ -87,7 +83,7 @@ def register_turn(
if not message_id:
return
- with _lock:
+ with process_state().lock:
if message_id in _settled:
return
entry = _entry(message_id)
@@ -120,7 +116,7 @@ def report_memory_writes(
if not message_id:
return
- with _lock:
+ with process_state().lock:
if message_id in _settled:
return
entry = _entry(message_id)
@@ -154,8 +150,8 @@ def acknowledge_durable_settlement(message_id: str) -> None:
"""
if not message_id:
return
- with _lock:
- entry = _pending.pop(message_id, None)
+ with process_state().lock:
+ entry = process_state().pending.pop(message_id, None)
if entry is not None and entry.timer is not None:
entry.timer.cancel()
_remember_settled(message_id)
@@ -163,8 +159,8 @@ def acknowledge_durable_settlement(message_id: str) -> None:
def _settle(message_id: str, *, timed_out: bool = False) -> None:
"""Build the summary and persist it. Never raises — this runs post-response."""
- with _lock:
- entry = _pending.pop(message_id, None)
+ with process_state().lock:
+ entry = process_state().pending.pop(message_id, None)
if entry is None or message_id in _settled:
return # already settled
if entry.timer is not None:
@@ -182,8 +178,8 @@ def _settle(message_id: str, *, timed_out: bool = False) -> None:
)
try:
- from core.evolution.settlement import settle_turn
- from core.evolution.settlement_store import persist_summary
+ from core.evolution.settlement.settlement import settle_turn
+ from core.evolution.settlement.settlement_store import persist_summary
summary = settle_turn(
message_id=message_id,
@@ -198,9 +194,22 @@ def _settle(message_id: str, *, timed_out: bool = False) -> None:
def reset_for_tests() -> None:
- with _lock:
- for entry in _pending.values():
+ with process_state().lock:
+ for entry in process_state().pending.values():
if entry.timer is not None:
entry.timer.cancel()
- _pending.clear()
+ process_state().pending.clear()
_settled.clear()
+
+
+@dataclass
+class SettlementRunnerState:
+ lock: object = field(default_factory=lambda: threading.Lock())
+ pending: Dict[str, _PendingTurn] = field(default_factory=lambda: {})
+
+
+_STATE = SettlementRunnerState()
+
+
+def process_state() -> SettlementRunnerState:
+ return _STATE
diff --git a/src/backend/core/evolution/settlement_store.py b/src/backend/core/evolution/settlement/settlement_store.py
similarity index 98%
rename from src/backend/core/evolution/settlement_store.py
rename to src/backend/core/evolution/settlement/settlement_store.py
index 1b6f26b8..869fbfe4 100644
--- a/src/backend/core/evolution/settlement_store.py
+++ b/src/backend/core/evolution/settlement/settlement_store.py
@@ -21,7 +21,7 @@
from typing import Any, Dict, Optional
from core.db.engine import SessionLocal
-from core.evolution.settlement import EvolutionSummary
+from core.evolution.settlement.settlement import EvolutionSummary
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/evolution/skill_gen.py b/src/backend/core/evolution/skill_gen.py
deleted file mode 100644
index 7b29e75d..00000000
--- a/src/backend/core/evolution/skill_gen.py
+++ /dev/null
@@ -1,521 +0,0 @@
-"""Writing a skill from evidence: evidence pack → LLM draft → machine validation.
-
-What a materialised skill used to be:
-
- ## 步骤
- 1. `search_web`
- 2. `fetch_page`
-
-A list of tool names, rendered by string concatenation. It states no applicability
-condition worth the name, no parameters, no checkpoint for "did this step work",
-no failure mode. Measured against a real model, the only part of that document
-that carried any benefit was the ordering-constraint section; the step list was a
-*harm* source, generalising this skill's order into a task family where the
-reverse is correct, 5 times out of 5.
-
-So the model writes the document. But it writes only the document — the three
-stages are separated precisely so that generation never becomes an authority:
-
-* **Stage A (code)** assembles the evidence: what was asked, what worked, what
- failed and where, the ordering rules with their scopes, and the procedural
- memories behind the pattern. Deciding *what a skill may be written from* stays
- in code.
-* **Stage B (model)** drafts SKILL.md under hard constraints stated in the
- prompt.
-* **Stage C (code)** validates every one of those constraints and rejects the
- draft if any fails, with up to :data:`MAX_ATTEMPTS` regeneration attempts
- carrying the specific violations back.
-
-If stage B could widen a tool grant, self-evolution would be a prompt-injection
-away from privilege escalation. It cannot: stage C intersects the draft's tools
-with the evidence's tools and refuses anything else, and the same document then
-goes through the IR's own escalation gate before it can be persisted.
-"""
-
-from __future__ import annotations
-
-import logging
-import re
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-logger = logging.getLogger(__name__)
-
-MAX_ATTEMPTS = 2
-# Long enough for steps, checkpoints and failure modes; short enough that a
-# skill cannot start competing with the user's own request for the model's
-# attention once it is opened.
-MAX_BODY_CHARS = 4000
-MIN_BODY_CHARS = 120
-DRAFT_TIMEOUT_S = 60
-
-# A skill body must not contain executable content. Self-modifying code is the
-# one capability this design refuses outright, and a fenced shell block inside a
-# document the agent reads and follows is executable content with extra steps.
-_CODE_FENCE = re.compile(r"```(?:\w+)?\s*\n", re.MULTILINE)
-_URL = re.compile(r"https?://", re.IGNORECASE)
-_FRONTMATTER_NAME = re.compile(r"^[a-z0-9_-]+$")
-_TOOL_MENTION = re.compile(r"`([A-Za-z_][A-Za-z0-9_.-]*)`")
-
-REJECT_NO_FRONTMATTER = "missing_frontmatter"
-REJECT_BAD_NAME = "invalid_skill_id"
-REJECT_UNKNOWN_TOOL = "tool_not_in_evidence"
-REJECT_CODE_BLOCK = "contains_code_block"
-REJECT_EXTERNAL_URL = "contains_external_url"
-REJECT_TOO_LONG = "body_too_long"
-REJECT_TOO_SHORT = "body_too_short"
-REJECT_NO_SCOPE = "missing_scope_section"
-REJECT_NO_CHECKPOINTS = "missing_checkpoints"
-REJECT_EMPTY = "empty_draft"
-
-
-@dataclass
-class EvidencePack:
- """Everything stage B is allowed to see, and nothing else."""
-
- skill_id: str
- objectives: List[str] = field(default_factory=list)
- tools: List[str] = field(default_factory=list)
- successful_sequences: List[List[str]] = field(default_factory=list)
- failures: List[Dict[str, Any]] = field(default_factory=list)
- ordering_constraints: List[Dict[str, Any]] = field(default_factory=list)
- procedures: List[Dict[str, str]] = field(default_factory=list)
- task_types: List[str] = field(default_factory=list)
- excluded_task_types: List[str] = field(default_factory=list)
- support: int = 0
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "skill_id": self.skill_id,
- "objectives": self.objectives[:10],
- "tools": list(self.tools),
- "successful_sequences": self.successful_sequences[:6],
- "failures": self.failures[:6],
- "ordering_constraints": self.ordering_constraints,
- "procedures": self.procedures[:10],
- "task_types": self.task_types,
- "excluded_task_types": self.excluded_task_types,
- "support": self.support,
- }
-
-
-@dataclass
-class DraftResult:
- ok: bool
- body: str = ""
- display_name: str = ""
- description: str = ""
- violations: List[str] = field(default_factory=list)
- attempts: int = 0
- # ``True`` when no draft could be produced because the model was unreachable
- # — an operational state, not a rejection, and the two must not look alike
- # in the console.
- generator_unavailable: bool = False
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "ok": self.ok,
- "display_name": self.display_name,
- "description": self.description,
- "violations": list(self.violations),
- "attempts": self.attempts,
- "generator_unavailable": self.generator_unavailable,
- "body_chars": len(self.body),
- }
-
-
-# ── Stage A: the evidence pack ───────────────────────────────────────────────
-
-
-def build_evidence_pack(
- *,
- skill_id: str,
- episodes: Sequence[Dict[str, Any]],
- ordering_constraints: Sequence[Dict[str, Any]] = (),
- procedures: Sequence[Dict[str, str]] = (),
-) -> EvidencePack:
- """Assemble what a skill may be written from.
-
- Failures are included deliberately. A document distilled only from successes
- describes a happy path and says nothing about the step where things actually
- go wrong, which is the part a reader needs most.
- """
- objectives: List[str] = []
- tools: List[str] = []
- sequences: List[List[str]] = []
- failures: List[Dict[str, Any]] = []
- task_types: List[str] = []
-
- for episode in episodes:
- objective = str(episode.get("objective") or "").strip()
- if objective and objective not in objectives:
- objectives.append(objective)
- task_type = str(episode.get("task_type") or "").strip()
- if task_type and task_type not in task_types:
- task_types.append(task_type)
-
- sequence = [str(t) for t in (episode.get("tool_sequence") or [])]
- for tool in sequence:
- if tool not in tools:
- tools.append(tool)
- if episode.get("verdict") == "success":
- if sequence and sequence not in sequences:
- sequences.append(sequence)
- elif episode.get("verdict") == "failed":
- failures.append(
- {
- "objective": objective[:160],
- "tool_sequence": sequence[:12],
- "failed_at": str(episode.get("failed_tool") or ""),
- "error": str(episode.get("error") or "")[:200],
- }
- )
-
- excluded = sorted(
- {
- str(name)
- for rule in ordering_constraints
- for name in (rule.get("contradicted_in") or [])
- if str(name)
- }
- )
- return EvidencePack(
- skill_id=skill_id,
- objectives=objectives,
- tools=tools,
- successful_sequences=sequences,
- failures=failures,
- ordering_constraints=list(ordering_constraints),
- procedures=[dict(p) for p in procedures],
- task_types=[t for t in task_types if t not in excluded],
- excluded_task_types=excluded,
- support=len(episodes),
- )
-
-
-# ── Stage B: the draft ───────────────────────────────────────────────────────
-
-_DRAFT_PROMPT = """你在为一个 AI 助手撰写一份「技能文档」(SKILL.md)。这份文档会被模型在遇到同类任务时读取并遵照执行。
-
-# 你拿到的证据
-
-## 用户实际提出过的请求({objective_count} 条)
-{objectives}
-
-## 成功执行时用过的工具
-{tools}
-
-## 成功的调用顺序
-{sequences}
-
-## 失败的执行(这些是最宝贵的部分,说明坑在哪)
-{failures}
-
-## 已验证的顺序约束
-{constraints}
-
-## 这些任务里沉淀下来的做法/口径
-{procedures}
-
-# 硬性要求(违反任何一条,文档会被机器校验打回)
-
-1. **只能引用上面「成功执行时用过的工具」里出现过的工具名**,一个都不能新增、不能猜。
-2. **开头第一段必须是「## 不适用范围」**,写明本技能不适用于哪些场景。{exclusion_hint}
-3. 必须有「## 适用条件」,写清楚**遇到什么样的请求才该用本技能**——要具体到能和别的任务区分开,不能写"当需要用到这些工具时"这种同义反复。
-4. 必须有「## 步骤」,每一步都要写清楚:做什么、**关键参数怎么给**、**怎么判断这一步做对了**(检查点)。
-5. 必须有「## 常见失败与处理」,基于上面的失败证据写。
-6. **禁止**出现任何代码块(``` 包裹的内容)、禁止出现外部链接(http/https)。
-7. 全文不超过 {max_chars} 字符。
-8. 用中文撰写,语气是给同事写操作手册,不要写"本技能由AI生成"之类的话。
-
-{retry_hint}
-
-# 输出格式
-
-严格按下面结构输出,不要加任何额外说明,不要用代码块包裹整篇文档:
-
-DISPLAY_NAME: <一行技能名,≤20字,说明这个技能干什么>
-DESCRIPTION: <一行描述,≤80字,说明什么时候该用它——模型只看这一行决定要不要打开本技能>
-BODY:
-## 不适用范围
-...
-## 适用条件
-...
-## 步骤
-...
-## 常见失败与处理
-...
-"""
-
-
-def _format_list(items: Sequence[Any], empty: str = "(无)") -> str:
- if not items:
- return empty
- lines = []
- for item in items:
- if isinstance(item, (list, tuple)):
- lines.append("- " + " → ".join(str(x) for x in item))
- elif isinstance(item, dict):
- lines.append("- " + ";".join(f"{k}: {v}" for k, v in item.items() if v))
- else:
- lines.append(f"- {item}")
- return "\n".join(lines)
-
-
-def _build_prompt(pack: EvidencePack, violations: Sequence[str]) -> str:
- exclusion_hint = (
- f"证据显示本技能在以下任务类型中被推翻,必须写进不适用范围:{'、'.join(pack.excluded_task_types)}。"
- if pack.excluded_task_types
- else "如果证据没有显示明确的排除场景,就写明本技能只适用于「适用条件」描述的场景,其余场景请忽略本技能。"
- )
- retry_hint = ""
- if violations:
- retry_hint = (
- "# 上一版被打回的原因(必须修正)\n"
- + "\n".join(f"- {v}" for v in violations)
- + "\n"
- )
- return _DRAFT_PROMPT.format(
- objective_count=len(pack.objectives),
- objectives=_format_list(pack.objectives[:10]),
- tools=_format_list([f"`{t}`" for t in pack.tools]),
- sequences=_format_list(pack.successful_sequences[:6]),
- failures=_format_list(pack.failures[:6]),
- constraints=_format_list(
- [
- f"先 {c.get('before')} 再 {c.get('after')}"
- + (
- f"(在 {'、'.join(str(x) for x in (c.get('validated_in') or []))} 中验证)"
- if c.get("validated_in")
- else ""
- )
- for c in pack.ordering_constraints
- if not (c.get("contradicted_in") or [])
- ]
- ),
- procedures=_format_list(
- [
- p.get("rule", "") + (f"(原因:{p['why']})" if p.get("why") else "")
- for p in pack.procedures[:10]
- ]
- ),
- exclusion_hint=exclusion_hint,
- max_chars=MAX_BODY_CHARS,
- retry_hint=retry_hint,
- )
-
-
-def _parse_draft(raw: str) -> Tuple[str, str, str]:
- """``(display_name, description, body)`` from the model's reply."""
- display_name = ""
- description = ""
- body = ""
- text = (raw or "").strip()
- if text.startswith("```"):
- lines = text.split("\n")
- text = "\n".join(lines[1:])
- if text.rstrip().endswith("```"):
- text = text.rstrip()[:-3]
-
- body_marker = re.search(r"^BODY:\s*$", text, re.MULTILINE)
- head = text[: body_marker.start()] if body_marker else text
- if body_marker:
- body = text[body_marker.end() :].strip()
-
- for line in head.splitlines():
- if line.upper().startswith("DISPLAY_NAME:"):
- display_name = line.split(":", 1)[1].strip()
- elif line.upper().startswith("DESCRIPTION:"):
- description = line.split(":", 1)[1].strip()
- return display_name, description, body
-
-
-# ── Stage C: validation ──────────────────────────────────────────────────────
-
-
-def validate_draft(
- *,
- skill_id: str,
- display_name: str,
- description: str,
- body: str,
- allowed_tools: Sequence[str],
-) -> List[str]:
- """Every constraint stage B was told about, checked. Returns violations."""
- violations: List[str] = []
-
- if not body.strip():
- return [REJECT_EMPTY]
- if not display_name.strip() or not description.strip():
- violations.append(REJECT_NO_FRONTMATTER)
- if not _FRONTMATTER_NAME.match(skill_id or ""):
- # The loader treats ``name`` as the id and rejects anything outside
- # ``[a-z0-9_-]``. A skill that fails this still appears in the capability
- # list — that metadata comes from table columns — while loading returns
- # nothing, so it is present and inert, which is the hardest failure of
- # all to notice.
- violations.append(REJECT_BAD_NAME)
-
- if _CODE_FENCE.search(body):
- violations.append(REJECT_CODE_BLOCK)
- if _URL.search(body):
- violations.append(REJECT_EXTERNAL_URL)
- if len(body) > MAX_BODY_CHARS:
- violations.append(REJECT_TOO_LONG)
- if len(body) < MIN_BODY_CHARS:
- violations.append(REJECT_TOO_SHORT)
-
- if "## 不适用范围" not in body:
- violations.append(REJECT_NO_SCOPE)
- else:
- # The exclusion has to precede anything the model could copy: measured,
- # withholding a contradicted rule was not enough, because the step list
- # generalises on its own.
- scope_at = body.index("## 不适用范围")
- earlier_section = re.search(r"^## ", body[:scope_at], re.MULTILINE)
- if earlier_section:
- violations.append(REJECT_NO_SCOPE)
- if "## 常见失败与处理" not in body and "## 步骤" not in body:
- violations.append(REJECT_NO_CHECKPOINTS)
-
- allowed = {str(t) for t in allowed_tools}
- mentioned = {m for m in _TOOL_MENTION.findall(body)}
- # Only identifiers that look like tool names are checked: prose in backticks
- # (a field name, a value) must not be mistaken for a tool grant, and a real
- # tool name always appears verbatim in the evidence.
- unknown = sorted(
- name
- for name in mentioned
- if name not in allowed and "_" in name and name.islower()
- )
- if unknown:
- violations.append(f"{REJECT_UNKNOWN_TOOL}: {unknown}")
-
- return violations
-
-
-def render_skill_markdown(
- *,
- skill_id: str,
- display_name: str,
- description: str,
- body: str,
- support: int,
- rationale: str = "",
-) -> str:
- """Wrap a validated body in the frontmatter the loader requires.
-
- ``name`` is the skill **id**, not the label — writing the human label there
- made every materialised skill unparseable.
- """
- provenance = (
- f"本技能由能力进化流水线从 {support} 条历史执行证据中沉淀而成,经机器校验后由人工批准。"
- )
- if rationale:
- provenance += f"\n\n沉淀依据:{rationale}"
- return f"""---
-name: {skill_id}
-display_name: {display_name}
-description: {description}
----
-
-# {display_name}
-
-{description}
-
-{body.strip()}
-
-## 来源
-
-{provenance}
-"""
-
-
-# ── The three stages, in order ───────────────────────────────────────────────
-
-
-async def draft_skill(pack: EvidencePack) -> DraftResult:
- """Draft and validate, retrying with the specific violations.
-
- Returns ``ok=False`` rather than raising. A skill that cannot be written to
- the required standard must produce no candidate at all: shipping a draft
- that failed validation would put exactly the document the constraints exist
- to prevent in front of the model.
- """
- from core.memory.extractors._base import run_llm_with_prompt
-
- violations: List[str] = []
- for attempt in range(1, MAX_ATTEMPTS + 1):
- prompt = _build_prompt(pack, violations)
- raw = await run_llm_with_prompt(prompt, timeout_s=DRAFT_TIMEOUT_S, max_tokens=2400)
- if raw is None:
- return DraftResult(
- ok=False,
- violations=["generator_unavailable"],
- attempts=attempt,
- generator_unavailable=True,
- )
-
- display_name, description, body = _parse_draft(raw)
- violations = validate_draft(
- skill_id=pack.skill_id,
- display_name=display_name,
- description=description,
- body=body,
- allowed_tools=pack.tools,
- )
- if not violations:
- return DraftResult(
- ok=True,
- body=body,
- display_name=display_name[:60],
- description=description[:200].replace("\n", " "),
- attempts=attempt,
- )
- logger.info(
- "[skill-gen] draft attempt %d rejected for %s: %s",
- attempt, pack.skill_id, violations,
- )
-
- return DraftResult(ok=False, violations=violations, attempts=MAX_ATTEMPTS)
-
-
-async def generate_skill_document(
- *,
- skill_id: str,
- episodes: Sequence[Dict[str, Any]],
- ordering_constraints: Sequence[Dict[str, Any]] = (),
- procedures: Sequence[Dict[str, str]] = (),
- rationale: str = "",
-) -> Tuple[Optional[Dict[str, str]], DraftResult]:
- """All three stages. Returns ``(document, result)``; document is ``None`` on failure."""
- pack = build_evidence_pack(
- skill_id=skill_id,
- episodes=episodes,
- ordering_constraints=ordering_constraints,
- procedures=procedures,
- )
- if not pack.tools and not pack.procedures:
- return None, DraftResult(ok=False, violations=["no_evidence_to_write_from"])
-
- result = await draft_skill(pack)
- if not result.ok:
- return None, result
-
- return (
- {
- "skill_id": skill_id,
- "display_name": result.display_name,
- "description": result.description,
- "content": render_skill_markdown(
- skill_id=skill_id,
- display_name=result.display_name,
- description=result.description,
- body=result.body,
- support=pack.support,
- rationale=rationale,
- ),
- "allowed_tools": list(pack.tools),
- },
- result,
- )
diff --git a/src/backend/core/extensions/__init__.py b/src/backend/core/extensions/__init__.py
deleted file mode 100644
index 6d56959a..00000000
--- a/src/backend/core/extensions/__init__.py
+++ /dev/null
@@ -1,69 +0,0 @@
-"""Optional feature modules that plug into core through registered hooks.
-
-Core never imports a module's implementation. A module contributes behaviour by
-registering hooks from its ``setup()`` at startup; when the module is disabled
-it registers nothing and core behaves as if the module were not installed.
-
-Shared modules are loaded in every edition. Additional modules are an edition concern (``core.extensions.edition``); whether
-an installed module is switched on is the module's own decision, usually an env
-flag read inside its ``setup()``.
-"""
-
-from __future__ import annotations
-
-from importlib import import_module
-
-from core.extensions.registry import (
- AgentProfile,
- agent_profile,
- channel_routes,
- is_enabled,
- lifecycle_hooks,
- mark_enabled,
- register_agent_profile,
- register_channel_route,
- register_lifecycle,
- register_router,
- register_sandbox_route,
- routers,
- sandbox_route,
-)
-from core.infra.logging import get_logger
-
-logger = get_logger(__name__)
-
-_loaded = False
-
-
-def load_extensions() -> None:
- """Import shared and edition modules and let it register itself. Idempotent."""
- global _loaded
- if _loaded:
- return
- _loaded = True
- from core.extensions.edition import EDITION_EXTENSIONS
-
- for module_name in ("personal_agent.extension", *EDITION_EXTENSIONS):
- module = import_module(module_name)
- if module.setup():
- mark_enabled(module.NAME)
- logger.info("extension_enabled", extension=module.NAME)
- else:
- logger.info("extension_disabled", extension=module.NAME)
-
-
-__all__ = [
- "AgentProfile",
- "agent_profile",
- "channel_routes",
- "is_enabled",
- "lifecycle_hooks",
- "load_extensions",
- "register_agent_profile",
- "register_channel_route",
- "register_lifecycle",
- "register_router",
- "register_sandbox_route",
- "routers",
- "sandbox_route",
-]
diff --git a/src/backend/core/extensions/registry.py b/src/backend/core/extensions/registry.py
deleted file mode 100644
index 64669d69..00000000
--- a/src/backend/core/extensions/registry.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""Hook tables that optional modules fill in at startup.
-
-Every table starts empty. Core consults them at its extension points (agent
-assembly, sandbox routing, inbound channel messages, app lifecycle, routers),
-so an unregistered module leaves those code paths exactly as they were.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any, Awaitable, Callable, Optional
-
-
-@dataclass(frozen=True)
-class AgentProfile:
- """A dedicated agent built by a module instead of the ordinary chat assembly.
-
- Selected by ``AgentRequest.profile``; ``AgentRequest.profile_ref`` carries the
- module's own identifier (for example the task the agent works on).
- """
-
- # (request) -> system prompt; replaces the whole chat prompt assembly.
- system_prompt: Callable[[Any], Awaitable[str]]
- # (toolkit, request, loader=...) -> None; replaces the native chat tools.
- register_tools: Callable[..., None]
- # (request) -> extra policy middlewares appended to the stack.
- middlewares: Callable[[Any], list] = field(default=lambda request: [])
- # Only function tools whose name passes are exposed; None keeps all.
- tool_filter: Optional[Callable[[str], bool]] = None
- # False: MCP servers come only from the request's explicit list, never catalog defaults.
- # False also prevents history and skill dependencies from widening explicit bindings.
- inherit_default_mcps: bool = True
- # False: the profile's own system prompt lists its skills and how to read them; the
- # chat skill section (which points to view_text_file) is left out.
- native_skill_prompt: bool = True
-
-
-_enabled: set[str] = set()
-_agent_profiles: dict[str, AgentProfile] = {}
-_sandbox_routes: list[tuple[str, Callable[[], Any]]] = []
-_channel_routes: list[tuple[Callable[[Any, Any], bool], Callable[..., Awaitable[None]]]] = []
-_lifecycle: list[tuple[Callable[[], Awaitable[None]], Callable[[], Awaitable[None]]]] = []
-_routers: list[tuple[Any, Optional[str]]] = []
-
-
-def mark_enabled(name: str) -> None:
- _enabled.add(name)
-
-
-def is_enabled(name: str) -> bool:
- return name in _enabled
-
-
-def register_agent_profile(name: str, profile: AgentProfile) -> None:
- _agent_profiles[name] = profile
-
-
-def agent_profile(name: str) -> AgentProfile:
- profile = _agent_profiles.get(name)
- if profile is None:
- raise ValueError(f"Agent profile {name!r} is not enabled")
- return profile
-
-
-def register_sandbox_route(session_prefix: str, provider: Callable[[], Any]) -> None:
- """Sandbox sessions whose id starts with ``session_prefix`` use ``provider()``."""
- _sandbox_routes.append((session_prefix, provider))
-
-
-def sandbox_route(session_id: str) -> Optional[Callable[[], Any]]:
- for prefix, provider in _sandbox_routes:
- if session_id.startswith(prefix):
- return provider
- return None
-
-
-def register_channel_route(
- handles: Callable[[Any, Any], bool],
- admit: Callable[..., Awaitable[None]],
-) -> None:
- """``handles(db, conn)`` claims an inbound bot message; ``admit(conn, msg, adapter)`` takes it."""
- _channel_routes.append((handles, admit))
-
-
-def channel_routes() -> tuple:
- return tuple(_channel_routes)
-
-
-def register_lifecycle(
- start: Callable[[], Awaitable[None]],
- stop: Callable[[], Awaitable[None]],
-) -> None:
- """Background work started on each service worker and stopped on shutdown."""
- _lifecycle.append((start, stop))
-
-
-def lifecycle_hooks() -> tuple:
- return tuple(_lifecycle)
-
-
-def register_router(router: Any, policy_key: Optional[str] = None) -> None:
- """An API router mounted with the same edition policy as ``EE_ROUTERS`` entries."""
- _routers.append((router, policy_key))
-
-
-def routers() -> tuple:
- return tuple(_routers)
diff --git a/src/backend/core/files/__init__.py b/src/backend/core/files/__init__.py
new file mode 100644
index 00000000..20c0b545
--- /dev/null
+++ b/src/backend/core/files/__init__.py
@@ -0,0 +1 @@
+"""File parsing, artifact access, spaces and storage contracts."""
diff --git a/src/backend/core/files/artifact_bytes.py b/src/backend/core/files/artifact_bytes.py
new file mode 100644
index 00000000..f2bb4479
--- /dev/null
+++ b/src/backend/core/files/artifact_bytes.py
@@ -0,0 +1,75 @@
+"""Shared model/file contracts without execution-layer dependencies."""
+from __future__ import annotations
+import logging
+from typing import Any, Dict, Optional
+logger = logging.getLogger(__name__)
+def is_image(f: Dict[str, Any]) -> bool:
+ mime = (f.get("mime_type") or "").lower()
+ return mime.startswith("image/")
+
+def download_artifact_bytes(
+ file_id: str,
+ fallback_name: str,
+ log_prefix: str,
+ user_id: Optional[str] = None,
+) -> Optional[bytes]:
+ """Resolve an artifact's storage key and download its raw bytes.
+
+ Returns None on any failure (missing artifact, storage error, etc.) and
+ logs at WARNING. Used by the image and xlsx-preview hook branches that
+ both need raw bytes (not parsed text).
+
+ When ``user_id`` is supplied, the artifact's owner is verified via
+ :func:`load_artifact_meta` before any storage round-trip. This is the
+ defense-in-depth gate for hook-level reads: the agent context is built
+ from ``request.attachments[].file_id`` (client-supplied), and the
+ backend boundary doesn't re-verify ownership before reaching the
+ hook. Without this check, a forged ``file_id`` belonging to another
+ user would have its raw bytes downloaded and silently materialized
+ into the agent's memory (image base64 or xlsx preview block). The
+ matching ``fetch_parsed_text`` already does the same check for parsed
+ text — this brings the binary paths to parity.
+
+ ``user_id=None`` preserves the legacy "trust the caller" behavior
+ used by ``core.llm.tools.read_artifact_tool._probe_xlsx_sheet_names`` /
+ ``_probe_pptx_slide_count`` where ``fetch_parsed_text(user_id=...)``
+ is the authoritative gate downstream.
+ """
+ from core.files.artifacts.reader import load_artifact_meta, resolve_artifact_storage
+ from core.infra.exceptions import StorageError
+ from core.files.storage import get_storage
+
+ if user_id:
+ meta = load_artifact_meta(file_id, user_id=user_id)
+ if meta is None:
+ logger.warning(
+ "%s: artifact %s denied for user %s (not owner / missing / deleted)",
+ log_prefix,
+ file_id,
+ user_id,
+ )
+ return None
+
+ from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref
+ if is_local_file_ref(file_id):
+ item = authorized_reference(file_id, user_id)
+ if not item:
+ return None
+ try:
+ from pathlib import Path
+ return Path(item["path"]).read_bytes()
+ except OSError:
+ return None
+
+ storage_key, _ = resolve_artifact_storage(file_id, fallback_name)
+ if not storage_key:
+ return None
+
+ try:
+ return get_storage().download_bytes(storage_key)
+ except StorageError as e:
+ logger.warning(f"{log_prefix}: storage download failed for {file_id}: {e}")
+ return None
+ except Exception as e:
+ logger.warning(f"{log_prefix}: unexpected download error for {file_id}: {e}")
+ return None
diff --git a/src/backend/core/artifacts/__init__.py b/src/backend/core/files/artifacts/__init__.py
similarity index 96%
rename from src/backend/core/artifacts/__init__.py
rename to src/backend/core/files/artifacts/__init__.py
index aa617bbd..26f96740 100644
--- a/src/backend/core/artifacts/__init__.py
+++ b/src/backend/core/files/artifacts/__init__.py
@@ -1,2 +1 @@
"""Artifact storage helpers."""
-
diff --git a/src/backend/core/services/artifact_backfill.py b/src/backend/core/files/artifacts/backfill.py
similarity index 95%
rename from src/backend/core/services/artifact_backfill.py
rename to src/backend/core/files/artifacts/backfill.py
index ec94cfcc..5e3d12fb 100644
--- a/src/backend/core/services/artifact_backfill.py
+++ b/src/backend/core/files/artifacts/backfill.py
@@ -3,13 +3,13 @@
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from core.db.models import Artifact, ChatMessage, ChatSession
-from core.content.artifact_refs import extract_file_refs, infer_artifact_type, require_artifact_storage_key
-from core.services.artifact_edition import artifact_scope_fields
+from core.files.artifacts.refs import extract_file_refs, infer_artifact_type, require_artifact_storage_key
+from core.files.edition import artifact_scope_fields
logger = logging.getLogger(__name__)
_BACKFILL_BATCH = 500
-def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int:
+def backfill_artifacts_from_messages(user_id: str, db: Session) -> int:
"""Scan historical messages for file references not yet in the Artifact table.
Source priority depends on whether the message was written under the
@@ -99,9 +99,7 @@ def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int:
# Pass B — insert what is missing. A chat with no project can only ever
# resolve to the empty scope, so skip the lookup entirely for those; it used
# to cost one ChatSession round-trip per referenced message.
- from core.services.project_scope import ( # local: avoid top-level cycle
- project_scope_from_chat_id,
- )
+ from core.projects.edition import project_scope_from_chat_id
rootless_fields = artifact_scope_fields(None)
scope_cache: Dict[str, Dict[str, Optional[str]]] = {}
diff --git a/src/backend/core/files/artifacts/generated.py b/src/backend/core/files/artifacts/generated.py
new file mode 100644
index 00000000..474a9381
--- /dev/null
+++ b/src/backend/core/files/artifacts/generated.py
@@ -0,0 +1,210 @@
+"""Persist generated artifacts and resolve authorized artifact inputs."""
+import base64
+import logging
+from pathlib import Path
+from typing import Any, Optional
+from core.config.settings import settings
+
+logger = logging.getLogger(__name__)
+MAX_ARTIFACT_FILE_SIZE = settings.sandbox.artifact_max_bytes
+
+def store_generated_files(
+ files_data: list[dict[str, Any]],
+ *,
+ user_id: Optional[str],
+ source: str,
+ extra_metadata: Optional[dict[str, Any]] = None,
+) -> list[dict[str, Any]]:
+ """Persist sidecar-generated files and return normalized artifact refs."""
+ if not files_data:
+ return []
+
+ try:
+ from core.files.artifacts.store import save_artifact_bytes
+ except Exception as exc:
+ logger.warning("artifact store unavailable for %s: %s", source, exc)
+ return []
+
+ refs: list[dict[str, Any]] = []
+ for fd in files_data:
+ content_b64 = fd.get("content_b64", "")
+ if not content_b64:
+ continue
+
+ name = str(fd.get("name", "output")).strip() or "output"
+ mime_type = (
+ str(fd.get("mime_type", "application/octet-stream")).strip()
+ or "application/octet-stream"
+ )
+ try:
+ content = base64.b64decode(content_b64)
+ except Exception:
+ logger.warning("skip invalid base64 artifact payload: %s", name)
+ continue
+
+ metadata: dict[str, Any] = {"source": source}
+ if user_id:
+ metadata["user_id"] = user_id
+ if extra_metadata:
+ metadata.update(extra_metadata)
+
+ try:
+ item = save_artifact_bytes(
+ content=content,
+ name=name,
+ mime_type=mime_type,
+ extension=Path(name).suffix.lstrip("."),
+ metadata=metadata,
+ )
+ except Exception as exc:
+ logger.warning("failed to persist generated artifact %s: %s", name, exc)
+ continue
+
+ refs.append(
+ {
+ "file_id": item["file_id"],
+ "name": item.get("name", name),
+ "url": f"/files/{item['file_id']}",
+ "mime_type": item.get("mime_type", mime_type),
+ "size": item.get("size", len(content)),
+ "storage_key": item.get("storage_key"),
+ }
+ )
+
+ return refs
+
+
+def store_generated_file_path(
+ file_path: str | Path,
+ *,
+ name: str,
+ mime_type: str,
+ user_id: Optional[str],
+ source: str,
+ extra_metadata: Optional[dict[str, Any]] = None,
+) -> dict[str, Any] | None:
+ """Persist a generated file from disk and return a normalized artifact ref."""
+ try:
+ from core.files.artifacts.store import save_artifact_file
+ except Exception as exc:
+ logger.warning("artifact store unavailable for %s: %s", source, exc)
+ return None
+
+ metadata: dict[str, Any] = {"source": source}
+ if user_id:
+ metadata["user_id"] = user_id
+ if extra_metadata:
+ metadata.update(extra_metadata)
+
+ try:
+ item = save_artifact_file(
+ file_path=file_path,
+ name=name,
+ mime_type=mime_type,
+ extension=Path(name).suffix.lstrip("."),
+ metadata=metadata,
+ )
+ except Exception as exc:
+ logger.warning("failed to persist generated artifact %s: %s", name, exc)
+ return None
+
+ return {
+ "file_id": item["file_id"],
+ "name": item.get("name", name),
+ "url": f"/files/{item['file_id']}",
+ "mime_type": item.get("mime_type", mime_type),
+ "size": item.get("size", 0),
+ "storage_key": item.get("storage_key"),
+ }
+
+
+def resolve_artifact_files(
+ artifact_refs: dict[str, str],
+ user_id: str | None,
+) -> tuple[dict[str, str] | None, str | None]:
+ """Resolve artifact: references to base64-encoded binary content.
+
+ Tries DB lookup first, falls back to local artifact store index.
+ Returns (files_b64_dict, error_message).
+ """
+ if not artifact_refs:
+ return None, None
+
+ try:
+ from core.files.artifacts.refs import resolve_artifact_storage_key
+ from core.files.storage.factory import get_storage
+ except Exception as exc:
+ return None, f"artifact 解析依赖不可用: {exc}"
+
+ result: dict[str, str] = {}
+ storage = get_storage()
+
+ for filename, artifact_id in artifact_refs.items():
+ from core.files.artifacts.local_reference import is_local_file_ref
+ if is_local_file_ref(artifact_id):
+ from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes
+ file_bytes = _download_artifact_bytes(artifact_id, filename, "artifact input", user_id=user_id)
+ if file_bytes is None:
+ return None, f"artifact '{artifact_id}' 不存在或无权访问"
+ if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE:
+ return None, f"artifact 文件 '{filename}' 过大"
+ result[filename] = base64.b64encode(file_bytes).decode("ascii")
+ continue
+ storage_key = None
+ owner_ok = True
+
+ # Try 1: DB lookup
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.repository import ArtifactRepository
+
+ db = SessionLocal()
+ try:
+ repo = ArtifactRepository(db)
+ art = repo.get_by_id(artifact_id)
+ if art:
+ if user_id and art.user_id != user_id:
+ owner_ok = False
+ else:
+ storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key)
+ finally:
+ db.close()
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools._tool_helpers:_resolve_artifact_files', fallback_error)
+
+ if not owner_ok:
+ return None, f"无权访问 artifact '{artifact_id}'"
+
+ # Try 2: local artifact store fallback
+ if not storage_key:
+ try:
+ from core.files.artifacts.store import get_artifact
+
+ item = get_artifact(artifact_id)
+ if item:
+ item_user = (item.get("metadata") or {}).get("user_id")
+ if user_id and item_user and item_user != user_id:
+ return None, f"无权访问 artifact '{artifact_id}'"
+ storage_key = item.get("storage_key")
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools._tool_helpers:_resolve_artifact_files', fallback_error)
+
+ if not storage_key:
+ return None, f"artifact '{artifact_id}' 不存在或已删除"
+
+ try:
+ file_bytes = storage.download_bytes(storage_key)
+ except Exception as exc:
+ return None, f"artifact 文件 '{filename}' 读取失败: {exc}"
+
+ if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE:
+ return None, (
+ f"artifact 文件 '{filename}' 过大: "
+ f"{len(file_bytes)} bytes > {MAX_ARTIFACT_FILE_SIZE} bytes"
+ )
+
+ result[filename] = base64.b64encode(file_bytes).decode("ascii")
+
+ return result or None, None
diff --git a/src/backend/core/content/artifact_key_repair.py b/src/backend/core/files/artifacts/key_repair.py
similarity index 100%
rename from src/backend/core/content/artifact_key_repair.py
rename to src/backend/core/files/artifacts/key_repair.py
diff --git a/src/backend/core/artifacts/local_reference.py b/src/backend/core/files/artifacts/local_reference.py
similarity index 89%
rename from src/backend/core/artifacts/local_reference.py
rename to src/backend/core/files/artifacts/local_reference.py
index c7dbc103..15dc2ae6 100644
--- a/src/backend/core/artifacts/local_reference.py
+++ b/src/backend/core/files/artifacts/local_reference.py
@@ -26,16 +26,17 @@ def _project_root(project) -> Path:
def project_file_path(path: str, scope, user_id: str | None, session_id: str | None = None) -> str:
- from core.llm.tools._paths import to_physical_path
+ from core.sandbox.workspace.resolution import to_physical_path
return to_physical_path(path, user_id, session_id=session_id, scope=scope)
def _allows_live_access(path: str, user_id: str, action: str = "read") -> bool:
- from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET, resolve_approval_mode
+ from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET
+ from core.sandbox.permissions.approval import resolve_approval_mode
from core.sandbox.workspace.paths import WORKSPACE
from core.sandbox.permissions.evaluator import evaluate_local_path
- from core.services.local_grant_service import grants_for_gate, policy_for_gate
+ from core.harness.registry.local_files import grants_for_gate, policy_for_gate
import os
verdict = evaluate_local_path(
@@ -47,14 +48,14 @@ def _allows_live_access(path: str, user_id: str, action: str = "read") -> bool:
platform="windows" if os.name == "nt" else "posix",
)
ticket = CURRENT_PERMISSION_TICKET.get()
- return verdict.decision == "allow" or bool(ticket and ticket.authorizes_path(path, action))
+ return verdict.decision == "allow" or bool(ticket and ticket.active and ticket.authorizes_path(path, action))
def reference_project_file(path: str, *, scope, user_id: str, name: str = "") -> dict:
from core.config.local_mode import local_mode_enabled
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
- from core.artifacts.store import _record_artifact, _now_iso
+ from core.projects.source import ProjectSourceService
+ from core.files.artifacts.store import _record_artifact, _now_iso
if not local_mode_enabled() or not scope or not scope.is_local or not user_id:
raise HTTPException(400, "仅本机项目支持按文件路径交付")
@@ -73,7 +74,7 @@ def reference_project_file(path: str, *, scope, user_id: str, name: str = "") ->
raise HTTPException(403, "文件必须位于当前本机项目目录内")
if not source.is_file():
raise HTTPException(404, "文件已移动或删除")
- from core.llm.tool_permissions import require_local_path_permission
+ from core.sandbox.permissions.dispatch_models import require_local_path_permission
require_local_path_permission(str(source), "read")
relative = source.relative_to(root).as_posix()
@@ -104,7 +105,7 @@ def resolve_project_reference(item: dict) -> dict | None:
"""Revalidate the live binding and symlink boundary on every read/open/preview."""
from core.config.local_mode import local_mode_enabled
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
if not local_mode_enabled():
return None
@@ -139,7 +140,7 @@ def is_project_file_path(path: str, scope) -> bool:
def authorized_reference(file_id: str, user_id: str | None) -> dict | None:
- from core.artifacts.store import get_artifact
+ from core.files.artifacts.store import get_artifact
if not user_id or not is_local_file_ref(file_id):
return None
@@ -152,8 +153,8 @@ def authorized_reference(file_id: str, user_id: str | None) -> dict | None:
def overwrite_reference(file_id: str, user_id: str, content: bytes, db) -> dict:
import os
import tempfile
- from core.services.project_source import ProjectSourceService
- from core.services.local_snapshot_service import snapshot
+ from core.projects.source import ProjectSourceService
+ from core.harness.registry.local_files import snapshot
item = authorized_reference(file_id, user_id)
if not item:
@@ -193,9 +194,9 @@ def overwrite_reference(file_id: str, user_id: str, content: bytes, db) -> dict:
def reference_local_file(path: str, *, scope, user_id: str, session_id: str | None, name: str = "") -> dict:
from core.config.local_mode import local_mode_enabled
from core.config.settings import settings
- from core.llm.tools._paths import workspace_directory
- from core.llm.tool_permissions import require_local_path_permission
- from core.artifacts.store import _write_record_local, _now_iso
+ from core.sandbox.workspace.resolution import workspace_directory
+ from core.sandbox.permissions.dispatch_models import require_local_path_permission
+ from core.files.artifacts.store import _write_record_local, _now_iso
if not local_mode_enabled():
raise HTTPException(400, "仅本机模式支持按文件路径交付")
@@ -236,7 +237,7 @@ def reference_local_file(path: str, *, scope, user_id: str, session_id: str | No
def resolve_session_reference(item: dict) -> dict | None:
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
meta = item.get("metadata") or {}
try:
diff --git a/src/backend/core/services/artifact_location.py b/src/backend/core/files/artifacts/location.py
similarity index 52%
rename from src/backend/core/services/artifact_location.py
rename to src/backend/core/files/artifacts/location.py
index 4edde2c2..decad0d0 100644
--- a/src/backend/core/services/artifact_location.py
+++ b/src/backend/core/files/artifacts/location.py
@@ -1,10 +1,8 @@
-"""Logical personal-file locations for already-authorized CE artifacts."""
+"""Logical UI locations for already-authorized artifacts; never expose storage keys."""
from pathlib import PurePosixPath
-
from fastapi import HTTPException
-
-from core.db import models
+from core.files.edition import artifact_location_scope, artifact_location_root
def artifact_location(db, artifact):
@@ -12,20 +10,26 @@ def artifact_location(db, artifact):
return {"kind": "unfiled", "root_name": None, "relative_path": None}
if artifact.deleted_at is not None:
raise HTTPException(404, "文件已移动或删除")
- folder_id = artifact.user_folder_id
+ kind, folder_id, folder_model, owner_field, owner_id = artifact_location_scope(artifact)
parts, visited = [], set()
while folder_id:
- if folder_id in visited or len(visited) >= 128:
+ if folder_id in visited or len(visited) >= 128 or folder_model is None:
raise HTTPException(409, "文件位置暂时无法解析")
visited.add(folder_id)
- folder = db.get(models.UserFolder, folder_id)
- if folder is None or folder.deleted_at is not None or folder.user_id != artifact.user_id:
+ folder = db.get(folder_model, folder_id)
+ if (
+ folder is None
+ or folder.deleted_at is not None
+ or getattr(folder, owner_field) != owner_id
+ ):
raise HTTPException(404, "文件位置不可用")
parts.append(folder.name)
folder_id = folder.parent_folder_id
+ root_name = artifact_location_root(db, artifact)
+ # A filename is a display label, never interpreted as an absolute path.
name = str(artifact.filename or artifact.title).replace(chr(92), "/").rsplit("/", 1)[-1]
return {
- "kind": "personal",
- "root_name": None,
+ "kind": kind,
+ "root_name": root_name,
"relative_path": str(PurePosixPath(*reversed(parts), name)),
}
diff --git a/src/backend/core/content/artifact_reader.py b/src/backend/core/files/artifacts/reader.py
similarity index 91%
rename from src/backend/core/content/artifact_reader.py
rename to src/backend/core/files/artifacts/reader.py
index 62923d9b..7c04def2 100644
--- a/src/backend/core/content/artifact_reader.py
+++ b/src/backend/core/files/artifacts/reader.py
@@ -57,13 +57,14 @@ def resolve_artifact_storage(
filename = fallback_filename
try:
- from core.artifacts.store import get_artifact
+ from core.files.artifacts.store import get_artifact
item = get_artifact(file_id)
if item and item.get("storage_key"):
storage_key = item["storage_key"]
filename = item.get("name") or filename
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.files.artifacts.reader:resolve_artifact_storage', fallback_error)
if not storage_key:
try:
@@ -100,14 +101,14 @@ def fetch_parsed_text(
if not file_id:
return ""
- from core.artifacts.local_reference import authorized_reference, is_local_file_ref
+ from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref
if is_local_file_ref(file_id):
item = authorized_reference(file_id, user_id)
if not item:
return ""
try:
from pathlib import Path
- from core.content.file_parser import parse_file
+ from core.files.content.file_parser import parse_file
# Live files must never use the immutable-artifact parsed-text cache.
return parse_file(Path(item["path"]).read_bytes(), item["name"]) or ""
except (OSError, RuntimeError, ValueError):
@@ -139,9 +140,9 @@ def fetch_parsed_text(
filename = art.filename or "file"
try:
- from core.content.file_parser import parse_file
+ from core.files.content.file_parser import parse_file
from core.infra.exceptions import StorageError
- from core.storage import get_storage
+ from core.files.storage import get_storage
file_bytes = (
prefetched_bytes
@@ -164,7 +165,7 @@ def fetch_parsed_text(
return ""
if parsed:
- from core.content.artifact_summary import build_summary_from_text
+ from core.files.artifacts.summary import build_summary_from_text
art.parsed_text = parsed
art.parsed_at = utc_now()
@@ -190,7 +191,7 @@ def load_artifact_meta(file_id: str, user_id: Optional[str] = None) -> Optional[
"""Return lightweight artifact metadata (no parsed_text), or None if inaccessible."""
if not file_id:
return None
- from core.artifacts.local_reference import authorized_reference, is_local_file_ref
+ from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref
if is_local_file_ref(file_id):
item = authorized_reference(file_id, user_id)
if not item:
diff --git a/src/backend/core/content/artifact_refs.py b/src/backend/core/files/artifacts/refs.py
similarity index 93%
rename from src/backend/core/content/artifact_refs.py
rename to src/backend/core/files/artifacts/refs.py
index 078f7b34..6a4de499 100644
--- a/src/backend/core/content/artifact_refs.py
+++ b/src/backend/core/files/artifacts/refs.py
@@ -1,7 +1,7 @@
"""Artifact reference helpers — pure data utilities over tool-result payloads.
Relocated out of ``api/routes/v1/artifacts.py`` so that lower layers
-(``core.llm.tools``, ``core.services.artifact_service``, ``routing.*``) can use
+(``core.llm.tools``, ``core.files.artifacts.service``, ``routing.*``) can use
them without importing an API route module (which previously forced a
``core/routing → api`` upward dependency). The API route now re-exports these.
"""
@@ -33,7 +33,7 @@ def resolve_artifact_storage_key(file_id: str, storage_key: Optional[str] = None
return storage_key
try:
- from core.artifacts.store import get_artifact
+ from core.files.artifacts.store import get_artifact
item = get_artifact(file_id)
if item and item.get("storage_key"):
@@ -51,8 +51,8 @@ def require_artifact_storage_key(
key = resolve_artifact_storage_key(file_id, storage_key)
if (not key or key == f"artifacts/{file_id}") and filename and size:
from types import SimpleNamespace
- from core.content.artifact_key_repair import verified_legacy_key
- from core.storage import get_storage
+ from core.files.artifacts.key_repair import verified_legacy_key
+ from core.files.storage import get_storage
repaired = verified_legacy_key(SimpleNamespace(
artifact_id=file_id, storage_key=f"artifacts/{file_id}",
@@ -62,7 +62,7 @@ def require_artifact_storage_key(
if not key:
raise ValueError(f"Artifact {file_id} has no authoritative storage key")
if key == f"artifacts/{file_id}":
- from core.storage import get_storage
+ from core.files.storage import get_storage
if not get_storage().exists(key):
raise ValueError(f"Artifact {file_id} has an unverified placeholder storage key")
diff --git a/src/backend/core/services/artifact_service.py b/src/backend/core/files/artifacts/service.py
similarity index 93%
rename from src/backend/core/services/artifact_service.py
rename to src/backend/core/files/artifacts/service.py
index 234753fe..42732f6c 100644
--- a/src/backend/core/services/artifact_service.py
+++ b/src/backend/core/files/artifacts/service.py
@@ -5,14 +5,14 @@
import uuid
from typing import TYPE_CHECKING, Any, Dict, List, Optional
-from core.content.artifact_refs import infer_artifact_type, require_artifact_storage_key
+from core.files.artifacts.refs import infer_artifact_type, require_artifact_storage_key
from core.db.models import Artifact as ArtifactModel
from core.db.repository import ArtifactRepository
-from core.services.artifact_edition import artifact_scope_fields
+from core.files.edition import artifact_scope_fields
from sqlalchemy.orm import Session
if TYPE_CHECKING:
- from core.services.project_scope import ProjectScope
+ from core.projects.edition import ProjectScope
logger = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ def store_bytes_as_artifact(
check_name(db, user_id, user_folder_id, filename)
artifact_id = artifact_id or f"ua_{uuid.uuid4().hex[:16]}"
storage_key = artifact_upload_key(user_id, artifact_id, filename)
- from core.storage import get_storage
+ from core.files.storage import get_storage
storage_url = get_storage().upload_bytes(content, storage_key)
meta: Dict[str, Any] = {"source": source}
@@ -143,8 +143,7 @@ def get_artifact(self, artifact_id: str, user_id: str) -> Optional[Dict[str, Any
# ── AI-generated artifact persistence (used by chat route + routing) ──────────
-# Relocated from ``api/routes/v1/chats/__init__.py`` so lower layers (``core.llm.tool``,
-# ``routing.*``) can persist artifacts without importing an API route module.
+# Lives in core so tools and routing can persist artifacts without importing an API route.
def persist_artifacts(
@@ -161,7 +160,7 @@ def persist_artifacts(
Strict mode: callers MUST pass only the list of files the agent
explicitly pinned via ``pin_to_workspace`` (e.g. the result of
- ``core.llm.workspace.get_pinned()``). Intermediate tool outputs that
+ ``core.llm.runtime.workspace.get_pinned()``). Intermediate tool outputs that
the agent did not pin are deliberately NOT persisted — the user's
"我的空间" should only show deliverables, not every transient docx
produced by `word_replace_text` etc.
@@ -179,7 +178,7 @@ def persist_artifacts(
workflow.py's finally had already reset the ContextVar, causing project
outputs to land in the wrong scope.
"""
- from core.artifacts.local_reference import is_local_file_ref
+ from core.files.artifacts.local_reference import is_local_file_ref
# Local project cards are live file references, not MySpace storage objects.
collected = [a for a in collected if not is_local_file_ref(a.get("file_id", ""))]
diff --git a/src/backend/core/artifacts/store.py b/src/backend/core/files/artifacts/store.py
similarity index 96%
rename from src/backend/core/artifacts/store.py
rename to src/backend/core/files/artifacts/store.py
index eaec62fd..3e245fb2 100644
--- a/src/backend/core/artifacts/store.py
+++ b/src/backend/core/files/artifacts/store.py
@@ -20,6 +20,7 @@
import logging
import os
import shutil
+from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Optional
@@ -30,7 +31,7 @@
# ── Local paths ──────────────────────────────────────────────────────────────
# Prefer STORAGE_PATH (usually /app/storage inside the container) to avoid creating
# a no-permission directory under /app; fall back to the project root result/ when unset.
-_PROJECT_ROOT = Path(__file__).resolve().parents[3]
+_PROJECT_ROOT = Path(__file__).resolve().parents[4]
_STORAGE_BASE = os.getenv("STORAGE_PATH", "").strip()
_BASE_DIR = Path(_STORAGE_BASE).expanduser() if _STORAGE_BASE else (_PROJECT_ROOT / "result")
_STORE_DIR = (_BASE_DIR / "artifacts").resolve()
@@ -61,7 +62,6 @@
_OSS_LEGACY_INDEX_KEY = "artifacts/_index.json"
# Bucket directories already created in this process (see _ensure_bucket).
-_known_buckets: set = set()
# ── Helper: get the current STORAGE_TYPE ──────────────────────────────────
@@ -71,7 +71,7 @@ def _storage_type() -> str:
# ── Helper: get the OSS backend (lazy import to avoid circular deps) ────────────────
def _get_oss_storage():
- from core.storage import get_storage
+ from core.files.storage import get_storage
return get_storage()
@@ -125,10 +125,10 @@ def _ensure_bucket(path: Path) -> None:
syscall repeated thousands of times during a migration. Keyed on the real path
so redirecting _STORE_DIR (tests, a moved STORAGE_PATH) still creates it.
"""
- if path in _known_buckets:
+ if path in process_state().known_buckets:
return
path.mkdir(parents=True, exist_ok=True)
- _known_buckets.add(path)
+ process_state().known_buckets.add(path)
def _write_record_local(item: Dict[str, Any]) -> str:
@@ -271,7 +271,7 @@ def save_artifact_bytes(
# fail-safe — returns the input untouched for non-SVG or on any parse error.
if "svg" in mime_type.lower() or ext.lower() == "svg":
try:
- from core.content.svg_fit import fit_svg_viewbox
+ from core.files.content.svg_fit import fit_svg_viewbox
content = fit_svg_viewbox(content)
except Exception as exc: # never block a save on the normaliser
@@ -410,7 +410,7 @@ def get_artifact(file_id: str) -> Optional[Dict[str, Any]]:
return None
if (item.get("metadata") or {}).get("source") == "local_project_reference":
- from .local_reference import resolve_project_reference
+ from core.files.artifacts.local_reference import resolve_project_reference
return resolve_project_reference(item)
@@ -443,3 +443,15 @@ def prepare_records() -> int:
if written or _has_records():
return written
return restore_records_from_oss()
+
+
+@dataclass
+class StoreState:
+ known_buckets: set = field(default_factory=lambda: set())
+
+
+_STATE = StoreState()
+
+
+def process_state() -> StoreState:
+ return _STATE
diff --git a/src/backend/core/content/artifact_summary.py b/src/backend/core/files/artifacts/summary.py
similarity index 97%
rename from src/backend/core/content/artifact_summary.py
rename to src/backend/core/files/artifacts/summary.py
index 2cadd220..8849a393 100644
--- a/src/backend/core/content/artifact_summary.py
+++ b/src/backend/core/files/artifacts/summary.py
@@ -15,7 +15,6 @@
import io
import logging
import os
-from typing import Optional
logger = logging.getLogger(__name__)
@@ -130,7 +129,7 @@ def build_summary(file_bytes: bytes, filename: str, mime_type: str = "") -> str:
def _summary_text_document(file_bytes: bytes, filename: str) -> str:
"""Parse and truncate first 500 chars."""
- from core.content.file_parser import parse_file
+ from core.files.content.file_parser import parse_file
text = parse_file(file_bytes, filename) or ""
if not text:
@@ -278,8 +277,9 @@ def _summary_pptx(file_bytes: bytes, filename: str) -> str:
try:
if slide.shapes.title is not None:
title = (slide.shapes.title.text or "").strip()
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.files.artifacts.summary:_summary_pptx', fallback_error)
if not title:
title = "(无标题)"
titles.append(f" 第 {i+1} 页: {title[:60]}")
diff --git a/src/backend/core/services/artifact_text_service.py b/src/backend/core/files/artifacts/text.py
similarity index 92%
rename from src/backend/core/services/artifact_text_service.py
rename to src/backend/core/files/artifacts/text.py
index d99240b3..40efa571 100644
--- a/src/backend/core/services/artifact_text_service.py
+++ b/src/backend/core/files/artifacts/text.py
@@ -9,11 +9,11 @@
from fastapi import HTTPException
from sqlalchemy import update
-from core.auth.permissions_iface import resolve_artifact_access
+from core.auth.edition import resolve_artifact_access
from core.db.models import Artifact
from core.infra.time import utc_now
-from core.services.artifact_edition import artifact_access_metadata
-from core.services.project_source import reserve_sqlite_writer
+from core.files.edition import artifact_access_metadata
+from core.projects.source import reserve_sqlite_writer
MAX_TEXT_BYTES = 2 * 1024 * 1024
TEXT_EXTENSIONS = frozenset(
@@ -91,7 +91,7 @@ def _document(self, artifact, level, data):
}
def read(self, file_id, actor):
- from core.storage import get_storage
+ from core.files.storage import get_storage
artifact, level = self._authorized(file_id, actor)
try:
@@ -101,7 +101,7 @@ def read(self, file_id, actor):
return self._document(artifact, level, data)
def save(self, file_id, actor, content, revision):
- from core.storage import get_storage
+ from core.files.storage import get_storage
try:
data = content.encode("utf-8")
@@ -148,8 +148,9 @@ def save(self, file_id, actor, content, revision):
self.db.rollback()
try:
storage.delete(key)
- except Exception:
- pass # Immutable orphan can be collected; never delete the last good bytes.
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.files.artifacts.text:save', fallback_error)
raise HTTPException(409, "文件已被修改,请保留草稿并重新打开文件后合并")
self.db.commit()
self.db.refresh(artifact)
diff --git a/src/backend/core/files/content/__init__.py b/src/backend/core/files/content/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/core/files/content/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/content/archive_browser.py b/src/backend/core/files/content/archive_browser.py
similarity index 99%
rename from src/backend/core/content/archive_browser.py
rename to src/backend/core/files/content/archive_browser.py
index 17bc1f18..801b1109 100644
--- a/src/backend/core/content/archive_browser.py
+++ b/src/backend/core/files/content/archive_browser.py
@@ -10,7 +10,7 @@
import zipfile
import zlib
-from core.content.archive_limits import check_zip_directory, open_bounded_xz
+from core.files.content.archive_limits import check_zip_directory, open_bounded_xz
from collections import Counter
from contextlib import ExitStack
from pathlib import Path, PurePosixPath
diff --git a/src/backend/core/content/archive_limits.py b/src/backend/core/files/content/archive_limits.py
similarity index 100%
rename from src/backend/core/content/archive_limits.py
rename to src/backend/core/files/content/archive_limits.py
diff --git a/src/backend/core/files/content/branding.py b/src/backend/core/files/content/branding.py
new file mode 100644
index 00000000..d4e6e33f
--- /dev/null
+++ b/src/backend/core/files/content/branding.py
@@ -0,0 +1,299 @@
+"""Product branding and page configuration stored as content."""
+from __future__ import annotations
+import re
+from datetime import datetime, timezone
+from typing import Any
+from core.config.settings import settings
+from core.db.models import ContentBlock
+from sqlalchemy.orm import Session
+from core.files.content import content_blocks_constants
+
+def normalize_app_config(payload: Any) -> dict[str, Any]:
+ """Normalize app_config payload to the canonical `{"apps": [...]}` shape.
+
+ An empty list (admin deleted all apps) is respected as "no apps" and no longer
+ falls back to any default entries.
+ """
+ if not isinstance(payload, dict):
+ return {"apps": list(content_blocks_constants.DEFAULT_APP_CONFIG["apps"])}
+
+ apps_raw = payload.get("apps")
+ if isinstance(apps_raw, list):
+ apps: list[dict[str, Any]] = []
+ for item in apps_raw:
+ if not isinstance(item, dict):
+ continue
+ app_id = str(item.get("id") or "").strip()
+ if not app_id:
+ continue
+ apps.append(
+ {
+ "id": app_id,
+ "enabled": bool(item.get("enabled", True)),
+ "name": str(item.get("name") or app_id),
+ "description": str(item.get("description") or ""),
+ "url": str(item.get("url") or ""),
+ "icon": str(item.get("icon") or ""),
+ }
+ )
+ return {"apps": apps}
+
+ return {"apps": list(content_blocks_constants.DEFAULT_APP_CONFIG["apps"])}
+
+
+def default_payload(alias: str) -> Any:
+ if alias == "page_config":
+ return content_blocks_constants.DEFAULT_PAGE_CONFIG
+ if alias == "app_config":
+ return content_blocks_constants.DEFAULT_APP_CONFIG
+ if alias == "prompt_versions":
+ return content_blocks_constants.DEFAULT_PROMPT_VERSIONS
+ return {} if alias in content_blocks_constants.DICT_PAYLOAD_BLOCKS else []
+
+
+def seed_page_config_if_missing(db: Session) -> bool:
+ """Insert default page_config row if it does not yet exist. Returns True if inserted."""
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ if row:
+ return False
+ db.add(
+ ContentBlock(
+ id="page_config",
+ payload=content_blocks_constants.DEFAULT_PAGE_CONFIG,
+ updated_at=datetime.now(timezone.utc),
+ updated_by="system_seed",
+ )
+ )
+ db.commit()
+ return True
+
+
+def enforce_ce_branding(db: Session) -> bool:
+ """Keep the fixed CE product identity and auth policy consistent.
+
+ ``page_config`` survives image upgrades, so merely changing the code/env
+ defaults does not repair an already-running CE instance. CE does not expose
+ the content-management console and therefore has no supported per-instance
+ branding override or self-service registration; normalize both the small
+ set of branding fields and the registration switch on startup. EE
+ deployments are never touched.
+ """
+ if settings.edition.edition != "ce":
+ return False
+
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ if row is None or not isinstance(row.payload, dict):
+ return False
+
+ product_name = "HugAgentOS"
+ payload = dict(row.payload)
+ branding = dict(payload.get("branding") or {})
+ navigation = dict(payload.get("navigation") or {})
+ auth = dict(payload.get("auth") or {})
+ admin_header = dict(navigation.get("admin_header") or {})
+ admin_platform = dict(navigation.get("admin_platform") or {})
+
+ desired_branding = {
+ "product_name": product_name,
+ "product_subtitle": "AI 智能助手",
+ "page_title": product_name,
+ "hero_title": f"你好,我是 {product_name}",
+ }
+ changed = any(branding.get(key) != value for key, value in desired_branding.items())
+ changed = changed or admin_header.get("title") != f"{product_name} — 后台管理"
+ changed = changed or admin_platform.get("product_name") != product_name
+ changed = changed or auth.get("allow_register") is not False
+ if not changed:
+ return False
+
+ branding.update(desired_branding)
+ admin_header["title"] = f"{product_name} — 后台管理"
+ admin_platform["product_name"] = product_name
+ navigation["admin_header"] = admin_header
+ navigation["admin_platform"] = admin_platform
+ auth["allow_register"] = False
+ payload["branding"] = branding
+ payload["navigation"] = navigation
+ payload["auth"] = auth
+ row.payload = payload
+ row.updated_at = datetime.now(timezone.utc)
+ row.updated_by = "system_seed"
+ from sqlalchemy.orm.attributes import flag_modified
+
+ flag_modified(row, "payload")
+ db.commit()
+ return True
+
+
+def backfill_navigation_entries(db: Session) -> int:
+ """Patch existing page_config row to add navigation entries listed in
+ ``_NAV_BACKFILL_ENTRIES`` if they are missing.
+
+ Returns the number of *individual fields* mutated (across all entries
+ and all three locations: sidebar_items / panel_titles / panel_subtitles).
+ Zero means nothing changed (idempotent steady state).
+ """
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ if not row:
+ return 0 # seed_page_config_if_missing handles fresh installs
+ payload = row.payload if isinstance(row.payload, dict) else {}
+ nav = payload.get("navigation")
+ if not isinstance(nav, dict):
+ return 0 # malformed payload — leave it alone, admin UI will surface
+
+ default_nav = content_blocks_constants.DEFAULT_PAGE_CONFIG["navigation"]
+
+ changed = 0
+ for entry in content_blocks_constants.NAV_BACKFILL_ENTRIES:
+ key = entry["key"]
+ bucket_field = "menu_items" if entry["bucket"] == "menu" else "sidebar_items"
+ items = nav.get(bucket_field)
+ # Present in *either* list counts as placed: the operator may have moved the entry
+ # between buckets on purpose, and re-seeding it would duplicate it across both.
+ already_placed = any(
+ key in value
+ for field in ("sidebar_items", "menu_items")
+ if isinstance(value := nav.get(field), list)
+ )
+ if isinstance(items, list) and not already_placed:
+ anchor = entry.get("insert_after")
+ new_items = list(items)
+ if anchor and anchor in new_items:
+ new_items.insert(new_items.index(anchor) + 1, key)
+ else:
+ new_items.append(key)
+ nav[bucket_field] = new_items
+ changed += 1
+
+ # Copy comes from DEFAULT_PAGE_CONFIG so it is never re-typed here.
+ for field in ("panel_titles", "panel_subtitles"):
+ target = nav.get(field)
+ text = default_nav[field].get(key)
+ if isinstance(target, dict) and key not in target and text:
+ target[key] = text
+ changed += 1
+
+ # 旧出厂文案改名(能力中心统一表述),与上面的补齐共用一次提交
+ for field, renames in content_blocks_constants.LEGACY_PANEL_TEXT_RENAMES.items():
+ target = nav.get(field)
+ if not isinstance(target, dict):
+ continue
+ for key, (legacy_values, new_text) in renames.items():
+ if target.get(key) in legacy_values:
+ target[key] = new_text
+ changed += 1
+
+ if changed:
+ payload["navigation"] = nav
+ row.payload = payload
+ row.updated_at = datetime.now(timezone.utc)
+ # Mark updated_by so audit can distinguish automatic backfill vs manual edits
+ row.updated_by = "system_seed"
+ # SQLAlchemy JSONB does not auto-detect mutation — flag explicitly
+ from sqlalchemy.orm.attributes import flag_modified
+
+ flag_modified(row, "payload")
+ db.commit()
+ return changed
+
+
+def get_branding_info() -> dict[str, str]:
+ """Read the branding fields from page_config (product_name + logo_url), falling back to defaults on failure.
+
+ Reused by server-side rendering paths (e.g. the SSO login page); manages its
+ own DB session internally.
+ """
+ defaults = {
+ "product_name": content_blocks_constants.DEFAULT_PAGE_CONFIG["branding"]["product_name"],
+ "logo_url": content_blocks_constants.DEFAULT_PAGE_CONFIG["branding"]["logo_url"],
+ }
+ try:
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ try:
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ payload = (row.payload if row else {}) or {}
+ branding = payload.get("branding", {}) if isinstance(payload, dict) else {}
+ return {
+ "product_name": branding.get("product_name") or defaults["product_name"],
+ "logo_url": branding.get("logo_url") or defaults["logo_url"],
+ }
+ finally:
+ db.close()
+ except Exception:
+ return defaults
+
+
+def is_register_allowed() -> bool:
+ """Return whether self-service local-account registration is enabled.
+
+ CE is permanently single-account and therefore always returns ``False``.
+ Other editions read ``page_config.auth.allow_register`` and keep the legacy
+ default of ``True`` when the setting is missing or cannot be read. The value
+ gates both the server-rendered UI and registration submissions.
+ """
+ if settings.edition.edition == "ce":
+ return False
+
+ try:
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ try:
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ payload = (row.payload if row else {}) or {}
+ auth = payload.get("auth", {}) if isinstance(payload, dict) else {}
+ val = auth.get("allow_register") if isinstance(auth, dict) else None
+ return True if val is None else bool(val)
+ finally:
+ db.close()
+ except Exception:
+ return True
+
+
+def get_admin_platform_info() -> dict[str, str]:
+ """Read page_config.navigation.admin_platform (admin platform brand + per-page labels).
+
+ When admin_platform is missing, derive the brand from the legacy
+ admin_header.title (shaped like " — 后台管理") as the product_name
+ fallback, then fall back to code defaults — consistent with the frontend
+ mergeAdminPlatform normalization. Reused by server-side rendering paths
+ (Swagger / ReDoc doc page titles etc.); manages its own DB session internally.
+ """
+ ap_def = content_blocks_constants.DEFAULT_PAGE_CONFIG["navigation"]["admin_platform"]
+ defaults = {
+ "product_name": ap_def["product_name"],
+ "content_label": ap_def["content_label"],
+ "config_label": ap_def["config_label"],
+ "apidoc_label": ap_def["apidoc_label"],
+ }
+ try:
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ try:
+ row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first()
+ payload = (row.payload if row else {}) or {}
+ nav = payload.get("navigation", {}) if isinstance(payload, dict) else {}
+ ap = nav.get("admin_platform") if isinstance(nav, dict) else None
+ ap = ap if isinstance(ap, dict) else {}
+ header = nav.get("admin_header") if isinstance(nav, dict) else None
+ derived = ""
+ if isinstance(header, dict) and isinstance(header.get("title"), str):
+ derived = re.split(r"\s+[—-]\s+", header["title"])[0].strip()
+
+ def pick(key: str, fallback: str) -> str:
+ v = ap.get(key)
+ return v.strip() if isinstance(v, str) and v.strip() else fallback
+
+ return {
+ "product_name": pick("product_name", derived or defaults["product_name"]),
+ "content_label": pick("content_label", defaults["content_label"]),
+ "config_label": pick("config_label", defaults["config_label"]),
+ "apidoc_label": pick("apidoc_label", defaults["apidoc_label"]),
+ }
+ finally:
+ db.close()
+ except Exception:
+ return defaults
diff --git a/src/backend/core/files/content/content_blocks.py b/src/backend/core/files/content/content_blocks.py
new file mode 100644
index 00000000..d7d2e5d5
--- /dev/null
+++ b/src/backend/core/files/content/content_blocks.py
@@ -0,0 +1,183 @@
+"""Export and import content blocks and the prompt pool as snapshots."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Mapping
+from core.db.models import ContentBlock
+from sqlalchemy.orm import Session
+from core.files.content import branding
+from core.files.content import content_blocks_constants
+
+class ContentSnapshotError(ValueError):
+ """Raised when a docs content snapshot is invalid."""
+
+
+def serialize_block(row: ContentBlock | None, alias: str | None = None) -> dict[str, Any]:
+ return {
+ "payload": row.payload if row else branding.default_payload(alias or ""),
+ "updated_at": row.updated_at.isoformat() if row and row.updated_at else None,
+ "updated_by": row.updated_by if row else None,
+ }
+
+
+def build_docs_snapshot(
+ db: Session,
+ block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP,
+) -> dict[str, Any]:
+ """Build a portable JSON snapshot for content blocks.
+
+ Defaults to the docs blocks; pass ``block_map=PROMPT_BLOCK_MAP`` (or use
+ ``build_prompt_snapshot``) to snapshot the system prompt pool instead.
+ """
+ rows = db.query(ContentBlock).filter(ContentBlock.id.in_(list(block_map.values()))).all()
+ row_map = {row.id: row for row in rows}
+
+ return {
+ "schema_version": content_blocks_constants.SNAPSHOT_SCHEMA_VERSION,
+ "exported_at": datetime.now(timezone.utc).isoformat(),
+ "blocks": {
+ alias: serialize_block(row_map.get(db_id), alias) for alias, db_id in block_map.items()
+ },
+ }
+
+
+def parse_snapshot_datetime(value: Any) -> datetime | None:
+ if value in (None, ""):
+ return None
+ if not isinstance(value, str):
+ raise ContentSnapshotError("updated_at must be an ISO datetime string")
+ try:
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise ContentSnapshotError(f"Invalid datetime: {value}") from exc
+
+
+def normalize_docs_snapshot(
+ snapshot: Mapping[str, Any],
+ block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP,
+) -> dict[str, Any]:
+ """Validate and normalize an incoming content snapshot."""
+ if not isinstance(snapshot, Mapping):
+ raise ContentSnapshotError("Snapshot body must be a JSON object")
+
+ schema_version = snapshot.get("schema_version", content_blocks_constants.SNAPSHOT_SCHEMA_VERSION)
+ if schema_version != content_blocks_constants.SNAPSHOT_SCHEMA_VERSION:
+ raise ContentSnapshotError(
+ f"Unsupported schema_version: {schema_version}. Expected {content_blocks_constants.SNAPSHOT_SCHEMA_VERSION}"
+ )
+
+ raw_blocks = snapshot.get("blocks")
+ if not isinstance(raw_blocks, Mapping):
+ raise ContentSnapshotError("Snapshot.blocks must be an object")
+
+ unknown_blocks = set(raw_blocks.keys()) - set(block_map.keys()) - content_blocks_constants.RETIRED_BLOCK_ALIASES
+ if unknown_blocks:
+ raise ContentSnapshotError(
+ f"Unknown blocks in snapshot: {', '.join(sorted(unknown_blocks))}"
+ )
+
+ normalized_blocks: dict[str, dict[str, Any]] = {}
+ for alias in block_map:
+ raw_block = raw_blocks.get(alias)
+ if raw_block is None:
+ continue
+ if not isinstance(raw_block, Mapping):
+ raise ContentSnapshotError(f"Snapshot block '{alias}' must be an object")
+
+ payload = raw_block.get("payload", branding.default_payload(alias))
+ expected_type = dict if alias in content_blocks_constants.DICT_PAYLOAD_BLOCKS else list
+ if not isinstance(payload, expected_type):
+ kind = "object" if expected_type is dict else "list"
+ raise ContentSnapshotError(f"Snapshot block '{alias}'.payload must be a {kind}")
+
+ normalized_blocks[alias] = {
+ "payload": payload,
+ "updated_at": parse_snapshot_datetime(raw_block.get("updated_at")),
+ "updated_by": raw_block.get("updated_by"),
+ }
+
+ if not normalized_blocks:
+ raise ContentSnapshotError("Snapshot does not contain any importable content blocks")
+
+ exported_at = snapshot.get("exported_at")
+ if exported_at not in (None, "") and not isinstance(exported_at, str):
+ raise ContentSnapshotError("Snapshot.exported_at must be a string")
+
+ return {
+ "schema_version": content_blocks_constants.SNAPSHOT_SCHEMA_VERSION,
+ "exported_at": exported_at,
+ "blocks": normalized_blocks,
+ }
+
+
+def import_docs_snapshot(
+ db: Session,
+ snapshot: Mapping[str, Any],
+ *,
+ overwrite: bool = True,
+ default_updated_by: str | None = None,
+ block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP,
+) -> dict[str, Any]:
+ """Import a content snapshot into the current database."""
+ normalized = normalize_docs_snapshot(snapshot, block_map=block_map)
+ imported: list[str] = []
+ skipped: list[str] = []
+
+ for alias, block in normalized["blocks"].items():
+ db_id = block_map[alias]
+ row = db.query(ContentBlock).filter(ContentBlock.id == db_id).first()
+ if row and not overwrite:
+ skipped.append(alias)
+ continue
+
+ updated_at = block["updated_at"] or datetime.now(timezone.utc)
+ updated_by = block["updated_by"] or default_updated_by
+
+ if row:
+ row.payload = block["payload"]
+ row.updated_at = updated_at
+ row.updated_by = updated_by
+ else:
+ row = ContentBlock(
+ id=db_id,
+ payload=block["payload"],
+ updated_at=updated_at,
+ updated_by=updated_by,
+ )
+ db.add(row)
+ imported.append(alias)
+
+ db.commit()
+
+ return {
+ "schema_version": normalized["schema_version"],
+ "imported": imported,
+ "skipped": skipped,
+ "count": len(imported),
+ "overwrite": overwrite,
+ }
+
+
+def build_prompt_snapshot(db: Session) -> dict[str, Any]:
+ """Build a portable snapshot of the system prompt pool (prompt_versions).
+
+ Same format as the docs snapshot, but scoped to PROMPT_BLOCK_MAP so it can
+ be migrated across environments independently of docs/page config.
+ """
+ return build_docs_snapshot(db, block_map=content_blocks_constants.PROMPT_BLOCK_MAP)
+
+
+def import_prompt_snapshot(
+ db: Session,
+ snapshot: Mapping[str, Any],
+ *,
+ overwrite: bool = True,
+ default_updated_by: str | None = None,
+) -> dict[str, Any]:
+ """Import a system prompt pool snapshot (counterpart of build_prompt_snapshot)."""
+ return import_docs_snapshot(
+ db,
+ snapshot,
+ overwrite=overwrite,
+ default_updated_by=default_updated_by,
+ block_map=content_blocks_constants.PROMPT_BLOCK_MAP,
+ )
diff --git a/src/backend/core/files/content/content_blocks_constants.py b/src/backend/core/files/content/content_blocks_constants.py
new file mode 100644
index 00000000..c46a3a41
--- /dev/null
+++ b/src/backend/core/files/content/content_blocks_constants.py
@@ -0,0 +1,181 @@
+"""Block maps and schema version for content snapshots."""
+from __future__ import annotations
+from typing import Any
+from core.config.settings import settings
+
+BRAND_NAME = settings.branding.product_name
+
+
+DOCS_BLOCK_MAP = {
+ "updates": "docs_updates",
+ "capabilities": "docs_capabilities",
+ "prompt_hub": "prompt_hub",
+ "page_config": "page_config",
+ "app_config": "app_config",
+}
+
+
+RETIRED_BLOCK_ALIASES = {"homepage_shortcuts"}
+
+
+DICT_PAYLOAD_BLOCKS = {"page_config", "prompt_versions", "app_config"}
+
+
+PROMPT_VERSIONS_BLOCK_ID = "prompt_versions"
+
+
+PROMPT_BLOCK_MAP = {
+ "prompt_versions": PROMPT_VERSIONS_BLOCK_ID,
+ "prompt_hub": "prompt_hub",
+}
+
+
+SNAPSHOT_SCHEMA_VERSION = 1
+
+
+DEFAULT_PAGE_CONFIG: dict[str, Any] = {
+ "branding": {
+ "product_name": BRAND_NAME,
+ "product_subtitle": "AI 智能助手",
+ "logo_url": "/home/header.svg",
+ "favicon_url": "/icon.png",
+ "page_title": BRAND_NAME,
+ "hero_title": f"你好,我是 {BRAND_NAME}",
+ "hero_subtitle": "今天想从哪里开始?",
+ "disclaimer": "本平台生成内容由AI大模型生成,不构成任何建议;涉及业务决策请以权威信息为准。",
+ },
+ "homepage": {
+ # The homepage uses the transparent standalone brand mark rather than
+ # the square sidebar logo, mirroring the lighter Qwen Office hero.
+ "show_logo": True,
+ "logo_url": "/icon.png",
+ "show_suggestions": True,
+ "suggested_questions": [
+ "分析新能源汽车产业链的关键环节与代表企业",
+ "对比两份产业政策,提炼支持方向和申报条件",
+ "查询一家企业的基本情况、经营风险和产业链位置",
+ "从这份材料中提炼核心观点和待办事项",
+ "分析这组经营数据的趋势、异常和可能原因",
+ "检索相关政策依据,并给出可核验的来源",
+ ],
+ },
+ "navigation": {
+ "panel_titles": {
+ "ability_center": "能力中心",
+ "skills": "技能",
+ "agents": "智能体",
+ "mcp": "连接器",
+ "kb": "知识库",
+ "docs": "更新记录",
+ "app_center": "应用中心",
+ "automation": "定时任务",
+ "sites": "站点",
+ "projects": "项目",
+ "lab": "实验室",
+ "settings": "系统设置",
+ "my_space": "我的空间",
+ },
+ "panel_subtitles": {
+ "ability_center": "智能体基础能力管理,包含智能体、技能、连接器与插件",
+ "skills": "启用/停用技能,并查看详细介绍、输入输出与示例。",
+ "agents": "选择与启用智能体,并查看其职责边界与路由提示。",
+ "mcp": "管理连接器服务,并查看其作用范围与可靠性影响。",
+ "kb": "浏览知识库、查看文档列表,并支持文档内检索。",
+ "docs": "查看功能更新、能力中心与平台说明。",
+ "app_center": "基于 AI 能力的场景化智能应用",
+ "automation": "按计划自动执行任务,也可随时手动触发。在任意对话中描述你想定期做的事,即可快速创建",
+ "sites": "在对话里描述需求,AI 生成完整网站并一键发布,由平台托管、凭链接即可访问",
+ "projects": "把对话、文件和指令打包成专属工作空间",
+ "lab": "AI 能力实验性应用",
+ "settings": "",
+ "my_space": "",
+ },
+ "admin_header": {
+ "title": f"{BRAND_NAME} — 后台管理",
+ "subtitle": "后台管理",
+ },
+ # Unified branding + per-page labels for the admin platform (content
+ # management / system config / API docs) — product_name drives the header
+ # brand and browser tab title of all three admin pages, replacing the old
+ # admin_header.
+ "admin_platform": {
+ "product_name": BRAND_NAME,
+ "content_label": "内容管理",
+ "config_label": "系统配置",
+ "apidoc_label": "接口文档",
+ },
+ # 与前端 utils/pageConfigDefaults.ts 的 DEFAULT_SIDEBAR_ITEMS / DEFAULT_MENU_ITEMS 保持一致。
+ # 这只是新装环境的出厂值——已存在的部署以 DB 现有取值为准,由 backfill_navigation_entries
+ # 补齐新增条目,管理员随后可在 /config「页面配置」里自由编排。
+ "sidebar_items": ["ability_center", "automation", "sites", "my_space"],
+ "menu_items": ["settings", "app_center", "projects", "lab"],
+ },
+ "texts": {
+ "input_placeholder": "请输入您的问题…",
+ "input_placeholder_agent": "请输入您的问题…",
+ "search_placeholder": "搜索对话",
+ "btn_new_chat": "新建对话",
+ "btn_logout": "退出",
+ "history_label": "历史对话",
+ "sidebar_empty_state": "暂无对话记录",
+ "search_no_results": "无匹配结果",
+ "dialog_logout_confirm_title": "确认退出登录?",
+ "dialog_logout_confirm_content": "退出登录不会丢失任何数据,重新登录后可继续使用。",
+ "dialog_logout_confirm_ok": "退出登录",
+ "recommend_banner_text": "",
+ },
+ "defaults": {
+ # Initial mode on every user login / new chat:
+ # - chat_mode: 'turbo' / 'fast' / 'medium' / 'high' / 'max' (recommended)
+ # - thinking_mode: bool (legacy field, fallback compat; true→medium, false→fast)
+ "chat_mode": "fast",
+ "thinking_mode": False,
+ },
+ "auth": {
+ # Whether the login page shows the "register" entry (tab + register subpage).
+ # When off, the login page keeps only login, and backend register submissions
+ # are rejected outright (not just hidden on the frontend).
+ "allow_register": True,
+ },
+}
+
+
+DEFAULT_APP_CONFIG: dict[str, Any] = {
+ "apps": [],
+}
+
+
+DEFAULT_PROMPT_VERSIONS: dict[str, Any] = {
+ "active": {},
+ "versions": [],
+}
+
+
+NAV_BACKFILL_ENTRIES: list[dict[str, Any]] = [
+ {"key": "projects", "bucket": "menu", "insert_after": "app_center"},
+ {"key": "automation", "bucket": "sidebar", "insert_after": "ability_center"},
+ {"key": "sites", "bucket": "sidebar", "insert_after": "automation"},
+]
+
+
+LEGACY_PANEL_TEXT_RENAMES: dict[str, dict[str, tuple[tuple[str, ...], str]]] = {
+ "panel_titles": {
+ "skills": (("技能库",), "技能"),
+ "agents": (("子智能体",), "智能体"),
+ "plugins": (("插件库",), "插件"),
+ "mcp": (("MCP工具库", "MCP 工具库", "MCP 工具"), "连接器"),
+ },
+ "panel_subtitles": {
+ "agents": (
+ ("选择与启用子智能体,并查看其职责边界与路由提示。",),
+ "选择与启用智能体,并查看其职责边界与路由提示。",
+ ),
+ "mcp": (
+ (
+ "管理 MCP 工具服务,并查看其作用范围与可靠性影响。",
+ "管理 MCP 连接器服务,并查看其作用范围与可靠性影响。",
+ ),
+ "管理连接器服务,并查看其作用范围与可靠性影响。",
+ ),
+ },
+}
diff --git a/src/backend/core/files/content/file_formats.py b/src/backend/core/files/content/file_formats.py
new file mode 100644
index 00000000..b5d6748d
--- /dev/null
+++ b/src/backend/core/files/content/file_formats.py
@@ -0,0 +1,33 @@
+"""File extensions the parsers accept."""
+from __future__ import annotations
+
+
+EXT_MAP: dict[str, str] = {
+ ".pdf": "pdf",
+ ".docx": "docx",
+ ".doc": "doc_wps",
+ ".wps": "doc_wps",
+ ".txt": "txt",
+ ".xlsx": "xlsx",
+ ".xls": "xls",
+ ".csv": "csv",
+ ".pptx": "pptx",
+}
+
+
+RAW_TEXT_EXTS = {
+ ".html", ".htm",
+ ".md", ".markdown",
+ ".json", ".jsonl", ".ndjson",
+ ".yaml", ".yml",
+ ".toml", ".ini", ".cfg",
+ ".xml", ".svg",
+ ".log",
+ ".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".bash",
+ ".css", ".scss",
+ ".tex", ".rst",
+ ".sql",
+}
+
+
+SUPPORTED_EXTENSIONS = list(EXT_MAP.keys()) + sorted(RAW_TEXT_EXTS)
diff --git a/src/backend/core/files/content/file_parser.py b/src/backend/core/files/content/file_parser.py
new file mode 100644
index 00000000..333fbc57
--- /dev/null
+++ b/src/backend/core/files/content/file_parser.py
@@ -0,0 +1,41 @@
+"""Dispatch an uploaded file to the parser for its extension."""
+from __future__ import annotations
+import os
+from typing import Optional
+from core.files.content import file_formats
+from core.files.content import pdf_parser
+from core.files.content import presentation_parser
+from core.files.content import spreadsheet_parser
+from core.files.content import text_parser
+from core.files.content import word_parser
+
+def parse_file(file_bytes: bytes, filename: str) -> Optional[str]:
+ """
+ Dispatch to the correct parser by file extension.
+
+ Returns extracted text/markdown, or None if the format is unsupported.
+ Raises RuntimeError with a Chinese message on parse failure.
+ """
+ suffix = os.path.splitext(filename.lower())[1]
+ kind = file_formats.EXT_MAP.get(suffix)
+
+ if kind == "pdf":
+ return pdf_parser.parse_pdf(file_bytes, filename)
+ elif kind == "docx":
+ return word_parser.parse_docx(file_bytes)
+ elif kind == "doc_wps":
+ return word_parser.parse_doc_wps(file_bytes, suffix)
+ elif kind == "txt":
+ return text_parser.parse_txt(file_bytes)
+ elif kind == "xlsx":
+ return spreadsheet_parser.parse_xlsx(file_bytes)
+ elif kind == "xls":
+ return spreadsheet_parser.parse_xls(file_bytes)
+ elif kind == "csv":
+ return spreadsheet_parser.parse_csv(file_bytes)
+ elif kind == "pptx":
+ return presentation_parser.parse_pptx(file_bytes)
+ elif suffix in file_formats.RAW_TEXT_EXTS:
+ return text_parser.parse_raw_text(file_bytes)
+ else:
+ return None
diff --git a/src/backend/core/content/file_validation.py b/src/backend/core/files/content/file_validation.py
similarity index 100%
rename from src/backend/core/content/file_validation.py
rename to src/backend/core/files/content/file_validation.py
diff --git a/src/backend/core/content/office.py b/src/backend/core/files/content/office.py
similarity index 100%
rename from src/backend/core/content/office.py
rename to src/backend/core/files/content/office.py
diff --git a/src/backend/core/content/office_preview_cache.py b/src/backend/core/files/content/office_preview_cache.py
similarity index 98%
rename from src/backend/core/content/office_preview_cache.py
rename to src/backend/core/files/content/office_preview_cache.py
index fc763d90..f05c7d41 100644
--- a/src/backend/core/content/office_preview_cache.py
+++ b/src/backend/core/files/content/office_preview_cache.py
@@ -14,7 +14,7 @@
import tempfile
import time
-from core.content.office import find_libreoffice_binary
+from core.files.content.office import find_libreoffice_binary
CACHE_BYTES = 256 * 1024 * 1024
CACHE_ENTRIES = 128
diff --git a/src/backend/core/files/content/parser_config.py b/src/backend/core/files/content/parser_config.py
new file mode 100644
index 00000000..2d5f4434
--- /dev/null
+++ b/src/backend/core/files/content/parser_config.py
@@ -0,0 +1,47 @@
+"""Settings for the external file-parsing service."""
+from __future__ import annotations
+import os
+
+def svc_get(key: str, default: str = "") -> str:
+ """Read from SystemConfigService (DB-first), fall back to env."""
+ try:
+ from core.config.store.system_config import SystemConfigService
+ val = SystemConfigService.get_instance().get(key)
+ if val is not None:
+ return val.strip()
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.files.content.parser_config:svc_get', fallback_error)
+ return (os.getenv(ENV_MAP.get(key, ""), default) or default).strip()
+
+
+def cfg_api_url() -> str:
+ return svc_get("file_parser.api_url")
+
+
+def cfg_timeout() -> int:
+ try:
+ return int(svc_get("file_parser.timeout", "60"))
+ except ValueError:
+ return 60
+
+
+def cfg_parse_params() -> dict:
+ return {
+ "lang_list": svc_get("file_parser.lang_list", "ch"),
+ "backend": svc_get("file_parser.backend", "pipeline"),
+ "parse_method": svc_get("file_parser.parse_method", "auto"),
+ "formula_enable": svc_get("file_parser.formula_enable", "true"),
+ "table_enable": svc_get("file_parser.table_enable", "true"),
+ }
+
+
+ENV_MAP = {
+ "file_parser.api_url": "FILE_PARSER_API_URL",
+ "file_parser.timeout": "FILE_PARSER_TIMEOUT",
+ "file_parser.lang_list": "FILE_PARSER_LANG_LIST",
+ "file_parser.backend": "FILE_PARSER_BACKEND",
+ "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD",
+ "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE",
+ "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE",
+}
diff --git a/src/backend/core/files/content/pdf_parser.py b/src/backend/core/files/content/pdf_parser.py
new file mode 100644
index 00000000..073eabd0
--- /dev/null
+++ b/src/backend/core/files/content/pdf_parser.py
@@ -0,0 +1,125 @@
+"""Parse PDFs through the external layout service."""
+from __future__ import annotations
+import base64
+import binascii
+import json
+from dataclasses import dataclass, field
+from typing import Optional
+import requests
+from core.files.content import parser_config
+
+@dataclass
+class RichParseResult:
+ """Layout-aware parse output: markdown plus the media the layout engine cut out.
+
+ ``markdown`` is byte-for-byte what :func:`parse_pdf` returns, so the text path is
+ unaffected. ``images`` maps the path used inside the markdown (``images/.jpg``
+ in a ```` link) to the decoded bytes, and ``blocks`` is the parsed
+ ``content_list`` — one entry per layout block, images carrying ``image_caption`` /
+ ``bbox`` / ``page_idx``.
+ """
+
+ markdown: str
+ images: dict[str, bytes] = field(default_factory=dict)
+ blocks: list[dict] = field(default_factory=list)
+
+
+def post_parse(file_bytes: bytes, filename: str, mime: str, *, rich: bool) -> dict:
+ """POST to the external parse service and return the single document's result dict.
+
+ ``rich`` additionally requests the media and layout payloads. They are opt-in
+ because the images come back base64-inlined in the JSON body — several MB for an
+ image-heavy PDF — which the chat-attachment path has no use for.
+ """
+ api_url = parser_config.cfg_api_url()
+ if not api_url:
+ raise RuntimeError("FILE_PARSER_API_URL 未配置,无法解析 PDF 文件")
+
+ timeout = parser_config.cfg_timeout()
+ data = parser_config.cfg_parse_params()
+ if rich:
+ data = {**data, "return_images": "true", "return_content_list": "true"}
+
+ try:
+ resp = requests.post(
+ api_url,
+ files={"files": (filename, file_bytes, mime)},
+ data=data,
+ timeout=timeout,
+ )
+ resp.raise_for_status()
+ except requests.exceptions.Timeout:
+ raise RuntimeError(f"PDF 解析服务超时({timeout}s),请稍后重试")
+ except requests.exceptions.RequestException as e:
+ raise RuntimeError(f"PDF 解析服务请求失败: {e}")
+
+ results = (resp.json() or {}).get("results", {})
+ if not results:
+ raise RuntimeError("PDF 解析服务返回结果为空")
+ return results[next(iter(results))] or {}
+
+
+def decode_data_uri(value: str) -> Optional[bytes]:
+ """Decode one ``images`` entry. The service returns ``data:image/jpeg;base64,...``.
+
+ A bare base64 payload (no data-URI prefix) is accepted too, so a service variant
+ that omits the prefix still works. Undecodable entries are dropped rather than
+ failing the whole document.
+ """
+ if not isinstance(value, str) or not value:
+ return None
+ payload = value.split(",", 1)[1] if value.startswith("data:") else value
+ try:
+ return base64.b64decode(payload, validate=False)
+ except (binascii.Error, ValueError):
+ return None
+
+
+def parse_content_list(raw) -> list[dict]:
+ """Normalise ``content_list`` — the service returns it as a JSON **string**."""
+ if isinstance(raw, list):
+ return [b for b in raw if isinstance(b, dict)]
+ if isinstance(raw, str) and raw.strip():
+ try:
+ data = json.loads(raw)
+ except ValueError:
+ return []
+ if isinstance(data, list):
+ return [b for b in data if isinstance(b, dict)]
+ return []
+
+
+def parse_pdf(file_bytes: bytes, filename: str = "file.pdf") -> str:
+ """Parse PDF via external file-parser API. Returns markdown text."""
+ content = post_parse(file_bytes, filename, "application/pdf", rich=False).get("md_content", "")
+ if not content:
+ raise RuntimeError("PDF 解析服务返回内容为空")
+ return content
+
+
+def parse_pdf_rich(file_bytes: bytes, filename: str = "file.pdf") -> RichParseResult:
+ """Parse PDF keeping the figures the layout engine cut out.
+
+ Same call as :func:`parse_pdf` plus ``return_images`` / ``return_content_list``.
+ Used by knowledge-base indexing to build image assets; the chat-attachment path
+ stays on the text-only call.
+ """
+ result = post_parse(file_bytes, filename, "application/pdf", rich=True)
+ markdown = result.get("md_content", "")
+ if not markdown:
+ raise RuntimeError("PDF 解析服务返回内容为空")
+
+ images: dict[str, bytes] = {}
+ for name, value in (result.get("images") or {}).items():
+ decoded = decode_data_uri(value)
+ if decoded:
+ # The markdown references the file under an ``images/`` prefix while the
+ # payload is keyed by bare filename; index both so lookups by either work.
+ images[str(name)] = decoded
+ images.setdefault(f"images/{name}", decoded)
+
+ return RichParseResult(
+ markdown=markdown,
+ images=images,
+ blocks=parse_content_list(result.get("content_list")),
+ )
diff --git a/src/backend/core/files/content/presentation_parser.py b/src/backend/core/files/content/presentation_parser.py
new file mode 100644
index 00000000..76a56824
--- /dev/null
+++ b/src/backend/core/files/content/presentation_parser.py
@@ -0,0 +1,63 @@
+"""Parse PPTX files into markdown."""
+from __future__ import annotations
+import io
+
+def open_pptx(file_bytes: bytes):
+ """Open a Presentation from bytes; raise RuntimeError with a Chinese
+ install hint if python-pptx is missing."""
+ try:
+ from pptx import Presentation
+ except ImportError:
+ raise RuntimeError(
+ "python-pptx 未安装,无法解析 PPTX 文件(pip install python-pptx)"
+ )
+ return Presentation(io.BytesIO(file_bytes))
+
+
+def extract_slide_text(slide) -> str:
+ # Iterate in author/Z order so the rendered output mirrors what a
+ # reader sees (title → body → footers), since python-pptx exposes
+ # shapes in that order on the slide.shapes collection.
+ parts: list[str] = []
+ for shape in slide.shapes:
+ if not getattr(shape, "has_text_frame", False):
+ continue
+ try:
+ text = shape.text_frame.text
+ except Exception:
+ continue
+ text = (text or "").strip()
+ if text:
+ parts.append(text)
+ return "\n".join(parts)
+
+
+def parse_pptx(file_bytes: bytes) -> str:
+ """Parse PPTX bytes → markdown with one ``## Slide N`` section per slide."""
+ p = open_pptx(file_bytes)
+ sections: list[str] = []
+ for idx, slide in enumerate(p.slides):
+ body = extract_slide_text(slide) or "(此页无文本)"
+ sections.append(f"## Slide {idx + 1}\n\n{body}")
+ if not sections:
+ raise RuntimeError("PPTX 文件无幻灯片")
+ return "\n\n".join(sections)
+
+
+def parse_pptx_slide_count(file_bytes: bytes) -> int:
+ """Cheap probe: return the slide count without extracting any text."""
+ return len(open_pptx(file_bytes).slides)
+
+
+def parse_pptx_slide(file_bytes: bytes, slide_index: int) -> str:
+ """Return only the named slide's text (0-indexed). Raises RuntimeError
+ on out-of-range index."""
+ slides = open_pptx(file_bytes).slides
+ total = len(slides)
+ if total == 0:
+ raise RuntimeError("PPTX 文件无幻灯片")
+ if slide_index < 0 or slide_index >= total:
+ raise RuntimeError(
+ f"slide_index 越界: {slide_index}(共 {total} 页,可选 0..{total - 1})"
+ )
+ return extract_slide_text(slides[slide_index]) or "(此页无文本)"
diff --git a/src/backend/core/files/content/spreadsheet_parser.py b/src/backend/core/files/content/spreadsheet_parser.py
new file mode 100644
index 00000000..f263805f
--- /dev/null
+++ b/src/backend/core/files/content/spreadsheet_parser.py
@@ -0,0 +1,332 @@
+"""Parse XLSX files into markdown tables and bounded previews."""
+from __future__ import annotations
+import csv
+import io
+import os
+import subprocess
+import tempfile
+from typing import Optional
+from core.files.content.office import find_libreoffice_binary
+from core.infra.proc import no_window_kwargs
+from core.files.content import text_parser
+
+def deduplicate_headers(header_raw: list[str]) -> list[str]:
+ """Disambiguate empty/duplicate header names so {placeholder} rendering
+ downstream is unambiguous (e.g., two ``""`` columns become ``col``/``col_1``).
+ """
+ seen: dict[str, int] = {}
+ cleaned: list[str] = []
+ for h in header_raw:
+ base = h if h else "col"
+ if base in seen:
+ seen[base] += 1
+ cleaned.append(f"{base}_{seen[base]}")
+ else:
+ seen[base] = 0
+ cleaned.append(base)
+ return cleaned
+
+
+def rows_to_markdown(header: list[str], data_rows: list[list[str]],
+ sheet_title: str | None = None) -> str:
+ """Convert header + data rows into a markdown table string."""
+ lines: list[str] = []
+ if sheet_title:
+ lines.append(f"## Sheet: {sheet_title}")
+ lines.append("")
+ lines.append("| " + " | ".join(header) + " |")
+ lines.append("|" + "|".join("---" for _ in header) + "|")
+ for row in data_rows:
+ lines.append("| " + " | ".join(row) + " |")
+ return "\n".join(lines)
+
+
+def open_xlsx(file_bytes: bytes):
+ """Open a workbook in read-only mode; raise RuntimeError with a Chinese
+ install hint if openpyxl is missing."""
+ try:
+ import openpyxl
+ except ImportError:
+ raise RuntimeError("openpyxl 未安装,无法解析 XLSX 文件(pip install openpyxl)")
+ return openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
+
+
+def xlsx_sheet_to_markdown(sheet) -> str | None:
+ """Render one openpyxl sheet to a markdown table, or None if empty.
+ Returns None (not raise) so callers can decide whether emptiness is an
+ error (single-sheet mode) or just a skip (whole-workbook mode)."""
+ rows = list(sheet.iter_rows(values_only=True))
+ if not rows:
+ return None
+ header = [str(c or "") for c in rows[0]]
+ data_rows = [
+ [str(c or "") for c in row]
+ for row in rows[1:]
+ if not all(c is None for c in row)
+ ]
+ return rows_to_markdown(header, data_rows, sheet.title)
+
+
+def parse_xlsx(file_bytes: bytes) -> str:
+ """Parse XLSX bytes → markdown tables (one per sheet)."""
+ wb = open_xlsx(file_bytes)
+ try:
+ sections = [md for sheet in wb.worksheets if (md := xlsx_sheet_to_markdown(sheet))]
+ finally:
+ wb.close()
+ if not sections:
+ raise RuntimeError("XLSX 文件无有效数据")
+ return "\n\n".join(sections)
+
+
+def parse_xlsx_sheet_names(file_bytes: bytes) -> list[str]:
+ """Return the list of sheet names in workbook order. Cheap probe: does
+ not iterate any rows, just reads workbook structure."""
+ wb = open_xlsx(file_bytes)
+ try:
+ return list(wb.sheetnames)
+ finally:
+ wb.close()
+
+
+def parse_xlsx_single_sheet(file_bytes: bytes, sheet_name: str) -> str:
+ """Parse only the named sheet → markdown table. Raises RuntimeError if
+ the sheet does not exist or is empty."""
+ wb = open_xlsx(file_bytes)
+ try:
+ if sheet_name not in wb.sheetnames:
+ raise RuntimeError(
+ f"sheet 不存在: '{sheet_name}'。可选 sheet: {wb.sheetnames}"
+ )
+ md = xlsx_sheet_to_markdown(wb[sheet_name])
+ if md is None:
+ raise RuntimeError(f"sheet '{sheet_name}' 无数据")
+ return md
+ finally:
+ wb.close()
+
+
+def default_xlsx_max_cell_chars() -> int:
+ """Bound spreadsheet preview cells; the default remains 2000 characters."""
+ try:
+ return max(50, int(os.getenv("XLSX_PREVIEW_MAX_CELL_CHARS", "2000")))
+ except ValueError:
+ return 2000
+
+
+def parse_xlsx_preview(
+ file_bytes: bytes,
+ *,
+ char_budget: int = 4000,
+ max_columns: int = 40,
+ max_cell_chars: Optional[int] = None,
+) -> dict:
+ """Parse XLSX into a bounded markdown preview + accurate total counts.
+
+ Produces what the main agent's file_context hook should inject for an
+ uploaded spreadsheet: a markdown table containing as many rows as fit
+ within ``char_budget``, plus the *real* total_rows / total_columns so
+ the agent knows the full size without seeing every row.
+
+ The total_rows count walks the entire sheet (cheap — read_only mode
+ streams the first column only). The preview stops as soon as the
+ cumulative markdown length exceeds ``char_budget``.
+
+ Returns:
+ ``{
+ "header": list[str], # column names (post-dedup)
+ "preview_md": str, # markdown table, ≤ char_budget chars
+ "total_rows": int, # all data rows in sheet 1
+ "total_columns": int, # raw column count (before max_columns)
+ "preview_rows": int, # rows actually included in preview_md
+ "truncated_columns": int, # cols dropped to fit max_columns
+ "truncated_cells": int, # cells truncated to max_cell_chars
+ }``
+
+ Reads only the first sheet.
+ """
+ if max_cell_chars is None:
+ max_cell_chars = default_xlsx_max_cell_chars()
+
+ try:
+ import openpyxl
+ except ImportError:
+ raise RuntimeError("openpyxl 未安装,无法解析 XLSX 文件(pip install openpyxl)")
+
+ wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
+ try:
+ sheet = wb.worksheets[0] if wb.worksheets else None
+ if sheet is None:
+ raise RuntimeError("XLSX 文件无 sheet")
+
+ rows_iter = sheet.iter_rows(values_only=True)
+ try:
+ header_row = next(rows_iter)
+ except StopIteration:
+ raise RuntimeError("XLSX 文件无数据行") from None
+
+ total_columns = len(header_row)
+ truncated_columns = max(0, total_columns - max_columns)
+ usable_cols = min(total_columns, max_columns)
+
+ header = deduplicate_headers([
+ str(c).strip() if c is not None else f"col{idx + 1}"
+ for idx, c in enumerate(header_row[:usable_cols])
+ ])
+
+ # Markdown header + separator (always present, counts toward budget)
+ md_lines: list[str] = [
+ "| " + " | ".join(header) + " |",
+ "|" + "|".join("---" for _ in header) + "|",
+ ]
+ running_chars = sum(len(ln) + 1 for ln in md_lines) # +1 for newline
+
+ preview_rows = 0
+ truncated_cells = 0
+ total_rows_seen = 0
+ budget_reached = False
+
+ for row in rows_iter:
+ total_rows_seen += 1
+ if all(c is None for c in row):
+ continue
+
+ if budget_reached:
+ continue # keep counting total_rows accurately
+
+ cells: list[str] = []
+ for val in row[:usable_cols]:
+ if val is None:
+ cells.append("")
+ continue
+ s = str(val).replace("|", "\\|").replace("\n", " ")
+ if len(s) > max_cell_chars:
+ s = s[:max_cell_chars] + "…"
+ truncated_cells += 1
+ cells.append(s)
+
+ # Pad short rows so the markdown table stays well-formed
+ while len(cells) < len(header):
+ cells.append("")
+
+ line = "| " + " | ".join(cells) + " |"
+ line_cost = len(line) + 1
+ if running_chars + line_cost > char_budget and preview_rows > 0:
+ # We have at least one row already; respect the budget.
+ budget_reached = True
+ continue
+ md_lines.append(line)
+ running_chars += line_cost
+ preview_rows += 1
+
+ return {
+ "header": header,
+ "preview_md": "\n".join(md_lines),
+ "total_rows": total_rows_seen,
+ "total_columns": total_columns,
+ "preview_rows": preview_rows,
+ "truncated_columns": truncated_columns,
+ "truncated_cells": truncated_cells,
+ }
+ finally:
+ wb.close()
+
+
+def parse_xls(file_bytes: bytes) -> str:
+ """Parse legacy XLS bytes → markdown tables.
+
+ Tries xlrd first; falls back to LibreOffice → XLSX → openpyxl.
+ """
+ try:
+ import xlrd
+ wb = xlrd.open_workbook(file_contents=file_bytes)
+ sections: list[str] = []
+ for sheet in wb.sheets():
+ if sheet.nrows == 0:
+ continue
+ header = [str(sheet.cell_value(0, c) or "") for c in range(sheet.ncols)]
+ data_rows = []
+ for r in range(1, sheet.nrows):
+ row = [str(sheet.cell_value(r, c) or "") for c in range(sheet.ncols)]
+ if all(v == "" for v in row):
+ continue
+ data_rows.append(row)
+ sections.append(rows_to_markdown(header, data_rows, sheet.name))
+ if not sections:
+ raise RuntimeError("XLS 文件无有效数据")
+ return "\n\n".join(sections)
+ except ImportError:
+ pass # xlrd not installed, try LibreOffice fallback
+
+ # Fallback: LibreOffice → XLSX → openpyxl
+ xlsx_bytes = convert_xls_to_xlsx(file_bytes)
+ return parse_xlsx(xlsx_bytes)
+
+
+def convert_xls_to_xlsx(file_bytes: bytes) -> bytes:
+ """Use LibreOffice headless to convert XLS → XLSX."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ input_path = os.path.join(tmpdir, "input.xls")
+ with open(input_path, "wb") as f:
+ f.write(file_bytes)
+
+ try:
+ result = subprocess.run(
+ [
+ find_libreoffice_binary() or "libreoffice",
+ "--headless",
+ "--convert-to", "xlsx",
+ "--outdir", tmpdir,
+ input_path,
+ ],
+ capture_output=True,
+ text=True,
+ timeout=120,
+ **no_window_kwargs(),
+ )
+ except FileNotFoundError:
+ raise RuntimeError(
+ "xlrd 和 LibreOffice 均不可用,无法解析 XLS 文件"
+ "(pip install xlrd 或安装 LibreOffice)"
+ )
+ except subprocess.TimeoutExpired:
+ raise RuntimeError("LibreOffice 转换 XLS → XLSX 超时")
+
+ output_path = os.path.join(tmpdir, "input.xlsx")
+ if not os.path.exists(output_path):
+ raise RuntimeError(f"LibreOffice 转换 XLS 失败: {result.stderr[:300]}")
+
+ with open(output_path, "rb") as f:
+ return f.read()
+
+
+def parse_csv(file_bytes: bytes) -> str:
+ """Parse CSV bytes → markdown table."""
+ # Decode with encoding detection
+ text = text_parser.parse_txt(file_bytes)
+
+ # Sniff dialect
+ try:
+ sample = text[:8192]
+ dialect = csv.Sniffer().sniff(sample)
+ except csv.Error:
+ dialect = csv.excel
+
+ reader = csv.reader(io.StringIO(text), dialect)
+ rows = list(reader)
+
+ if not rows:
+ raise RuntimeError("CSV 文件无有效数据")
+
+ header = [c.strip() for c in rows[0]]
+ data_rows = []
+ for row in rows[1:]:
+ if all(c.strip() == "" for c in row):
+ continue
+ # Pad or trim to match header length
+ padded = [c.strip() for c in row]
+ while len(padded) < len(header):
+ padded.append("")
+ data_rows.append(padded[:len(header)])
+
+ return rows_to_markdown(header, data_rows)
diff --git a/src/backend/core/content/svg_fit.py b/src/backend/core/files/content/svg_fit.py
similarity index 97%
rename from src/backend/core/content/svg_fit.py
rename to src/backend/core/files/content/svg_fit.py
index 86ca9c1e..b4f436d5 100644
--- a/src/backend/core/content/svg_fit.py
+++ b/src/backend/core/files/content/svg_fit.py
@@ -142,8 +142,9 @@ def _walk(el, tx: float, ty: float, font_size: float, acc: _Bounds) -> None:
w = _f(el, "width"); h = _f(el, "height")
if w or h:
acc.add(tx + x, ty + y, tx + x + w, ty + y + h)
- except Exception: # never let one weird node abort the whole fit
- pass
+ except Exception as fallback_error: # never let one weird node abort the whole fit
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.files.content.svg_fit:_walk', fallback_error)
for child in el:
_walk(child, tx, ty, cur_fs, acc)
diff --git a/src/backend/core/files/content/text_parser.py b/src/backend/core/files/content/text_parser.py
new file mode 100644
index 00000000..a98766ef
--- /dev/null
+++ b/src/backend/core/files/content/text_parser.py
@@ -0,0 +1,21 @@
+"""Decode plain-text files."""
+from __future__ import annotations
+
+
+def parse_txt(file_bytes: bytes) -> str:
+ """Decode text file bytes, trying UTF-8 then GBK then latin-1."""
+ for enc in ("utf-8", "gbk", "latin-1"):
+ try:
+ return file_bytes.decode(enc)
+ except UnicodeDecodeError:
+ continue
+ return file_bytes.decode("utf-8", errors="replace")
+
+
+def parse_raw_text(file_bytes: bytes) -> str:
+ """UTF-8 decode for plain-text formats (HTML/MD/code/etc.).
+
+ No structural extraction—just the source bytes as a string. Replaces
+ undecodable bytes so we always return something usable.
+ """
+ return file_bytes.decode("utf-8", errors="replace")
diff --git a/src/backend/core/files/content/word_parser.py b/src/backend/core/files/content/word_parser.py
new file mode 100644
index 00000000..6b0e3917
--- /dev/null
+++ b/src/backend/core/files/content/word_parser.py
@@ -0,0 +1,77 @@
+"""Parse DOCX, DOC and WPS files into markdown."""
+from __future__ import annotations
+import os
+import subprocess
+import tempfile
+from core.files.content.office import find_libreoffice_binary, find_pandoc_binary
+from core.infra.proc import no_window_kwargs
+
+def docx_bytes_to_markdown(docx_bytes: bytes) -> str:
+ """Convert DOCX bytes to markdown string via pandoc."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ input_path = os.path.join(tmpdir, "input.docx")
+ with open(input_path, "wb") as f:
+ f.write(docx_bytes)
+
+ try:
+ result = subprocess.run(
+ [find_pandoc_binary() or "pandoc", input_path, "-t", "markdown", "--wrap=none"],
+ capture_output=True,
+ text=True,
+ timeout=60,
+ **no_window_kwargs(),
+ )
+ except FileNotFoundError:
+ raise RuntimeError("pandoc 未安装,无法解析 Word 文档")
+ except subprocess.TimeoutExpired:
+ raise RuntimeError("pandoc 转换超时")
+
+ if result.returncode != 0:
+ raise RuntimeError(f"pandoc 转换失败: {result.stderr[:300]}")
+
+ return result.stdout
+
+
+def parse_docx(file_bytes: bytes) -> str:
+ """Parse DOCX bytes → markdown via pandoc."""
+ return docx_bytes_to_markdown(file_bytes)
+
+
+def convert_to_docx_bytes(file_bytes: bytes, suffix: str) -> bytes:
+ """Use LibreOffice headless to convert DOC/WPS bytes → DOCX bytes."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ input_path = os.path.join(tmpdir, f"input{suffix}")
+ with open(input_path, "wb") as f:
+ f.write(file_bytes)
+
+ try:
+ result = subprocess.run(
+ [
+ find_libreoffice_binary() or "libreoffice",
+ "--headless",
+ "--convert-to", "docx",
+ "--outdir", tmpdir,
+ input_path,
+ ],
+ capture_output=True,
+ text=True,
+ timeout=120,
+ **no_window_kwargs(),
+ )
+ except FileNotFoundError:
+ raise RuntimeError("LibreOffice 未安装,无法解析 DOC/WPS 文件")
+ except subprocess.TimeoutExpired:
+ raise RuntimeError("LibreOffice 转换超时")
+
+ output_path = os.path.join(tmpdir, "input.docx")
+ if not os.path.exists(output_path):
+ raise RuntimeError(f"LibreOffice 转换失败: {result.stderr[:300]}")
+
+ with open(output_path, "rb") as f:
+ return f.read()
+
+
+def parse_doc_wps(file_bytes: bytes, suffix: str) -> str:
+ """Parse DOC/WPS via LibreOffice → DOCX → pandoc markdown."""
+ docx_bytes = convert_to_docx_bytes(file_bytes, suffix)
+ return docx_bytes_to_markdown(docx_bytes)
diff --git a/src/backend/core/files/edition.py b/src/backend/core/files/edition.py
new file mode 100644
index 00000000..7f51dc7b
--- /dev/null
+++ b/src/backend/core/files/edition.py
@@ -0,0 +1,98 @@
+"""Personal ownership, mounts and artifact policy for Community Edition."""
+from __future__ import annotations
+
+def project_directory(scope):
+ return ""
+
+def project_directory_by_id(project_id):
+ return ""
+
+def workspace_instructions():
+ return ""
+
+def is_organization_path(scope, actor, path):
+ return False
+
+def resolve_alias(path):
+ return None
+
+def resolve_project_path(path, user_id, root, scope):
+ return None
+
+def is_shared_scope(scope):
+ return False
+
+def context_directory(context):
+ return ""
+
+def source_directory(project):
+ return ""
+
+def folder_scope_label(kind):
+ return "我的空间"
+
+def command_instructions(scope):
+ return ""
+
+from typing import Any, Optional
+
+def artifact_list_scope() -> str:
+ return "personal"
+
+def extend_artifact_item(artifact, item: dict) -> dict:
+ return item
+
+def personal_artifact_create_fields() -> dict[str, None]:
+ return {}
+
+def artifact_scope_fields(scope: Any) -> dict[str, Optional[str]]:
+ return {
+ "user_folder_id": (
+ scope.root_folder_id or None if scope is not None and scope.is_personal else None
+ )
+ }
+
+def artifact_scope_folder_id(scope: Any, artifact: Any) -> Optional[str]:
+ return artifact.user_folder_id
+
+def can_access_artifact(db: Any, user_id: str, artifact: Any) -> bool:
+ return str(artifact.user_id) == str(user_id)
+
+def artifact_access_metadata(artifact: Any) -> dict[str, Optional[str]]:
+ return {"owner_id": artifact.user_id}
+
+def can_access_artifact_metadata(db: Any, user_id: str, metadata: dict[str, Any]) -> bool:
+ owner_id = metadata.get("owner_id") or metadata.get("user_id")
+ return owner_id is None or str(owner_id) == str(user_id)
+
+def upload_target_matches(artifact: Any, scope: str, folder_id: Optional[str]) -> bool:
+ return scope == "personal" and artifact.user_folder_id == folder_id
+
+workspace_mount_names = ("myspace", "skills")
+
+def team_grants(db, actor):
+ return ()
+
+def session_entrypoint(entrypoint, user_id):
+ return list(entrypoint)
+
+def project_location(db, project):
+ return None
+
+def matches_scope(project, scope):
+ return scope.kind == project.kind and scope.root_folder_id == project.linked_folder_id
+
+def owner_space(column, owner):
+ raise ValueError("Unsupported directory owner")
+
+def revoke_computers(actor):
+ return None
+
+
+def artifact_location_scope(artifact):
+ from core.db.models import UserFolder
+ return "personal", artifact.user_folder_id, UserFolder, "user_id", artifact.user_id
+
+
+def artifact_location_root(db, artifact):
+ return None
diff --git a/src/backend/core/services/folder_batch.py b/src/backend/core/files/folder_batch.py
similarity index 69%
rename from src/backend/core/services/folder_batch.py
rename to src/backend/core/files/folder_batch.py
index 7196e419..81d0c382 100644
--- a/src/backend/core/services/folder_batch.py
+++ b/src/backend/core/files/folder_batch.py
@@ -1,47 +1,12 @@
"""Create a bounded manifest in one transaction; edition models are supplied by callers."""
-import hashlib
import uuid
-from threading import Lock
from core.db.repository import AuditLogRepository
-from sqlalchemy import event, or_
+from sqlalchemy import or_
from sqlalchemy.orm import Session
-_SQLITE_LOCKS = [Lock() for _ in range(64)]
-
-
-def _release_sqlite_locks(session, transaction):
- if transaction.parent is None:
- for lock in session.info.pop("folder_upload_locks", []):
- lock.release()
-
-
-def lock_folder_owner(db: Session, owner_model, owner_column: str, owner_id: str) -> None:
- if db.get_bind().dialect.name == "sqlite":
- # SQLite ignores FOR UPDATE. Its supported single-process deployment still
- # serves requests on multiple threads, so retain a bounded process lock
- # until commit/rollback/close, including early-return and error paths.
- if not db.in_transaction():
- db.begin()
- if not db.info.get("folder_upload_lock_listener"):
- event.listen(db, "after_transaction_end", _release_sqlite_locks)
- db.info["folder_upload_lock_listener"] = True
- digest = hashlib.sha256((owner_model.__tablename__ + ":" + owner_id).encode()).digest()
- lock = _SQLITE_LOCKS[int.from_bytes(digest[:2], "big") % len(_SQLITE_LOCKS)]
- held = db.info.setdefault("folder_upload_locks", [])
- if lock not in held:
- lock.acquire()
- held.append(lock)
- # NO KEY UPDATE does not conflict with FK KEY SHARE taken by artifact inserts.
- owner = (
- db.query(owner_model)
- .filter(getattr(owner_model, owner_column) == owner_id)
- .with_for_update(key_share=True)
- .first()
- )
- if owner is None:
- raise ValueError("文件夹所属空间不存在")
+from core.db.repository.folder import lock_folder_owner
def create_folder_batch(
@@ -104,10 +69,10 @@ def create_folder_batch(
siblings[(parent_id, name)] = row
mapping[path] = row.folder_id
db.flush()
- from core.spaces.files import SpaceFiles
- from core.spaces.projects import folder_path
- from core.spaces.edition_access import owner_space
- from core.storage import get_storage
+ from core.files.spaces.files import SpaceFiles
+ from core.files.spaces.projects import folder_path
+ from core.files.edition import owner_space
+ from core.files.storage import get_storage
if owner_column == "user_id":
if owner_id != actor:
raise ValueError("Directory owner does not match the account")
diff --git a/src/backend/core/services/personal_file_operations.py b/src/backend/core/files/personal_operations.py
similarity index 86%
rename from src/backend/core/services/personal_file_operations.py
rename to src/backend/core/files/personal_operations.py
index dd356c3e..6cfaaff8 100644
--- a/src/backend/core/services/personal_file_operations.py
+++ b/src/backend/core/files/personal_operations.py
@@ -1,14 +1,14 @@
"""Save personal conversation attachments into mounted directories."""
from core.db.models import Artifact
from core.infra.time import utc_now
-from core.services.artifact_edition import is_personal_artifact
-from core.spaces.files import SpaceFiles
-from core.storage import get_storage
+from core.db.artifact_scope import is_personal_artifact
+from core.files.spaces.files import SpaceFiles
+from core.files.storage import get_storage
class PersonalFileOperations:
def _save_artifact(self, artifact_id, target_folder_id, actor, *, move):
- from core.services.user_folder_service import FolderResult
+ from core.projects.user_folders import FolderResult
artifact = self.db.query(Artifact).filter(
Artifact.artifact_id == artifact_id, Artifact.deleted_at.is_(None),
).first()
@@ -18,7 +18,7 @@ def _save_artifact(self, artifact_id, target_folder_id, actor, *, move):
if target_folder_id and target is None:
return FolderResult(False, "目标文件夹不存在")
name = artifact.filename
- from core.storage.entries import validate_key
+ from core.files.storage.entries import validate_key
try:
validate_key(name)
if "/" in name:
diff --git a/src/backend/core/spaces/__init__.py b/src/backend/core/files/spaces/__init__.py
similarity index 100%
rename from src/backend/core/spaces/__init__.py
rename to src/backend/core/files/spaces/__init__.py
diff --git a/src/backend/core/spaces/access.py b/src/backend/core/files/spaces/access.py
similarity index 95%
rename from src/backend/core/spaces/access.py
rename to src/backend/core/files/spaces/access.py
index 7b3dacb6..0c44f5cb 100644
--- a/src/backend/core/spaces/access.py
+++ b/src/backend/core/files/spaces/access.py
@@ -6,7 +6,7 @@
import re
from fastapi import HTTPException
-from core.storage.entries import validate_key
+from core.files.storage.entries import validate_key
def identifier(value: str) -> str:
if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value):
@@ -49,7 +49,7 @@ def resolve_grants(db, actor: str) -> tuple[SpaceGrant, ...]:
actor = identifier(actor)
if actor.startswith("api_ak_"):
return ()
- from core.spaces.edition_access import team_grants
+ from core.files.edition import team_grants
return (personal_grant(actor), *team_grants(db, actor))
def require_grant(db, actor: str, space_id: str, *, write=False) -> SpaceGrant:
diff --git a/src/backend/core/spaces/files.py b/src/backend/core/files/spaces/files.py
similarity index 96%
rename from src/backend/core/spaces/files.py
rename to src/backend/core/files/spaces/files.py
index 122dc31d..07a7a937 100644
--- a/src/backend/core/spaces/files.py
+++ b/src/backend/core/files/spaces/files.py
@@ -1,9 +1,9 @@
"""File operations within an authenticated space grant."""
from fastapi import HTTPException
-from core.spaces.access import require_grant
-from core.spaces.leases import authorized_operation
-from core.spaces.search import search_directory
-from core.spaces.tree import ensure_parents, tree_entries, copy_tree, remove_tree
+from core.files.spaces.access import require_grant
+from core.files.spaces.leases import authorized_operation
+from core.files.spaces.search import search_directory
+from core.files.spaces.tree import ensure_parents, tree_entries, copy_tree, remove_tree
MAX_TRANSFER_BYTES = 50 * 1024 * 1024
diff --git a/src/backend/core/spaces/leases.py b/src/backend/core/files/spaces/leases.py
similarity index 86%
rename from src/backend/core/spaces/leases.py
rename to src/backend/core/files/spaces/leases.py
index ced7dde5..a4d98850 100644
--- a/src/backend/core/spaces/leases.py
+++ b/src/backend/core/files/spaces/leases.py
@@ -1,6 +1,8 @@
"""Serialize file access, sandbox creation and permission changes per account."""
+from __future__ import annotations
from contextlib import contextmanager, asynccontextmanager
import asyncio
+from dataclasses import dataclass, field
from datetime import timedelta
from functools import wraps
import hashlib
@@ -9,9 +11,6 @@
from fastapi import HTTPException
from sqlalchemy import text
-_local_guard = threading.Lock()
-_local_locks = {}
-
@contextmanager
def space_lock(identity, *, engine=None):
@@ -25,8 +24,8 @@ def space_lock(identity, *, engine=None):
connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": key})
yield
else:
- with _local_guard:
- lock = _local_locks.setdefault((str(engine.url), key), threading.Lock())
+ with process_state().local_guard:
+ lock = process_state().local_locks.setdefault((str(engine.url), key), threading.Lock())
if not lock.acquire(timeout=30):
raise HTTPException(503, "文件权限正在更新,请重试")
try:
@@ -37,7 +36,7 @@ def space_lock(identity, *, engine=None):
def revoke_sandboxes(actor):
from core.config.settings import settings
- from core.spaces.edition_revocation import revoke_computers
+ from core.files.edition import revoke_computers
revoke_computers(actor)
if settings.sandbox.provider != "opensandbox":
return
@@ -100,3 +99,16 @@ async def async_space_lock(identity):
yield
finally:
await asyncio.to_thread(guard.__exit__, None, None, None)
+
+
+@dataclass
+class LeasesState:
+ local_guard: object = field(default_factory=lambda: threading.Lock())
+ local_locks: object = field(default_factory=lambda: {})
+
+
+_STATE = LeasesState()
+
+
+def process_state() -> LeasesState:
+ return _STATE
diff --git a/src/backend/core/spaces/mounts.py b/src/backend/core/files/spaces/mounts.py
similarity index 78%
rename from src/backend/core/spaces/mounts.py
rename to src/backend/core/files/spaces/mounts.py
index e6d836ba..dc7c0c5d 100644
--- a/src/backend/core/spaces/mounts.py
+++ b/src/backend/core/files/spaces/mounts.py
@@ -3,8 +3,8 @@
import base64
from pathlib import Path
from fastapi import HTTPException
-from core.spaces.access import fingerprint, resolve_grants
-from core.storage.entries import validate_key
+from core.files.spaces.access import fingerprint, resolve_grants
+from core.files.storage.entries import validate_key
MOUNT_REVISION_TAG = "hugagent-space-revision"
@@ -31,14 +31,12 @@ def mount_revision(grants):
def provider_volume(grant):
from opensandbox.models.sandboxes import Host, OSSFS, Volume
from core.config.settings import settings
- from core.storage import get_storage
backend = os.getenv("STORAGE_TYPE", "local").lower()
name = "space-" + fingerprint((grant,))[:24]
if backend == "local":
root = settings.storage.root.resolve()
path = root / grant.prefix
- get_storage().make_directory(grant.prefix)
if path.is_symlink() or not path.resolve().is_relative_to(root):
raise HTTPException(503, "Invalid local space root")
host_root = settings.sandbox.opensandbox_host_storage_path
@@ -58,8 +56,6 @@ def provider_volume(grant):
key_prefix = os.getenv("OSS_KEY_PREFIX", "").strip("/") if backend == "oss" else ""
path = "/".join(part for part in (key_prefix, grant.prefix.rstrip("/")) if part)
validate_key(path)
- if grant.writable:
- get_storage().make_directory(grant.prefix)
options = ["hugagent_driver=s3fs", "allow_other", "uid=1000", "gid=1000",
"umask=0022", "stat_cache_expire=0", "attr_timeout=0", "entry_timeout=0",
"negative_timeout=0"]
@@ -79,9 +75,21 @@ def prepare_volumes(actor):
from core.db.engine import SessionLocal
with SessionLocal() as db:
grants = resolve_grants(db, actor)
- revision = mount_revision(grants)
- volumes = [provider_volume(grant) for grant in grants]
- db.expire_all()
- if mount_revision(resolve_grants(db, actor)) != revision:
- raise HTTPException(409, "Space permissions changed while preparing volumes")
- return volumes, {MOUNT_REVISION_TAG: revision}
+ return [provider_volume(grant) for grant in grants], {MOUNT_REVISION_TAG: mount_revision(grants)}
+
+
+def provision_volumes(actor):
+ """Initialize directory roots once, at sandbox creation."""
+ from core.files.storage import get_storage
+ volumes, metadata = prepare_volumes(actor)
+ # Grants are represented by the authorized volume specifications. Resolve
+ # writable prefixes once for initialization, then verify the same revision.
+ from core.db.engine import SessionLocal
+ with SessionLocal() as db:
+ grants = resolve_grants(db, actor)
+ if mount_revision(grants) != metadata[MOUNT_REVISION_TAG]:
+ raise HTTPException(409, "Space permissions changed before provisioning")
+ for grant in grants:
+ if grant.writable or os.getenv("STORAGE_TYPE", "local").lower() == "local":
+ get_storage().make_directory(grant.prefix)
+ return volumes, metadata
diff --git a/src/backend/core/spaces/projects.py b/src/backend/core/files/spaces/projects.py
similarity index 94%
rename from src/backend/core/spaces/projects.py
rename to src/backend/core/files/spaces/projects.py
index a191fa79..0f5b3fca 100644
--- a/src/backend/core/spaces/projects.py
+++ b/src/backend/core/files/spaces/projects.py
@@ -3,8 +3,8 @@
import mimetypes
from fastapi import HTTPException
from core.db.models import UserFolder
-from core.spaces.access import namespace
-from core.storage.entries import validate_key
+from core.files.spaces.access import namespace
+from core.files.storage.entries import validate_key
def folder_path(db, model, owner_filter, folder_id):
@@ -28,7 +28,7 @@ def project_location(db, project):
if project.kind == "personal" and project.linked_folder_id:
path = folder_path(db, UserFolder, {"user_id": project.owner_user_id}, project.linked_folder_id)
return "personal", path, namespace("users", project.owner_user_id) + path + "/"
- from core.spaces.edition_access import project_location as edition_location
+ from core.files.edition import project_location as edition_location
location = edition_location(db, project)
if location is not None:
return location
diff --git a/src/backend/core/spaces/recovery.py b/src/backend/core/files/spaces/recovery.py
similarity index 94%
rename from src/backend/core/spaces/recovery.py
rename to src/backend/core/files/spaces/recovery.py
index 027753e4..b7604d6f 100644
--- a/src/backend/core/spaces/recovery.py
+++ b/src/backend/core/files/spaces/recovery.py
@@ -5,8 +5,8 @@
import httpx
from fastapi import HTTPException
-from core.spaces.leases import async_space_lock
-from core.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes
+from core.files.spaces.leases import async_space_lock
+from core.files.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes
EPOCH_TAG = "hugagent-mount-epoch"
diff --git a/src/backend/core/spaces/sandbox.py b/src/backend/core/files/spaces/sandbox.py
similarity index 73%
rename from src/backend/core/spaces/sandbox.py
rename to src/backend/core/files/spaces/sandbox.py
index 652eb804..7940d6c4 100644
--- a/src/backend/core/spaces/sandbox.py
+++ b/src/backend/core/files/spaces/sandbox.py
@@ -3,8 +3,8 @@
from fastapi import HTTPException
from core.sandbox.processes.dispatch import _on_service_loop
-from core.spaces.access import resolve_grants
-from core.spaces.mounts import MOUNT_REVISION_TAG, mount_revision
+from core.files.spaces.access import resolve_grants
+from core.files.spaces.mounts import MOUNT_REVISION_TAG, mount_revision
def current_revision(actor):
@@ -14,14 +14,14 @@ def current_revision(actor):
class SpaceSandboxMixin:
- from core.spaces.edition_access import workspace_mount_names
+ from core.files.edition import workspace_mount_names
def _session_entrypoint(self, user_id):
- from core.spaces.edition_access import session_entrypoint
+ from core.files.edition import session_entrypoint
return session_entrypoint(super()._session_entrypoint(user_id), user_id)
async def _create_jupyter_sandbox(self, *, from_snapshot_id=None, user_id=None):
- from core.spaces.leases import async_space_lock
+ from core.files.spaces.leases import async_space_lock
if not user_id:
return await super()._create_jupyter_sandbox(from_snapshot_id=from_snapshot_id)
async with async_space_lock(user_id):
@@ -35,7 +35,7 @@ async def _grants_match(self, session):
if (info.metadata or {}).get(MOUNT_REVISION_TAG) != revision:
return False
from core.config.settings import settings
- from core.spaces.recovery import ensure_mounts
+ from core.files.spaces.recovery import ensure_mounts
await ensure_mounts(session.sandbox, session.user_id, info,
domain=settings.sandbox.opensandbox_domain,
api_key=settings.sandbox.opensandbox_api_key)
@@ -55,18 +55,10 @@ async def _resolve_bound_session(self, session_id, user_id=None):
async def ensure_user_workspace(self, session_id, user_id):
await self._get_or_create_session(session_id, user_id)
- @_on_service_loop
- async def write_stdin(
- self, session_id, *, sandbox_session_id, user_id=None, chars="", yield_time_ms=60000
- ):
- await self._get_or_create_session(sandbox_session_id, user_id)
- return await super().write_stdin(
- session_id, sandbox_session_id=sandbox_session_id, user_id=user_id,
- chars=chars, yield_time_ms=yield_time_ms,
- )
-
async def _renew_in_background(self, session_id, session):
- if session.user_id and not await self._grants_match(session):
+ if session.user_id and (await asyncio.to_thread(current_revision, session.user_id)) != (
+ (await session.sandbox.get_info()).metadata or {}
+ ).get(MOUNT_REVISION_TAG):
await self._destroy_session(session_id, session, reuse_sandbox=False)
return
await super()._renew_in_background(session_id, session)
diff --git a/src/backend/core/spaces/search.py b/src/backend/core/files/spaces/search.py
similarity index 97%
rename from src/backend/core/spaces/search.py
rename to src/backend/core/files/spaces/search.py
index f3a36346..02784067 100644
--- a/src/backend/core/spaces/search.py
+++ b/src/backend/core/files/spaces/search.py
@@ -5,7 +5,7 @@
import time
from fastapi import HTTPException
-from core.storage.entries import DirectoryPage, page_limit
+from core.files.storage.entries import DirectoryPage, page_limit
BATCH_SIZE = 1000
SCAN_SECONDS = 20
diff --git a/src/backend/core/spaces/text.py b/src/backend/core/files/spaces/text.py
similarity index 92%
rename from src/backend/core/spaces/text.py
rename to src/backend/core/files/spaces/text.py
index f31337f8..f3cae529 100644
--- a/src/backend/core/spaces/text.py
+++ b/src/backend/core/files/spaces/text.py
@@ -2,8 +2,8 @@
import hashlib
from fastapi import HTTPException
-from core.services.artifact_text_service import decode_text, MAX_TEXT_BYTES, TEXT_EXTENSIONS
-from core.spaces.leases import authorized_operation
+from core.files.artifacts.text import decode_text, MAX_TEXT_BYTES, TEXT_EXTENSIONS
+from core.files.spaces.leases import authorized_operation
class SpaceText:
diff --git a/src/backend/core/spaces/tree.py b/src/backend/core/files/spaces/tree.py
similarity index 100%
rename from src/backend/core/spaces/tree.py
rename to src/backend/core/files/spaces/tree.py
diff --git a/src/backend/core/storage/__init__.py b/src/backend/core/files/storage/__init__.py
similarity index 56%
rename from src/backend/core/storage/__init__.py
rename to src/backend/core/files/storage/__init__.py
index dc8b02b4..9e5d14a4 100644
--- a/src/backend/core/storage/__init__.py
+++ b/src/backend/core/files/storage/__init__.py
@@ -1,17 +1,12 @@
"""Object storage backend abstraction layer.
Re-exports all public symbols from the split modules for backwards compatibility
-with ``from core.storage import get_storage, generate_storage_key`` etc.
+with ``from core.files.storage import get_storage, generate_storage_key`` etc.
"""
-from core.storage.protocol import StorageBackend
-from core.storage.local import LocalStorageBackend
-from core.storage.factory import (
- get_storage_backend,
- get_storage,
- generate_storage_key,
- get_storage_category_for_resource,
-)
+from core.files.storage.protocol import StorageBackend
+from core.files.storage.local import LocalStorageBackend
+from core.files.storage.factory import get_storage_backend, get_storage, generate_storage_key, get_storage_category_for_resource
__all__ = [
"StorageBackend",
@@ -24,12 +19,12 @@
# Cloud backends re-exported on demand (seam C4): the CE tree physically lacks s3/oss, so the import must not crash.
try:
- from core.storage.s3 import S3StorageBackend # noqa: F401
+ from core.files.storage.s3 import S3StorageBackend # noqa: F401
__all__.append("S3StorageBackend")
except ModuleNotFoundError:
pass
try:
- from core.storage.oss import OSSStorageBackend # noqa: F401
+ from core.files.storage.oss import OSSStorageBackend # noqa: F401
__all__.append("OSSStorageBackend")
except ModuleNotFoundError:
pass
diff --git a/src/backend/core/storage/entries.py b/src/backend/core/files/storage/entries.py
similarity index 100%
rename from src/backend/core/storage/entries.py
rename to src/backend/core/files/storage/entries.py
diff --git a/src/backend/core/storage/factory.py b/src/backend/core/files/storage/factory.py
similarity index 81%
rename from src/backend/core/storage/factory.py
rename to src/backend/core/files/storage/factory.py
index 4268d0f2..ccd914cb 100644
--- a/src/backend/core/storage/factory.py
+++ b/src/backend/core/files/storage/factory.py
@@ -1,8 +1,10 @@
"""Storage backend factory and utility functions."""
+from __future__ import annotations
from core.infra.time import utc_now
import logging
import os
+from dataclasses import dataclass
from datetime import datetime
from typing import Optional
@@ -18,8 +20,8 @@ def secure_filename(filename: str) -> str:
sanitized = sanitized.strip("._")
return sanitized or "file"
-from core.storage.protocol import StorageBackend
-from core.storage.local import LocalStorageBackend
+from core.files.storage.protocol import StorageBackend
+from core.files.storage.local import LocalStorageBackend
logger = logging.getLogger(__name__)
@@ -40,13 +42,13 @@ def get_storage_backend() -> StorageBackend:
storage_type = os.getenv("STORAGE_TYPE", "local").lower()
if storage_type == "oss":
- from core.storage.oss import OSSStorageBackend
+ from core.files.storage.oss import OSSStorageBackend
return OSSStorageBackend()
elif storage_type == "s3":
- from core.storage.s3 import S3StorageBackend
+ from core.files.storage.s3 import S3StorageBackend
return S3StorageBackend()
elif storage_type == "obs":
- from core.storage.obs import OBSStorageBackend
+ from core.files.storage.obs import OBSStorageBackend
return OBSStorageBackend()
elif storage_type == "local":
return LocalStorageBackend()
@@ -105,14 +107,24 @@ def get_storage_category_for_resource(resource_type: str) -> str:
# Module-level storage instance (lazy-loaded)
-_storage_instance: Optional[StorageBackend] = None
def get_storage() -> StorageBackend:
"""
Get the global storage backend instance (lazy-loaded singleton).
"""
- global _storage_instance
- if _storage_instance is None:
- _storage_instance = get_storage_backend()
- return _storage_instance
+ if process_state().storage_instance is None:
+ process_state().storage_instance = get_storage_backend()
+ return process_state().storage_instance
+
+
+@dataclass
+class FactoryState:
+ storage_instance: Optional[StorageBackend] = None
+
+
+_STATE = FactoryState()
+
+
+def process_state() -> FactoryState:
+ return _STATE
diff --git a/src/backend/core/storage/local.py b/src/backend/core/files/storage/local.py
similarity index 96%
rename from src/backend/core/storage/local.py
rename to src/backend/core/files/storage/local.py
index 3f9b551c..ae7f9f75 100644
--- a/src/backend/core/storage/local.py
+++ b/src/backend/core/files/storage/local.py
@@ -7,10 +7,10 @@
from pathlib import Path
from core.infra.exceptions import StorageError
-from core.storage.protocol import StorageBackend
-from core.storage.local_directory import LocalDirectoryOperations
-from core.storage.entries import validate_key
-from core.storage import local_paths
+from core.files.storage.protocol import StorageBackend
+from core.files.storage.local_directory import LocalDirectoryOperations
+from core.files.storage.entries import validate_key
+from core.files.storage import local_paths
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/storage/local_directory.py b/src/backend/core/files/storage/local_directory.py
similarity index 94%
rename from src/backend/core/storage/local_directory.py
rename to src/backend/core/files/storage/local_directory.py
index 49684fbc..ce60b674 100644
--- a/src/backend/core/storage/local_directory.py
+++ b/src/backend/core/files/storage/local_directory.py
@@ -2,8 +2,8 @@
import os
import stat
from datetime import datetime, timezone
-from core.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key
-from core.storage.local_paths import directory_fd, file_stat, parent_fd
+from core.files.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key
+from core.files.storage.local_paths import directory_fd, file_stat, parent_fd
def entry(name, key, info):
diff --git a/src/backend/core/storage/local_paths.py b/src/backend/core/files/storage/local_paths.py
similarity index 98%
rename from src/backend/core/storage/local_paths.py
rename to src/backend/core/files/storage/local_paths.py
index 45630a12..d1a1a3c1 100644
--- a/src/backend/core/storage/local_paths.py
+++ b/src/backend/core/files/storage/local_paths.py
@@ -5,7 +5,7 @@
import uuid
from pathlib import Path
-from core.storage.entries import validate_key
+from core.files.storage.entries import validate_key
@contextmanager
diff --git a/src/backend/core/storage/protocol.py b/src/backend/core/files/storage/protocol.py
similarity index 100%
rename from src/backend/core/storage/protocol.py
rename to src/backend/core/files/storage/protocol.py
diff --git a/src/backend/core/storage/s3.py b/src/backend/core/files/storage/s3.py
similarity index 98%
rename from src/backend/core/storage/s3.py
rename to src/backend/core/files/storage/s3.py
index ff3231a9..b697a66d 100644
--- a/src/backend/core/storage/s3.py
+++ b/src/backend/core/files/storage/s3.py
@@ -5,8 +5,8 @@
from pathlib import Path
from core.infra.exceptions import StorageError
-from core.storage.protocol import StorageBackend
-from core.storage.s3_directory import S3DirectoryOperations
+from core.files.storage.protocol import StorageBackend
+from core.files.storage.s3_directory import S3DirectoryOperations
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/storage/s3_directory.py b/src/backend/core/files/storage/s3_directory.py
similarity index 96%
rename from src/backend/core/storage/s3_directory.py
rename to src/backend/core/files/storage/s3_directory.py
index 2531530c..f61b80bf 100644
--- a/src/backend/core/storage/s3_directory.py
+++ b/src/backend/core/files/storage/s3_directory.py
@@ -1,5 +1,5 @@
"""S3 directory operations shared by S3-compatible services."""
-from core.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key
+from core.files.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key
class S3DirectoryOperations:
def list_directory(self, prefix, *, cursor=None, limit=200):
diff --git a/src/backend/core/services/upload_retry.py b/src/backend/core/files/upload_retry.py
similarity index 92%
rename from src/backend/core/services/upload_retry.py
rename to src/backend/core/files/upload_retry.py
index d4feb439..270004a1 100644
--- a/src/backend/core/services/upload_retry.py
+++ b/src/backend/core/files/upload_retry.py
@@ -5,8 +5,8 @@
from uuid import UUID
from core.db.models import Artifact, UserShadow
-from core.services.artifact_edition import upload_target_matches
-from core.services.folder_batch import lock_folder_owner
+from core.files.edition import upload_target_matches
+from core.db.repository.folder import lock_folder_owner
from fastapi import HTTPException
diff --git a/src/backend/core/capabilities/errors.py b/src/backend/core/harness/capability_errors.py
similarity index 100%
rename from src/backend/core/capabilities/errors.py
rename to src/backend/core/harness/capability_errors.py
diff --git a/src/backend/core/harness/context_items.py b/src/backend/core/harness/context_items.py
new file mode 100644
index 00000000..8654de64
--- /dev/null
+++ b/src/backend/core/harness/context_items.py
@@ -0,0 +1,6 @@
+"""Framework-neutral context additions emitted by execution policies."""
+
+def reminder_item(text, origin, *, priority=850, token_budget=4000):
+ return dict(content=f"\n{text}\n ",
+ kind="reminder", origin=origin, trust="system",
+ priority=priority, token_budget=token_budget)
diff --git a/src/backend/core/harness/events.py b/src/backend/core/harness/events.py
index 9a9a117c..f54932fb 100644
--- a/src/backend/core/harness/events.py
+++ b/src/backend/core/harness/events.py
@@ -5,12 +5,15 @@
import asyncio
import copy
import inspect
-from collections.abc import Awaitable, Callable, Mapping
+from collections import deque
+from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import UTC, datetime
from types import MappingProxyType
from typing import Any
+from core.harness.observation_writer import observation_writer
+
def freeze_value(value: Any) -> Any:
"""Recursively detach and freeze a value before exposing it to consumers."""
@@ -67,6 +70,28 @@ def create(
EventStore = Callable[[Event], Event]
+def _detach(event: Event) -> Event:
+ return Event(
+ run_id=event.run_id,
+ event_type=event.event_type,
+ phase=event.phase,
+ payload=freeze_value(event.payload),
+ created_at=event.created_at,
+ event_seq=event.event_seq,
+ )
+
+
+def store_events(store: EventStore, events: Sequence[Event]) -> tuple[Event, ...]:
+ """Write a batch through ``store.store_many`` when the store supports it."""
+ store_many = getattr(store, "store_many", None)
+ if store_many is not None:
+ return tuple(store_many(events))
+ return tuple(store(event) for event in events)
+
+
+MEMORY_EVENT_LIMIT = 256
+
+
class EventSink:
"""Append first, then notify observers with immutable detached events.
@@ -85,20 +110,15 @@ def __init__(
self._store_timeout_ms = max(1, int(store_timeout_ms))
self._subscriber_timeout_ms = max(1, int(subscriber_timeout_ms))
self._handlers: list[EventHandler] = []
- self._memory: list[Event] = []
+ # Recent events for in-process inspection; durable history lives in the store.
+ self._memory: deque[Event] = deque(maxlen=MEMORY_EVENT_LIMIT)
def subscribe(self, handler: EventHandler) -> None:
self._handlers.append(handler)
async def append(self, event: Event) -> Event:
- detached = Event(
- run_id=event.run_id,
- event_type=event.event_type,
- phase=event.phase,
- payload=freeze_value(event.payload),
- created_at=event.created_at,
- event_seq=event.event_seq,
- )
+ """Store, then notify: observers see the durable sequence number."""
+ detached = _detach(event)
recorded = detached
if self._store is not None:
try:
@@ -111,15 +131,25 @@ async def append(self, event: Event) -> Event:
# channel. Durable stores log their own failures.
recorded = detached
self._memory.append(recorded)
+ await self._notify(recorded)
+ return recorded
+
+ async def publish(self, event: Event) -> Event:
+ """Record without waiting for the store.
+
+ The durable write is queued for the background observation writer, so
+ the returned and observed event carries no ``event_seq``.
+ """
+ detached = _detach(event)
+ if self._store is not None:
+ observation_writer().submit(self._store, store_events, detached.run_id, (detached,))
+ self._memory.append(detached)
+ await self._notify(detached)
+ return detached
+
+ async def _notify(self, recorded: Event) -> None:
for handler in tuple(self._handlers):
- consumer_copy = Event(
- run_id=recorded.run_id,
- event_type=recorded.event_type,
- phase=recorded.phase,
- payload=freeze_value(recorded.payload),
- created_at=recorded.created_at,
- event_seq=recorded.event_seq,
- )
+ consumer_copy = _detach(recorded)
try:
async def notify(
@@ -144,7 +174,6 @@ def call_handler() -> Any:
)
except Exception: # noqa: BLE001, S112
continue
- return recorded
def events(self) -> tuple[Event, ...]:
return tuple(self._memory)
diff --git a/src/backend/core/llm/execution_manifest.py b/src/backend/core/harness/execution_manifest.py
similarity index 99%
rename from src/backend/core/llm/execution_manifest.py
rename to src/backend/core/harness/execution_manifest.py
index 98c14c97..e49e15b8 100644
--- a/src/backend/core/llm/execution_manifest.py
+++ b/src/backend/core/harness/execution_manifest.py
@@ -17,7 +17,7 @@
from pathlib import Path
from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
-from core.immutable import FrozenDict, freeze_json, thaw_json
+from core.infra.immutable import FrozenDict, freeze_json, thaw_json
MANIFEST_SCHEMA_VERSION = "harness.execution-manifest.v1"
_PUBLIC_CONTEXT_REFS = {
diff --git a/src/backend/core/harness/hooks.py b/src/backend/core/harness/hooks.py
index 83ec2555..94ab6256 100644
--- a/src/backend/core/harness/hooks.py
+++ b/src/backend/core/harness/hooks.py
@@ -12,12 +12,14 @@
from typing import Any
from core.harness.events import Event, EventSink, freeze_value, thaw_value
-from core.harness.usage import AttemptUsage, UsageAttempt, UsageRecorder
+from core.harness.observation_writer import observation_writer
+from core.harness.usage import AttemptUsage, UsageAttempt, UsageRecorder, record_attempt_batch
class HookStage(str, Enum):
BEFORE_RUN = "before_run"
TRANSFORM_CONTEXT = "transform_context"
+ AFTER_CONTEXT = "after_context"
BEFORE_MODEL = "before_model"
AFTER_MODEL = "after_model"
BEFORE_TOOL = "before_tool"
@@ -41,24 +43,25 @@ class FailureMode(str, Enum):
STAGE_MUTABLE_FIELDS: Mapping[HookStage, frozenset[str]] = MappingProxyType(
{
- HookStage.BEFORE_RUN: frozenset(),
- HookStage.TRANSFORM_CONTEXT: frozenset({"messages", "context"}),
+ HookStage.BEFORE_RUN: frozenset({"model_selection"}),
+ HookStage.TRANSFORM_CONTEXT: frozenset({"messages", "context", "context_items", "tool_choice", "steer_delivery"}),
# AgentScope reaches on_model_call only after HugAgentOS has assembled and
# bound the execution manifest. Rewriting provider input here would make
# the evidence bundle describe different inputs than the real request.
+ HookStage.AFTER_CONTEXT: frozenset(),
HookStage.BEFORE_MODEL: frozenset(),
HookStage.AFTER_MODEL: frozenset(),
# AgentScope reaches on_acting after schema and permission checks. A
# rewrite here would bypass both, so the call is observation-only.
- HookStage.BEFORE_TOOL: frozenset(),
- # Tool chunks may already be visible by this point, so results are
- # read-only here; policies that rewrite calls must act BEFORE_TOOL.
- HookStage.AFTER_TOOL: frozenset(),
+ HookStage.BEFORE_TOOL: frozenset({"ontology_runtime", "context_items", "tool_response"}),
+ # Tool results may change only while buffered; run() rejects a rewrite
+ # when the adapter has already exposed any result chunks.
+ HookStage.AFTER_TOOL: frozenset({"context_items", "tool_result"}),
# Source history is read-only because the checkpoint watermark/hash
# must continue to describe exactly the rows being covered.
HookStage.BEFORE_COMPACTION: frozenset({"budget"}),
HookStage.AFTER_COMPACTION: frozenset({"replacement"}),
- HookStage.BEFORE_FINISH: frozenset(),
+ HookStage.BEFORE_FINISH: frozenset({"pin_file_ids", "continue_run"}),
}
)
@@ -131,7 +134,7 @@ class HookSpec:
stage: HookStage
handler: HookHandler
priority: int = 100
- timeout_ms: int = 1_000
+ timeout_ms: int | None = 1_000
failure_mode: FailureMode = FailureMode.CLOSED
mutable_fields: frozenset[str] = field(default_factory=frozenset)
@@ -174,11 +177,9 @@ def __init__(
*,
event_sink: EventSink | None = None,
usage_recorder: UsageRecorder | None = None,
- usage_timeout_ms: int = 1_000,
) -> None:
self.event_sink = event_sink or EventSink()
self.usage_recorder = usage_recorder
- self.usage_timeout_ms = max(1, int(usage_timeout_ms))
self._hooks: dict[HookStage, list[tuple[int, int, HookSpec]]] = {}
self._registration_seq = 0
@@ -195,52 +196,55 @@ def register(self, spec: HookSpec) -> None:
bucket.append((spec.priority, self._registration_seq, spec))
bucket.sort(key=lambda item: (item[0], item[1]))
+ def registered_specs(self) -> tuple[HookSpec, ...]:
+ """Expose the ordered policy inventory without mutable registry internals."""
+ return tuple(spec for stage in HookStage
+ for _, _, spec in self._hooks.get(stage, ()))
+
async def _call(self, spec: HookSpec, invocation: Invocation) -> Decision:
- async def invoke() -> Decision:
- if inspect.iscoroutinefunction(spec.handler):
- result = await spec.handler(invocation)
+ # Synchronous handlers are in-memory policies and run inline: a thread
+ # hop plus timeout would let executor saturation reject or skip them.
+ # Policies that block on I/O must be async and own their offloading.
+ result = spec.handler(invocation)
+ if inspect.isawaitable(result):
+ # Interactive policies own their timeout and must retain caller cancellation.
+ if spec.timeout_ms is None:
+ result = await result
else:
- result = await asyncio.to_thread(spec.handler, invocation)
- if inspect.isawaitable(result):
- result = await result
- if result is None:
- return Decision()
- if not isinstance(result, Decision):
- raise HookContractError(
- f"hook {spec.name} returned {type(result).__name__}"
- )
- return result
-
- return await asyncio.wait_for(invoke(), timeout=max(1, spec.timeout_ms) / 1_000)
+ result = await asyncio.wait_for(result, timeout=max(1, spec.timeout_ms) / 1_000)
+ if result is None:
+ return Decision()
+ if not isinstance(result, Decision):
+ raise HookContractError(f"hook {spec.name} returned {type(result).__name__}")
+ return result
async def run(self, invocation: Invocation) -> HookOutcome:
current = invocation
executed: list[str] = []
terminal = Decision()
- await self.event_sink.append(
- Event.create(
- run_id=current.run_id,
- event_type="hook_stage_started",
- phase=current.stage.value,
- payload={"operation_name": current.operation_name},
- )
- )
- for _, _, spec in tuple(self._hooks.get(current.stage, ())):
+ hooks = tuple(self._hooks.get(current.stage, ()))
+ if not hooks:
+ return HookOutcome(current, terminal, ())
+ attempts: list[dict[str, Any]] = []
+ usage: list[UsageAttempt] = []
+ cancellation: asyncio.CancelledError | None = None
+ for _, _, spec in hooks:
started = time.monotonic()
status = "success"
decision = Decision()
- cancellation: asyncio.CancelledError | None = None
try:
decision = await self._call(spec, current)
- if decision.kind == DecisionKind.MODIFY:
+ if decision.patch:
invalid = set(decision.patch) - set(spec.mutable_fields)
if invalid:
raise HookContractError(
f"hook {spec.name} modified undeclared fields: "
f"{sorted(invalid)}"
)
+ if "tool_result" in decision.patch and current.data.get("result_exposed"):
+ raise HookContractError("cannot rewrite an exposed tool result")
current = current.patched(decision.patch)
- elif decision.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}:
+ if decision.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}:
terminal = decision
except asyncio.CancelledError as exc:
status = "cancelled"
@@ -255,57 +259,56 @@ async def run(self, invocation: Invocation) -> HookOutcome:
terminal = Decision.reject(f"hook {spec.name} failed: {exc}")
latency_ms = int((time.monotonic() - started) * 1_000)
executed.append(spec.name)
- if self.usage_recorder is not None:
- try:
- await asyncio.wait_for(
- asyncio.to_thread(
- self.usage_recorder.record_attempt,
- UsageAttempt(
- run_id=current.run_id,
- kind="hook",
- operation_name=spec.name,
- status=status,
- latency_ms=latency_ms,
- usage=decision.usage,
- metadata={"stage": current.stage.value},
- ),
- ),
- timeout=self.usage_timeout_ms / 1_000,
- )
- except Exception: # noqa: BLE001, S110
- pass
- await self.event_sink.append(
- Event.create(
+ usage.append(
+ UsageAttempt(
run_id=current.run_id,
- event_type="hook_attempt_finished",
- phase=current.stage.value,
- payload={
- "hook": spec.name,
- "status": status,
- "decision": terminal.kind.value
- if terminal.kind != DecisionKind.CONTINUE
- else decision.kind.value,
- "latency_ms": latency_ms,
- },
+ kind="hook",
+ operation_name=spec.name,
+ status=status,
+ latency_ms=latency_ms,
+ usage=decision.usage,
+ metadata={"stage": current.stage.value},
)
)
- if terminal.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}:
+ attempts.append(
+ {
+ "hook": spec.name,
+ "status": status,
+ "decision": terminal.kind.value
+ if terminal.kind != DecisionKind.CONTINUE
+ else decision.kind.value,
+ "latency_ms": latency_ms,
+ }
+ )
+ if terminal.kind in {DecisionKind.REJECT, DecisionKind.PAUSE} or cancellation is not None:
break
- if cancellation is not None:
- raise cancellation
- await self.event_sink.append(
+ await self._record_stage(current, terminal, executed, attempts, usage)
+ if cancellation is not None:
+ raise cancellation
+ return HookOutcome(current, terminal, tuple(executed))
+
+ async def _record_stage(self, current, terminal, executed, attempts, usage) -> None:
+ """Queue one usage batch and one event per stage, not one per hook.
+
+ Both are written by the background observation writer; the stage never
+ waits for the database and a failed write cannot change its outcome.
+ """
+ if self.usage_recorder is not None and current.run_id:
+ observation_writer().submit(self.usage_recorder, record_attempt_batch, current.run_id, usage)
+ await self.event_sink.publish(
Event.create(
run_id=current.run_id,
event_type="hook_stage_finished",
phase=current.stage.value,
payload={
+ "operation_name": current.operation_name,
"decision": terminal.kind.value,
"executed_hooks": executed,
+ "attempts": attempts,
},
)
)
- return HookOutcome(current, terminal, tuple(executed))
async def enforce(self, invocation: Invocation) -> Invocation:
outcome = await self.run(invocation)
diff --git a/src/backend/core/services/harness_ledger.py b/src/backend/core/harness/ledger.py
similarity index 53%
rename from src/backend/core/services/harness_ledger.py
rename to src/backend/core/harness/ledger.py
index bb5cb2d7..d346f0cb 100644
--- a/src/backend/core/services/harness_ledger.py
+++ b/src/backend/core/harness/ledger.py
@@ -39,18 +39,59 @@ def _insert_cursor_if_missing(db, model: type[Any], values: dict[str, Any]) -> N
db.execute(statement)
-def _reserve(db, model: type[Any], run_id: str, column) -> int:
+def _reserve(db, model: type[Any], run_id: str, column, count: int = 1) -> int:
+ """Reserve ``count`` consecutive sequence numbers and return the last one."""
_insert_cursor_if_missing(db, model, {"run_id": run_id, column.key: 0})
value = db.execute(
update(model)
.where(model.run_id == run_id)
- .values({column.key: column + 1})
+ .values({column.key: column + count})
.returning(column)
.execution_options(synchronize_session=False)
).scalar_one()
return int(value)
+def _previous_failed_attempt(db, attempt: UsageAttempt) -> int | None:
+ """A tool effect's retry links to its latest failed physical attempt."""
+ if attempt.kind != "tool" or not attempt.effect_id:
+ return None
+ previous = (
+ db.query(HarnessUsageAttempt)
+ .filter(
+ HarnessUsageAttempt.run_id == attempt.run_id,
+ HarnessUsageAttempt.kind == attempt.kind,
+ HarnessUsageAttempt.operation_name == attempt.operation_name,
+ HarnessUsageAttempt.status.in_(("failed", "timeout")),
+ HarnessUsageAttempt.effect_id == attempt.effect_id,
+ )
+ .order_by(HarnessUsageAttempt.attempt_seq.desc())
+ .first()
+ )
+ return int(previous.attempt_seq) if previous is not None else None
+
+
+def _attempt_row(attempt: UsageAttempt, seq: int, retry_of: int | None) -> HarnessUsageAttempt:
+ return HarnessUsageAttempt(
+ attempt_id=f"hat_{uuid.uuid4().hex}",
+ run_id=attempt.run_id,
+ attempt_seq=seq,
+ kind=attempt.kind,
+ operation_name=attempt.operation_name,
+ provider=attempt.provider,
+ model=attempt.model,
+ effect_id=attempt.effect_id or None,
+ prompt_tokens=max(0, int(attempt.usage.prompt_tokens)),
+ completion_tokens=max(0, int(attempt.usage.completion_tokens)),
+ cache_read_tokens=max(0, int(attempt.usage.cache_read_tokens)),
+ cache_write_tokens=max(0, int(attempt.usage.cache_write_tokens)),
+ latency_ms=max(0, int(attempt.latency_ms)),
+ status=attempt.status,
+ retry_of=retry_of,
+ attempt_metadata=thaw_value(attempt.metadata),
+ )
+
+
class HarnessUsageLedger:
"""Append attempts and derive aggregates without mutable summary rows."""
@@ -67,48 +108,35 @@ def _validate(attempt: UsageAttempt) -> None:
raise ValueError(f"invalid attempt status: {attempt.status}")
def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt:
- self._validate(attempt)
+ return self.record_attempts((attempt,))[0]
+
+ def record_attempts(self, attempts) -> tuple[UsageAttempt, ...]:
+ """Append attempts of one run in a single transaction and cursor update."""
+ attempts = tuple(attempts)
+ if not attempts:
+ return ()
+ for attempt in attempts:
+ self._validate(attempt)
+ run_id = attempts[0].run_id
+ if any(attempt.run_id != run_id for attempt in attempts):
+ raise ValueError("a usage batch must belong to one run")
with self._session_factory() as db:
- seq = _reserve(
+ last = _reserve(
db,
HarnessUsageCursor,
- attempt.run_id,
+ run_id,
HarnessUsageCursor.next_attempt_seq,
+ len(attempts),
)
- retry_of = attempt.retry_of
- if retry_of is None and attempt.kind == "tool" and attempt.effect_id:
- query = db.query(HarnessUsageAttempt).filter(
- HarnessUsageAttempt.run_id == attempt.run_id,
- HarnessUsageAttempt.kind == attempt.kind,
- HarnessUsageAttempt.operation_name == attempt.operation_name,
- HarnessUsageAttempt.status.in_(("failed", "timeout")),
- )
- query = query.filter(HarnessUsageAttempt.effect_id == attempt.effect_id)
- previous = query.order_by(
- HarnessUsageAttempt.attempt_seq.desc()
- ).first()
- retry_of = int(previous.attempt_seq) if previous is not None else None
- row = HarnessUsageAttempt(
- attempt_id=f"hat_{uuid.uuid4().hex}",
- run_id=attempt.run_id,
- attempt_seq=seq,
- kind=attempt.kind,
- operation_name=attempt.operation_name,
- provider=attempt.provider,
- model=attempt.model,
- effect_id=attempt.effect_id or None,
- prompt_tokens=max(0, int(attempt.usage.prompt_tokens)),
- completion_tokens=max(0, int(attempt.usage.completion_tokens)),
- cache_read_tokens=max(0, int(attempt.usage.cache_read_tokens)),
- cache_write_tokens=max(0, int(attempt.usage.cache_write_tokens)),
- latency_ms=max(0, int(attempt.latency_ms)),
- status=attempt.status,
- retry_of=retry_of,
- attempt_metadata=thaw_value(attempt.metadata),
- )
- db.add(row)
+ recorded = []
+ for seq, attempt in enumerate(attempts, start=last - len(attempts) + 1):
+ retry_of = attempt.retry_of
+ if retry_of is None:
+ retry_of = _previous_failed_attempt(db, attempt)
+ db.add(_attempt_row(attempt, seq, retry_of))
+ recorded.append(replace(attempt, attempt_seq=seq, retry_of=retry_of))
db.commit()
- return replace(attempt, attempt_seq=seq, retry_of=retry_of)
+ return tuple(recorded)
def attempts(self, run_id: str) -> tuple[UsageAttempt, ...]:
with self._session_factory() as db:
@@ -167,32 +195,39 @@ def __init__(self, session_factory=SessionLocal) -> None:
self._session_factory = session_factory
def __call__(self, event: Event) -> Event:
- if not event.run_id:
- return event
+ return self.store_many((event,))[0]
+
+ def store_many(self, events) -> tuple[Event, ...]:
+ """Append events in one transaction, one cursor update per run."""
+ recorded = list(events)
+ by_run: dict[str, list[int]] = {}
+ for index, event in enumerate(recorded):
+ if event.run_id:
+ by_run.setdefault(event.run_id, []).append(index)
+ if not by_run:
+ return tuple(recorded)
with self._session_factory() as db:
- seq = _reserve(
- db,
- HarnessEventCursor,
- event.run_id,
- HarnessEventCursor.next_event_seq,
- )
- db.add(
- HarnessEventLog(
- event_id=f"hev_{uuid.uuid4().hex}",
- run_id=event.run_id,
- event_seq=seq,
- event_type=event.event_type,
- phase=event.phase,
- payload=thaw_value(event.payload),
- created_at=event.created_at,
+ for run_id, indexes in by_run.items():
+ last = _reserve(
+ db,
+ HarnessEventCursor,
+ run_id,
+ HarnessEventCursor.next_event_seq,
+ len(indexes),
)
- )
+ for seq, index in enumerate(indexes, start=last - len(indexes) + 1):
+ event = recorded[index]
+ db.add(
+ HarnessEventLog(
+ event_id=f"hev_{uuid.uuid4().hex}",
+ run_id=run_id,
+ event_seq=seq,
+ event_type=event.event_type,
+ phase=event.phase,
+ payload=thaw_value(event.payload),
+ created_at=event.created_at,
+ )
+ )
+ recorded[index] = replace(event, payload=freeze_value(event.payload), event_seq=seq)
db.commit()
- return Event(
- run_id=event.run_id,
- event_type=event.event_type,
- phase=event.phase,
- payload=freeze_value(event.payload),
- created_at=event.created_at,
- event_seq=seq,
- )
+ return tuple(recorded)
diff --git a/src/backend/core/harness/log_audit.py b/src/backend/core/harness/log_audit.py
new file mode 100644
index 00000000..7340a7cd
--- /dev/null
+++ b/src/backend/core/harness/log_audit.py
@@ -0,0 +1,144 @@
+"""Write sandbox tool calls as security audit events."""
+from __future__ import annotations
+from datetime import datetime
+from typing import Any, Dict, Optional
+from sqlalchemy.orm import Session
+from core.db.engine import SessionLocal
+from core.infra.logging import get_logger
+from core.harness import log_context
+
+logger = get_logger(__name__)
+
+async def resolve_sandbox_id(session_id: Optional[str]) -> Optional[str]:
+ """Look up the current sandbox instance id from the sandbox provider by session key (default = chat_id). Pure query; None on failure."""
+ if not session_id:
+ return None
+ try:
+ from core.sandbox import get_sandbox_provider
+
+ return await get_sandbox_provider().current_sandbox_id(session_id)
+ except (
+ Exception
+ ): # noqa: BLE001 — audit enrichment is best-effort, must never affect the main path
+ return None
+
+
+def audit_status(raw: Optional[str]) -> str:
+ return AUDIT_STATUS_MAP.get(raw or "success", "failure")
+
+
+def insert_sandbox_audit(
+ *,
+ action: str,
+ sandbox_id: Any,
+ user_id: Optional[str],
+ trace_id: Optional[str],
+ details: Dict[str, Any],
+ status: str,
+ created_at: Optional[datetime],
+) -> None:
+ """Write one resource=sandbox security audit row. Best-effort: failure only degrades to a log entry, never affects the main path."""
+ from core.db.repository import AuditLogRepository
+
+ db: Session = SessionLocal()
+ try:
+ # Reuse the audit repository's write (has its own add/commit/rollback; audit failure doesn't block the main path).
+ AuditLogRepository(db).create(
+ {
+ "user_id": user_id
+ or None, # empty string would trip the FK constraint; normalize to NULL
+ "action": action,
+ "resource_type": "sandbox",
+ "resource_id": str(sandbox_id)[:64],
+ "sandbox_id": str(sandbox_id)[:128],
+ "details": details,
+ "trace_id": trace_id,
+ "status": status,
+ "created_at": created_at or log_context.now_utc(),
+ }
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning("sandbox_audit_write_failed", error=str(exc), sandbox=str(sandbox_id))
+ finally:
+ db.close()
+
+
+def write_sandbox_audit_sync(record: Dict[str, Any]) -> None:
+ """Write one sandbox-type tool call as a security audit event (resource=sandbox / sandbox instance id).
+
+ Separate from tool-call logging: audits let "Security Management → Audit
+ Logs" distinguish and filter by sandbox instance. details only summarizes
+ the input args (redacted + truncated), never dumps full results.
+ """
+ sandbox_id = record.get("sandbox_id")
+ if not sandbox_id:
+ return
+ tool_name = record.get("tool_name")
+ action = SANDBOX_AUDIT_ACTIONS.get(tool_name, "sandbox.tool.call")
+ args_safe = log_context.sanitize_tool_args(record.get("tool_args"))
+ details: Dict[str, Any] = {"tool": tool_name}
+ if isinstance(args_safe, dict):
+ cmd = args_safe.get("command")
+ if cmd is not None:
+ preview, _ = log_context.truncate_text(str(cmd), 2000)
+ details["command"] = preview
+ # File/search tools: record path and pattern (not large fields like content) to trace what was changed/read.
+ for k in ("file_path", "filename", "path", "pattern", "artifact_id", "name"):
+ if args_safe.get(k) is not None:
+ details[k] = args_safe[k]
+ if record.get("error_message"):
+ details["error"], _ = log_context.truncate_text(str(record["error_message"]), 500)
+
+ insert_sandbox_audit(
+ action=action,
+ sandbox_id=sandbox_id,
+ user_id=record.get("user_id"),
+ trace_id=record.get("trace_id"),
+ details=details,
+ status=audit_status(record.get("status")),
+ created_at=record.get("created_at"),
+ )
+
+
+def write_skill_sandbox_audit_sync(record: Dict[str, Any]) -> None:
+ """Skill script execution (run_script) = running code in the sandbox; write one sandbox.skill.run audit.
+
+ Triggered only by run_script (view / auto_load do not execute code in the
+ sandbox); skipped when there is no sandbox instance.
+ """
+ sandbox_id = record.get("sandbox_id")
+ if not sandbox_id:
+ return
+ details: Dict[str, Any] = {
+ "skill_id": record.get("skill_id"),
+ "skill_name": record.get("skill_name"),
+ "script_name": record.get("script_name"),
+ "script_language": record.get("script_language"),
+ }
+ if record.get("error_message"):
+ details["error"], _ = log_context.truncate_text(str(record["error_message"]), 500)
+
+ insert_sandbox_audit(
+ action="sandbox.skill.run",
+ sandbox_id=sandbox_id,
+ user_id=record.get("user_id"),
+ trace_id=record.get("trace_id"),
+ details=details,
+ status=audit_status(record.get("status")),
+ created_at=record.get("created_at"),
+ )
+
+
+SANDBOX_AUDIT_ACTIONS = {
+ "Bash": "sandbox.bash.exec",
+ "sandbox_put_artifact": "sandbox.artifact.put",
+ "sandbox_get_artifact": "sandbox.artifact.get",
+ "Write": "sandbox.file.write",
+ "Edit": "sandbox.file.edit",
+ "Read": "sandbox.file.read",
+ "Grep": "sandbox.file.grep",
+ "Glob": "sandbox.file.glob",
+}
+
+
+AUDIT_STATUS_MAP = {"success": "success", "error": "error"}
diff --git a/src/backend/core/harness/log_context.py b/src/backend/core/harness/log_context.py
new file mode 100644
index 00000000..d8474190
--- /dev/null
+++ b/src/backend/core/harness/log_context.py
@@ -0,0 +1,161 @@
+"""Shared helpers for call logs: ids, truncation and the current log scope."""
+from __future__ import annotations
+import json
+import os
+import uuid
+from contextlib import contextmanager
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterator, Optional
+from sqlalchemy.orm import Session
+from core.db.models import LocalUser, UserShadow
+from core.infra.data_masking import mask_sensitive_data
+from core.infra.logging import chat_id_var, trace_id_var, user_id_var
+from core.infra.logging import get_logger
+logger = get_logger(__name__)
+
+def new_id() -> str:
+ return uuid.uuid4().hex
+
+
+def now_utc() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+def truncate_text(text: Optional[str], limit: int) -> tuple[Optional[str], bool]:
+ if text is None:
+ return None, False
+ if not isinstance(text, str):
+ text = str(text)
+ raw = text.encode("utf-8", errors="ignore")
+ if len(raw) <= limit:
+ return text, False
+ return raw[:limit].decode("utf-8", errors="ignore") + "\n…[truncated]", True
+
+
+def truncate_json(payload: Any, limit: int) -> tuple[Any, bool]:
+ if payload is None:
+ return None, False
+ try:
+ encoded = json.dumps(payload, ensure_ascii=False, default=str).encode(
+ "utf-8", errors="ignore"
+ )
+ except Exception:
+ return {"_repr": str(payload)[:limit]}, True
+ if len(encoded) <= limit:
+ return payload, False
+ preview = encoded[: limit // 2].decode("utf-8", errors="ignore")
+ return {"_truncated": True, "_preview": preview}, True
+
+
+def context_ids() -> Dict[str, str]:
+ return {
+ "trace_id": trace_id_var.get() or "",
+ "user_id": user_id_var.get() or "",
+ "chat_id": chat_id_var.get() or "",
+ }
+
+
+def fetch_username(db: Session, user_id: str) -> Optional[str]:
+ """The "user name" used in logs: prefer the local account display name
+ (nickname → real_name), fall back to the login account username.
+
+ When the audit log panel shows this column, operators should see a
+ recognizable person name (e.g. "张三"), not the login account (e.g.
+ "zhangsan01") — the latter would not match the display names shown
+ elsewhere in the UI.
+ """
+ if not user_id:
+ return None
+ try:
+ row = (
+ db.query(UserShadow.username, LocalUser.nickname, LocalUser.real_name)
+ .outerjoin(LocalUser, LocalUser.user_id == UserShadow.user_id)
+ .filter(UserShadow.user_id == user_id)
+ .one_or_none()
+ )
+ if not row:
+ return None
+ return row.nickname or row.real_name or row.username
+ except Exception as exc: # noqa: BLE001
+ logger.debug("username_lookup_failed", user_id=user_id, error=str(exc))
+ return None
+
+
+@contextmanager
+def subagent_scope(subagent_log_id: Optional[str], source: str = "subagent") -> Iterator[None]:
+ tok_id = subagent_log_id_var.set(subagent_log_id)
+ tok_src = source_var.set(source)
+ try:
+ yield
+ finally:
+ subagent_log_id_var.reset(tok_id)
+ source_var.reset(tok_src)
+
+
+@contextmanager
+def skill_scope(source: str = "skill") -> Iterator[None]:
+ tok = source_var.set(source)
+ try:
+ yield
+ finally:
+ source_var.reset(tok)
+
+
+def set_current_message_id(message_id: str) -> None:
+ current_message_id.set(message_id or "")
+
+
+def current_subagent_log_id() -> Optional[str]:
+ return subagent_log_id_var.get()
+
+
+def current_source() -> str:
+ return source_var.get() or "main_agent"
+
+
+def effective_status(declared: Optional[str], result: Any) -> str:
+ """声明的状态优先,但"成功"要经得起载荷检验。
+
+ 只认**顶层** error,且只认 dict / 能解析成 dict 的 JSON 串——正文里出现 error 字样的
+ 正常内容(比如搜到一篇讲错误码的网页)不该被误判成故障。
+ """
+ status = declared or "success"
+ if status != "success":
+ return status
+ from core.chat.tool_log import _payload_carries_error
+
+ return "error" if _payload_carries_error(result) else "success"
+
+
+def sanitize_tool_args(value: Any) -> Any:
+ from core.runs.tool_effects.arguments import canonical_tool_args
+
+ if isinstance(value, dict):
+ return canonical_tool_args(value)[2]
+ return mask_sensitive_data(value, field_patterns=REDACT_FIELDS)
+
+
+source_var: ContextVar[str] = ContextVar("log_current_source", default="main_agent")
+
+
+subagent_log_id_var: ContextVar[Optional[str]] = ContextVar(
+ "log_current_subagent", default=None
+)
+
+
+current_message_id: ContextVar[str] = ContextVar("log_current_message", default="")
+
+
+REDACT_FIELDS = [
+ s.strip()
+ for s in os.getenv(
+ "LOG_REDACT_FIELDS",
+ "password,token,api_key,apikey,authorization,secret,access_key",
+ ).split(",")
+ if s.strip()
+]
+
+
+# Shared request correlation survives model gateway token rotation.
+model_call_id = ContextVar("desktop_model_call_id", default="")
diff --git a/src/backend/core/harness/log_records.py b/src/backend/core/harness/log_records.py
new file mode 100644
index 00000000..66dc0eda
--- /dev/null
+++ b/src/backend/core/harness/log_records.py
@@ -0,0 +1,212 @@
+"""Synchronous writers for tool, sub-agent and skill call logs."""
+from __future__ import annotations
+from typing import Any, Dict
+from sqlalchemy.orm import Session
+from core.db.engine import SessionLocal
+from core.db.models import SkillCallLog, SubAgentCallLog, ToolCallLog, ToolEffectLedger
+from core.infra.data_masking import mask_sensitive_data
+from core.infra.logging import get_logger
+from core.harness import log_context
+from core.harness import log_writer_constants
+
+logger = get_logger(__name__)
+
+def write_tool_call_sync(record: Dict[str, Any]) -> None:
+ if not log_writer_constants.TOOL_LOG_ENABLED:
+ return
+ db: Session = SessionLocal()
+ try:
+ args_safe = log_context.sanitize_tool_args(record.get("tool_args"))
+ args_safe, _ = log_context.truncate_json(args_safe, log_writer_constants.MAX_RESULT_BYTES)
+ result_safe, result_trunc = log_context.truncate_json(record.get("tool_result"), log_writer_constants.MAX_RESULT_BYTES)
+ username = log_context.fetch_username(db, record.get("user_id", ""))
+ effect = None
+ effect_id = str(record.get("effect_id") or "")
+ run_id = str(record.get("run_id") or "")
+ tool_call_id = str(record.get("tool_call_id") or "")
+ if effect_id:
+ effect = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one_or_none()
+ )
+ if effect is None and run_id and tool_call_id:
+ effect = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.run_id == run_id,
+ ToolEffectLedger.tool_call_id == tool_call_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .order_by(ToolEffectLedger.event_id.desc())
+ .first()
+ )
+ # ToolCallLog stays a replaceable query projection. The pre-allocated
+ # result id makes retries update the same row, while ToolEffectLedger
+ # remains the append-only recovery authority.
+ row_id = (
+ (effect.result_id if effect is not None else None)
+ or record.get("result_id")
+ or record.get("id")
+ or log_context.new_id()
+ )
+ row = db.get(ToolCallLog, row_id) or ToolCallLog(id=row_id)
+ row.effect_id = effect.effect_id if effect is not None else record.get("effect_id")
+ row.trace_id = record.get("trace_id")
+ row.chat_id = record.get("chat_id")
+ row.message_id = record.get("message_id")
+ row.user_id = record.get("user_id")
+ row.user_name = username
+ row.tool_name = record["tool_name"]
+ row.tool_display_name = record.get("tool_display_name")
+ row.tool_call_id = record.get("tool_call_id")
+ row.mcp_server = record.get("mcp_server")
+ row.tool_args = args_safe
+ row.tool_result = result_safe
+ row.result_truncated = bool(result_trunc)
+ # A top-level error payload is a failed projection even when an MCP
+ # adapter declared the transport call successful.
+ row.status = log_context.effective_status(record.get("status"), record.get("tool_result"))
+ row.error_message = record.get("error_message")
+ row.duration_ms = record.get("duration_ms")
+ row.source = record.get("source") or "main_agent"
+ row.subagent_log_id = record.get("subagent_log_id")
+ row.skill_log_id = record.get("skill_log_id")
+ row.sandbox_id = record.get("sandbox_id")
+ row.started_at = record.get("started_at")
+ row.created_at = record.get("created_at") or log_context.now_utc()
+ db.add(row)
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ db.rollback()
+ logger.warning("tool_call_log_write_failed", error=str(exc), tool=record.get("tool_name"))
+ finally:
+ db.close()
+
+
+def write_subagent_create_sync(record: Dict[str, Any]) -> None:
+ if not log_writer_constants.SUBAGENT_LOG_ENABLED:
+ return
+ db: Session = SessionLocal()
+ try:
+ input_safe, _ = log_context.truncate_json(record.get("input_messages"), log_writer_constants.MAX_RESULT_BYTES)
+ username = log_context.fetch_username(db, record.get("user_id", ""))
+ row = SubAgentCallLog(
+ id=record["id"],
+ trace_id=record.get("trace_id"),
+ chat_id=record.get("chat_id"),
+ message_id=record.get("message_id"),
+ user_id=record.get("user_id"),
+ user_name=username,
+ subagent_id=record.get("subagent_id"),
+ subagent_name=record["subagent_name"],
+ subagent_type=record.get("subagent_type"),
+ plan_id=record.get("plan_id"),
+ step_id=record.get("step_id"),
+ step_index=record.get("step_index"),
+ step_title=record.get("step_title"),
+ model=record.get("model"),
+ input_messages=input_safe,
+ status="running",
+ parent_subagent_log_id=record.get("parent_subagent_log_id"),
+ started_at=record.get("started_at") or log_context.now_utc(),
+ created_at=log_context.now_utc(),
+ )
+ db.add(row)
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ db.rollback()
+ logger.warning(
+ "subagent_log_create_failed", error=str(exc), name=record.get("subagent_name")
+ )
+ finally:
+ db.close()
+
+
+def write_subagent_finish_sync(
+ subagent_log_id: str,
+ status: str,
+ updates: Dict[str, Any],
+) -> None:
+ if not log_writer_constants.SUBAGENT_LOG_ENABLED:
+ return
+ db: Session = SessionLocal()
+ try:
+ row = db.query(SubAgentCallLog).filter(SubAgentCallLog.id == subagent_log_id).one_or_none()
+ if row is None:
+ return
+ row.status = status
+ if "output_content" in updates:
+ output, _ = log_context.truncate_text(updates.get("output_content"), log_writer_constants.MAX_RESULT_BYTES)
+ row.output_content = output
+ if "intermediate_steps" in updates:
+ steps_safe, _ = log_context.truncate_json(updates.get("intermediate_steps"), log_writer_constants.MAX_RESULT_BYTES)
+ row.intermediate_steps = steps_safe
+ if "token_usage" in updates:
+ row.token_usage = updates.get("token_usage")
+ if "tool_calls_count" in updates:
+ row.tool_calls_count = updates.get("tool_calls_count")
+ if "skill_calls_count" in updates:
+ row.skill_calls_count = updates.get("skill_calls_count")
+ if "error_message" in updates:
+ row.error_message = updates.get("error_message")
+ if "duration_ms" in updates:
+ row.duration_ms = updates.get("duration_ms")
+ row.completed_at = log_context.now_utc()
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ db.rollback()
+ logger.warning("subagent_log_finish_failed", error=str(exc), id=subagent_log_id)
+ finally:
+ db.close()
+
+
+def write_skill_call_sync(record: Dict[str, Any]) -> None:
+ if not log_writer_constants.SKILL_LOG_ENABLED:
+ return
+ db: Session = SessionLocal()
+ try:
+ args_safe = mask_sensitive_data(record.get("script_args"), field_patterns=log_context.REDACT_FIELDS)
+ args_safe, _ = log_context.truncate_json(args_safe, log_writer_constants.MAX_RESULT_BYTES)
+ stdout_safe, trunc_out = log_context.truncate_text(record.get("script_stdout"), log_writer_constants.MAX_STDOUT_BYTES)
+ stderr_safe, trunc_err = log_context.truncate_text(record.get("script_stderr"), log_writer_constants.MAX_STDOUT_BYTES)
+ stdin_safe, _ = log_context.truncate_text(record.get("script_stdin"), log_writer_constants.MAX_STDOUT_BYTES)
+ username = log_context.fetch_username(db, record.get("user_id", ""))
+ row = SkillCallLog(
+ id=record.get("id") or log_context.new_id(),
+ trace_id=record.get("trace_id"),
+ chat_id=record.get("chat_id"),
+ message_id=record.get("message_id"),
+ user_id=record.get("user_id"),
+ user_name=username,
+ skill_id=record["skill_id"],
+ skill_name=record.get("skill_name"),
+ skill_version=record.get("skill_version"),
+ skill_source=record.get("skill_source"),
+ invocation_type=record.get("invocation_type") or "auto_load",
+ script_name=record.get("script_name"),
+ script_language=record.get("script_language"),
+ script_args=args_safe,
+ script_stdin=stdin_safe,
+ script_stdout=stdout_safe,
+ script_stderr=stderr_safe,
+ output_truncated=bool(trunc_out or trunc_err),
+ exit_code=record.get("exit_code"),
+ status=record.get("status") or "success",
+ error_message=record.get("error_message"),
+ duration_ms=record.get("duration_ms"),
+ source=record.get("source") or log_context.current_source(),
+ subagent_log_id=record.get("subagent_log_id") or log_context.current_subagent_log_id(),
+ started_at=record.get("started_at"),
+ created_at=record.get("created_at") or log_context.now_utc(),
+ )
+ db.add(row)
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ db.rollback()
+ logger.warning("skill_call_log_write_failed", error=str(exc), skill=record.get("skill_id"))
+ finally:
+ db.close()
diff --git a/src/backend/core/harness/log_writer.py b/src/backend/core/harness/log_writer.py
new file mode 100644
index 00000000..a4ccb852
--- /dev/null
+++ b/src/backend/core/harness/log_writer.py
@@ -0,0 +1,121 @@
+"""Async entry points that schedule call-log writes off the event loop."""
+from __future__ import annotations
+import asyncio
+from typing import Any, Dict, List, Optional
+from core.infra.background import spawn
+from core.infra.logging import get_logger
+from core.harness import log_audit
+from core.harness import log_context
+from core.harness import log_records
+from core.harness import log_writer_constants
+
+logger = get_logger(__name__)
+
+async def write_tool_call(record: Dict[str, Any]) -> None:
+ record.setdefault("source", log_context.current_source())
+ record.setdefault("subagent_log_id", log_context.current_subagent_log_id())
+ ctx = log_context.context_ids()
+ record.setdefault("trace_id", ctx["trace_id"])
+ record.setdefault("user_id", ctx["user_id"])
+ record.setdefault("chat_id", ctx["chat_id"])
+ record.setdefault("message_id", log_context.current_message_id.get())
+ # Sandbox-type tools: resolve the sandbox instance id they run in (session
+ # key default = chat_id). Awaiting in place on this async path — provider
+ # reads its active-session table (synchronous in-memory read, no side
+ # effects) — is more reliable than passing a ContextVar across tasks.
+ if record.get("tool_name") in log_writer_constants.SANDBOX_TOOL_NAMES:
+ if not record.get("sandbox_id"):
+ record["sandbox_id"] = await log_audit.resolve_sandbox_id(record.get("chat_id"))
+ # Security audit is independent of the tool-call log switch: write the audit first, then the (optional) tool log.
+ if log_writer_constants.SANDBOX_AUDIT_ENABLED:
+ try:
+ await asyncio.to_thread(log_audit.write_sandbox_audit_sync, record)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("sandbox_audit_task_failed", error=str(exc))
+ if not log_writer_constants.TOOL_LOG_ENABLED:
+ return
+ try:
+ await asyncio.to_thread(log_records.write_tool_call_sync, record)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("tool_call_log_task_failed", error=str(exc))
+
+
+async def start_subagent_log(record: Dict[str, Any]) -> str:
+ log_id = record.get("id") or log_context.new_id()
+ record["id"] = log_id
+ if not log_writer_constants.SUBAGENT_LOG_ENABLED:
+ return log_id
+ ctx = log_context.context_ids()
+ record.setdefault("trace_id", ctx["trace_id"])
+ record.setdefault("user_id", ctx["user_id"])
+ record.setdefault("chat_id", ctx["chat_id"])
+ record.setdefault("message_id", log_context.current_message_id.get())
+ record.setdefault("parent_subagent_log_id", log_context.current_subagent_log_id())
+ record.setdefault("started_at", log_context.now_utc())
+ try:
+ await asyncio.to_thread(log_records.write_subagent_create_sync, record)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("subagent_log_start_failed", error=str(exc))
+ return log_id
+
+
+async def finish_subagent_log(
+ subagent_log_id: str,
+ *,
+ status: str = "success",
+ output_content: Optional[str] = None,
+ intermediate_steps: Optional[List[Dict[str, Any]]] = None,
+ token_usage: Optional[Dict[str, int]] = None,
+ tool_calls_count: int = 0,
+ skill_calls_count: int = 0,
+ error_message: Optional[str] = None,
+ duration_ms: Optional[int] = None,
+) -> None:
+ if not log_writer_constants.SUBAGENT_LOG_ENABLED or not subagent_log_id:
+ return
+ updates = {
+ "output_content": output_content,
+ "intermediate_steps": intermediate_steps,
+ "token_usage": token_usage,
+ "tool_calls_count": tool_calls_count,
+ "skill_calls_count": skill_calls_count,
+ "error_message": error_message,
+ "duration_ms": duration_ms,
+ }
+ try:
+ await asyncio.to_thread(log_records.write_subagent_finish_sync, subagent_log_id, status, updates)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("subagent_log_finish_failed", error=str(exc))
+
+
+async def write_skill_call(record: Dict[str, Any]) -> str:
+ log_id = record.get("id") or log_context.new_id()
+ record["id"] = log_id
+ ctx = log_context.context_ids()
+ record.setdefault("trace_id", ctx["trace_id"])
+ record.setdefault("user_id", ctx["user_id"])
+ record.setdefault("chat_id", ctx["chat_id"])
+ record.setdefault("message_id", log_context.current_message_id.get())
+ # Skill script execution (run_script) = running code in the sandbox → write a security audit (independent of the skill log switch).
+ if log_writer_constants.SANDBOX_AUDIT_ENABLED and record.get("invocation_type") == "run_script":
+ if not record.get("sandbox_id"):
+ record["sandbox_id"] = await log_audit.resolve_sandbox_id(record.get("chat_id"))
+ try:
+ await asyncio.to_thread(log_audit.write_skill_sandbox_audit_sync, record)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("skill_sandbox_audit_task_failed", error=str(exc))
+ if not log_writer_constants.SKILL_LOG_ENABLED:
+ return log_id
+ try:
+ await asyncio.to_thread(log_records.write_skill_call_sync, record)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("skill_call_log_task_failed", error=str(exc))
+ return log_id
+
+
+def schedule_tool_call_write(record: Dict[str, Any]) -> None:
+ spawn(write_tool_call(record), name="log_writer.tool_call")
+
+
+def schedule_skill_call_write(record: Dict[str, Any]) -> None:
+ spawn(write_skill_call(record), name="log_writer.skill_call")
diff --git a/src/backend/core/harness/log_writer_constants.py b/src/backend/core/harness/log_writer_constants.py
new file mode 100644
index 00000000..ecaf3e65
--- /dev/null
+++ b/src/backend/core/harness/log_writer_constants.py
@@ -0,0 +1,32 @@
+"""Switches and size limits for call logging."""
+from __future__ import annotations
+import os
+
+TOOL_LOG_ENABLED = os.getenv("TOOL_CALL_LOG_ENABLED", "true").lower() == "true"
+
+
+SUBAGENT_LOG_ENABLED = os.getenv("SUBAGENT_LOG_ENABLED", "true").lower() == "true"
+
+
+SKILL_LOG_ENABLED = os.getenv("SKILL_LOG_ENABLED", "true").lower() == "true"
+
+
+SANDBOX_AUDIT_ENABLED = os.getenv("SANDBOX_AUDIT_ENABLED", "true").lower() == "true"
+
+
+MAX_RESULT_BYTES = int(os.getenv("LOG_MAX_RESULT_BYTES", 64 * 1024))
+
+
+MAX_STDOUT_BYTES = int(os.getenv("LOG_MAX_STDOUT_BYTES", 64 * 1024))
+
+
+SANDBOX_TOOL_NAMES = {
+ "Bash",
+ "sandbox_put_artifact",
+ "sandbox_get_artifact",
+ "Write",
+ "Edit",
+ "Read",
+ "Grep",
+ "Glob",
+}
diff --git a/src/backend/core/harness/observation_writer.py b/src/backend/core/harness/observation_writer.py
new file mode 100644
index 00000000..ad630370
--- /dev/null
+++ b/src/backend/core/harness/observation_writer.py
@@ -0,0 +1,134 @@
+"""Bounded background persistence for hook-path observability records.
+
+Hook stages and framework events are recorded on every model and tool step.
+Waiting for their database writes would put observability on the execution
+critical path, so they are queued here and written by one daemon thread.
+``submit`` never blocks: a full queue drops the record and logs the loss.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import queue
+import threading
+import time
+from collections.abc import Callable, Sequence
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+MAX_PENDING = 10_000
+MAX_BATCH = 500
+DROP_LOG_EVERY = 1_000
+
+BatchWrite = Callable[[Any, Sequence[Any]], Any]
+
+
+class ObservationWriter:
+ """Drain ``(sink, write, run_id, records)`` items in per-sink, per-run batches."""
+
+ def __init__(self, *, max_pending: int = MAX_PENDING, max_batch: int = MAX_BATCH) -> None:
+ self._max_pending = max(1, int(max_pending))
+ self._max_batch = max(1, int(max_batch))
+ self._lock = threading.Lock()
+ self._queue: queue.Queue | None = None
+ self._pid: int | None = None
+ self.dropped = 0
+
+ def _ensure_started(self) -> queue.Queue:
+ with self._lock:
+ # A forked worker inherits the queue but not the thread.
+ if self._queue is None or self._pid != os.getpid():
+ self._queue = queue.Queue(self._max_pending)
+ self._pid = os.getpid()
+ threading.Thread(
+ target=self._run,
+ args=(self._queue,),
+ name="harness-observation-writer",
+ daemon=True,
+ ).start()
+ return self._queue
+
+ def submit(self, sink: Any, write: BatchWrite, run_id: str, records: Sequence[Any]) -> bool:
+ """Queue ``write(sink, records)``; return False when the record was dropped."""
+ if not records:
+ return True
+ try:
+ self._ensure_started().put_nowait((sink, write, run_id, tuple(records)))
+ return True
+ except queue.Full:
+ with self._lock:
+ self.dropped += 1
+ dropped = self.dropped
+ if dropped == 1 or dropped % DROP_LOG_EVERY == 0:
+ logger.warning(
+ "Observation queue full (%d pending); dropped %d record batch(es) so far",
+ self._max_pending,
+ dropped,
+ )
+ return False
+
+ def _run(self, pending: queue.Queue) -> None:
+ while True:
+ items = [pending.get()]
+ while len(items) < self._max_batch:
+ try:
+ items.append(pending.get_nowait())
+ except queue.Empty:
+ break
+ try:
+ self._write(items)
+ finally:
+ for _ in items:
+ pending.task_done()
+
+ @staticmethod
+ def _write(items) -> None:
+ # Order is preserved within a sink and run, which is all sequence
+ # allocation depends on; one call per group lets stores batch a run.
+ groups: dict[tuple[int, BatchWrite, str], tuple[Any, list]] = {}
+ for sink, write, run_id, records in items:
+ groups.setdefault((id(sink), write, run_id), (sink, []))[1].extend(records)
+ for (_, write, run_id), (sink, records) in groups.items():
+ try:
+ write(sink, records)
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ "Observation write failed for run %s (%d record(s)): %s",
+ run_id,
+ len(records),
+ exc,
+ )
+
+ def flush(self, timeout: float = 5.0) -> bool:
+ """Wait until every queued record is written; False on timeout."""
+ pending = self._queue
+ if pending is None or self._pid != os.getpid():
+ return True
+ deadline = time.monotonic() + max(0.0, timeout)
+ with pending.all_tasks_done:
+ while pending.unfinished_tasks:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return False
+ pending.all_tasks_done.wait(remaining)
+ return True
+
+
+_WRITER = ObservationWriter()
+
+
+def observation_writer() -> ObservationWriter:
+ return _WRITER
+
+
+def flush_observations(timeout: float = 5.0) -> bool:
+ flushed = _WRITER.flush(timeout)
+ if not flushed:
+ logger.warning("Observation writes still pending after %.1fs flush", timeout)
+ return flushed
+
+
+atexit.register(flush_observations)
diff --git a/src/backend/core/harness/registry/__init__.py b/src/backend/core/harness/registry/__init__.py
new file mode 100644
index 00000000..486ec361
--- /dev/null
+++ b/src/backend/core/harness/registry/__init__.py
@@ -0,0 +1,79 @@
+"""Optional feature modules that plug into core through registered hooks.
+
+Core never imports a module's implementation. A module contributes behaviour by
+registering hooks from its ``setup()`` at startup; when the module is disabled
+it registers nothing and core behaves as if the module were not installed.
+
+Shared modules are loaded in every edition. Additional modules are an edition concern (``core.harness.registry.edition``); whether
+an installed module is switched on is the module's own decision, usually an env
+flag read inside its ``setup()``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from importlib import import_module
+
+from core.harness.registry.user_cleanup import register_user_cleanup
+from core.harness.registry.hook_factories import hook_specs, register_hook_factory
+
+from core.harness.registry.registry import AgentProfile, NativeToolContext, register_tool, install_tools, SandboxCredentials, register_sandbox_credentials, sandbox_credentials, agent_profile, channel_routes, is_enabled, lifecycle_hooks, mark_enabled, register_agent_profile, register_channel_route, register_lifecycle, register_router, register_sandbox_route, routers, sandbox_route
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+_loaded = False
+
+
+def load_models(installed: Iterable[str]) -> None:
+ """Register installed schemas independently of runtime enablement."""
+ for package in installed:
+ import_module(f"{package}.models")
+
+
+def load_extensions(installed: Iterable[str]) -> None:
+ """Import shared and edition modules and let it register itself. Idempotent."""
+ global _loaded
+ if _loaded:
+ return
+ _loaded = True
+ from core.harness.registry.edition import EDITION_EXTENSIONS
+
+ installed = tuple(installed)
+ load_models(installed)
+ from core.harness.registry.schema_lifecycle import load_schema_lifecycle
+ load_schema_lifecycle(installed)
+ for module_name in (*(f"{name}.extension" for name in installed), *EDITION_EXTENSIONS):
+ module = import_module(module_name)
+ if module.setup():
+ mark_enabled(module.NAME)
+ logger.info("extension_enabled", extension=module.NAME)
+ else:
+ logger.info("extension_disabled", extension=module.NAME)
+
+
+__all__ = [
+ "register_user_cleanup",
+ "hook_specs",
+ "register_hook_factory",
+ "AgentProfile",
+ "NativeToolContext",
+ "register_tool",
+ "install_tools",
+ "SandboxCredentials",
+ "register_sandbox_credentials",
+ "sandbox_credentials",
+ "agent_profile",
+ "channel_routes",
+ "is_enabled",
+ "lifecycle_hooks",
+ "load_extensions",
+ "load_models",
+ "register_agent_profile",
+ "register_channel_route",
+ "register_lifecycle",
+ "register_router",
+ "register_sandbox_route",
+ "routers",
+ "sandbox_route",
+]
diff --git a/src/backend/core/harness/registry/authentication.py b/src/backend/core/harness/registry/authentication.py
new file mode 100644
index 00000000..a02bcbad
--- /dev/null
+++ b/src/backend/core/harness/registry/authentication.py
@@ -0,0 +1,39 @@
+"""Optional host authentication before the ordinary cookie/session resolver."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+
+def register_host_authentication(
+ claims: Callable[[Any], bool], resolver: Callable[[Any, Any], Any]
+) -> None:
+ """Install a host identity resolver.
+
+ ``claims(request)`` must answer from the request alone, without I/O: it is
+ asked on every authenticated request, and only a request it claims pays for
+ ``resolver(request, db)`` (database work, run off the event loop).
+ """
+ registration_state().claims = claims
+ registration_state().resolver = resolver
+
+
+def host_user_resolver(request) -> Callable[[Any, Any], Any] | None:
+ """The host resolver when one is registered and claims ``request``, else None."""
+ state = registration_state()
+ if state.resolver is None or state.claims is None or not state.claims(request):
+ return None
+ return state.resolver
+
+
+@dataclass
+class RegistrationState:
+ claims: Callable[[Any], bool] | None = None
+ resolver: Callable[[Any, Any], Any] | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/build_governance.py b/src/backend/core/harness/registry/build_governance.py
new file mode 100644
index 00000000..1fd4907b
--- /dev/null
+++ b/src/backend/core/harness/registry/build_governance.py
@@ -0,0 +1,51 @@
+"""Governance policy applied before capability definitions are persisted."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable
+
+
+@dataclass
+class BuildGovernance:
+ ensure: Callable | None = None
+ validate: Callable | None = None
+ tag_options: Callable | None = None
+ import_policy: Callable | None = None
+
+
+@lru_cache(maxsize=1)
+def build_governance() -> BuildGovernance:
+ return BuildGovernance()
+
+
+def register_build_governance(*, ensure, validate, tag_options, import_policy):
+ services = build_governance()
+ services.ensure, services.validate = ensure, validate
+ services.tag_options, services.import_policy = tag_options, import_policy
+
+
+def ensure_ontology_build_valid(*args, **kwargs):
+ ensure = build_governance().ensure
+ if ensure is None:
+ raise RuntimeError("Capability build governance is not registered")
+ return ensure(*args, **kwargs)
+
+
+def validate_build(db, **kwargs):
+ validate = build_governance().validate
+ if validate is None:
+ raise RuntimeError("Capability build validator is not registered")
+ return validate(db, **kwargs)
+
+
+def list_asset_tag_options(db, asset_type):
+ options = build_governance().tag_options
+ if options is None:
+ raise RuntimeError("Capability tag governance is not registered")
+ return options(db, asset_type)
+
+
+def resolve_plugin_import_ontology_validation(*args, **kwargs):
+ policy = build_governance().import_policy
+ if policy is None:
+ raise RuntimeError("Capability import governance is not registered")
+ return policy(*args, **kwargs)
diff --git a/src/backend/core/harness/registry/cache_invalidation.py b/src/backend/core/harness/registry/cache_invalidation.py
new file mode 100644
index 00000000..02c7286b
--- /dev/null
+++ b/src/backend/core/harness/registry/cache_invalidation.py
@@ -0,0 +1,27 @@
+"""Named host cache invalidations invoked by domain mutations."""
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from functools import lru_cache
+from threading import RLock
+
+
+@dataclass
+class CacheInvalidators:
+ callbacks: dict[tuple[str, str], Callable[[], None]] = field(default_factory=dict)
+ lock: RLock = field(default_factory=RLock)
+
+ def register(self, stage: str, name: str, callback: Callable[[], None]) -> None:
+ with self.lock:
+ self.callbacks[stage, name] = callback
+
+ def run(self, stage: str) -> None:
+ with self.lock:
+ callbacks = tuple(callback for (phase, _), callback in self.callbacks.items() if phase == stage)
+ for callback in callbacks:
+ callback()
+
+
+@lru_cache(maxsize=1)
+def cache_invalidators() -> CacheInvalidators:
+ """Process-owned registration state; callbacks are installed by the host."""
+ return CacheInvalidators()
diff --git a/src/backend/core/harness/registry/capability_management.py b/src/backend/core/harness/registry/capability_management.py
new file mode 100644
index 00000000..564fb6c0
--- /dev/null
+++ b/src/backend/core/harness/registry/capability_management.py
@@ -0,0 +1,27 @@
+"""Installed host implementation of local capability management."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable
+
+
+@dataclass
+class ManagerRegistration:
+ execute: Callable | None = None
+
+
+@lru_cache(maxsize=1)
+def manager_registration() -> ManagerRegistration:
+ return ManagerRegistration()
+
+
+def register_manager_executor(execute: Callable) -> None:
+ manager_registration().execute = execute
+
+
+def execute_manager(*, manager, server_id, headers, cloud_backed, tool_name, input_schema, arguments):
+ execute = manager_registration().execute
+ if execute is None:
+ raise RuntimeError("Local capability manager is not registered")
+ return execute(manager=manager, server_id=server_id, headers=headers,
+ cloud_backed=cloud_backed, tool_name=tool_name,
+ input_schema=input_schema, arguments=arguments)
diff --git a/src/backend/core/harness/registry/capability_revisions.py b/src/backend/core/harness/registry/capability_revisions.py
new file mode 100644
index 00000000..ede0293a
--- /dev/null
+++ b/src/backend/core/harness/registry/capability_revisions.py
@@ -0,0 +1,34 @@
+"""Authorized capability revision snapshots supplied by the installed host."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Protocol
+
+
+class CapabilityRevisions(Protocol):
+ def model(self, db, kind, key): ...
+ def snapshot_in_session(self, db, user, kind, key): ...
+ def snapshot(self, user, kind, key): ...
+ def commit(self, user, kind, key, body): ...
+ def revision(self, files): ...
+ def record_id(self, prefix, user, kind, key): ...
+
+
+@dataclass
+class RevisionRegistration:
+ provider: CapabilityRevisions | None = None
+
+
+@lru_cache(maxsize=1)
+def revision_registration() -> RevisionRegistration:
+ return RevisionRegistration()
+
+
+def register_capability_revisions(provider: CapabilityRevisions) -> None:
+ revision_registration().provider = provider
+
+
+def capability_revisions() -> CapabilityRevisions:
+ provider = revision_registration().provider
+ if provider is None:
+ raise RuntimeError("Capability revision service is not registered")
+ return provider
diff --git a/src/backend/core/harness/registry/capability_runtime.py b/src/backend/core/harness/registry/capability_runtime.py
new file mode 100644
index 00000000..95326b57
--- /dev/null
+++ b/src/backend/core/harness/registry/capability_runtime.py
@@ -0,0 +1,74 @@
+"""Frozen capability authorization supplied by the execution host."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+from core.harness.capability_errors import ViewUnavailable
+
+
+class CapabilityRuntime(Protocol):
+ def pin_agent_definition(self, run_id, user_id, agent, *, scope_id=""): ...
+ def view_for_execution(self, run_id, user_id, *, scope_id=""): ...
+ def get(self, run_id: str, scope_id: str = "") -> Any: ...
+ def validate(self, run: Any, *, user_id: str | None = None,
+ execution_plane: str = "local", only_skill: str | None = None) -> None: ...
+ def validate_activation(self, run: Any, nodes: Any, *, available_mcp=()) -> None: ...
+ def require_root_tool_scope(self, run_id: str, user_id: str, tool_call_id: str, tool_name: str) -> None: ...
+ def child_scope(self, parent_scope: str, kind: str, *identities: str) -> str: ...
+
+
+def register_capability_runtime(provider: CapabilityRuntime) -> None:
+ pass
+ registration_state().provider = provider
+
+
+def require_runtime() -> CapabilityRuntime:
+ if registration_state().provider is None:
+ raise ViewUnavailable("Execution host capability runtime is not registered")
+ return registration_state().provider
+
+
+def get(run_id: str, scope_id: str = ""):
+ return require_runtime().get(run_id, scope_id=scope_id)
+
+
+def validate(run, *, user_id=None, execution_plane="local", only_skill=None):
+ return require_runtime().validate(run, user_id=user_id, execution_plane=execution_plane, only_skill=only_skill)
+
+
+def validate_activation(run, nodes, *, available_mcp=()):
+ return require_runtime().validate_activation(run, nodes, available_mcp=available_mcp)
+
+
+def require_root_tool_scope(run_id, user_id, tool_call_id, tool_name):
+ return require_runtime().require_root_tool_scope(run_id, user_id, tool_call_id, tool_name)
+
+
+def child_scope(parent_scope, kind, *identities):
+ # Scope identities are used even on cloud hosts without device capabilities.
+ import hashlib
+ import json
+ from core.harness.capability_errors import IntegrityFailed
+ if not kind or not identities or any(not str(value or "").strip() for value in identities):
+ raise IntegrityFailed("capability scope requires durable invocation identities")
+ material = json.dumps([str(parent_scope or ""), str(kind), *map(str, identities)], separators=(",", ":"))
+ return "s_" + hashlib.sha256(material.encode()).hexdigest()
+
+
+def pin_agent_definition(run_id, user_id, agent, *, scope_id=""):
+ return require_runtime().pin_agent_definition(run_id, user_id, agent, scope_id=scope_id)
+
+
+def view_for_execution(run_id, user_id, *, scope_id=""):
+ return require_runtime().view_for_execution(run_id, user_id, scope_id=scope_id)
+
+
+@dataclass
+class RegistrationState:
+ provider: CapabilityRuntime | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/catalog_options.py b/src/backend/core/harness/registry/catalog_options.py
new file mode 100644
index 00000000..e7a70220
--- /dev/null
+++ b/src/backend/core/harness/registry/catalog_options.py
@@ -0,0 +1,26 @@
+"""Optional domain-owned option lists used by agent administration."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+
+def register_catalog_options(kind: str, provider: Callable[[Any], list[dict]]) -> None:
+ registration_state().providers[kind] = provider
+
+
+def catalog_options(kind: str, db: Any) -> list[dict]:
+ provider = registration_state().providers.get(kind)
+ return provider(db) if provider is not None else []
+
+
+@dataclass
+class RegistrationState:
+ providers: dict[str, Callable[[Any], list[dict]]] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/chat_execution.py b/src/backend/core/harness/registry/chat_execution.py
new file mode 100644
index 00000000..3c8f558a
--- /dev/null
+++ b/src/backend/core/harness/registry/chat_execution.py
@@ -0,0 +1,27 @@
+"""Top-level chat execution supplied to durable chat workers by the host."""
+from collections.abc import AsyncIterator, Callable
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+
+@dataclass
+class ChatExecution:
+ stream: Callable[..., AsyncIterator[Any]] | None = None
+
+
+@lru_cache(maxsize=1)
+def chat_execution() -> ChatExecution:
+ return ChatExecution()
+
+
+def register_chat_execution(stream: Callable[..., AsyncIterator[Any]]) -> None:
+ chat_execution().stream = stream
+
+
+def stream_chat_workflow(*, raw_user_message, session_messages, user_message, context):
+ stream = chat_execution().stream
+ if stream is None:
+ raise RuntimeError("Chat execution workflow is not registered")
+ return stream(raw_user_message=raw_user_message, session_messages=session_messages,
+ user_message=user_message, context=context)
diff --git a/src/backend/core/harness/registry/chat_history.py b/src/backend/core/harness/registry/chat_history.py
new file mode 100644
index 00000000..5dc083ba
--- /dev/null
+++ b/src/backend/core/harness/registry/chat_history.py
@@ -0,0 +1,21 @@
+"""Composition contract for user-scoped history tools."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable
+
+@dataclass
+class HistoryRegistration:
+ factory: Callable | None = None
+
+@lru_cache(maxsize=1)
+def history_registration():
+ return HistoryRegistration()
+
+def register_chat_history(factory: Callable):
+ history_registration().factory = factory
+
+def history_operations(**scope):
+ factory = history_registration().factory
+ if factory is None:
+ raise RuntimeError("Chat history services are not registered")
+ return factory(**scope)
diff --git a/src/backend/core/harness/registry/compaction.py b/src/backend/core/harness/registry/compaction.py
new file mode 100644
index 00000000..355237be
--- /dev/null
+++ b/src/backend/core/harness/registry/compaction.py
@@ -0,0 +1,62 @@
+"""Conversation-owned compaction and checkpoint services for the model runtime."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable
+
+
+@dataclass
+class CompactionServices:
+ run: Callable | None = None
+ mid_turn: Callable | None = None
+ token_limit: Callable | None = None
+ trigger_ratio: Callable | None = None
+ checkpoint: Callable | None = None
+
+
+@lru_cache(maxsize=1)
+def compaction_services() -> CompactionServices:
+ return CompactionServices()
+
+
+def register_compaction_services(*, run, mid_turn, token_limit, trigger_ratio, checkpoint):
+ services = compaction_services()
+ services.run = run
+ services.mid_turn = mid_turn
+ services.token_limit = token_limit
+ services.trigger_ratio = trigger_ratio
+ services.checkpoint = checkpoint
+
+
+async def run_compaction(*args, **kwargs):
+ run = compaction_services().run
+ if run is None:
+ raise RuntimeError("Conversation compaction is not registered")
+ return await run(*args, **kwargs)
+
+
+async def run_mid_turn_compaction(*args, **kwargs):
+ run = compaction_services().mid_turn
+ if run is None:
+ raise RuntimeError("Mid-turn compaction is not registered")
+ return await run(*args, **kwargs)
+
+
+def resolve_token_limit(context_window, *, ratio=None):
+ resolve = compaction_services().token_limit
+ if resolve is None:
+ raise RuntimeError("Compaction token policy is not registered")
+ return resolve(context_window, ratio=ratio)
+
+
+def resolve_trigger_ratio():
+ resolve = compaction_services().trigger_ratio
+ if resolve is None:
+ raise RuntimeError("Compaction trigger policy is not registered")
+ return resolve()
+
+
+def create_context_checkpoint(**kwargs):
+ factory = compaction_services().checkpoint
+ if factory is None:
+ raise RuntimeError("Context checkpoint service is not registered")
+ return factory(**kwargs)
diff --git a/src/backend/core/harness/registry/device_catalog.py b/src/backend/core/harness/registry/device_catalog.py
new file mode 100644
index 00000000..ee18da46
--- /dev/null
+++ b/src/backend/core/harness/registry/device_catalog.py
@@ -0,0 +1,79 @@
+"""Optional execution-device catalog projected into domain selection."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Protocol, Any
+
+
+class DeviceCatalog(Protocol):
+ def account_plugin_installations(self, user_id): ...
+ def owned_identities(self, user_id): ...
+ def resolve_plugin(self, slug, user_id, *, install_id=None): ...
+ def active(self) -> bool: ...
+ def agent_entries(self) -> list[dict]: ...
+ def plugin_entries(self, user_id=None) -> list[dict]: ...
+ def catalog_overlay(self, user_id=None) -> dict: ...
+ def installations(self, kind: str, user_id=None) -> list[Any]: ...
+ def merge_items(self, base, projected, *, key): ...
+
+
+@dataclass
+class CatalogRegistration:
+ provider: DeviceCatalog | None = None
+
+
+@lru_cache(maxsize=1)
+def catalog_registration() -> CatalogRegistration:
+ return CatalogRegistration()
+
+
+def register_device_catalog(provider: DeviceCatalog) -> None:
+ catalog_registration().provider = provider
+
+
+def active() -> bool:
+ provider = catalog_registration().provider
+ return bool(provider and provider.active())
+
+
+def agent_entries():
+ provider = catalog_registration().provider
+ return provider.agent_entries() if provider else []
+
+
+def plugin_entries(user_id=None):
+ provider = catalog_registration().provider
+ return provider.plugin_entries(user_id) if provider else []
+
+
+def catalog_overlay(user_id=None):
+ provider = catalog_registration().provider
+ return provider.catalog_overlay(user_id) if provider else {}
+
+
+def installations(kind, user_id=None):
+ provider = catalog_registration().provider
+ return provider.installations(kind, user_id=user_id) if provider else []
+
+
+def merge_items(base, projected, *, key):
+ provider = catalog_registration().provider
+ if provider is None:
+ if projected:
+ raise RuntimeError("Device catalog is not registered")
+ return base
+ return provider.merge_items(base, projected, key=key)
+
+
+def resolve_plugin(slug, user_id, *, install_id=None):
+ provider = catalog_registration().provider
+ return provider.resolve_plugin(slug, user_id, install_id=install_id) if provider else None
+
+
+def account_plugin_installations(user_id):
+ provider = catalog_registration().provider
+ return provider.account_plugin_installations(user_id) if provider else []
+
+
+def owned_identities(user_id):
+ provider = catalog_registration().provider
+ return provider.owned_identities(user_id) if provider else set()
diff --git a/src/backend/core/extensions/edition.py b/src/backend/core/harness/registry/edition.py
similarity index 100%
rename from src/backend/core/extensions/edition.py
rename to src/backend/core/harness/registry/edition.py
diff --git a/src/backend/core/harness/registry/evaluation.py b/src/backend/core/harness/registry/evaluation.py
new file mode 100644
index 00000000..4c82baaa
--- /dev/null
+++ b/src/backend/core/harness/registry/evaluation.py
@@ -0,0 +1,72 @@
+"""Reserved execution policy installed by the evaluation host."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+class EvaluationPolicy(Protocol):
+ def apply_context(self, context: dict) -> dict: ...
+ def parse_scope(self, **identity: Any) -> Any: ...
+ def project_agent(self, agent: Any) -> Any: ...
+ def current_paths(self) -> Any: ...
+ def register_tools(self, toolkit: Any, scope: Any, **options: Any) -> None: ...
+ async def authorize(self, request: Any, user_id: str, *, agent_scoped: bool = False) -> None: ...
+
+
+def register_evaluation_policy(policy: EvaluationPolicy) -> None:
+ pass
+ registration_state().policy = policy
+
+
+def is_evaluation_session(value):
+ return isinstance(value, str) and value.startswith("eval_")
+
+
+def require_policy():
+ if registration_state().policy is None:
+ raise ValueError("Reserved evaluation execution is not installed")
+ return registration_state().policy
+
+
+def apply_evaluation_scope(context):
+ if registration_state().policy is None:
+ if any(is_evaluation_session(context.get(k)) for k in ("chat_id", "sandbox_session_id")):
+ require_policy()
+ return context
+ return registration_state().policy.apply_context(context)
+
+
+def parse_evaluation_scope(**identity):
+ if registration_state().policy is None and not any(is_evaluation_session(identity.get(k)) for k in ("chat_id", "session_id")):
+ return None
+ return require_policy().parse_scope(**identity)
+
+
+def evaluation_user_agent(agent):
+ return require_policy().project_agent(agent)
+
+
+def current_path_rules():
+ return registration_state().policy.current_paths() if registration_state().policy is not None else None
+
+
+def register_evaluation_tools(toolkit, scope, **options):
+ require_policy().register_tools(toolkit, scope, **options)
+
+
+async def authorize(request, user_id, *, agent_scoped=False):
+ if not is_evaluation_session(request.chat_id):
+ return
+ await require_policy().authorize(request, user_id, agent_scoped=agent_scoped)
+
+
+@dataclass
+class RegistrationState:
+ policy: EvaluationPolicy | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/execution_account.py b/src/backend/core/harness/registry/execution_account.py
new file mode 100644
index 00000000..8c9cd4ed
--- /dev/null
+++ b/src/backend/core/harness/registry/execution_account.py
@@ -0,0 +1,35 @@
+"""Host account authorization for account-scoped capability selection."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Protocol
+
+class ExecutionAccount(Protocol):
+ def current_account_profile(self) -> str | None: ...
+ def current_local_user_id(self) -> str | None: ...
+ def account_authorized_for(self, user_id: str | None) -> bool: ...
+
+
+def register_execution_account(provider: ExecutionAccount | None) -> None:
+ pass
+ registration_state().provider = provider
+
+def current_account_profile() -> str | None:
+ return registration_state().provider.current_account_profile() if registration_state().provider is not None else None
+
+def current_local_user_id() -> str | None:
+ return registration_state().provider.current_local_user_id() if registration_state().provider is not None else None
+
+def account_authorized_for(user_id: str | None) -> bool:
+ return bool(registration_state().provider is not None and registration_state().provider.account_authorized_for(user_id))
+
+
+@dataclass
+class RegistrationState:
+ provider: ExecutionAccount | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/execution_catalog.py b/src/backend/core/harness/registry/execution_catalog.py
new file mode 100644
index 00000000..fd90b656
--- /dev/null
+++ b/src/backend/core/harness/registry/execution_catalog.py
@@ -0,0 +1,51 @@
+"""Host-owned execution catalog selection, optional on cloud-only hosts."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+class ExecutionCatalog(Protocol):
+ def local_plugin_configs(self, user_id, enabled_only=True) -> dict: ...
+ def bridge_context(self) -> dict[str, Any] | None: ...
+ def apply_to_enabled_mcp_ids(self, ids: list[str] | None) -> list[str] | None: ...
+ def apply_to_enabled_skill_ids(self, ids: list[str] | None) -> list[str] | None: ...
+ def cloud_gateway_mcp_configs(self, enabled_mcp_ids=None, *, resolution_out=None, unavailable_out=None) -> dict: ...
+
+
+def register_execution_catalog(provider: ExecutionCatalog | None) -> None:
+ pass
+ registration_state().provider = provider
+
+def bridge_context():
+ return registration_state().provider.bridge_context() if registration_state().provider is not None else None
+
+def apply_to_enabled_mcp_ids(ids):
+ return registration_state().provider.apply_to_enabled_mcp_ids(ids) if registration_state().provider is not None else ids
+
+def apply_to_enabled_skill_ids(ids):
+ return registration_state().provider.apply_to_enabled_skill_ids(ids) if registration_state().provider is not None else ids
+
+def cloud_gateway_mcp_configs(enabled_mcp_ids=None, *, resolution_out=None, unavailable_out=None):
+ if registration_state().provider is None:
+ return {}
+ return registration_state().provider.cloud_gateway_mcp_configs(enabled_mcp_ids, resolution_out=resolution_out, unavailable_out=unavailable_out)
+
+
+def local_plugin_configs(user_id, enabled_only=True):
+ return registration_state().provider.local_plugin_configs(user_id, enabled_only=enabled_only) if registration_state().provider else {}
+
+
+def authorizes_manager(user_id, server_id, manager):
+ config = local_plugin_configs(user_id).get(server_id)
+ return bool(config and config.get("gateway_plugin") == manager)
+
+
+@dataclass
+class RegistrationState:
+ provider: ExecutionCatalog | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/execution_identity.py b/src/backend/core/harness/registry/execution_identity.py
new file mode 100644
index 00000000..dc0308d2
--- /dev/null
+++ b/src/backend/core/harness/registry/execution_identity.py
@@ -0,0 +1,78 @@
+"""Host-owned execution identity contract for core model and tool operations."""
+from __future__ import annotations
+from contextlib import AbstractContextManager
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+class ExecutionIdentity(Protocol):
+ def context_headers(self, *, model=False) -> dict[str, str]: ...
+ def notify_cloud_changed(self) -> None: ...
+ def get_state(self) -> dict[str, Any] | None: ...
+ def state_fingerprint(self, state: dict[str, Any] | None) -> str: ...
+ def require_current_account(self, state: dict[str, Any]) -> None: ...
+ def ensure_current_authorization(self, state: dict[str, Any] | None = None) -> None: ...
+ def cloud_headers(self, state: dict[str, Any]) -> dict[str, str]: ...
+ def account_scope(self, state: dict[str, Any]) -> AbstractContextManager: ...
+ def clear_state(self) -> None: ...
+
+
+def register_execution_identity(provider: ExecutionIdentity | None) -> None:
+ pass
+ registration_state().provider = provider
+
+
+def require_provider() -> ExecutionIdentity:
+ if registration_state().provider is None:
+ raise RuntimeError("Execution identity provider is not registered")
+ return registration_state().provider
+
+
+def get_state() -> dict[str, Any] | None:
+ return registration_state().provider.get_state() if registration_state().provider is not None else None
+
+
+def state_fingerprint(state: dict[str, Any] | None) -> str:
+ if state is None and registration_state().provider is None:
+ return ""
+ return require_provider().state_fingerprint(state)
+
+
+def require_current_account(state: dict[str, Any]) -> None:
+ require_provider().require_current_account(state)
+
+
+def ensure_current_authorization(state: dict[str, Any] | None = None) -> None:
+ require_provider().ensure_current_authorization(state)
+
+
+def cloud_headers(state: dict[str, Any]) -> dict[str, str]:
+ return require_provider().cloud_headers(state)
+
+
+def account_scope(state: dict[str, Any]) -> AbstractContextManager:
+ return require_provider().account_scope(state)
+
+
+def clear_state() -> None:
+ require_provider().clear_state()
+
+
+def context_headers(*, model=False):
+ return registration_state().provider.context_headers(model=model) if registration_state().provider else {}
+
+
+def notify_cloud_changed():
+ return require_provider().notify_cloud_changed()
+
+
+@dataclass
+class RegistrationState:
+ provider: ExecutionIdentity | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/hook_factories.py b/src/backend/core/harness/registry/hook_factories.py
new file mode 100644
index 00000000..11b41778
--- /dev/null
+++ b/src/backend/core/harness/registry/hook_factories.py
@@ -0,0 +1,69 @@
+"""Per-run hooks contributed by installed modules."""
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from core.harness.hooks import HookSpec
+
+HookFactory = Callable[[Any], Iterable["HookSpec"]]
+
+
+def register_hook_factory(factory: HookFactory) -> None:
+ """Register once; invoke afresh for each agent assembly."""
+ if factory not in registration_state().factories:
+ registration_state().factories.append(factory)
+
+
+def hook_specs(request: Any) -> Iterator[HookSpec]:
+ for factory in tuple(registration_state().factories):
+ yield from factory(request)
+
+
+# Run-domain policies are created at assembly time, never shared between agents.
+from dataclasses import dataclass, field
+from functools import lru_cache
+
+
+@dataclass
+class RunHookFactories:
+ steering: Callable[[], Any] | None = None
+ command_completion: Callable[[], Iterable[HookSpec]] | None = None
+
+
+@lru_cache(maxsize=1)
+def run_hook_factories() -> RunHookFactories:
+ return RunHookFactories()
+
+
+def register_run_hook_factories(*, steering, command_completion) -> None:
+ factories = run_hook_factories()
+ factories.steering = steering
+ factories.command_completion = command_completion
+
+
+def create_steering_policy():
+ factory = run_hook_factories().steering
+ if factory is None:
+ raise RuntimeError("Run steering policy is not registered")
+ return factory()
+
+
+def command_completion_hooks():
+ factory = run_hook_factories().command_completion
+ if factory is None:
+ raise RuntimeError("Command completion hooks are not registered")
+ return factory()
+
+
+@dataclass
+class RegistrationState:
+ factories: list[HookFactory] = field(default_factory=list)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/host_runtime.py b/src/backend/core/harness/registry/host_runtime.py
new file mode 100644
index 00000000..199a56a6
--- /dev/null
+++ b/src/backend/core/harness/registry/host_runtime.py
@@ -0,0 +1,13 @@
+"""Host policy for operations that must be forwarded to a remote execution plane."""
+from collections.abc import Callable
+
+_remote_execution_required: Callable[[], bool] = lambda: False
+
+
+def register_remote_execution_policy(predicate: Callable[[], bool]) -> None:
+ global _remote_execution_required
+ _remote_execution_required = predicate
+
+
+def requires_remote_execution() -> bool:
+ return bool(_remote_execution_required())
diff --git a/src/backend/core/harness/registry/job_operations.py b/src/backend/core/harness/registry/job_operations.py
new file mode 100644
index 00000000..8348949b
--- /dev/null
+++ b/src/backend/core/harness/registry/job_operations.py
@@ -0,0 +1,21 @@
+"""Host binding for scoped job action operations."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable
+
+@dataclass
+class JobRegistration:
+ factory: Callable | None = None
+
+@lru_cache(maxsize=1)
+def job_registration():
+ return JobRegistration()
+
+def register_job_operations(factory: Callable):
+ job_registration().factory = factory
+
+def job_operations(**scope):
+ factory = job_registration().factory
+ if factory is None:
+ raise RuntimeError("Job operations are not registered")
+ return factory(**scope)
diff --git a/src/backend/core/harness/registry/local_files.py b/src/backend/core/harness/registry/local_files.py
new file mode 100644
index 00000000..fa6f13ca
--- /dev/null
+++ b/src/backend/core/harness/registry/local_files.py
@@ -0,0 +1,65 @@
+"""Host-owned file grants and recovery snapshots used by core operations."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Protocol, Any
+
+
+class LocalSecurityProvider(Protocol):
+ def grants_for_gate(self) -> list[Any]: ...
+ def policy_for_gate(self, approval_mode: str) -> Any: ...
+
+
+class LocalSnapshotProvider(Protocol):
+ def snapshot(self, real_path: str) -> str | None: ...
+ def maybe_snapshot_local(self, real_path: str) -> str | None: ...
+
+
+@dataclass
+class LocalFileServices:
+ security: LocalSecurityProvider | None = None
+ snapshots: LocalSnapshotProvider | None = None
+
+
+@lru_cache(maxsize=1)
+def local_file_services() -> LocalFileServices:
+ return LocalFileServices()
+
+
+def register_local_file_services(security: LocalSecurityProvider,
+ snapshots: LocalSnapshotProvider) -> None:
+ services = local_file_services()
+ services.security = security
+ services.snapshots = snapshots
+
+
+def grants_for_gate() -> list[Any]:
+ provider = local_file_services().security
+ if provider is None:
+ raise RuntimeError("Local file security provider is not registered")
+ return provider.grants_for_gate()
+
+
+def policy_for_gate(approval_mode: str) -> Any:
+ provider = local_file_services().security
+ if provider is None:
+ raise RuntimeError("Local file security provider is not registered")
+ return provider.policy_for_gate(approval_mode)
+
+
+def snapshot(real_path: str) -> str | None:
+ provider = local_file_services().snapshots
+ if provider is None:
+ raise RuntimeError("Local file snapshot provider is not registered")
+ return provider.snapshot(real_path)
+
+
+def maybe_snapshot_local(real_path: str) -> str | None:
+ from core.config.local_mode import local_mode_enabled
+ if not local_mode_enabled():
+ return None
+ provider = local_file_services().snapshots
+ if provider is None:
+ raise RuntimeError("Local file snapshot provider is not registered")
+ return provider.maybe_snapshot_local(real_path)
diff --git a/src/backend/core/harness/registry/local_publication.py b/src/backend/core/harness/registry/local_publication.py
new file mode 100644
index 00000000..f5a35e5e
--- /dev/null
+++ b/src/backend/core/harness/registry/local_publication.py
@@ -0,0 +1,69 @@
+"""Host-owned publication of local skill views and installation revisions."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+class LocalPublication(Protocol):
+ def management_lock_path(self): ...
+ def provision_plugin_assets(self, slug): ...
+ def migrate_local_plugins(self): ...
+ def migrate_local_skills(self): ...
+ async def ensure_native_servers(self, configs): ...
+ def publish_local_plugin(self, definition, *, owner_user_id): ...
+ def remove_local_plugin(self, slug): ...
+ def skill_backend(self, *, priority=150, local=False): ...
+ def merge_skill_infos(self, groups, *, backend_hash): ...
+ def view_generation(self): ...
+ def skill_root(self): ...
+ def merge_visible_agents(self, user_id, rows): ...
+ def account_agent_definition(self, agent_id): ...
+ def publish_local_agent(self, definition): ...
+ def remove_local_agent(self, agent_id): ...
+ def rebuild_device_view(self) -> Any: ...
+ def rebuild_user_view(self, user_id: str) -> Any: ...
+ def rebuild_views(self, user_id: str | None) -> Any: ...
+ def remove_local_skill(self, skill_id: str) -> bool: ...
+ def prune_local(self, owners: dict) -> int: ...
+ def publish_local_skill(self, *args: Any, **kwargs: Any) -> Any: ...
+ def device_view_dir(self) -> Any: ...
+ def user_view_dir(self, user_id: str | None) -> Any: ...
+ def stamp_installation(self, db: Any, install_id: str, stamp: Any) -> None: ...
+ def claim_revision(self, db: Any, install_id: str, expected_revision: str) -> int: ...
+
+
+def register_local_publication(provider: LocalPublication) -> None:
+ pass
+ registration_state().provider = provider
+
+def local_publication() -> LocalPublication:
+ if registration_state().provider is None:
+ raise RuntimeError("Host must register local publication before changing local capabilities")
+ return registration_state().provider
+
+
+def provision_plugin_assets(slug):
+ return local_publication().provision_plugin_assets(slug)
+
+
+def migrate_local_plugins():
+ return local_publication().migrate_local_plugins()
+
+
+def migrate_local_skills():
+ return local_publication().migrate_local_skills()
+
+
+async def ensure_native_servers(configs):
+ return await local_publication().ensure_native_servers(configs)
+
+
+@dataclass
+class RegistrationState:
+ provider: LocalPublication | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/marketplace_catalog.py b/src/backend/core/harness/registry/marketplace_catalog.py
new file mode 100644
index 00000000..e74d634e
--- /dev/null
+++ b/src/backend/core/harness/registry/marketplace_catalog.py
@@ -0,0 +1,44 @@
+"""Marketplace-owned listings, grants and installation operations."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+class MarketplaceCatalog(Protocol):
+ def plugin_metadata(self, db: Any, slug: str) -> dict: ...
+ def mcp_version(self, db: Any, slug: str) -> tuple[Any, Any]: ...
+ def normalize_auth_config(self, config: Any, schema: list) -> dict: ...
+ def installation_secrets(self, version: Any, credentials: dict, existing: dict, *, auth_method: str) -> dict: ...
+ async def install_market_item(self, slug: str, **kwargs: Any) -> dict: ...
+ def market_package(self, db: Any, slug: str) -> Any: ...
+ def publish_package(self, db: Any, slug: str, fields: dict, *, created_by: str | None, now: Any) -> str: ...
+ def delete_package(self, db: Any, slug: str) -> None: ...
+ def excluded_skills(self, db: Any, slug: str) -> set[str]: ...
+ def exclude_skill(self, db: Any, slug: str, skill_name: str, *, created_by: str | None) -> bool: ...
+ def suspended_mcp_ids(self, db: Any, ids: list[str]) -> set[str]: ...
+ def list_marketplace_skills(self, db: Any) -> list[dict]: ...
+ def list_marketplace_agents(self, db: Any) -> list[dict]: ...
+ def list_plugins(self, db: Any, user_id: str | None) -> list[dict]: ...
+ def install_marketplace_skill(self, db: Any, slug: str, *, owner_user_id: str | None) -> dict: ...
+ def install_marketplace_agent(self, db: Any, slug: str, *, owner_user_id: str | None) -> dict: ...
+
+
+def register_marketplace_catalog(provider: MarketplaceCatalog) -> None:
+ pass
+ registration_state().provider = provider
+
+def marketplace_catalog() -> MarketplaceCatalog:
+ if registration_state().provider is None:
+ raise RuntimeError("Host must register the marketplace catalog")
+ return registration_state().provider
+
+
+@dataclass
+class RegistrationState:
+ provider: MarketplaceCatalog | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/mcp_clients.py b/src/backend/core/harness/registry/mcp_clients.py
new file mode 100644
index 00000000..65ca4de3
--- /dev/null
+++ b/src/backend/core/harness/registry/mcp_clients.py
@@ -0,0 +1,21 @@
+"""Host-owned SDK client factory used by capability connectivity probes."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Callable, Any
+
+@dataclass
+class ClientFactory:
+ create: Callable[..., Any] | None = None
+
+@lru_cache(maxsize=1)
+def client_factory() -> ClientFactory:
+ return ClientFactory()
+
+def register_mcp_client_factory(factory: Callable[..., Any]) -> None:
+ client_factory().create = factory
+
+def discover_tools(*args, **kwargs):
+ factory = client_factory().create
+ if factory is None:
+ raise RuntimeError("MCP client factory is not registered")
+ return factory(*args, **kwargs)
diff --git a/src/backend/core/harness/registry/model_credentials.py b/src/backend/core/harness/registry/model_credentials.py
new file mode 100644
index 00000000..85410553
--- /dev/null
+++ b/src/backend/core/harness/registry/model_credentials.py
@@ -0,0 +1,70 @@
+"""Host-selected request authorization; no execution-environment imports."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Protocol
+
+
+def register_request_header_resolver(resolver: Callable | None) -> None:
+ pass
+ registration_state().resolver = resolver
+
+
+def prepare_request_headers(api_key, base_url, captured_state=None):
+ if registration_state().resolver is not None:
+ return registration_state().resolver(api_key, base_url, captured_state)
+ return {"Content-Type": "application/json", "Authorization": "Bearer " + api_key}
+
+
+class ModelCredentialProvider(Protocol):
+ """Host contract for account-bound references and request authorization."""
+ def is_reference(self, value): ...
+ def bind(self, reference, base_url): ...
+ def request_hook(self, reference, base_url): ...
+ def sync_client_kwargs(self, reference, base_url, *, timeout=600): ...
+
+
+def register_model_credential_provider(provider):
+ pass
+ registration_state().provider = provider
+
+
+def require_provider():
+ if registration_state().provider is None:
+ from core.harness.capability_errors import CloudUnavailable
+ raise CloudUnavailable("Model credential provider is not registered")
+ return registration_state().provider
+
+
+def is_reference(value):
+ if registration_state().provider is not None:
+ return registration_state().provider.is_reference(value)
+ # A persisted reference must never be sent as a literal bearer credential.
+ return isinstance(value, str) and value.startswith("desktop-capability:")
+
+
+def bind(reference, base_url):
+ return require_provider().bind(reference, base_url)
+
+
+def request_hook(reference, base_url):
+ return require_provider().request_hook(reference, base_url)
+
+
+def sync_client_kwargs(reference, base_url, *, timeout=600):
+ if not is_reference(reference):
+ return {}
+ return require_provider().sync_client_kwargs(reference, base_url, timeout=timeout)
+
+
+@dataclass
+class RegistrationState:
+ resolver: Callable | None = None
+ provider: ModelCredentialProvider | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/model_services.py b/src/backend/core/harness/registry/model_services.py
new file mode 100644
index 00000000..590e2069
--- /dev/null
+++ b/src/backend/core/harness/registry/model_services.py
@@ -0,0 +1,59 @@
+"""Host-provided model operations used by capability and evolution domains."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Callable
+
+
+def register_model_services(*, ranker: Callable[..., list[str]], prompt_reader: Callable[[], list[tuple[str, dict[str, Any]]]], prompt_renderer: Callable, text_agent: Callable, environment: Callable) -> None:
+ pass
+ registration_state().ranker = ranker
+ registration_state().prompt_reader = prompt_reader
+ registration_state().prompt_renderer = prompt_renderer
+ registration_state().text_agent = text_agent
+ registration_state().environment = environment
+
+
+def rank_skills(**kwargs: Any) -> list[str]:
+ if registration_state().ranker is None:
+ raise RuntimeError("Model services are not registered")
+ return registration_state().ranker(**kwargs)
+
+
+def active_prompt_versions() -> list[tuple[str, dict[str, Any]]]:
+ if registration_state().prompt_reader is None:
+ raise RuntimeError("Model services are not registered")
+ return registration_state().prompt_reader()
+
+
+def render_prompt_part(kind, part):
+ if registration_state().prompt_renderer is None:
+ raise RuntimeError("Prompt rendering service is not registered")
+ return registration_state().prompt_renderer(kind, part)
+
+
+async def run_text_agent(prompt, **context):
+ if registration_state().text_agent is None:
+ raise RuntimeError("Text review agent is not registered")
+ return await registration_state().text_agent(prompt, **context)
+
+
+def model_environment():
+ if registration_state().environment is None:
+ raise RuntimeError("Model configuration environment is not registered")
+ return registration_state().environment()
+
+
+@dataclass
+class RegistrationState:
+ ranker: Callable[..., list[str]] | None = None
+ prompt_reader: Callable[[], list[tuple[str, dict[str, Any]]]] | None = None
+ prompt_renderer: Callable | None = None
+ text_agent: Callable | None = None
+ environment: Callable | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/package_sources.py b/src/backend/core/harness/registry/package_sources.py
new file mode 100644
index 00000000..ce2f485c
--- /dev/null
+++ b/src/backend/core/harness/registry/package_sources.py
@@ -0,0 +1,48 @@
+"""Source and upgrade contracts for plugin packages."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+class PackageSource(Protocol):
+ def normalize(self, db: Any, slug: str) -> Any: ...
+ def archive(self, db: Any, slug: str) -> bytes | None: ...
+
+
+class PackageUpgradePolicy(Protocol):
+ def lock_configuration(self, db: Any) -> None: ...
+ def prepare(self, db: Any, existing: Any, package: Any, source: str, approved_revision: str | None) -> Any: ...
+ def reserve_local(self, existing: Any, revision: str, files: Any) -> None: ...
+ def prepare_local(self, db: Any, current: Any, prepared: Any) -> Any: ...
+
+
+def register_package_source(name: str, source: PackageSource) -> None:
+ registration_state().sources[name] = source
+
+
+def package_source(name: str) -> PackageSource:
+ return registration_state().sources[name]
+
+
+def register_package_upgrade_policy(policy: PackageUpgradePolicy) -> None:
+ pass
+ registration_state().upgrade_policy = policy
+
+
+def package_upgrade_policy() -> PackageUpgradePolicy:
+ if registration_state().upgrade_policy is None:
+ raise RuntimeError("Host must register the package upgrade policy before installing plugins")
+ return registration_state().upgrade_policy
+
+
+@dataclass
+class RegistrationState:
+ sources: dict[str, PackageSource] = field(default_factory=dict)
+ upgrade_policy: PackageUpgradePolicy | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/package_views.py b/src/backend/core/harness/registry/package_views.py
new file mode 100644
index 00000000..e4692e5d
--- /dev/null
+++ b/src/backend/core/harness/registry/package_views.py
@@ -0,0 +1,66 @@
+"""Host-owned package interface validation, presentation and enablement."""
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+
+@dataclass
+class PackageViews:
+ project: Callable | None = None
+ normalize: Callable | None = None
+ find: Callable | None = None
+ enabled: Callable | None = None
+ workspace_only: Callable | None = None
+ set_enabled: Callable | None = None
+
+
+@lru_cache(maxsize=1)
+def package_views() -> PackageViews:
+ return PackageViews()
+
+
+def register_package_view(project, *, normalize, find, enabled, workspace_only, set_enabled) -> None:
+ services = package_views()
+ services.project = project
+ services.normalize = normalize
+ services.find = find
+ services.enabled = enabled
+ services.workspace_only = workspace_only
+ services.set_enabled = set_enabled
+
+
+def public_package_view(declaration: Any, *, slug: str) -> dict[str, Any]:
+ project = package_views().project
+ return project(declaration, slug=slug) if project is not None else {}
+
+
+def normalize_ui(declaration):
+ normalize = package_views().normalize
+ if normalize is None:
+ raise RuntimeError("Package interface validator is not registered")
+ return normalize(declaration)
+
+
+def find_module(stored, module_id):
+ find = package_views().find
+ if find is None:
+ raise RuntimeError("Package interface resolver is not registered")
+ return find(stored, module_id)
+
+
+def enabled(db, row, user_id):
+ predicate = package_views().enabled
+ return bool(predicate and predicate(db, row, user_id))
+
+
+def workspace_only(row):
+ predicate = package_views().workspace_only
+ return bool(predicate and predicate(row))
+
+
+def set_enabled(db, row, value, user_id=None):
+ update = package_views().set_enabled
+ if update is None:
+ raise RuntimeError("Package interface enablement policy is not registered")
+ return update(db, row, value, user_id=user_id)
diff --git a/src/backend/core/harness/registry/registry.py b/src/backend/core/harness/registry/registry.py
new file mode 100644
index 00000000..9b7798ca
--- /dev/null
+++ b/src/backend/core/harness/registry/registry.py
@@ -0,0 +1,169 @@
+"""Hook tables that optional modules fill in at startup.
+
+Every table starts empty. Core consults them at its extension points (agent
+assembly, sandbox routing, inbound channel messages, app lifecycle, routers),
+so an unregistered module leaves those code paths exactly as they were.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Optional
+
+
+@dataclass(frozen=True)
+class AgentProfile:
+ """A dedicated agent built by a module instead of the ordinary chat assembly.
+
+ Selected by ``AgentRequest.profile``; ``AgentRequest.profile_ref`` carries the
+ module's own identifier (for example the task the agent works on).
+ """
+
+ # (request) -> system prompt; replaces the whole chat prompt assembly.
+ system_prompt: Callable[[Any], Awaitable[str]]
+ # (toolkit, request, loader=...) -> None; replaces the native chat tools.
+ register_tools: Callable[..., None]
+ # Framework-neutral per-agent policies and optional steering source.
+ hooks: Callable[[Any], list] = field(default=lambda request: [])
+ steering: Callable[[Any], Any] = field(default=lambda request: None)
+ # Only function tools whose name passes are exposed; None keeps all.
+ tool_filter: Optional[Callable[[str], bool]] = None
+ # False: MCP servers come only from the request's explicit list, never catalog defaults.
+ # False also prevents history and skill dependencies from widening explicit bindings.
+ inherit_default_mcps: bool = True
+ # False: the profile's own system prompt lists its skills and how to read them; the
+ # chat skill section (which points to view_text_file) is left out.
+ native_skill_prompt: bool = True
+
+
+def mark_enabled(name: str) -> None:
+ registration_state().enabled.add(name)
+
+
+def is_enabled(name: str) -> bool:
+ return name in registration_state().enabled
+
+
+def register_agent_profile(name: str, profile: AgentProfile) -> None:
+ registration_state().agent_profiles[name] = profile
+
+
+def agent_profile(name: str) -> AgentProfile:
+ profile = registration_state().agent_profiles.get(name)
+ if profile is None:
+ raise ValueError(f"Agent profile {name!r} is not enabled")
+ return profile
+
+
+def register_sandbox_route(session_prefix: str, provider: Callable[[], Any]) -> None:
+ """Sandbox sessions whose id starts with ``session_prefix`` use ``provider()``."""
+ registration_state().sandbox_routes.append((session_prefix, provider))
+
+
+def sandbox_route(session_id: str) -> Optional[Callable[[], Any]]:
+ for prefix, provider in registration_state().sandbox_routes:
+ if session_id.startswith(prefix):
+ return provider
+ return None
+
+
+def register_channel_route(
+ handles: Callable[[Any, Any], bool],
+ admit: Callable[..., Awaitable[None]],
+) -> None:
+ """``handles(db, conn)`` claims an inbound bot message; ``admit(conn, msg, adapter)`` takes it."""
+ registration_state().channel_routes.append((handles, admit))
+
+
+def channel_routes() -> tuple:
+ return tuple(registration_state().channel_routes)
+
+
+def register_lifecycle(
+ start: Callable[[], Awaitable[None]],
+ stop: Callable[[], Awaitable[None]],
+) -> None:
+ """Background work started on each service worker and stopped on shutdown."""
+ registration_state().lifecycle.append((start, stop))
+
+
+def lifecycle_hooks() -> tuple:
+ return tuple(registration_state().lifecycle)
+
+
+def register_router(router: Any, policy_key: Optional[str] = None) -> None:
+ """An API router mounted with the same edition policy as ``EE_ROUTERS`` entries."""
+ registration_state().routers.append((router, policy_key))
+
+
+def routers() -> tuple:
+ return tuple(registration_state().routers)
+
+
+@dataclass(frozen=True)
+class SandboxCredentials:
+ """Contribution to cloud credentials; callbacks retain provider-specific SDK context."""
+ name: str
+ environment: Optional[Callable[[], dict[str, str]]] = None
+ mounts: Optional[Callable[[Optional[str]], list[Any]]] = None
+ before_command: Optional[Callable[[Any, Any, str, str], Awaitable[None]]] = None
+ after_command: Optional[Callable[[Any, Any, str, str], Awaitable[None]]] = None
+ prepare_order: int = 0
+
+
+def register_sandbox_credentials(provider: SandboxCredentials) -> None:
+ registration_state().sandbox_credentials[provider.name] = provider
+
+
+def sandbox_credentials() -> tuple[SandboxCredentials, ...]:
+ return tuple(registration_state().sandbox_credentials.values())
+
+
+@dataclass(frozen=True)
+class NativeToolContext:
+ request: Any
+ interactive: bool
+ is_channel_run: bool
+ sandbox_session_id: Any
+ project_scope: Any
+ skill_ids: Any
+ enabled_mcp_keys: Any
+ logger: Any
+
+
+@dataclass(frozen=True)
+class NativeToolContribution:
+ name: str
+ factory: Callable[[Any, NativeToolContext], Awaitable[None]]
+ enabled: Callable[[NativeToolContext], bool]
+ stage: str
+
+
+def register_tool(name, factory, enabled, *, stage):
+ """Register a conditional tool at an ordered native assembly stage."""
+ registration_state().tools[name] = NativeToolContribution(name, factory, enabled, stage)
+
+
+async def install_tools(stage, toolkit, context):
+ for contribution in tuple(registration_state().tools.values()):
+ if contribution.stage == stage and contribution.enabled(context):
+ await contribution.factory(toolkit, context)
+
+
+@dataclass
+class RegistrationState:
+ enabled: set[str] = field(default_factory=set)
+ agent_profiles: dict[str, AgentProfile] = field(default_factory=dict)
+ sandbox_routes: list[tuple[str, Callable[[], Any]]] = field(default_factory=list)
+ channel_routes: list[tuple[Callable[[Any, Any], bool], Callable[..., Awaitable[None]]]] = field(default_factory=list)
+ lifecycle: list[tuple[Callable[[], Awaitable[None]], Callable[[], Awaitable[None]]]] = field(default_factory=list)
+ routers: list[tuple[Any, Optional[str]]] = field(default_factory=list)
+ sandbox_credentials: dict[str, SandboxCredentials] = field(default_factory=dict)
+ tools: dict[str, NativeToolContribution] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/resource_owners.py b/src/backend/core/harness/registry/resource_owners.py
new file mode 100644
index 00000000..9edce5ac
--- /dev/null
+++ b/src/backend/core/harness/registry/resource_owners.py
@@ -0,0 +1,48 @@
+"""Installed application owners supply policy for generic plugin resources."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Callable
+
+
+@dataclass(frozen=True)
+class ResourceOwner:
+ binding: Callable[..., Any]
+ resolve_installation: Callable[..., Any]
+ exact_installation: Callable[..., Any]
+ authorize: Callable[..., Any]
+ observe: Callable[..., Any]
+ record: Callable[..., Any]
+ renew: Callable[..., Any]
+ arm: Callable[..., Any]
+ invalidate: Callable[..., Any]
+ sweep: Callable[..., Any]
+ collect: Callable[..., Any]
+ create: Callable[..., Any]
+ owned: Callable[..., Any]
+ lease: Callable[..., Any]
+ reserve: Callable[..., Any]
+ close: Callable[..., Any]
+ revoke_session: Callable[..., Any]
+
+
+def register_resource_owner(owner: ResourceOwner) -> None:
+ pass
+ registration_state().owner = owner
+
+
+def resource_owner() -> ResourceOwner:
+ if registration_state().owner is None:
+ raise RuntimeError("Host must register the application resource owner")
+ return registration_state().owner
+
+
+@dataclass
+class RegistrationState:
+ owner: ResourceOwner | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/retrieval_context.py b/src/backend/core/harness/registry/retrieval_context.py
new file mode 100644
index 00000000..ba24ff18
--- /dev/null
+++ b/src/backend/core/harness/registry/retrieval_context.py
@@ -0,0 +1,37 @@
+"""External retrieval context supplied by an installed domain."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass
+
+
+def register_retrieval_context(provider: Callable[[list[str]], tuple[dict, dict]] | None) -> None:
+ pass
+ registration_state().provider = provider
+
+
+def runtime_request_context(selected_ids: list[str]) -> tuple[dict, dict]:
+ return registration_state().provider(selected_ids) if registration_state().provider is not None else ({}, {})
+
+
+def register_retrieval_access_filter(provider: Callable) -> None:
+ pass
+ registration_state().access_filter = provider
+
+
+def filter_accessible_kb_ids(db, user_id: str, selected_ids: list[str]) -> list[str]:
+ if registration_state().access_filter is None:
+ raise RuntimeError("Retrieval access policy is not registered")
+ return registration_state().access_filter(db, user_id, selected_ids)
+
+
+@dataclass
+class RegistrationState:
+ provider: Callable[[list[str]], tuple[dict, dict]] | None = None
+ access_filter: Callable | None = None
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/run_cancellation.py b/src/backend/core/harness/registry/run_cancellation.py
new file mode 100644
index 00000000..49e51e6e
--- /dev/null
+++ b/src/backend/core/harness/registry/run_cancellation.py
@@ -0,0 +1,37 @@
+"""Run cancellation ports preserve the caller's transaction boundary."""
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+
+@dataclass
+class RunCancellationServices:
+ cancel: Callable[..., Awaitable[bool]] | None = None
+ cancel_in_session: Callable[..., Any] | None = None
+
+
+@lru_cache(maxsize=1)
+def cancellation_services() -> RunCancellationServices:
+ return RunCancellationServices()
+
+
+def register_run_cancellation(cancel: Callable[..., Awaitable[bool]], *,
+ cancel_in_session: Callable[..., Any]) -> None:
+ services = cancellation_services()
+ services.cancel = cancel
+ services.cancel_in_session = cancel_in_session
+
+
+async def cancel_run(run_id: str, *, user_id: str, stop_terminal_worker: bool = False) -> bool:
+ cancel = cancellation_services().cancel
+ if cancel is None:
+ raise RuntimeError("Run cancellation service is not registered")
+ return await cancel(run_id, user_id=user_id, stop_terminal_worker=stop_terminal_worker)
+
+
+def cancel_run_in_session(db, run, *, reason: str, now) -> None:
+ cancel = cancellation_services().cancel_in_session
+ if cancel is None:
+ raise RuntimeError("Transactional run cancellation service is not registered")
+ cancel(db, run, reason=reason, now=now)
diff --git a/src/backend/core/harness/registry/run_capabilities.py b/src/backend/core/harness/registry/run_capabilities.py
new file mode 100644
index 00000000..d241094c
--- /dev/null
+++ b/src/backend/core/harness/registry/run_capabilities.py
@@ -0,0 +1,114 @@
+"""Per-run capability contracts supplied by an installed execution environment."""
+from __future__ import annotations
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+class PreparedCapabilities(Protocol):
+ run_id: str
+ bindings: Mapping[str, Any]
+ dependency_report: dict
+ unavailable: dict
+
+ def validate_tool_call(self, server_id: str) -> bool: ...
+ def release(self) -> None: ...
+ def validate_selected_skills(self, skill_ids: list[str] | None) -> None: ...
+ def validate(self, *, user_id: str) -> None: ...
+ def record_tool_scope(self, tool_call_id: str, tool_name: str) -> None: ...
+ def child_scope(self, kind: str, *identities: str) -> str: ...
+ def skill_loader(self) -> Any: ...
+ def tool_guard(self, *, scope_id: str): ...
+ def bind_mcp(self, servers: dict, resolution: Any) -> dict: ...
+ def preflight(self, **context: Any) -> PreparedCapabilities: ...
+
+
+class RunCapabilityProvider(Protocol):
+ def cloud_plugin_selection(self, plugin_id, user_id): ...
+ def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids): ...
+ def plugin_skill_defaults(self, user_id, skill_ids): ...
+ def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None): ...
+ def available_skill_names(self, names, *, user_id): ...
+ def plugin_binding_ids(self, plugin_ids, *, user_id): ...
+ async def prepare(self, request: Any, **selection: Any) -> tuple[Any, Any, PreparedCapabilities | None]: ...
+
+
+class CloudCapabilities:
+ def cloud_plugin_selection(self, plugin_id, user_id):
+ return None
+
+ def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids):
+ return skill_ids, mcp_ids
+
+ def plugin_skill_defaults(self, user_id, skill_ids):
+ return skill_ids
+
+ def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None):
+ return None
+
+ def available_skill_names(self, names, *, user_id):
+ return list(names)
+
+ def plugin_binding_ids(self, plugin_ids, *, user_id):
+ return [], []
+
+ async def prepare(self, request, **selection):
+ return selection["_caps_skill_ids"], None, None
+
+
+_default = CloudCapabilities()
+
+
+def register_run_capability_provider(name, provider, *, enabled):
+ registration_state().providers[name] = (enabled, provider)
+
+
+def run_capability_provider() -> RunCapabilityProvider:
+ for enabled, provider in tuple(registration_state().providers.values()):
+ if enabled():
+ return provider
+ return _default
+
+
+def has_run_capabilities() -> bool:
+ return run_capability_provider() is not _default
+
+
+async def prepare_run_capabilities(request, **selection):
+ return await run_capability_provider().prepare(request, **selection)
+
+
+def ensure_cloud_ready(user_id, *, skill_keys=None, plugin_keys=None):
+ return run_capability_provider().ensure_ready(user_id, skill_keys=skill_keys, plugin_keys=plugin_keys)
+
+
+def filter_available_names(names, *, user_id):
+ return run_capability_provider().available_skill_names(names, user_id=user_id)
+
+
+def cloud_binding_ids(plugin_ids, *, user_id=None):
+ return run_capability_provider().plugin_binding_ids(plugin_ids, user_id=user_id)
+
+
+def plugin_skill_defaults(user_id, skill_ids):
+ return run_capability_provider().plugin_skill_defaults(user_id, skill_ids)
+
+
+def resolve_explicit_ids(user_id, skill_ids, mcp_ids):
+ return run_capability_provider().resolve_explicit_ids(user_id, skill_ids, mcp_ids)
+
+
+def cloud_plugin_selection(plugin_id, user_id):
+ return run_capability_provider().cloud_plugin_selection(plugin_id, user_id)
+
+
+@dataclass
+class RegistrationState:
+ providers: dict[str, tuple[Callable[[], bool], RunCapabilityProvider]] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/schema_lifecycle.py b/src/backend/core/harness/registry/schema_lifecycle.py
new file mode 100644
index 00000000..3e1a30d5
--- /dev/null
+++ b/src/backend/core/harness/registry/schema_lifecycle.py
@@ -0,0 +1,36 @@
+"""Synchronous installed-module lifecycle callbacks during local schema startup."""
+from __future__ import annotations
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+from importlib import import_module
+
+
+def register_schema_lifecycle(name: str, callback: Callable[[], None]) -> None:
+ """Register an idempotent local-SQLite initializer by stable owner name."""
+ registration_state().callbacks[name] = callback
+
+
+def load_schema_lifecycle(installed: Iterable[str]) -> None:
+ """Compose schema hooks without starting runtime workers or enabling features."""
+ for package in installed:
+ register = getattr(import_module(f"{package}.extension"), "register_schema", None)
+ if register is not None:
+ register()
+
+
+def initialize_local_schema() -> None:
+ """Run in startup order; initialization failures must block startup."""
+ for callback in tuple(registration_state().callbacks.values()):
+ callback()
+
+
+@dataclass
+class RegistrationState:
+ callbacks: dict[str, Callable[[], None]] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/skill_content.py b/src/backend/core/harness/registry/skill_content.py
new file mode 100644
index 00000000..e9bb7495
--- /dev/null
+++ b/src/backend/core/harness/registry/skill_content.py
@@ -0,0 +1,24 @@
+"""Installed integrations adapt their own skill text for the execution host."""
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from functools import lru_cache
+
+
+@dataclass
+class SkillContentAdapters:
+ adapters: dict[str, Callable[[str, str | None], str]] = field(default_factory=dict)
+
+
+@lru_cache(maxsize=1)
+def skill_content_adapters() -> SkillContentAdapters:
+ return SkillContentAdapters()
+
+
+def register_skill_content_adapter(name, adapter):
+ skill_content_adapters().adapters[name] = adapter
+
+
+def adapt_skill_content(content: str, user_id: str | None) -> str:
+ for adapter in tuple(skill_content_adapters().adapters.values()):
+ content = adapter(content, user_id)
+ return content
diff --git a/src/backend/core/harness/registry/system_configuration.py b/src/backend/core/harness/registry/system_configuration.py
new file mode 100644
index 00000000..8df46ff5
--- /dev/null
+++ b/src/backend/core/harness/registry/system_configuration.py
@@ -0,0 +1,31 @@
+"""Persistence contract for the configuration cache."""
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Protocol
+
+
+class SystemConfigurationStore(Protocol):
+ def set(self, key: str, value: str | None, updated_by: str): ...
+ def bulk_set(self, items: list[dict], updated_by: str): ...
+ def load(self) -> tuple[dict, dict]: ...
+
+
+@dataclass
+class ConfigurationRegistration:
+ store: SystemConfigurationStore | None = None
+
+
+@lru_cache(maxsize=1)
+def configuration_registration() -> ConfigurationRegistration:
+ return ConfigurationRegistration()
+
+
+def register_system_configuration_store(store: SystemConfigurationStore) -> None:
+ configuration_registration().store = store
+
+
+def system_configuration_store() -> SystemConfigurationStore:
+ store = configuration_registration().store
+ if store is None:
+ raise RuntimeError("System configuration persistence is not registered")
+ return store
diff --git a/src/backend/core/harness/registry/tool_operations.py b/src/backend/core/harness/registry/tool_operations.py
new file mode 100644
index 00000000..916bda30
--- /dev/null
+++ b/src/backend/core/harness/registry/tool_operations.py
@@ -0,0 +1,59 @@
+"""Host registrations for component-owned local dispatch and durable receipts."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+
+class RemoteOperationConflict(ValueError):
+ """The authenticated operation key is already bound to another request."""
+
+
+@dataclass(frozen=True)
+class ToolOperations:
+ receipt_tools: frozenset[str]
+ local_tools: frozenset[str]
+ bind: Callable[..., str]
+ lookup: Callable[..., Any]
+ remember: Callable[..., Any]
+ dispatch_local: Callable[..., Any]
+ prepare_arguments: Callable[..., dict] | None = None
+
+
+def register_tool_operations(component: str, operations: ToolOperations) -> None:
+ registration_state().components[component] = operations
+
+
+def receipt_operations(tool_name: str) -> ToolOperations | None:
+ return next((value for value in registration_state().components.values() if tool_name in value.receipt_tools), None)
+
+
+def local_tool_allowed(component: str, tool_name: str) -> bool:
+ provider = registration_state().components.get(component)
+ return provider is not None and tool_name in provider.local_tools
+
+
+async def local_tool_result(component, tool_name, arguments, headers, *, authorize=None):
+ provider = registration_state().components.get(component)
+ if provider is None or tool_name not in provider.local_tools:
+ return None
+ return await provider.dispatch_local(component, tool_name, arguments, headers, authorize=authorize)
+
+
+def prepare_tool_arguments(tool_name, arguments, user_id, chat_id):
+ provider = receipt_operations(tool_name)
+ if provider is None or provider.prepare_arguments is None:
+ return arguments
+ return provider.prepare_arguments(tool_name, arguments, user_id, chat_id)
+
+
+@dataclass
+class RegistrationState:
+ components: dict[str, ToolOperations] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/services/desktop_gateway_uploads.py b/src/backend/core/harness/registry/transfer_channels.py
similarity index 53%
rename from src/backend/core/services/desktop_gateway_uploads.py
rename to src/backend/core/harness/registry/transfer_channels.py
index 51faaf1e..fd736fd4 100644
--- a/src/backend/core/services/desktop_gateway_uploads.py
+++ b/src/backend/core/harness/registry/transfer_channels.py
@@ -8,7 +8,7 @@
from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
# Wire contract shared by the uploading client and the receiving gateway route.
@@ -27,35 +27,39 @@ class UploadChannel:
package: Callable[[Dict[str, Any], Dict[str, str]], Awaitable[Tuple[bytes, str]]]
-def _site_publish_channel() -> UploadChannel:
- from core.services.desktop_site_publish import package_local_site
+@dataclass(frozen=True)
+class UploadReceiver:
+ options_model: Any
+ max_bytes: int
+ publish: Callable[..., Any]
+
+
+def register_upload_channel(source_plugin, tool_name, factory, *, receiver=None):
+ registration_state().CHANNELS.setdefault(source_plugin, {})[tool_name] = factory
+ if receiver is not None:
+ registration_state().RECEIVERS[factory().endpoint] = receiver
- return UploadChannel(
- endpoint="site-publish",
- content_type="application/gzip",
- package=package_local_site,
- )
+def register_result_localizer(source_plugin, tool_name, localize):
+ registration_state().LOCALIZERS.setdefault(source_plugin, {})[tool_name] = localize
-# Providing plugin slug → its uploading tools. The slug is the fact the platform
-# registered for the server (``AdminMcpServer.source_plugin``); keying on it means
-# a shared and a per-user install of the same plugin resolve identically, and no
-# id string ever has to be taken apart. Values are factories so a channel's module
-# is imported only when that plugin's tool is actually invoked.
-_CHANNELS: Dict[str, Dict[str, Callable[[], UploadChannel]]] = {
- "sites": {"publish_site": _site_publish_channel},
-}
+
+def upload_receiver(endpoint):
+ receiver = registration_state().RECEIVERS.get(endpoint)
+ if receiver is None:
+ raise RuntimeError("gateway upload component is not registered")
+ return receiver
def upload_channel(source_plugin: str, tool_name: str) -> Optional[UploadChannel]:
"""The declared upload channel for this plugin's tool, if it has one."""
- factory = _CHANNELS.get(str(source_plugin or ""), {}).get(tool_name)
+ factory = registration_state().CHANNELS.get(str(source_plugin or ""), {}).get(tool_name)
return factory() if factory is not None else None
def endpoint_plugin(endpoint: str) -> Optional[str]:
"""Which providing plugin a gateway upload endpoint belongs to."""
- for slug, tools in _CHANNELS.items():
+ for slug, tools in registration_state().CHANNELS.items():
for factory in tools.values():
if factory().endpoint == endpoint:
return slug
@@ -64,12 +68,18 @@ def endpoint_plugin(endpoint: str) -> Optional[str]:
def result_localizer(source_plugin: str, tool_name: str):
"""Resolve component-owned receipt handling independently of upload transport."""
- if source_plugin != "sites":
- return None
- from core.services.desktop_site_publish import localize_application_result, localize_site_result
-
- return {
- "publish_site": localize_site_result,
- "manage_application": localize_application_result,
- "publish_mcp": localize_application_result,
- }.get(tool_name)
+ return registration_state().LOCALIZERS.get(source_plugin, {}).get(tool_name)
+
+
+@dataclass
+class RegistrationState:
+ CHANNELS: Dict[str, Dict[str, Callable[[], UploadChannel]]] = field(default_factory=dict)
+ LOCALIZERS: Dict[str, Dict[str, Callable[..., Any]]] = field(default_factory=dict)
+ RECEIVERS: Dict[str, UploadReceiver] = field(default_factory=dict)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/registry/user_cleanup.py b/src/backend/core/harness/registry/user_cleanup.py
new file mode 100644
index 00000000..862a4d14
--- /dev/null
+++ b/src/backend/core/harness/registry/user_cleanup.py
@@ -0,0 +1,27 @@
+"""Transactional cleanup contributions for ORM user deletion."""
+from __future__ import annotations
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+
+def register_user_cleanup(handler: Callable[[Any, Any], None]) -> None:
+ if handler not in registration_state().handlers:
+ registration_state().handlers.append(handler)
+
+
+def cleanup_user(session: Any, user: Any) -> None:
+ for handler in tuple(registration_state().handlers):
+ handler(session, user)
+
+
+@dataclass
+class RegistrationState:
+ handlers: list[Callable[[Any, Any], None]] = field(default_factory=list)
+
+
+_REGISTRATIONS = RegistrationState()
+
+
+def registration_state() -> RegistrationState:
+ return _REGISTRATIONS
diff --git a/src/backend/core/harness/tool_call_context.py b/src/backend/core/harness/tool_call_context.py
new file mode 100644
index 00000000..c0906a5d
--- /dev/null
+++ b/src/backend/core/harness/tool_call_context.py
@@ -0,0 +1,5 @@
+"""Execution identity shared by nested tool calls and run entry points."""
+from contextvars import ContextVar
+
+CURRENT_TOOL_CALL_ID: ContextVar[str] = ContextVar("jx_current_tool_call_id", default="")
+CURRENT_RUN_BINDING: ContextVar[tuple[str, str] | None] = ContextVar("jx_current_run_binding", default=None)
diff --git a/src/backend/core/harness/tool_effect_context.py b/src/backend/core/harness/tool_effect_context.py
new file mode 100644
index 00000000..78d3f461
--- /dev/null
+++ b/src/backend/core/harness/tool_effect_context.py
@@ -0,0 +1,18 @@
+"""Invocation identity shared by tool execution and transport adapters."""
+from __future__ import annotations
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass(frozen=True)
+class ToolInvocationContext:
+ effect_id: str
+ result_id: str
+ idempotency_key: str
+ run_id: str
+ fencing_owner: str
+
+
+CURRENT_TOOL_EFFECT: ContextVar[Optional[ToolInvocationContext]] = ContextVar(
+ "jx_current_tool_effect", default=None
+)
diff --git a/src/backend/core/harness/usage.py b/src/backend/core/harness/usage.py
index 230a0c9f..d6ad31fb 100644
--- a/src/backend/core/harness/usage.py
+++ b/src/backend/core/harness/usage.py
@@ -4,7 +4,7 @@
import asyncio
import threading
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any, Protocol
@@ -50,33 +50,56 @@ def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt:
"""Append one attempt and return it with its allocated sequence."""
+def record_attempt_batch(recorder: UsageRecorder, attempts: Sequence[UsageAttempt]) -> tuple[UsageAttempt, ...]:
+ batch = getattr(recorder, "record_attempts", None)
+ if batch is not None:
+ return tuple(batch(attempts))
+ return tuple(recorder.record_attempt(attempt) for attempt in attempts)
+
+
def attempt_status_for_exception(exc: BaseException) -> str:
"""Classify provider/library timeout types without importing each SDK."""
name = type(exc).__name__.lower()
return "timeout" if isinstance(exc, TimeoutError) or "timeout" in name else "failed"
-async def record_usage_safely(
+async def record_attempts_safely(
recorder: UsageRecorder,
- attempt: UsageAttempt,
+ attempts: Sequence[UsageAttempt],
*,
timeout_ms: int = 1_000,
-) -> UsageAttempt | None:
- """Bound observability I/O so it cannot become an execution dependency."""
+) -> tuple[UsageAttempt, ...] | None:
+ """Bound observability I/O so it cannot become an execution dependency.
+
+ All attempts go to the recorder in one call, so batching recorders write
+ them in one transaction.
+ """
+ if not attempts:
+ return ()
try:
return await asyncio.wait_for(
- asyncio.to_thread(recorder.record_attempt, attempt),
+ asyncio.to_thread(record_attempt_batch, recorder, tuple(attempts)),
timeout=max(1, int(timeout_ms)) / 1_000,
)
except Exception: # noqa: BLE001
return None
+async def record_usage_safely(
+ recorder: UsageRecorder,
+ attempt: UsageAttempt,
+ *,
+ timeout_ms: int = 1_000,
+) -> UsageAttempt | None:
+ recorded = await record_attempts_safely(recorder, (attempt,), timeout_ms=timeout_ms)
+ return recorded[0] if recorded else None
+
+
class MemoryUsageRecorder:
"""Thread-safe append-only recorder for unit tests and embedded runtimes."""
def __init__(self) -> None:
- self._lock = threading.Lock()
+ self._lock = threading.RLock()
self._rows: dict[str, list[UsageAttempt]] = {}
def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt:
@@ -91,6 +114,10 @@ def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt:
rows.append(recorded)
return recorded
+ def record_attempts(self, attempts: Sequence[UsageAttempt]) -> tuple[UsageAttempt, ...]:
+ with self._lock:
+ return tuple(self.record_attempt(attempt) for attempt in attempts)
+
def attempts(self, run_id: str) -> tuple[UsageAttempt, ...]:
with self._lock:
return tuple(self._rows.get(run_id, ()))
diff --git a/src/backend/core/infra/background.py b/src/backend/core/infra/background.py
index fdb2628f..2083485f 100644
--- a/src/backend/core/infra/background.py
+++ b/src/backend/core/infra/background.py
@@ -18,14 +18,13 @@
from __future__ import annotations
import asyncio
+from dataclasses import dataclass, field
from typing import Any, Coroutine, Set
from core.infra.logging import get_logger
logger = get_logger(__name__)
-_tasks: Set[asyncio.Task] = set()
-
def spawn(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Task:
"""把 *coro* 挂到当前事件循环后台执行,返回值不必接收。
@@ -42,16 +41,27 @@ def spawn(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Task:
) from None
task = loop.create_task(coro, name=name)
- _tasks.add(task)
+ process_state().tasks.add(task)
task.add_done_callback(_forget)
return task
def _forget(task: asyncio.Task) -> None:
- _tasks.discard(task)
+ process_state().tasks.discard(task)
if task.cancelled():
return
exc = task.exception()
if exc is not None:
logger.warning("background_task_failed", task=task.get_name(), error=str(exc))
+
+@dataclass
+class BackgroundState:
+ tasks: Set[asyncio.Task] = field(default_factory=lambda: set())
+
+
+_STATE = BackgroundState()
+
+
+def process_state() -> BackgroundState:
+ return _STATE
diff --git a/src/backend/core/infra/byte_range.py b/src/backend/core/infra/byte_range.py
new file mode 100644
index 00000000..0cdd13f2
--- /dev/null
+++ b/src/backend/core/infra/byte_range.py
@@ -0,0 +1,71 @@
+"""HTTP single-range responses for content already held in memory.
+
+Starlette's ``FileResponse`` answers ``Range`` requests by itself; routes that
+return ``Response(bytes)`` do not, so browsers cannot seek inside audio/video
+served by them. Multi-range requests are answered with the full body (200),
+which RFC 9110 allows.
+"""
+
+import re
+from typing import Mapping, Optional
+
+from starlette.responses import Response
+
+_SINGLE_RANGE = re.compile(r"bytes=(\d*)-(\d*)", re.IGNORECASE)
+
+
+def _resolve(range_header: str, size: int) -> Optional[tuple[int, int]]:
+ """Return the inclusive ``(start, end)`` span, ``None`` to send the full body,
+ or raise ``ValueError`` when the range cannot be satisfied."""
+ match = _SINGLE_RANGE.fullmatch(range_header.strip())
+ if not match or match.groups() == ("", ""):
+ return None
+ first, last = match.groups()
+ if first and last and int(first) > int(last):
+ # An invalid range-spec is ignored (RFC 9110 §14.2), not refused.
+ return None
+ if first:
+ start = int(first)
+ end = min(int(last), size - 1) if last else size - 1
+ else:
+ suffix = int(last)
+ if suffix == 0:
+ raise ValueError("empty suffix range")
+ start, end = max(size - suffix, 0), size - 1
+ if start >= size:
+ raise ValueError("range outside content")
+ return start, end
+
+
+def is_continuation_range(range_header: Optional[str]) -> bool:
+ """True for a range that resumes mid-file — the follow-up requests a media
+ player sends while seeking, which should not be audited as new reads."""
+ match = _SINGLE_RANGE.fullmatch((range_header or "").strip())
+ return bool(match and match.group(1) and int(match.group(1)) > 0)
+
+
+def byte_range_response(
+ data: bytes,
+ range_header: Optional[str],
+ *,
+ media_type: str,
+ headers: Mapping[str, str],
+) -> Response:
+ """Serve ``data`` whole (200), as one slice (206) or as unsatisfiable (416)."""
+ base = {**headers, "Accept-Ranges": "bytes"}
+ if not range_header:
+ return Response(data, media_type=media_type, headers=base)
+ size = len(data)
+ try:
+ span = _resolve(range_header, size)
+ except ValueError:
+ return Response(status_code=416, headers={**base, "Content-Range": f"bytes */{size}"})
+ if span is None:
+ return Response(data, media_type=media_type, headers=base)
+ start, end = span
+ return Response(
+ data[start : end + 1],
+ status_code=206,
+ media_type=media_type,
+ headers={**base, "Content-Range": f"bytes {start}-{end}/{size}"},
+ )
diff --git a/src/backend/core/infra/crypto.py b/src/backend/core/infra/crypto.py
index 77b9aefc..72eff767 100644
--- a/src/backend/core/infra/crypto.py
+++ b/src/backend/core/infra/crypto.py
@@ -20,6 +20,7 @@
import hashlib
import logging
import os
+from dataclasses import dataclass
from typing import Optional
logger = logging.getLogger(__name__)
@@ -27,38 +28,34 @@
# Local-dev fallback key: used only when neither EMAIL_SECRET_KEY nor ADMIN_TOKEN is configured; emits a warning.
# In production always configure EMAIL_SECRET_KEY (or at least ADMIN_TOKEN), otherwise moving/reinstalling machines makes old ciphertext undecryptable.
_DEV_FALLBACK = "hugagent-email-dev-secret-please-override"
-_warned = False
def _deployment_secret() -> str:
- global _warned
sec = (os.environ.get("EMAIL_SECRET_KEY") or "").strip()
if sec:
return sec
sec = (os.environ.get("ADMIN_TOKEN") or "").strip()
if sec:
return sec
- if not _warned:
+ if not process_state().warned:
logger.warning(
"[crypto] 未配置 EMAIL_SECRET_KEY / ADMIN_TOKEN,邮箱授权码加密使用 dev 兜底密钥;"
"生产请务必配置 EMAIL_SECRET_KEY,否则重装/换机后旧凭据无法解密。"
)
- _warned = True
+ process_state().warned = True
return _DEV_FALLBACK
# The derived key comes from the deployment-level secret and is constant at runtime → lazily construct it once and reuse, avoiding the SHA256+Fernet init on every encrypt/decrypt.
-_fernet_cache = None
def _fernet():
- global _fernet_cache
- if _fernet_cache is None:
+ if process_state().fernet_cache is None:
from cryptography.fernet import Fernet
key = base64.urlsafe_b64encode(hashlib.sha256(_deployment_secret().encode("utf-8")).digest())
- _fernet_cache = Fernet(key)
- return _fernet_cache
+ process_state().fernet_cache = Fernet(key)
+ return process_state().fernet_cache
def encrypt_secret(plaintext: str) -> str:
@@ -75,3 +72,16 @@ def decrypt_secret(token: Optional[str]) -> Optional[str]:
except Exception as exc: # noqa: BLE001 (InvalidToken etc. are all downgraded to None)
logger.warning("[crypto] decrypt_secret failed: %s", type(exc).__name__)
return None
+
+
+@dataclass
+class CryptoState:
+ warned: object = False
+ fernet_cache: object = None
+
+
+_STATE = CryptoState()
+
+
+def process_state() -> CryptoState:
+ return _STATE
diff --git a/src/backend/core/capabilities/junction.py b/src/backend/core/infra/directory_links.py
similarity index 92%
rename from src/backend/core/capabilities/junction.py
rename to src/backend/core/infra/directory_links.py
index 2ebfc25d..b7482ac0 100644
--- a/src/backend/core/capabilities/junction.py
+++ b/src/backend/core/infra/directory_links.py
@@ -46,7 +46,7 @@ def wrapped(*args, **kwargs):
return decorate
-def _strip_windows_prefix(raw: str) -> str:
+def strip_windows_prefix(raw: str) -> str:
if raw.startswith("\\\\?\\UNC\\"):
return "\\\\" + raw[8:]
if raw.startswith(_WIN_PREFIX):
@@ -54,7 +54,7 @@ def _strip_windows_prefix(raw: str) -> str:
return raw
-def _extended_windows_path(raw: str) -> str:
+def extended_windows_path(raw: str) -> str:
"""Extended Win32 paths work without changing machine-wide path policy."""
raw = ntpath.abspath(raw)
if raw.startswith(_WIN_PREFIX):
@@ -64,18 +64,18 @@ def _extended_windows_path(raw: str) -> str:
return _WIN_PREFIX + raw
-def _native(path: Path) -> Path:
- return Path(_extended_windows_path(str(path))) if _IS_WINDOWS else path
+def native_path(path: Path) -> Path:
+ return Path(extended_windows_path(str(path))) if _IS_WINDOWS else path
def _resolved(path: Path) -> Path:
- raw = os.path.realpath(_native(path))
- return Path(_strip_windows_prefix(raw) if _IS_WINDOWS else raw)
+ raw = os.path.realpath(native_path(path))
+ return Path(strip_windows_prefix(raw) if _IS_WINDOWS else raw)
def _lstat(path: Path) -> Optional[os.stat_result]:
try:
- return os.lstat(_native(path))
+ return os.lstat(native_path(path))
except FileNotFoundError:
return None
@@ -98,9 +98,9 @@ def read_directory_link(path: Path) -> Optional[Path]:
"""Absolute target of a link, or None when ``path`` is not a link."""
if not is_directory_link(path):
return None
- raw = os.readlink(_native(path))
+ raw = os.readlink(native_path(path))
if _IS_WINDOWS:
- raw = _strip_windows_prefix(raw)
+ raw = strip_windows_prefix(raw)
target = Path(raw)
if not target.is_absolute():
target = path.parent / target
@@ -128,12 +128,12 @@ def assert_target_allowed(target: Path, allowed_roots: Iterable[Path]) -> None:
def remove_directory_link(path: Path) -> None:
"""Remove the link itself; never follows it, never deletes a real directory."""
if not is_directory_link(path):
- if _native(path).exists():
+ if native_path(path).exists():
raise LinkError(f"{path} is a real directory, not a link; refusing to remove")
return
if _IS_WINDOWS:
# Junctions and directory symlinks are removed with rmdir on NTFS.
- os.rmdir(_native(path))
+ os.rmdir(native_path(path))
else:
os.unlink(path)
@@ -181,7 +181,7 @@ def _create_windows_junction(link: Path, target: Path) -> None:
]
kernel.DeviceIoControl.restype = wintypes.BOOL
kernel.CloseHandle.argtypes = [wintypes.HANDLE]
- native_link = _native(link)
+ native_link = native_path(link)
native_link.mkdir()
handle = kernel.CreateFileW(str(native_link), 0x40000000, 7, None, 3, 0x02200000, None)
invalid = ctypes.c_void_p(-1).value
@@ -213,13 +213,13 @@ def _create_windows_junction(link: Path, target: Path) -> None:
@_link_errors("create")
def create_directory_link(link: Path, target: Path, *, allowed_roots: Iterable[Path]) -> None:
"""Create ``link`` → ``target``. ``target`` must exist and be under an allowed root."""
- if not _native(target).is_dir():
+ if not native_path(target).is_dir():
raise LinkError(f"link target does not exist or is not a directory: {target}")
assert_target_allowed(target, allowed_roots)
if _lstat(link) is not None:
raise LinkError(f"link path already exists: {link}")
- if not _native(link.parent).is_dir():
- _native(link.parent).mkdir(parents=True, exist_ok=True)
+ if not native_path(link.parent).is_dir():
+ native_path(link.parent).mkdir(parents=True, exist_ok=True)
if _IS_WINDOWS:
try:
_create_windows_junction(link, _resolved(target))
@@ -248,7 +248,7 @@ def ensure_directory_link(link: Path, target: Path, *, allowed_roots: Iterable[P
return True
wanted = Path(os.path.normpath(_resolved(target)))
assert_target_allowed(wanted, allowed_roots)
- if not _native(wanted).is_dir():
+ if not native_path(wanted).is_dir():
raise LinkError(f"link target does not exist or is not a directory: {wanted}")
current = read_directory_link(link)
if current is not None:
diff --git a/src/backend/core/infra/dns_cache.py b/src/backend/core/infra/dns_cache.py
index 90e75509..64b3a90c 100644
--- a/src/backend/core/infra/dns_cache.py
+++ b/src/backend/core/infra/dns_cache.py
@@ -29,67 +29,79 @@
import socket
import threading
import time
+from dataclasses import dataclass, field
from typing import Any, Dict, Optional, Tuple
DEFAULT_TTL_S = 300.0
MAX_ENTRIES = 256
-_lock = threading.Lock()
-_cache: Dict[Tuple[Any, ...], Tuple[float, Any]] = {}
-_original = None
+
_ttl_s = DEFAULT_TTL_S
def _store(key: Tuple[Any, ...], answer: Any, now: float) -> None:
- with _lock:
- if len(_cache) >= MAX_ENTRIES:
- for stale in [k for k, (expires, _) in _cache.items() if expires <= now]:
- del _cache[stale]
- if len(_cache) >= MAX_ENTRIES:
- _cache.clear()
- _cache[key] = (now + _ttl_s, answer)
+ with process_state().lock:
+ if len(process_state().cache) >= MAX_ENTRIES:
+ for stale in [k for k, (expires, _) in process_state().cache.items() if expires <= now]:
+ del process_state().cache[stale]
+ if len(process_state().cache) >= MAX_ENTRIES:
+ process_state().cache.clear()
+ process_state().cache[key] = (now + _ttl_s, answer)
def _cached_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
key = (host, port, family, type, proto, flags)
now = time.monotonic()
- with _lock:
- hit = _cache.get(key)
+ with process_state().lock:
+ hit = process_state().cache.get(key)
if hit is not None and hit[0] > now:
return hit[1]
# A failure is never cached: the exception propagates from here, so a name
# that starts resolving is picked up on the very next attempt.
- answer = _original(host, port, family, type, proto, flags)
+ answer = process_state().original(host, port, family, type, proto, flags)
_store(key, answer, now)
return answer
def install(ttl_s: Optional[float] = None) -> bool:
"""Install the cache process-wide. Idempotent; True the first time it takes."""
- global _original, _ttl_s
+ global _ttl_s
configured = os.getenv("HUGAGENT_DNS_CACHE_TTL_S")
_ttl_s = float(configured) if configured else float(DEFAULT_TTL_S if ttl_s is None else ttl_s)
- if _ttl_s <= 0 or _original is not None:
+ if _ttl_s <= 0 or process_state().original is not None:
return False
- _original = socket.getaddrinfo
+ process_state().original = socket.getaddrinfo
socket.getaddrinfo = _cached_getaddrinfo
return True
def uninstall() -> None:
- global _original
clear()
- if _original is not None:
- socket.getaddrinfo = _original
- _original = None
+ if process_state().original is not None:
+ socket.getaddrinfo = process_state().original
+ process_state().original = None
def clear() -> None:
- with _lock:
- _cache.clear()
+ with process_state().lock:
+ process_state().cache.clear()
def installed() -> bool:
- return _original is not None
+ return process_state().original is not None
+
+
+@dataclass
+class DnsCacheState:
+ lock: object = field(default_factory=lambda: threading.Lock())
+ cache: Dict[Tuple[Any, ...], Tuple[float, Any]] = field(default_factory=lambda: {})
+ original: object = None
+
+
+_STATE = DnsCacheState()
+
+
+def process_state() -> DnsCacheState:
+ return _STATE
diff --git a/src/backend/core/infra/failures.py b/src/backend/core/infra/failures.py
new file mode 100644
index 00000000..50c2e4fa
--- /dev/null
+++ b/src/backend/core/infra/failures.py
@@ -0,0 +1,13 @@
+"""Observable best-effort failures without logging payloads or credentials."""
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def record_best_effort_failure(operation: str, error: Exception) -> None:
+ """Preserve a caller's fallback while identifying the failed operation.
+
+ Exception messages and tracebacks can contain tokens or user content. Record
+ only the operation and exception class at this generic boundary.
+ """
+ logger.warning("best_effort_failure operation=%s error_type=%s", operation, type(error).__name__)
diff --git a/src/backend/core/capabilities/lockfile.py b/src/backend/core/infra/file_lock.py
similarity index 100%
rename from src/backend/core/capabilities/lockfile.py
rename to src/backend/core/infra/file_lock.py
diff --git a/src/backend/core/immutable.py b/src/backend/core/infra/immutable.py
similarity index 100%
rename from src/backend/core/immutable.py
rename to src/backend/core/infra/immutable.py
diff --git a/src/backend/core/infra/json_output.py b/src/backend/core/infra/json_output.py
new file mode 100644
index 00000000..7eabceaa
--- /dev/null
+++ b/src/backend/core/infra/json_output.py
@@ -0,0 +1,49 @@
+"""Parse structured model output with the existing fenced/prose fallbacks."""
+
+import json
+import re
+from typing import Any, Dict, List, Optional
+
+
+def parse_json_array_lenient(text: str) -> Optional[List[Any]]:
+ if not text:
+ return None
+ candidates = [text.strip()]
+ m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
+ if m:
+ candidates.append(m.group(1).strip())
+ s, e = text.find("["), text.rfind("]")
+ if s != -1 and e != -1 and e > s:
+ candidates.append(text[s : e + 1])
+ for c in candidates:
+ try:
+ obj = json.loads(c)
+ if isinstance(obj, list):
+ return obj
+ except (json.JSONDecodeError, ValueError):
+ continue
+ return None
+
+
+def parse_json_lenient(text: str) -> Optional[Dict[str, Any]]:
+ """Three-level lenient parsing (object), following plan_mode._parse_plan_json."""
+ for candidate in _json_candidates(text):
+ try:
+ obj = json.loads(candidate)
+ if isinstance(obj, dict):
+ return obj
+ except (json.JSONDecodeError, ValueError):
+ continue
+ return None
+
+
+def _json_candidates(text: str):
+ if not text:
+ return
+ yield text.strip()
+ m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
+ if m:
+ yield m.group(1).strip()
+ s, e = text.find("{"), text.rfind("}")
+ if s != -1 and e != -1 and e > s:
+ yield text[s : e + 1]
diff --git a/src/backend/core/infra/leader.py b/src/backend/core/infra/leader.py
index c91b284d..d3b28370 100644
--- a/src/backend/core/infra/leader.py
+++ b/src/backend/core/infra/leader.py
@@ -1,207 +1,138 @@
-"""Work that must happen in exactly one backend process.
-
-The backend serves requests from several uvicorn workers, but a large part of
-what it starts at boot is not per-worker work: reapers, schedulers, recovery
-passes, outbound channel connections and the seeding of default rows. Running
-those once per worker is not merely wasteful — a scheduler tick fires a task N
-times, a recovery pass fights itself over the same orphan rows, and two seeders
-race on the same unique key.
-
-Two shapes cover every such step, and both are expressed against the
-``hold``/``claim`` mutex in :mod:`core.infra.ephemeral`, so a deployment
-without Redis (the desktop's single-process backend) gets the same semantics
-from the in-process store it already uses for every other short-lived key.
-
-``Leadership``
- A renewed lease for the long-lived loops. One worker wins it and starts
- them; it keeps the lease alive for as long as it is healthy. If that worker
- dies, the lease lapses and the next worker to look takes over and starts
- them there — which is the property a bare "decide once at boot" check does
- not have, and the reason this is a loop rather than a single check.
-
-``run_once``
- A barrier for the one-shot boot steps. Exactly one worker runs the step;
- the others wait for it to finish before continuing, so no worker begins
- serving against half-seeded state.
-"""
-
+"""Lease-owned background work and fail-closed worker startup barriers."""
from __future__ import annotations
import asyncio
import contextlib
+import math
import os
import socket
import uuid
-from typing import Awaitable, Callable, Optional
+from typing import Awaitable, Callable
from core.infra.ephemeral import get_ephemeral_state
from core.infra.logging import get_logger
from core.infra.worker_count import resolve as resolved_workers
logger = get_logger(__name__)
-
-_LEADER_PREFIX = "jx:leader:"
-_ONCE_PREFIX = "jx:boot:"
-
-# The lease has to outlive a renewal that is merely slow — a worker pausing for
-# a GC or a busy disk must not hand its loops to a second process — while still
-# expiring soon enough that a *dead* worker's loops restart promptly. Renewal
-# runs at a third of it, so two consecutive renewals can be lost before the
-# lease actually lapses.
_LEASE_SECONDS = 45
-_RENEW_SECONDS = _LEASE_SECONDS / 3
-
-# How often a worker waiting on a boot barrier looks to see whether the worker
-# running the step has finished.
_POLL_SECONDS = 0.2
+Callback = Callable[[], Awaitable[None]]
def _token() -> str:
- """An identity no other process can accidentally present."""
- return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
+ return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex}"
def coordination_required() -> bool:
- """Whether this deployment actually runs more than one backend process.
-
- A single-process backend has nobody to coordinate with, and saying so is
- not an optimisation — it is the difference between booting and not: the
- desktop profile runs one process with no Redis at all, and neither it nor a
- deployment whose Redis is briefly down should acquire a dependency on a
- mutex that can only ever be uncontended.
-
- It asks :func:`~core.infra.worker_count.resolve` rather than reading
- ``WEB_CONCURRENCY``, and that distinction is load-bearing twice over. The
- resolved count already refuses multi-worker without Redis, which is exactly
- when :func:`get_ephemeral_state` would hand back the per-process store and
- this lease would quietly stop being a lease. And a launch path that never
- ran the entrypoint — ``cli.py``, ``api.app.main`` — would otherwise read the
- *requested* value and believe a single process is a cluster.
- """
return resolved_workers() > 1
class Leadership:
- """A renewed lease naming one process the owner of a role's singleton work."""
+ """Stop owned work before releasing the token; uncertainty revokes ownership."""
def __init__(self, role: str, *, lease_seconds: int = _LEASE_SECONDS) -> None:
- self._key = f"{_LEADER_PREFIX}{role}"
+ self._key = f"jx:leader:{role}"
self._role = role
self._token = _token()
self._lease = int(lease_seconds)
self._elected = False
- self._task: Optional[asyncio.Task] = None
- self._work: Optional[asyncio.Task] = None
+ self._task: asyncio.Task | None = None
+ self._work: asyncio.Task | None = None
+ self._on_deposed: Callback | None = None
async def _hold(self) -> bool:
if not coordination_required():
return True
try:
- return await get_ephemeral_state().hold(self._key, self._token, ttl=self._lease)
- except Exception as exc: # noqa: BLE001 - a store blip must not depose a healthy leader
+ state = get_ephemeral_state()
+ # Renewal must never recreate an expired lease.
+ operation = (
+ state.compare_exchange(self._key, self._token, self._token, ttl=self._lease)
+ if self._elected
+ else state.hold(self._key, self._token, ttl=self._lease)
+ )
+ return await asyncio.wait_for(operation, timeout=self._lease / 3)
+ except Exception as exc:
logger.warning("leader_hold_failed", role=self._role, error=str(exc))
- return self._elected
+ return False
+
+ async def _depose(self) -> None:
+ if not self._elected:
+ return
+ if self._work is not None:
+ if not self._work.done():
+ self._work.cancel()
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ await self._work
+ self._work = None
+ # A failed stop must not release ownership or permit a new local start.
+ if self._on_deposed is not None:
+ await self._on_deposed()
+ self._elected = False
+ if coordination_required():
+ try:
+ await get_ephemeral_state().release(self._key, self._token)
+ except Exception as exc:
+ logger.warning("leader_release_failed", role=self._role, error=str(exc))
- async def _supervise(
- self,
- on_elected: Callable[[], Awaitable[None]],
- on_deposed: Callable[[], Awaitable[None]],
- ) -> None:
+ async def _supervise(self, on_elected: Callback) -> None:
while True:
holding = await self._hold()
- if holding and not self._elected:
+ failed = self._work is not None and self._work.done() and (
+ self._work.cancelled() or self._work.exception() is not None
+ )
+ if self._elected and (not holding or failed):
+ logger.warning("leader_deposed", role=self._role, startup_failed=failed)
+ await self._depose()
+ elif holding and not self._elected:
self._elected = True
- logger.info("leader_elected", role=self._role, token=self._token)
- # Started, never awaited: the callback brings up every singleton
- # loop in turn and can easily outlast a lease. Awaiting it here
- # would stop renewing for that whole stretch, the lease would
- # lapse mid-startup, and a second worker would elect itself and
- # bring up a duplicate set — the exact outcome this exists to
- # prevent, now paid for twice.
self._work = asyncio.create_task(on_elected(), name=f"leader_work:{self._role}")
- if not coordination_required():
- # Uncontended by construction: nothing can take this role,
- # so there is nothing left to supervise. Returning keeps
- # single-process deployments — the default, and the
- # battery-powered desktop one — free of a forever timer.
- return
- elif not holding and self._elected:
- # Losing a lease we were renewing means this process stalled
- # past the whole lease; another worker owns the loops now and
- # ours have to stop, or both run.
- self._elected = False
- logger.warning("leader_deposed", role=self._role, token=self._token)
- await self._cancel_work()
- await on_deposed()
- await asyncio.sleep(_RENEW_SECONDS)
-
- async def _cancel_work(self) -> None:
- if self._work is None or self._work.done():
- return
- self._work.cancel()
- with contextlib.suppress(asyncio.CancelledError):
- await self._work
-
- def start(
- self,
- *,
- on_elected: Callable[[], Awaitable[None]],
- on_deposed: Callable[[], Awaitable[None]],
- ) -> None:
- self._task = asyncio.create_task(
- self._supervise(on_elected, on_deposed), name=f"leader:{self._role}"
- )
+ await asyncio.sleep(self._lease / 3)
+
+ def start(self, *, on_elected: Callback, on_deposed: Callback) -> None:
+ if self._task is not None:
+ raise RuntimeError("Leadership already started")
+ self._on_deposed = on_deposed
+ self._task = asyncio.create_task(self._supervise(on_elected), name=f"leader:{self._role}")
async def stop(self) -> None:
- if self._task is not None and not self._task.done():
- self._task.cancel()
+ if self._task is not None:
+ if not self._task.done():
+ self._task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._task
- # The elected work used to live in the deferred-startup task, which the
- # shutdown path cancels; now it is ours, so cancelling it is too.
- await self._cancel_work()
- if self._elected:
- self._elected = False
- try:
- await get_ephemeral_state().drop(self._key)
- except Exception as exc: # noqa: BLE001 - shutdown must not raise
- logger.warning("leader_release_failed", role=self._role, error=str(exc))
+ self._task = None
+ await self._depose()
async def run_once(
- name: str, step: Callable[[], Awaitable[None]], *, timeout: float = 300.0
+ name: str, step: Callback, *, timeout: float = 300.0
) -> None:
- """Run *step* in one worker of this boot; the rest wait for it to finish.
-
- The workers of a single boot share a parent — uvicorn's master — so its pid
- names this boot without any coordination between them. The done marker is
- deliberately short-lived: a later restart is *supposed* to run the boot
- steps again, and only workers of the same boot, which all start within a
- few seconds of one another, ever need to see it.
- """
+ """All workers observe success, failure or timeout of required initialization."""
if not coordination_required():
await step()
return
-
state = get_ephemeral_state()
- boot = f"{_ONCE_PREFIX}{socket.gethostname()}:{os.getppid()}:{name}"
+ boot = f"jx:boot:{socket.gethostname()}:{os.getppid()}:{name}"
lock, done = f"{boot}:lock", f"{boot}:done"
-
- if await state.claim(lock, ttl=int(timeout)):
- try:
- await step()
- finally:
- # The marker is written even when the step raised: a failed boot
- # step is this worker's problem to log, not a reason to hang every
- # other worker until the timeout.
- await state.put(done, "1", ttl=60)
- return
-
- for _ in range(int(timeout / _POLL_SECONDS)):
- if await state.get(done) is not None:
+ deadline = asyncio.get_running_loop().time() + timeout
+ while True:
+ outcome = await state.get(done)
+ if outcome == "ok":
+ return
+ if outcome is not None:
+ raise RuntimeError(f"Required startup step failed: {name}")
+ if await state.claim(lock, ttl=max(1, math.ceil(timeout) + 1)):
+ try:
+ await asyncio.wait_for(step(), timeout=timeout)
+ except BaseException:
+ await state.put(done, "failed", ttl=max(60, math.ceil(timeout) * 2))
+ raise
+ await state.put(done, "ok", ttl=max(60, math.ceil(timeout) * 2))
return
+ if asyncio.get_running_loop().time() >= deadline:
+ raise TimeoutError(f"Required startup step timed out: {name}")
await asyncio.sleep(_POLL_SECONDS)
- logger.warning("boot_step_wait_timeout", name=name, after_seconds=timeout)
__all__ = ["Leadership", "coordination_required", "run_once"]
diff --git a/src/backend/core/infra/local_runtime/__init__.py b/src/backend/core/infra/local_runtime/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/core/infra/local_runtime/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/infra/local_runtime/dynamic_servers.py b/src/backend/core/infra/local_runtime/dynamic_servers.py
new file mode 100644
index 00000000..0754ef8b
--- /dev/null
+++ b/src/backend/core/infra/local_runtime/dynamic_servers.py
@@ -0,0 +1,64 @@
+"""Own lifecycle and loop affinity for dynamically selected bundled sidecars."""
+import asyncio
+import sys
+
+_lock = asyncio.Lock()
+
+async def ensure_dynamic_servers(bindings):
+ import core.infra.local_runtime.startup as host
+
+ if not host._cloud_serves_tools():
+ return # The local-only launcher already owns every bundled MCP.
+ if not bindings:
+ return
+ # Isolated agents execute on worker loops. Process handles, the watchdog
+ # and the serialization lock remain on the backend's lifecycle loop.
+ owner_loop = (
+ host._WATCHDOG.get_loop() if host._WATCHDOG is not None else asyncio.get_running_loop()
+ )
+ if owner_loop is not asyncio.get_running_loop():
+ future = asyncio.run_coroutine_threadsafe(_start(bindings), owner_loop)
+ await asyncio.wrap_future(future)
+ return
+ await _start(bindings)
+
+
+async def _start(bindings):
+ import core.infra.local_runtime.startup as host
+ from extensions.mcp_servers._ports import package_name
+
+ async with _lock:
+ for sid, port in sorted(bindings):
+ label = "plugin_mcp:" + sid
+ running = next(
+ (p for name, p in host._PROCS if name == label and p.returncode is None), None
+ )
+ if running is not None:
+ continue
+ if await host._tcp_port_ready("127.0.0.1", port):
+ raise RuntimeError("local plugin MCP port is already owned by another process")
+ proc = await host._spawn(
+ label,
+ [
+ sys.executable,
+ "-m",
+ f"extensions.mcp_servers.{package_name(sid)}.server",
+ "--transport",
+ "streamable-http",
+ "--port",
+ str(port),
+ ],
+ )
+ if proc is None:
+ raise RuntimeError("local plugin MCP could not start")
+ try:
+ await host._wait_for_mcp_ports(
+ proc, {sid: port}, timeout=host._ready_timeout_seconds()
+ )
+ await host._list_mcp_tool_names(sid, port)
+ except BaseException:
+ from core.infra.local_runtime.lifecycle import discard
+
+ await discard(label, proc)
+ raise
+ host._start_watchdog()
diff --git a/src/backend/orchestration/local_sidecar_lifecycle.py b/src/backend/core/infra/local_runtime/lifecycle.py
similarity index 89%
rename from src/backend/orchestration/local_sidecar_lifecycle.py
rename to src/backend/core/infra/local_runtime/lifecycle.py
index 57c5a033..3221b0e5 100644
--- a/src/backend/orchestration/local_sidecar_lifecycle.py
+++ b/src/backend/core/infra/local_runtime/lifecycle.py
@@ -1,19 +1,28 @@
"""Lifecycle ownership shared by startup, lazy plugin activation and restart."""
+
from __future__ import annotations
+
import asyncio
import contextlib
import os
import signal
from typing import List, Optional
-from orchestration import local_subprocess as host
+
+import core.infra.local_runtime.startup as host
+import core.infra.local_runtime.runner_auth as runner_auth
_spawn_lock = asyncio.Lock()
-async def _spawn(label: str, argv: List[str], *, require_registered=False) -> Optional["asyncio.subprocess.Process"]:
+
+async def _spawn(
+ label: str, argv: List[str], *, require_registered=False
+) -> Optional["asyncio.subprocess.Process"]:
async with _spawn_lock:
if host._SHUTTING_DOWN or (require_registered and host._SPECS.get(label) != argv):
return None
- running = next((p for name, p in host._PROCS if name == label and p.returncode is None), None)
+ running = next(
+ (p for name, p in host._PROCS if name == label and p.returncode is None), None
+ )
if running is not None:
return running
return await _create(label, argv)
@@ -54,7 +63,7 @@ async def _start_script_runner(py: str) -> None:
# 关起来的那条命令(回环在沙箱里依然可达)。没有这个密钥,它再调一次 /processes/start
# 就能拿到一个不带任何约束的子进程,等于从自己所在的沙箱里走出来。密钥经环境变量
# 交给 sidecar,用户命令那一侧用的是白名单 env,拿不到它。
- os.environ[host.runner_auth.ENV_VAR] = host.runner_auth.ensure_token()
+ os.environ[runner_auth.ENV_VAR] = runner_auth.ensure_token()
runner_port = host._script_runner_port()
# 清理后目标端口仍有监听者 → 是外部程序在占用。不同品牌的桌面壳
# 应通过独立端口命名空间避免走到这里。
@@ -100,8 +109,7 @@ async def _start_script_runner(py: str) -> None:
if asyncio.get_event_loop().time() > runner_deadline:
await stop_local_sidecars()
raise RuntimeError(
- f"本机代码执行服务未就绪(127.0.0.1:{runner_port})——"
- "端口可能被其它程序占用"
+ f"本机代码执行服务未就绪(127.0.0.1:{runner_port})——" "端口可能被其它程序占用"
)
await asyncio.sleep(0.5)
@@ -114,9 +122,7 @@ async def _supervise_sidecars() -> None:
for index, (label, proc) in enumerate(list(host._PROCS)):
if proc.returncode is None:
continue
- host.logger.warning(
- "local_sidecar_exited", sidecar=label, returncode=proc.returncode
- )
+ host.logger.warning("local_sidecar_exited", sidecar=label, returncode=proc.returncode)
try:
host._PROCS.remove((label, proc))
except ValueError:
@@ -160,8 +166,9 @@ async def stop_local_sidecars() -> None:
except asyncio.TimeoutError:
try:
proc.kill()
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.infra.local_runtime.lifecycle:stop_local_sidecars', fallback_error)
host.logger.info("local_sidecar_stopped", sidecar=label)
host._PROCS.clear()
@@ -191,9 +198,7 @@ def _child_env() -> dict:
# single-machine profile.
env["MCP_HOST"] = "127.0.0.1"
env["MCP_BIND_HOST"] = "127.0.0.1"
- inherited_pythonpath = [
- entry for entry in env.get("PYTHONPATH", "").split(os.pathsep) if entry
- ]
+ inherited_pythonpath = [entry for entry in env.get("PYTHONPATH", "").split(os.pathsep) if entry]
pythonpath = [host._BACKEND_DIR]
pythonpath.extend(entry for entry in inherited_pythonpath if entry != host._BACKEND_DIR)
env["PYTHONPATH"] = os.pathsep.join(pythonpath)
diff --git a/src/backend/core/sandbox/providers/script_runner/auth.py b/src/backend/core/infra/local_runtime/runner_auth.py
similarity index 100%
rename from src/backend/core/sandbox/providers/script_runner/auth.py
rename to src/backend/core/infra/local_runtime/runner_auth.py
diff --git a/src/backend/orchestration/local_subprocess.py b/src/backend/core/infra/local_runtime/startup.py
similarity index 87%
rename from src/backend/orchestration/local_subprocess.py
rename to src/backend/core/infra/local_runtime/startup.py
index fa780c0a..35d92eb1 100644
--- a/src/backend/orchestration/local_subprocess.py
+++ b/src/backend/core/infra/local_runtime/startup.py
@@ -35,7 +35,6 @@
from core.config.settings import settings
from core.infra.logging import get_logger
from core.infra.proc import no_window_kwargs
-import core.sandbox.providers.script_runner.auth as runner_auth
logger = get_logger(__name__)
@@ -51,7 +50,7 @@
# of automatically adding ``src/backend`` (the CLI script's directory) to
# ``sys.path``. Keep the packaged source root explicit so the desktop runtime
# can import ``mcp_servers`` and ``services`` on every platform.
-_BACKEND_DIR = str(Path(__file__).resolve().parents[1])
+_BACKEND_DIR = str(Path(__file__).resolve().parents[3])
# These are not optional conveniences in the local/desktop product: they back
# the three plugins installed on the first zero-state boot. Keeping this
@@ -75,7 +74,6 @@
"services.script_runner_service.server:app",
)
-
def _packaged_install_root() -> Optional[str]:
"""打包桌面形态的安装根指纹;开发形态返回 None(跳过清理)。
@@ -91,7 +89,6 @@ def _packaged_install_root() -> Optional[str]:
return None
return exe[: idx + len(seg) - 1]
-
def _find_stale_sidecar_pids(
ps_output: str, *, root_marker: str, my_pid: int, fold_case: bool = False
) -> List[int]:
@@ -121,7 +118,6 @@ def _find_stale_sidecar_pids(
pids.append(pid)
return pids
-
async def _process_table() -> str:
"""`` `` 一行一个进程。Windows 没有 ps,走 CIM。"""
if os.name == "nt":
@@ -143,7 +139,6 @@ async def _process_table() -> str:
out, _ = await proc.communicate()
return out.decode("utf-8", "replace")
-
async def _describe_port_owner(port: int) -> str:
"""占用 loopback 端口的进程描述(``pid name``),查不到时返回空串。"""
try:
@@ -183,7 +178,6 @@ async def _describe_port_owner(port: int) -> str:
except Exception: # noqa: BLE001 — 诊断信息缺失不改变失败本身
return ""
-
def _listener_pid_from_netstat(output: str, port: int) -> Optional[int]:
"""从 ``netstat -ano`` 输出里取 loopback/任意地址上该端口的监听进程。"""
suffix = f":{port}"
@@ -199,7 +193,6 @@ def _listener_pid_from_netstat(output: str, port: int) -> Optional[int]:
continue
return None
-
async def _terminate_process_trees(pids: List[int]) -> None:
"""Windows 没有 SIGTERM;结束进程时连子进程树一起结束,MCP launcher 的
子服务才不会变成下一次启动要清理的孤儿。"""
@@ -216,7 +209,6 @@ async def _terminate_process_trees(pids: List[int]) -> None:
)
await proc.wait()
-
def _pid_alive(pid: int) -> bool:
try:
os.kill(pid, 0)
@@ -226,7 +218,6 @@ def _pid_alive(pid: int) -> bool:
return True
return True
-
async def _reap_stale_sidecars() -> None:
"""启动期一次性清掉上次会话遗留的孤儿 sidecar,再拉本会话自己的。
@@ -272,13 +263,10 @@ async def _reap_stale_sidecars() -> None:
if reaped and settings.sandbox.provider == "script_runner":
runner_port = _script_runner_port()
deadline = time.monotonic() + 3.0
- while time.monotonic() < deadline and await _tcp_port_ready(
- "127.0.0.1", runner_port
- ):
+ while time.monotonic() < deadline and await _tcp_port_ready("127.0.0.1", runner_port):
await asyncio.sleep(0.2)
logger.info("stale_sidecars_reaped", count=reaped)
-
def _script_runner_port() -> int:
"""Return the local runner port from the already-resolved settings URL."""
@@ -293,14 +281,11 @@ def _script_runner_port() -> int:
"127.0.0.1",
"localhost",
}:
- raise RuntimeError(
- "本机模式 SANDBOX_RUNNER_URL 必须指向 loopback:http://127.0.0.1:"
- )
+ raise RuntimeError("本机模式 SANDBOX_RUNNER_URL 必须指向 loopback:http://127.0.0.1:")
if port is None or port == 0:
raise RuntimeError(f"SANDBOX_RUNNER_URL 缺少端口:{raw!r}")
return port
-
def _ready_timeout_seconds() -> float:
raw = os.getenv("LOCAL_SIDECAR_READY_TIMEOUT_SECONDS", "30")
try:
@@ -308,17 +293,15 @@ def _ready_timeout_seconds() -> float:
except (TypeError, ValueError):
return 30.0
-
def _child_env():
- from orchestration.local_sidecar_lifecycle import _child_env as child_env
+ from core.infra.local_runtime.lifecycle import _child_env as child_env
return child_env()
-
async def _spawn(label, argv):
- from orchestration import local_sidecar_lifecycle as lifecycle
- return await lifecycle._spawn(label, argv)
+ import core.infra.local_runtime.lifecycle as lifecycle
+ return await lifecycle._spawn(label, argv)
async def _tcp_port_ready(host: str, port: int) -> bool:
try:
@@ -334,6 +317,32 @@ async def _tcp_port_ready(host: str, port: int) -> bool:
except (OSError, asyncio.TimeoutError):
return False
+async def _http_port_ready(host: str, port: int) -> bool:
+ """Wait for an HTTP response before disconnecting a new MCP listener.
+
+ A connect-and-close probe can abort Windows Proactor's pending AcceptEx and
+ permanently stop that listener. Receiving response headers proves the server
+ accepted the connection; a missing root route (404/405) is still ready.
+ """
+ writer = None
+ try:
+ async with asyncio.timeout(2.0):
+ reader, writer = await asyncio.open_connection(host, port)
+ writer.write(
+ f"HEAD / HTTP/1.1\r\nHost: {host}:{port}\r\n"
+ "Connection: close\r\n\r\n".encode("ascii")
+ )
+ await writer.drain()
+ headers = await reader.readuntil(b"\r\n\r\n")
+ return headers.startswith((b"HTTP/1.0 ", b"HTTP/1.1 "))
+ except (OSError, TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError):
+ return False
+ finally:
+ if writer is not None:
+ writer.close()
+ with contextlib.suppress(OSError, TimeoutError):
+ await asyncio.wait_for(writer.wait_closed(), timeout=0.5)
+
async def _wait_for_mcp_ports(
launcher: "asyncio.subprocess.Process",
@@ -348,7 +357,7 @@ async def _wait_for_mcp_ports(
if launcher.returncode is not None:
raise RuntimeError(f"MCP 启动器提前退出(exit={launcher.returncode})")
checks = await asyncio.gather(
- *(_tcp_port_ready("127.0.0.1", port) for port in pending.values())
+ *(_http_port_ready("127.0.0.1", port) for port in pending.values())
)
pending = {
server_id: port
@@ -361,22 +370,20 @@ async def _wait_for_mcp_ports(
details = ", ".join(f"{server_id}:{port}" for server_id, port in pending.items())
raise RuntimeError(f"MCP 服务未在 {timeout:.0f} 秒内就绪:{details}")
-
async def _list_mcp_tool_names(server_id: str, port: int) -> set[str]:
"""Use the production MCP client to verify a server's actual tool list."""
- from core.llm.mcp_pool import make_client
+ from core.harness.registry.mcp_clients import discover_tools
- client = make_client(
+ client = await discover_tools(
server_id,
{
"transport": "streamable_http",
"url": f"http://127.0.0.1:{port}/mcp/",
"transport_timeout": 5,
},
- is_stateful=False,
)
try:
- tools = await client.list_tools()
+ tools = client.tools
return {str(name) for tool in tools if (name := getattr(tool, "name", None))}
finally:
try:
@@ -388,9 +395,9 @@ async def _list_mcp_tool_names(server_id: str, port: int) -> set[str]:
task = asyncio.current_task()
if task is not None and task.cancelling():
raise
- except Exception: # noqa: BLE001 — readiness result is already known
- pass
-
+ except Exception as fallback_error: # noqa: BLE001 — readiness result is already known
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.infra.local_runtime.startup:_list_mcp_tool_names', fallback_error)
async def _verify_required_plugin_tools(ports: dict[str, int]) -> None:
"""Assert that every default plugin MCP exposes its contract tool."""
@@ -410,24 +417,22 @@ async def _verify(server_id: str, expected_tool: str) -> None:
)
)
-
def _cloud_serves_tools() -> bool:
"""混合模式(桌面壳注入了桥接密钥)下,工具全部来自云端 capability manifest:
本机执行面只负责本地项目的代码执行,不运行任何内置 MCP。"""
- from core.auth.desktop_bridge import bridge_enabled
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
return bridge_enabled()
-
def _start_watchdog():
- from orchestration import local_sidecar_lifecycle as lifecycle
- return lifecycle._start_watchdog()
+ import core.infra.local_runtime.lifecycle as lifecycle
+ return lifecycle._start_watchdog()
async def _start_script_runner(py: str):
- from orchestration import local_sidecar_lifecycle as lifecycle
- return await lifecycle._start_script_runner(py)
+ import core.infra.local_runtime.lifecycle as lifecycle
+ return await lifecycle._start_script_runner(py)
async def start_local_sidecars() -> None:
"""Spawn the script_runner sidecar, plus the MCP launcher for local-only installs."""
@@ -443,7 +448,9 @@ async def start_local_sidecars() -> None:
if _cloud_serves_tools():
await _start_script_runner(py)
_start_watchdog()
- logger.info("local_sidecars_ready", mode="cloud_tools", sidecars=[label for label, _ in _PROCS])
+ logger.info(
+ "local_sidecars_ready", mode="cloud_tools", sidecars=[label for label, _ in _PROCS]
+ )
return
from extensions.mcp_servers._launcher import PORTS as launcher_ports
@@ -489,12 +496,12 @@ async def start_local_sidecars() -> None:
_start_watchdog()
-
async def _supervise_sidecars():
- from orchestration import local_sidecar_lifecycle as lifecycle
- return await lifecycle._supervise_sidecars()
+ import core.infra.local_runtime.lifecycle as lifecycle
+ return await lifecycle._supervise_sidecars()
async def stop_local_sidecars():
- from orchestration import local_sidecar_lifecycle as lifecycle
+ import core.infra.local_runtime.lifecycle as lifecycle
+
return await lifecycle.stop_local_sidecars()
diff --git a/src/backend/core/infra/model_output.py b/src/backend/core/infra/model_output.py
new file mode 100644
index 00000000..157af94e
--- /dev/null
+++ b/src/backend/core/infra/model_output.py
@@ -0,0 +1,95 @@
+"""Shared reasoning-block cleanup and tolerant JSON extraction for model output."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, Optional
+
+
+THINK_BLOCK_RE = re.compile(r".*? ", re.DOTALL | re.IGNORECASE)
+
+
+_THINK_CLOSE_RE = re.compile(r" ", re.IGNORECASE)
+
+
+_THINK_OPEN_RE = re.compile(r"", re.IGNORECASE)
+
+
+def strip_think_blocks(text: str) -> str:
+ """Strip a reasoning model's think blocks and return the body.
+
+ Covers three shapes:
+ - closed ``... `` blocks (possibly multiple);
+ - only `` `` (some serving stacks swallow the opening tag into the chat template, with the body following the closing tag)
+ → take the content after the last closing tag;
+ - only ```` (output was truncated mid-thought by max_tokens, with no body after)
+ → discard everything from the opening tag onward.
+ """
+ t = THINK_BLOCK_RE.sub("", text)
+ m = None
+ for m in _THINK_CLOSE_RE.finditer(t):
+ pass
+ if m is not None:
+ t = t[m.end():]
+ m = _THINK_OPEN_RE.search(t)
+ if m is not None:
+ t = t[: m.start()]
+ return t.strip()
+
+
+def extract_first_json_object(text: str) -> Optional[str]:
+ start = text.find("{")
+ if start < 0:
+ return None
+ depth = 0
+ in_str = False
+ escape = False
+ for i, ch in enumerate(text[start:], start=start):
+ if in_str:
+ if escape:
+ escape = False
+ elif ch == "\\":
+ escape = True
+ elif ch == '"':
+ in_str = False
+ continue
+ if ch == '"':
+ in_str = True
+ elif ch == "{":
+ depth += 1
+ elif ch == "}":
+ depth -= 1
+ if depth == 0:
+ return text[start:i + 1]
+ return None
+
+
+def parse_strict_json(text: str) -> Optional[Dict[str, Any]]:
+ """Parse JSON tolerantly: strips blocks, ``` fences, and surrounding prose."""
+ if not text:
+ return None
+ # If the body is already valid JSON, return it directly to avoid corruption from stripping a think tag that happens to appear inside a string value
+ try:
+ parsed = json.loads(text.strip())
+ if isinstance(parsed, dict):
+ return parsed
+ except json.JSONDecodeError:
+ parsed = None
+ t = strip_think_blocks(text.strip())
+
+ if t.startswith("```"):
+ lines = [l for l in t.splitlines() if not l.strip().startswith("```")]
+ t = "\n".join(lines).strip()
+
+ try:
+ return json.loads(t)
+ except json.JSONDecodeError:
+ parsed = None
+
+ candidate = extract_first_json_object(t)
+ if candidate:
+ try:
+ return json.loads(candidate)
+ except json.JSONDecodeError:
+ return None
+ return None
diff --git a/src/backend/core/infra/probes/__init__.py b/src/backend/core/infra/probes/__init__.py
new file mode 100644
index 00000000..4d450022
--- /dev/null
+++ b/src/backend/core/infra/probes/__init__.py
@@ -0,0 +1 @@
+"""Health probes for the services the backend depends on."""
diff --git a/src/backend/core/services/edition_service_probe.py b/src/backend/core/infra/probes/edition.py
similarity index 100%
rename from src/backend/core/services/edition_service_probe.py
rename to src/backend/core/infra/probes/edition.py
diff --git a/src/backend/core/services/service_probes.py b/src/backend/core/infra/probes/service_probes.py
similarity index 89%
rename from src/backend/core/services/service_probes.py
rename to src/backend/core/infra/probes/service_probes.py
index 59b685c6..371428c5 100644
--- a/src/backend/core/services/service_probes.py
+++ b/src/backend/core/infra/probes/service_probes.py
@@ -38,27 +38,6 @@ def config_row_to_dict(cfg: dict, mask: bool = True) -> dict:
# ── After config changes: rebuild the MCP connection pool ────────────────────
-async def reinitialize_mcp_pool(source: str = "service_configs") -> None:
- """Invalidate MCP config cache and reinitialize stable connections.
-
- Called after system config changes so that MCP sub-processes pick up
- new env vars (e.g. updated API URLs/keys).
- """
- try:
- from core.services.mcp_service import McpServerConfigService
-
- mcp_svc = McpServerConfigService.get_instance()
- mcp_svc.invalidate_cache()
-
- from core.llm.mcp_pool import MCPConnectionPool
-
- pool = MCPConnectionPool.get_instance()
- if pool.is_initialized:
- new_configs = mcp_svc.get_all_servers()
- await pool.reinitialize_if_config_changed(new_configs)
- logger.info("[%s] MCP pool reinitialize triggered after config update", source)
- except Exception as exc:
- logger.warning("[%s] MCP pool reinitialize failed: %s", source, exc)
# ── Connectivity probes ──────────────────────────────────────────────────────
@@ -190,12 +169,12 @@ async def test_service_group(group_key: str) -> dict:
Supports knowledge_base / file_parser / internet_search; other groups return
"unsupported". Shared by service_configs (EE) and me_system (CE).
"""
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc = SystemConfigService.get_instance()
if group_key == "knowledge_base":
- from core.services.edition_service_probe import test_external_knowledge
+ from core.infra.probes.edition import test_external_knowledge
provider = (svc.get("knowledge_base.provider") or "").strip().lower()
if provider in {"custom", "local"}:
diff --git a/src/backend/core/infra/rate_limit.py b/src/backend/core/infra/rate_limit.py
deleted file mode 100644
index eb106ba3..00000000
--- a/src/backend/core/infra/rate_limit.py
+++ /dev/null
@@ -1,21 +0,0 @@
-"""HTTP rate limiter configuration. Circuit state lives in circuit_breaker."""
-
-from core.config.settings import settings
-from slowapi import Limiter
-from slowapi.util import get_remote_address
-
-
-def get_rate_limit_enabled() -> bool:
- return settings.rate_limit.enabled
-
-
-def get_rate_limit_storage() -> str:
- return settings.rate_limit.storage
-
-
-limiter = Limiter(
- key_func=get_remote_address,
- storage_uri=get_rate_limit_storage(),
- enabled=get_rate_limit_enabled(),
- headers_enabled=True,
-)
diff --git a/src/backend/core/infra/redis.py b/src/backend/core/infra/redis.py
index 05514d8b..f361e7a1 100644
--- a/src/backend/core/infra/redis.py
+++ b/src/backend/core/infra/redis.py
@@ -1,11 +1,12 @@
"""Event-loop-local Redis clients for session storage."""
+from __future__ import annotations
import asyncio
import threading
+from dataclasses import dataclass, field
from weakref import WeakKeyDictionary
import redis.asyncio as aioredis
-
from core.config.settings import settings
from core.infra.logging import get_logger
@@ -30,14 +31,13 @@
_stream_pools: WeakKeyDictionary[asyncio.AbstractEventLoop, aioredis.Redis] = (
WeakKeyDictionary()
)
-_redis_pools_lock = threading.Lock()
def redis_configured() -> bool:
"""Whether this deployment has a Redis to talk to.
The single predicate every store-agnostic seam branches on
- (:mod:`core.infra.ephemeral`, :mod:`orchestration.run_event_stream`). A
+ (:mod:`core.infra.ephemeral`, :mod:`core.runs.events.store`). A
deployment without Redis — the desktop's single-process backend — leaves
``REDIS_URL`` empty and those seams use their in-process backend instead.
"""
@@ -52,8 +52,7 @@ def _create_client(max_connections: int) -> tuple[aioredis.Redis, str]:
# in-process backend from redis_configured() long before reaching here,
# so arriving with no URL means a caller bypassed one.
raise RuntimeError(
- "REDIS_URL is not configured; use core.infra.ephemeral / "
- "orchestration.run_event_stream instead of get_redis()"
+ 'REDIS_URL is not configured; use core.infra.ephemeral / core.runs.events.store instead of get_redis()'
)
# NOTE: redis-py 8.0 changed the default socket_timeout from None -> 5s.
# The chat-stream follower blocks on `XREAD BLOCK 5000`; if the socket
@@ -79,7 +78,7 @@ def _get_pooled(
role: str,
) -> aioredis.Redis:
loop = asyncio.get_running_loop()
- with _redis_pools_lock:
+ with process_state().redis_pools_lock:
existing = pools.get(loop)
if existing is not None:
return existing
@@ -114,7 +113,7 @@ def get_redis(*, blocking: bool = False) -> aioredis.Redis:
Only call this where Redis is known to be configured. Deployments without
one (the desktop backend) leave ``REDIS_URL`` empty and reach the same
capabilities through :mod:`core.infra.ephemeral` and
- :mod:`orchestration.run_event_stream`, which pick an in-process backend.
+ :mod:`core.runs.events.store`, which pick an in-process backend.
There is deliberately no imitation Redis to fall back on: an imitation has
to track redis-py's wire protocol, and a mismatch shows up as a silent
stall rather than a clear failure.
@@ -131,7 +130,7 @@ def get_redis(*, blocking: bool = False) -> aioredis.Redis:
async def close_redis() -> None:
"""Close every Redis client owned by the current event loop."""
loop = asyncio.get_running_loop()
- with _redis_pools_lock:
+ with process_state().redis_pools_lock:
clients = [
pools.pop(loop, None) for pools in (_redis_pools, _stream_pools)
]
@@ -139,3 +138,15 @@ async def close_redis() -> None:
if client is not None:
await client.aclose()
logger.info("redis_pool_closed", event_loop=id(loop))
+
+
+@dataclass
+class RedisState:
+ redis_pools_lock: object = field(default_factory=lambda: threading.Lock())
+
+
+_STATE = RedisState()
+
+
+def process_state() -> RedisState:
+ return _STATE
diff --git a/src/backend/core/infra/runtime_state.py b/src/backend/core/infra/runtime_state.py
index af97dc56..d00de8e9 100644
--- a/src/backend/core/infra/runtime_state.py
+++ b/src/backend/core/infra/runtime_state.py
@@ -2,27 +2,39 @@
The FastAPI app starts several background workers (idle-session reaper,
automation scheduler, distillation scheduler, …) and stores their handles as
-module globals. Monitoring code (``core.services.security_service``) needs to
+module globals. Monitoring code (``features.security_console.console``) needs to
report whether they are running, but it must not import the API app module
(that would be a ``core → api`` upward dependency). The app registers handles
here on startup; monitoring reads them back through this neutral registry.
"""
+from __future__ import annotations
+from dataclasses import dataclass, field
from typing import Any, Dict
-_REGISTRY: Dict[str, Any] = {}
-
def register(name: str, handle: Any) -> None:
"""Register (or replace) a named background task/scheduler handle."""
- _REGISTRY[name] = handle
+ process_state().REGISTRY[name] = handle
def get(name: str) -> Any:
"""Return the handle registered under ``name`` (or None)."""
- return _REGISTRY.get(name)
+ return process_state().REGISTRY.get(name)
def clear() -> None:
"""Drop all registered handles (used on shutdown)."""
- _REGISTRY.clear()
+ process_state().REGISTRY.clear()
+
+
+@dataclass
+class RuntimeState:
+ REGISTRY: Dict[str, Any] = field(default_factory=lambda: {})
+
+
+_STATE = RuntimeState()
+
+
+def process_state() -> RuntimeState:
+ return _STATE
diff --git a/src/backend/core/chat/display_bounds.py b/src/backend/core/infra/text_bounds.py
similarity index 100%
rename from src/backend/core/chat/display_bounds.py
rename to src/backend/core/infra/text_bounds.py
diff --git a/src/backend/core/infra/time.py b/src/backend/core/infra/time.py
index 118562f9..a646ef02 100644
--- a/src/backend/core/infra/time.py
+++ b/src/backend/core/infra/time.py
@@ -1,17 +1,29 @@
"""Application instants use aware UTC; calendar schedules keep their explicit zone."""
from datetime import datetime, timezone
+from typing import Optional, overload
def utc_now() -> datetime:
return datetime.now(timezone.utc)
-def as_utc(value: datetime) -> datetime:
+@overload
+def as_utc(value: datetime) -> datetime: ...
+
+
+@overload
+def as_utc(value: None) -> None: ...
+
+
+def as_utc(value: Optional[datetime]) -> Optional[datetime]:
"""Normalize an instant; legacy timezone-less application values mean UTC.
+ ``None`` passes through so nullable columns can be normalized in place.
User-entered local calendar values must first be localized with their
declared business timezone, not passed here as naive timestamps.
"""
+ if value is None:
+ return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
diff --git a/src/backend/core/infra/worker_count.py b/src/backend/core/infra/worker_count.py
index f0d73ee7..6dc228a5 100644
--- a/src/backend/core/infra/worker_count.py
+++ b/src/backend/core/infra/worker_count.py
@@ -11,7 +11,7 @@
* ``SESSION_STORE=memory`` — a login handled by worker A is invisible to worker
B, so a browser is randomly signed out on roughly ``1 - 1/N`` of its requests.
* No Redis — the per-run SSE event log falls back to an in-process buffer
- (:mod:`orchestration.run_event_stream`), and so does the mutex behind
+ (:mod:`core.runs.events.store`), and so does the mutex behind
:mod:`core.infra.leader`, so neither the stream nor the leader lease would
actually cross a process boundary.
* ``DEPLOY_PROFILE=local`` — the no-Docker desktop profile hosts MCP and sandbox
@@ -33,6 +33,13 @@
only launch path: ``cli.py`` and ``api.app.main`` call ``uvicorn.run`` in
process, where nothing exported the resolved value. Deriving the answer instead
of trusting the environment keeps those paths from believing they are a cluster.
+
+Each worker also brings its own database pool, and that pool is not a separate
+knob: a process can only use as many connections as it has threads doing
+database work, so :func:`core.db.engine.thread_budget` derives the pool from
+``WORKER_THREADS`` (anyio's limiter) and the event loop's executor, and every
+worker applies those limits at startup. Postgres therefore needs
+``max_connections >= resolve() * (pool_size + overflow)`` for the backend alone.
"""
from __future__ import annotations
diff --git a/src/backend/orchestration/jobs/__init__.py b/src/backend/core/jobs/__init__.py
similarity index 100%
rename from src/backend/orchestration/jobs/__init__.py
rename to src/backend/core/jobs/__init__.py
diff --git a/src/backend/orchestration/jobs/agent.py b/src/backend/core/jobs/agent.py
similarity index 94%
rename from src/backend/orchestration/jobs/agent.py
rename to src/backend/core/jobs/agent.py
index c2b8a322..f0a9bc17 100644
--- a/src/backend/orchestration/jobs/agent.py
+++ b/src/backend/core/jobs/agent.py
@@ -58,9 +58,9 @@ def snapshot_parent_runtime(request, *, skill_ids, mcp_ids, sandbox_session_id,
def resolve_subagents(runtime: dict, user_id: str) -> list[dict]:
"""Recheck current accessibility; never add agents absent from the parent."""
from core.db.engine import SessionLocal
- from core.llm.builtin_subagents import merge_builtin_subagents
- from core.services.user_agent_service import UserAgentService
- from core.services.user_service import UserService
+ from core.capabilities.agents.builtin import merge_builtin_subagents
+ from core.capabilities.agents.edition import UserAgentService
+ from core.auth.users import UserService
ids = set(runtime.get("subagent_ids") or [])
if not ids:
@@ -86,11 +86,11 @@ async def run_derived_agent(
tools: list[str] | None = None,
):
"""Use the main factory and streaming loop with fresh task-local context."""
- from core.llm import workspace
+ from core.llm.runtime import workspace
from core.llm.factory import create_agent_executor
- from core.llm.mcp_manager import close_clients
- from core.services.chat_mode_service import ChatModeSpec
- from orchestration.streaming import StreamingAgent
+ from core.llm.mcp.manager import close_clients
+ from core.llm.runtime.streaming import StreamingAgent
+ from core.chat.mode_types import ChatModeSpec
if not isinstance(parent_runtime, dict):
raise ValueError("作业缺少主智能体运行配置,请重新提交工作流")
diff --git a/src/backend/orchestration/jobs/callback.py b/src/backend/core/jobs/callback.py
similarity index 100%
rename from src/backend/orchestration/jobs/callback.py
rename to src/backend/core/jobs/callback.py
diff --git a/src/backend/orchestration/jobs/files.py b/src/backend/core/jobs/files.py
similarity index 95%
rename from src/backend/orchestration/jobs/files.py
rename to src/backend/core/jobs/files.py
index 3712a2f9..4465df26 100644
--- a/src/backend/orchestration/jobs/files.py
+++ b/src/backend/core/jobs/files.py
@@ -3,17 +3,15 @@
import base64
from typing import Tuple
-from core.sandbox import ProcessRequest, get_sandbox_provider
-
def _job_directory(session_id: str, job_row_id: str) -> str:
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
return f"{workspace_directory(session_id)}/.job/{job_row_id}"
def _quote_path(path: str) -> str:
- from core.llm.tools._paths import path_rules
+ from core.sandbox.workspace.resolution import path_rules
return path_rules().quote_shell_path(path)
diff --git a/src/backend/orchestration/jobs/notifications.py b/src/backend/core/jobs/notifications.py
similarity index 81%
rename from src/backend/orchestration/jobs/notifications.py
rename to src/backend/core/jobs/notifications.py
index 78a681af..54a03057 100644
--- a/src/backend/orchestration/jobs/notifications.py
+++ b/src/backend/core/jobs/notifications.py
@@ -6,7 +6,7 @@
logger = logging.getLogger(__name__)
-def _emit_progress(chat_id: Optional[str], note: str) -> None:
+def emit_progress(chat_id: Optional[str], note: str) -> None:
"""活性信号:转译成 model_progress 喂主 run 的无活动看门狗。
长作业期间主对话没有任何可渲染输出,缺了这条会被 600s 看门狗当成卡死强杀。
@@ -17,7 +17,7 @@ def _emit_progress(chat_id: Optional[str], note: str) -> None:
if not chat_id:
return
try:
- from core.llm import _subagent_stream
+ from core.llm.subagents import stream as _subagent_stream
if not _subagent_stream.is_active(chat_id):
return
@@ -25,8 +25,9 @@ def _emit_progress(chat_id: Optional[str], note: str) -> None:
chat_id,
{"sub_type": "progress", "agent_id": "job", "agent_name": "批量作业", "note": note},
)
- except Exception: # noqa: BLE001 —— 活性信号是尽力而为,永远不该拖垮作业
- pass
+ except Exception as fallback_error: # noqa: BLE001 —— 活性信号是尽力而为,永远不该拖垮作业
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.jobs.notifications:_emit_progress', fallback_error)
def _emit_ui_progress(
@@ -51,12 +52,12 @@ def _emit_ui_progress(
if not chat_id:
return
try:
- from core.llm import _subagent_stream
+ from core.llm.subagents import stream as _subagent_stream
if not _subagent_stream.is_active(chat_id):
return
try:
- from core.llm.middlewares import CURRENT_TOOL_CALL_ID
+ from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID
parent_tool_id = CURRENT_TOOL_CALL_ID.get("") or ""
except Exception: # noqa: BLE001
@@ -82,14 +83,15 @@ def _emit_ui_progress(
"pending": int(stats.get("pending", 0) or 0),
},
)
- except Exception: # noqa: BLE001 —— 进度是尽力而为,永远不该拖垮作业
- pass
+ except Exception as fallback_error: # noqa: BLE001 —— 进度是尽力而为,永远不该拖垮作业
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.jobs.notifications:_emit_ui_progress', fallback_error)
async def _maybe_wake(job_row_id: str) -> None:
"""作业终态后叫醒会话(幂等;失败只记日志,不影响作业结果)。"""
try:
- from orchestration.job_wakeup import wake_on_job_finish
+ from core.chat.runtime.job_wakeup import wake_on_job_finish
await wake_on_job_finish(job_row_id)
except Exception as exc: # noqa: BLE001
@@ -101,7 +103,7 @@ async def _maybe_wake_progress(
) -> None:
"""中途播报进度(失败只记日志,绝不影响作业本身)。"""
try:
- from orchestration.job_wakeup import wake_on_job_progress
+ from core.chat.runtime.job_wakeup import wake_on_job_progress
await wake_on_job_progress(
job_row_id, stats=stats, budget_left=budget_left, stalled=stalled
diff --git a/src/backend/orchestration/jobs/owner.py b/src/backend/core/jobs/owner.py
similarity index 62%
rename from src/backend/orchestration/jobs/owner.py
rename to src/backend/core/jobs/owner.py
index a500bc3b..8db024bb 100644
--- a/src/backend/orchestration/jobs/owner.py
+++ b/src/backend/core/jobs/owner.py
@@ -1,46 +1,43 @@
"""Route job control onto the long-lived service loop, never a model worker loop."""
+from __future__ import annotations
import asyncio
import functools
import logging
import threading
+from dataclasses import dataclass, field
logger = logging.getLogger(__name__)
-_loop = None
-_guard = threading.Lock()
-_tasks = set()
def bind():
- global _loop
current = asyncio.get_running_loop()
- with _guard:
- if _loop is not None and _loop.is_running() and _loop is not current:
+ with process_state().guard:
+ if process_state().loop is not None and process_state().loop.is_running() and process_state().loop is not current:
raise RuntimeError("Job service is already bound to another live loop")
- _loop = current
+ process_state().loop = current
def service_loop():
- global _loop
from core.sandbox import get_sandbox_provider
- with _guard:
- if _loop is None or _loop.is_closed():
+ with process_state().guard:
+ if process_state().loop is None or process_state().loop.is_closed():
provider_loop = getattr(get_sandbox_provider(), "_service_loop", None)
- _loop = provider_loop
- if _loop is None:
+ process_state().loop = provider_loop
+ if process_state().loop is None:
raise RuntimeError("Job service is not initialized on the application loop")
- if not _loop.is_running():
+ if not process_state().loop.is_running():
raise RuntimeError("Job service loop is not running")
- return _loop
+ return process_state().loop
def _retain(coro):
task = asyncio.create_task(coro)
- _tasks.add(task)
+ process_state().tasks.add(task)
def done(finished):
- _tasks.discard(finished)
+ process_state().tasks.discard(finished)
if not finished.cancelled() and finished.exception():
logger.warning("Job control operation failed: %s", type(finished.exception()).__name__)
@@ -64,3 +61,17 @@ async def operation():
return await asyncio.shield(asyncio.wrap_future(future))
return wrapper
+
+
+@dataclass
+class OwnerState:
+ loop: object = None
+ guard: object = field(default_factory=lambda: threading.Lock())
+ tasks: object = field(default_factory=lambda: set())
+
+
+_STATE = OwnerState()
+
+
+def process_state() -> OwnerState:
+ return _STATE
diff --git a/src/backend/orchestration/jobs/process.py b/src/backend/core/jobs/process.py
similarity index 95%
rename from src/backend/orchestration/jobs/process.py
rename to src/backend/core/jobs/process.py
index d1985fa7..b8dc7d30 100644
--- a/src/backend/orchestration/jobs/process.py
+++ b/src/backend/core/jobs/process.py
@@ -7,9 +7,11 @@
from typing import Any
from core.sandbox import ProcessRequest, get_sandbox_provider
-from core.sandbox.providers.opensandbox.recovery import execution_reference
-from core.sandbox.providers.opensandbox.recovery import recover_execution
-from core.sandbox.providers.opensandbox.recovery import release_execution
+from core.sandbox.providers.opensandbox.recovery import (
+ execution_reference,
+ recover_execution,
+ release_execution,
+)
from . import state
from .callback import resolve_callback_base
@@ -82,8 +84,9 @@ async def receipt(self):
"failed",
):
return data
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.jobs.process:receipt', fallback_error)
return None
async def log(self):
diff --git a/src/backend/core/jobs/reminder_hook.py b/src/backend/core/jobs/reminder_hook.py
new file mode 100644
index 00000000..26375f33
--- /dev/null
+++ b/src/backend/core/jobs/reminder_hook.py
@@ -0,0 +1,80 @@
+"""Job-ledger reminder contributed by the execution host."""
+import asyncio
+from typing import Optional
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage
+
+
+class JobLedgerHook:
+ def __init__(self, *, chat_id: Optional[str], user_id: Optional[str]) -> None:
+ self._chat_id = chat_id or ""
+ self._user_id = user_id or ""
+ self._last_key: tuple | None = None
+
+ def _ledger_lines(self) -> list[str]:
+ from core.db.engine import SessionLocal
+ from core.db.models import Job
+ from core.jobs.store import JobService
+
+ with SessionLocal() as db:
+ jobs = (
+ db.query(Job)
+ .filter(Job.chat_id == self._chat_id, Job.user_id == self._user_id)
+ .order_by(Job.created_at.desc())
+ .limit(3)
+ .all()
+ )
+ svc = JobService(db)
+ lines = []
+ for job in jobs:
+ stats = svc.stats(job.job_id)
+ unsettled = int(stats.get("pending", 0)) + int(stats.get("failed", 0))
+ if unsettled <= 0 and job.status in ("completed", "cancelled"):
+ continue
+ lines.append(
+ f"- 作业「{job.name or job.job_id}」({job.job_id},状态 {job.status}):"
+ f"总计 {stats.get('total', 0)},已完成 {stats.get('done', 0)},"
+ f"查无 {stats.get('not_found', 0)},待办 {stats.get('pending', 0)},"
+ f"失败 {stats.get('failed', 0)}"
+ )
+ return lines
+
+ async def __call__(self, invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ runtime = invocation.data["runtime"]
+ if not self._chat_id:
+ return
+ key = (
+ runtime.get("reply_id", "") or "",
+ int(runtime.get("cur_iter", 0) or 0),
+ )
+ if key == self._last_key:
+ return
+ # HookBus runs sync handlers inline on the event loop; keep the ledger
+ # query off it.
+ lines = await asyncio.to_thread(self._ledger_lines)
+ if not lines:
+ return
+
+ self._last_key = key
+ reminder = (
+ "本会话有尚未结算完的批量作业(数字取自台账,是可查证的事实,不是估计):\n"
+ + "\n".join(lines)
+ + "\n\n还有待办或失败项时:用 run_job(action='resume', job_id=...) 续跑"
+ "(已完成的项不会重做),或先查 run_job(action='status', job_id=...) 看明细。"
+ "**不得**把未完成当作完成来交付;确实要收尾,也必须如实报出分母、完成数与未覆盖清单。"
+ )
+ return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), {
+ "content": f"\n{reminder}\n ",
+ "kind": "reminder", "origin": "harness:job_ledger", "trust": "system",
+ "priority": 850, "token_budget": 4000,
+ }]})
+
+
+
+def job_ledger_hooks(request):
+ if request.workflow_mode and not request.isolated and request.chat_id:
+ yield HookSpec("job_ledger", HookStage.TRANSFORM_CONTEXT,
+ JobLedgerHook(chat_id=request.chat_id, user_id=request.current_user_id),
+ priority=500, failure_mode=FailureMode.OPEN,
+ mutable_fields=frozenset({"context_items"}))
diff --git a/src/backend/orchestration/jobs/runner.py b/src/backend/core/jobs/runner.py
similarity index 100%
rename from src/backend/orchestration/jobs/runner.py
rename to src/backend/core/jobs/runner.py
diff --git a/src/backend/orchestration/job_runtime.py b/src/backend/core/jobs/runtime.py
similarity index 85%
rename from src/backend/orchestration/job_runtime.py
rename to src/backend/core/jobs/runtime.py
index d0334eab..b0265fa8 100644
--- a/src/backend/orchestration/job_runtime.py
+++ b/src/backend/core/jobs/runtime.py
@@ -1,19 +1,23 @@
"""Public job lifecycle API backed by service-owned managed sandbox processes."""
+from __future__ import annotations
import asyncio
import logging
+from dataclasses import dataclass, field
-from core.db.engine import SessionLocal
-from core.services.job_service import JobService
-from orchestration.jobs import owner, process, state, supervisor
+import core.jobs.owner as owner
+import core.jobs.process as process
+import core.jobs.state as state
+import core.jobs.supervisor as supervisor
+from core.jobs.store import JobService
+from core.db.threads import run_in_session
logger = logging.getLogger(__name__)
-_launches = {}
async def _launch(jid, user_id, *, resume=False):
attempt = state.begin_attempt(jid, user_id, resume=resume)
- _launches[jid] = asyncio.current_task()
+ process_state().launches[jid] = asyncio.current_task()
try:
execution = await asyncio.wait_for(process.launch(state.snapshot(jid)), timeout=90)
entry = supervisor.attach(execution)
@@ -34,7 +38,7 @@ async def _launch(jid, user_id, *, resume=False):
state.finish(jid, attempt, "failed", f"作业启动失败:{type(exc).__name__}: {exc}")
raise
finally:
- _launches.pop(jid, None)
+ process_state().launches.pop(jid, None)
@owner.owned
@@ -50,7 +54,7 @@ async def start_job(
start_params=None,
interpreter="${PY_BIN:-python3}",
):
- with SessionLocal() as db:
+ def _create_job(db):
job = JobService(db).create(
user_id=user_id,
chat_id=chat_id,
@@ -62,6 +66,8 @@ async def start_job(
start_params={**(start_params or {}), "interpreter": interpreter},
)
jid = job.job_id
+ return jid
+ jid = await run_in_session(_create_job)
return await _launch(jid, user_id)
@@ -83,14 +89,15 @@ async def cancel_job(job_row_id, *, user_id):
if not attempt:
# Legacy nohup jobs have no attributable process handle. Never broad-kill
# another job or claim to have stopped a process we cannot identify.
- with SessionLocal() as db:
+ def _cancel_legacy_job(db):
JobService(db).finish(
job_row_id, "cancelled", error="旧作业已撤销回调权限,进程状态无法确认"
)
+ await run_in_session(_cancel_legacy_job)
raise RuntimeError("旧作业已撤销回调权限,但无持久进程引用,无法确认停止;请勿自动重跑")
state.finish(job_row_id, attempt, "cancelled", "用户取消", require_owner=False)
state.take_control(job_row_id, attempt)
- launching = _launches.get(job_row_id)
+ launching = process_state().launches.get(job_row_id)
if launching:
await asyncio.gather(asyncio.shield(launching), return_exceptions=True)
row = state.snapshot(job_row_id)
@@ -122,7 +129,7 @@ async def resume_job(job_row_id, *, user_id, chat_id=None):
async def reap_orphan_jobs():
count = 0
for jid in state.claim_expired():
- if jid in supervisor._entries:
+ if jid in supervisor.process_state().entries:
continue
row = state.snapshot(jid)
attempt = row["meta"]["execution"]["attempt_id"]
@@ -141,7 +148,7 @@ async def reap_orphan_jobs():
"interrupted",
"执行实例失联,已撤销旧回调权限;确认原进程退出后才能 resume:" + str(exc),
)
- from orchestration.jobs.notifications import _maybe_wake
+ from core.jobs.notifications import _maybe_wake
await _maybe_wake(jid)
count += 1
@@ -164,3 +171,15 @@ async def run_job_reaper_loop():
await asyncio.sleep(15)
finally:
await supervisor.shutdown()
+
+
+@dataclass
+class RuntimeState:
+ launches: object = field(default_factory=lambda: {})
+
+
+_STATE = RuntimeState()
+
+
+def process_state() -> RuntimeState:
+ return _STATE
diff --git a/src/backend/orchestration/jobs/sdk.py b/src/backend/core/jobs/sdk.py
similarity index 100%
rename from src/backend/orchestration/jobs/sdk.py
rename to src/backend/core/jobs/sdk.py
diff --git a/src/backend/orchestration/jobs/state.py b/src/backend/core/jobs/state.py
similarity index 98%
rename from src/backend/orchestration/jobs/state.py
rename to src/backend/core/jobs/state.py
index 852fedce..513025d1 100644
--- a/src/backend/orchestration/jobs/state.py
+++ b/src/backend/core/jobs/state.py
@@ -5,7 +5,7 @@
from core.db.engine import SessionLocal
from core.db.models import Job
-from core.services.job_service import JobService, mint_token
+from core.jobs.store import JobService, mint_token
from sqlalchemy.orm.attributes import flag_modified
OWNER = uuid.uuid4().hex
diff --git a/src/backend/core/services/job_service.py b/src/backend/core/jobs/store.py
similarity index 100%
rename from src/backend/core/services/job_service.py
rename to src/backend/core/jobs/store.py
diff --git a/src/backend/orchestration/jobs/supervisor.py b/src/backend/core/jobs/supervisor.py
similarity index 88%
rename from src/backend/orchestration/jobs/supervisor.py
rename to src/backend/core/jobs/supervisor.py
index 4c29c8e4..0913658e 100644
--- a/src/backend/orchestration/jobs/supervisor.py
+++ b/src/backend/core/jobs/supervisor.py
@@ -1,4 +1,5 @@
"""Service-owned execution monitors: handshake, budgets, receipts and cancellation."""
+from __future__ import annotations
import asyncio
import logging
@@ -7,13 +8,11 @@
from datetime import datetime
from . import state
-from .notifications import _emit_progress, _emit_ui_progress, _maybe_wake, _maybe_wake_progress
+from core.jobs.notifications import emit_progress as _emit_progress, _emit_ui_progress, _maybe_wake, _maybe_wake_progress
logger = logging.getLogger(__name__)
STARTUP_SECONDS = 30.0
PROGRESS_WAKE_SECONDS = 300.0
-_entries = {}
-_notifications = set()
def _notify(coro):
@@ -24,8 +23,8 @@ async def deliver():
logger.warning("Job notification failed: %s", type(exc).__name__)
task = asyncio.create_task(deliver())
- _notifications.add(task)
- task.add_done_callback(_notifications.discard)
+ process_state().notifications.add(task)
+ task.add_done_callback(process_state().notifications.discard)
@dataclass
@@ -37,17 +36,17 @@ class Entry:
def attach(execution):
jid = execution.job_id
- if jid in _entries:
+ if jid in process_state().entries:
raise RuntimeError("Job already has an execution monitor")
entry = Entry(execution)
- _entries[jid] = entry
+ process_state().entries[jid] = entry
entry.task = asyncio.create_task(_monitor(entry))
entry.task.add_done_callback(lambda task: task.exception() if not task.cancelled() else None)
return entry
async def interrupt(jid):
- entry = _entries.get(jid)
+ entry = process_state().entries.get(jid)
if entry:
if not await entry.execution.stop():
raise RuntimeError("作业执行权已转移,无法确认停止")
@@ -56,7 +55,7 @@ async def interrupt(jid):
async def wait(jid, timeout=0):
- entry = _entries.get(jid)
+ entry = process_state().entries.get(jid)
if entry:
try:
if timeout > 0:
@@ -181,19 +180,32 @@ async def _monitor(entry):
state.finish(jid, attempt, "interrupted", "作业监控失联,原执行状态待确认")
finally:
entry.ready.set()
- if _entries.get(jid) is entry:
- _entries.pop(jid)
+ if process_state().entries.get(jid) is entry:
+ process_state().entries.pop(jid)
row = state.snapshot(jid)
_notify(_maybe_wake(jid))
return {"status": row["status"], "stats": row["stats"], "error": row["error"]}
async def shutdown():
- tasks = [entry.task for entry in list(_entries.values())]
+ tasks = [entry.task for entry in list(process_state().entries.values())]
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
- notices = list(_notifications)
+ notices = list(process_state().notifications)
for notice in notices:
notice.cancel()
await asyncio.gather(*notices, return_exceptions=True)
+
+
+@dataclass
+class SupervisorState:
+ entries: object = field(default_factory=lambda: {})
+ notifications: object = field(default_factory=lambda: set())
+
+
+_STATE = SupervisorState()
+
+
+def process_state() -> SupervisorState:
+ return _STATE
diff --git a/src/backend/core/jobs/tool_operations.py b/src/backend/core/jobs/tool_operations.py
new file mode 100644
index 00000000..aa62206a
--- /dev/null
+++ b/src/backend/core/jobs/tool_operations.py
@@ -0,0 +1,410 @@
+"""run_job 工具 —— 主对话智能体提交批量作业的唯一入口。
+
+定位:这不是「又一个固定形状的批处理工具」,而是一个**可编程的编排面**。智能体先用
+``write`` 把一段作业脚本写进沙箱,再用 ``run_job`` 提交;脚本里的控制流(循环、条件、
+分层取数、动态收敛、嵌套 map)都是普通 Python,平台不需要知道任务在做什么。
+
+与 Bash / read / write / glob / grep 同级原生注册,不走 catalog 开关——「对话内默认
+可用」就落在这里。
+"""
+
+from __future__ import annotations
+
+import json as _json
+import logging
+from typing import Any, Dict, List, Optional
+
+
+from core.sandbox.workspace.resolution import quote_shell_path as _quote_path
+from core.db.threads import run_db, run_in_session
+
+logger = logging.getLogger(__name__)
+
+# 前台等待(wait=True)的硬上限。超过就把作业转后台,不再阻塞这一轮对话。
+# 该等多久是模型自己判断的,而它对"这批要跑多久"的估计经常偏乐观——实测把一个 265 项、
+# 每项都要过模型的分类作业当成"小作业",前台一口气阻塞了 5 分钟,用户看不到进度、
+# 插不上话、也没法中途取消。提示词已经写过"几分钟以上就走后台"仍然拦不住,所以在
+# 机制上给前台等待封顶:等够 90 秒还没完,作业照跑,对话先还给用户。
+FOREGROUND_WAIT_CAP_S = 90.0
+
+
+def mark_wake_on_finish(job_id: str) -> None:
+ """转后台后要有人叫醒会话播报终态,否则作业跑完没人吭声。"""
+ try:
+ from core.db.engine import SessionLocal
+ from core.jobs.store import JobService
+
+ with SessionLocal() as db:
+ JobService(db).set_wake_on_finish(job_id, True)
+ except Exception: # noqa: BLE001 —— 叫醒是尽力而为,不该反过来弄挂工具调用
+ logger.warning("[run_job] mark wake_on_finish failed job=%s", job_id, exc_info=True)
+
+
+def detached_payload(job_id: str, *, resumed: bool = False) -> Dict[str, Any]:
+ """前台等超时 → 按后台语义返回(作业没停,只是不占着这一轮对话了)。"""
+ mark_wake_on_finish(job_id)
+ return {
+ "ok": True,
+ "job_id": job_id,
+ "status": "started",
+ "waited": False,
+ "detached": True,
+ "next": (
+ f"作业比预期久(前台已等满 {int(FOREGROUND_WAIT_CAP_S)} 秒),"
+ "**已自动转入后台继续跑,没有中断**。"
+ "现在就结束这轮回复:告诉用户作业仍在进行、"
+ "输入框上方的状态条会实时显示进度,期间可以继续聊别的、也可以随时取消。"
+ "作业跑完会自动叫醒你播报,不要在这里轮询 action='status' 干等,"
+ "也不要重新提交一遍(它还在跑)。"
+ ),
+ }
+
+
+
+def _resolve_job(db, job_id: str, user_id: str, chat_id: str):
+ """Return ``(job_id, None)`` for an owned job, else ``(None, error payload)``."""
+ from core.db.models import Job
+ from core.jobs.store import JobService
+
+ svc = JobService(db)
+ resolved = svc.get(job_id)
+ if (resolved is None or resolved.user_id != user_id) and not job_id.startswith("job_"):
+ cand = svc.get(f"job_{job_id}")
+ if cand is not None and cand.user_id == user_id:
+ resolved, job_id = cand, f"job_{job_id}"
+ if resolved is not None and resolved.user_id == user_id:
+ return job_id, None
+ rows = (
+ db.query(Job)
+ .filter(Job.user_id == user_id, Job.chat_id == chat_id)
+ .order_by(Job.created_at.desc())
+ .limit(5)
+ .all()
+ )
+ return None, {
+ "ok": False,
+ "error": f"job 不存在或无权访问: {job_id}",
+ "jobs_in_this_chat": [
+ {"job_id": r.job_id, "name": r.name, "status": r.status, "stats": svc.stats(r.job_id)}
+ for r in rows
+ ],
+ "hint": "用上面列表里的完整 job_id 重试(注意 job_ 前缀);"
+ "已经跑完的作业**不要重跑**,直接 export 取结果。",
+ }
+
+
+def _job_status(db, job_id: str, user_id: str) -> Dict[str, Any]:
+ from core.jobs.store import JobService
+
+ svc = JobService(db)
+ job = svc.get(job_id)
+ if job is None or job.user_id != user_id:
+ return {"ok": False, "error": "job 不存在或无权访问"}
+ return {
+ "ok": True,
+ "job_id": job_id,
+ "status": job.status,
+ "stats": svc.stats(job_id),
+ "usage": dict(job.usage or {}),
+ "budget_left": svc.budget_left(job_id),
+ "error": job.error_message,
+ }
+
+
+def _ledger_rows(db, job_id: str, user_id: str, status: str) -> Dict[str, Any]:
+ from core.jobs.store import JobService
+
+ svc = JobService(db)
+ job = svc.get(job_id)
+ if job is None or job.user_id != user_id:
+ return {"ok": False, "error": "job 不存在或无权访问"}
+ wanted = [s.strip() for s in (status or "").split(",") if s.strip()] or [
+ "done",
+ "not_found",
+ "failed",
+ "needs_review",
+ "pending",
+ ]
+ rows = []
+ for st in wanted:
+ rows.extend(svc.pending(job_id, status=st))
+ return {"rows": rows, "stats": svc.stats(job_id)}
+
+
+def job_operations(
+ *,
+ user_id: str,
+ chat_id: Optional[str],
+ sandbox_session_id: Optional[str],
+ allowed_tools: Optional[List[str]] = None,
+ model_name: Optional[str] = None,
+ model_provider_id: Optional[str] = None,
+ project_ctx: Optional[Dict[str, Any]] = None,
+ parent_runtime: Optional[Dict[str, Any]] = None,
+):
+ """Build scoped job operations.
+
+ ``parent_runtime`` 保存主智能体的能力配置;每个 agent() 使用独立上下文派生实例。
+ ``allowed_tools`` 保留脚本显式选择 MCP 的权限检查。
+ """
+
+ async def run_job(
+ action: str = "start",
+ script_path: str = "",
+ name: str = "",
+ job_id: str = "",
+ wait: bool = False,
+ # 默认必须是**带三方库**的那个解释器。沙箱里裸 python3 是干净的系统解释器,
+ # 连 openpyxl/pandas 都没有——默认值写成 python3 的直接后果是作业脚本第一行
+ # import 就 ModuleNotFoundError 当场失败(实测踩过)。$PY_BIN 指向预装全套数据
+ # 依赖的解释器,取不到时才退回 python3。
+ interpreter: str = "${PY_BIN:-python3}",
+ max_calls: int = 0,
+ max_seconds: int = 0,
+ concurrency: int = 8,
+ dest_path: str = "",
+ status: str = "",
+ progress_wake_sec: int = 300,
+ on_conflict: str = "block",
+ ) -> Dict[str, Any]:
+ import core.jobs.files as job_files
+ import core.jobs.runtime as job_runtime
+ from core.db.models import Job
+ from core.jobs.store import JobService
+
+ act = (action or "start").strip().lower()
+
+ if act == "start":
+ if not script_path:
+ return {"ok": False, "error": "script_path 必填:先用 write 把作业脚本写进沙箱"}
+ if not sandbox_session_id:
+ return {"ok": False, "error": "当前会话没有可用沙箱,无法提交作业"}
+
+ # 同一会话不许叠加在跑的作业。被进度唤醒后"改个脚本再交一份"是很自然的动作,
+ # 但旧作业不会自己消失:两份并存会同时烧预算、同时叫醒会话,用户在状态条上
+ # 也分不清哪份才算数。要么先 cancel,要么 resume(已完成的项不会重做)。
+ # 本会话已有在跑的作业时**默认**拦一道:默默叠加两份会双倍烧预算、双份叫醒会话,
+ # 用户在状态条上也分不清哪份算数。但这只是默认,不是禁令——确实要换新版本时
+ # 用 on_conflict 明说:'replace' 停掉旧的再跑新的,'parallel' 两份并行。
+ # 判断权留给调用方,工具只负责让"叠加"变成一个显式选择而不是意外。
+ conflict = (on_conflict or "block").strip().lower()
+ if chat_id and conflict != "parallel":
+ # 字段必须在 session 内取出:出了 with 块 ORM 实例就 detached,再读属性会炸
+ def _live_job(db):
+ live = (
+ db.query(Job)
+ .filter(
+ Job.chat_id == chat_id,
+ Job.status.in_(("pending", "running")),
+ )
+ .order_by(Job.created_at.desc())
+ .first()
+ )
+ live_id = live.job_id if live is not None else ""
+ live_name = (live.name or "未命名") if live is not None else ""
+ return live_id, live_name
+ live_id, live_name = await run_in_session(_live_job)
+ if live_id and conflict == "replace":
+ await job_runtime.cancel_job(live_id, user_id=user_id)
+ logger.info("[run_job] replaced live job %s in chat %s", live_id, chat_id)
+ elif live_id:
+ return {
+ "ok": False,
+ "error": (
+ f"本会话已有在跑的作业 {live_id}({live_name})。三条路选一条:"
+ "① 想用新脚本取代它 → 再调一次本工具并带 on_conflict='replace'"
+ "(自动停掉旧作业再提交新的);"
+ f"② 旧作业只是中断、脚本没问题 → run_job(action='resume', job_id='{live_id}')"
+ "断点续跑,已完成的项不会重做;"
+ "③ 确实需要两份同时跑 → on_conflict='parallel'。"
+ "默认拦下只是为了让叠加成为显式选择,不是不让跑。"
+ ),
+ "job_id": live_id,
+ "hint": "on_conflict=replace|parallel",
+ }
+
+ # 读脚本正文:既做存在性校验,也作为 job 的审计快照与 resume 依据。
+ # ⚠️ 必须走 base64,不能直接 `cat`:沙箱 execute 回传的 stdout 会丢换行,
+ # 脚本会被压成一行落地 → SyntaxError(实测踩过)。base64 -w0 保字节不变。
+ try:
+ code, out, err = await job_files._sbx_bash(
+ f"base64 -w0 {_quote_path(script_path)} 2>/dev/null || base64 -i {_quote_path(script_path)}",
+ session_id=sandbox_session_id,
+ user_id=user_id,
+ timeout=30,
+ )
+ except Exception as exc: # noqa: BLE001
+ return {"ok": False, "error": f"读取脚本失败: {exc}"}
+ if code != 0 or not (out or "").strip():
+ return {"ok": False, "error": f"脚本不存在或为空: {script_path} {(err or '')[:200]}"}
+ import base64 as _b64dec
+
+ try:
+ out = _b64dec.b64decode("".join((out or "").split())).decode("utf-8")
+ except Exception as exc: # noqa: BLE001
+ return {"ok": False, "error": f"脚本解码失败: {exc}"}
+ if not out.strip():
+ return {"ok": False, "error": f"脚本为空: {script_path}"}
+
+ budget: Dict[str, Any] = {"concurrency": concurrency}
+ if max_calls:
+ budget["max_calls"] = max_calls
+ if max_seconds:
+ budget["max_seconds"] = max_seconds
+
+ try:
+ jid = await job_runtime.start_job(
+ user_id=user_id,
+ chat_id=chat_id,
+ name=name or "批量作业",
+ script_path=script_path,
+ script_text=out,
+ session_id=sandbox_session_id,
+ budget=budget,
+ start_params={
+ "allowed_tools": list(allowed_tools or []),
+ "model_name": model_name,
+ "model_provider_id": model_provider_id,
+ "project_ctx": project_ctx,
+ "parent_runtime": parent_runtime,
+ # wait=False 时作业跑完没人叫醒本会话 → 打标,让驱动在终态入队一轮续跑
+ "wake_on_finish": not wait,
+ # 中途进度播报间隔(秒);0 = 只在终态叫一次
+ "progress_wake_sec": max(0, int(progress_wake_sec)),
+ },
+ interpreter=interpreter or "${PY_BIN:-python3}",
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[run_job] start failed: %s", exc)
+ return {"ok": False, "error": f"作业启动失败: {exc}"}
+
+ if not wait:
+ return {
+ "ok": True,
+ "job_id": jid,
+ "status": "started",
+ "waited": False,
+ "next": (
+ "作业已在后台开始。**现在就结束这轮回复**:告诉用户作业已提交、"
+ "输入框上方的状态条会实时显示进度(几分之几、失败数、已运行多久),"
+ "期间可以继续聊别的、也可以点状态条上的按钮取消。"
+ "作业跑完和中途都会自动叫醒你播报,不要在这里轮询 action='status' 干等。"
+ ),
+ }
+
+ res = await job_runtime.run_and_wait(
+ jid, chat_id=chat_id, detach_after=FOREGROUND_WAIT_CAP_S
+ )
+ if res.get("status") == "detached":
+ return await run_db(detached_payload, jid)
+ def _finished_job(db):
+ svc = JobService(db)
+ stats = svc.stats(jid)
+ job = svc.get(jid)
+ usage = dict(job.usage or {}) if job else {}
+ return stats, usage
+ stats, usage = await run_in_session(_finished_job)
+ payload = {
+ "ok": res.get("status") == "completed",
+ "job_id": jid,
+ "status": res.get("status"),
+ "stats": stats,
+ "usage": usage,
+ }
+ if res.get("error"):
+ payload["error"] = res["error"]
+ if stats.get("remaining", 0) > 0:
+ payload["note"] = (
+ f"仍有 {stats['remaining']} 项未完成——必须如实告知用户并给出未覆盖清单与"
+ f"续跑方式(run_job action=resume job_id={jid}),不得当作已完成。"
+ )
+ # 脚本崩溃时把 runner 日志尾巴带回来,省一轮排查
+ if res.get("status") == "failed" and sandbox_session_id:
+ try:
+ payload["runner_log"] = (
+ await job_files.read_runner_log(
+ jid, user_id=user_id, session_id=sandbox_session_id
+ )
+ )[-1500:]
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure("jobs.tool_operations:runner_log", fallback_error)
+ return payload
+
+ if not job_id:
+ return {"ok": False, "error": f"action={act} 需要 job_id"}
+
+ # job_id 容错 + 找不到时把本会话的作业列出来。
+ # 实测踩过:模型凭记忆抄 id 时漏了 "job_" 前缀,只回一句「job 不存在」它无从自救,
+ # 结果决定把 568 次子调用整个重跑一遍。错误信息必须自带下一步。
+ resolved_id, missing = await run_in_session(_resolve_job, job_id, user_id, chat_id)
+ if missing is not None:
+ return missing
+ job_id = resolved_id
+
+ if act == "status":
+ return await run_in_session(_job_status, job_id, user_id)
+
+ if act == "export":
+ # 台账在 DB,逐项结果**不进对话上下文**——所以导出的方式是写成沙箱里的
+ # JSONL 文件,之后用 Bash/python 处理(写 Excel、跑校验、喂评审)。
+ if not sandbox_session_id:
+ return {"ok": False, "error": "当前会话没有可用沙箱"}
+ ledger = await run_in_session(_ledger_rows, job_id, user_id, status)
+ if "error" in ledger:
+ return ledger
+ rows, stats = ledger["rows"], ledger["stats"]
+ dest = dest_path or f"jobs/{job_id}_ledger.jsonl"
+ body = "\n".join(_json.dumps(r, ensure_ascii=False) for r in rows)
+ # 分块写 + 读回校验:单条 Bash 携带大 base64 到约 170KB 会**静默失败**
+ # (exit=0、stderr 空、文件不存在)。568 行台账正好落在这个区间——历史上
+ # 因此出现过「导出报成功 rows=568,文件却从来没出现」。成功与否只认沙箱
+ # 里读回的真实字节数,绝不用 Python 侧的行数报平安。
+ try:
+ ok_write, detail = await job_files.write_sandbox_file(
+ dest, body, session_id=sandbox_session_id, user_id=user_id
+ )
+ except Exception as exc: # noqa: BLE001
+ return {"ok": False, "error": f"导出失败: {exc}"}
+ if not ok_write:
+ return {"ok": False, "error": f"导出失败(落盘未通过校验): {detail}"}
+ return {
+ "ok": True,
+ "job_id": job_id,
+ "path": dest,
+ "rows": len(rows),
+ "bytes": detail,
+ "stats": stats,
+ "hint": "每行一个 JSON:{key, status, payload, result, review, attempts}。"
+ "用 Bash/python 读这个文件写产物,不要把逐项内容读回对话。",
+ }
+
+ if act == "cancel":
+ ok = await job_runtime.cancel_job(job_id, user_id=user_id)
+ return {"ok": ok, "job_id": job_id, "status": "cancelled" if ok else "unknown"}
+
+ if act == "resume":
+ res = await job_runtime.resume_job(job_id, user_id=user_id, chat_id=chat_id)
+ if not res.get("ok"):
+ return {"ok": False, "error": res.get("error")}
+ if not wait:
+ return {"ok": True, "job_id": job_id, "status": "running", "waited": False}
+ done = await job_runtime.run_and_wait(
+ job_id, chat_id=chat_id, detach_after=FOREGROUND_WAIT_CAP_S
+ )
+ if done.get("status") == "detached":
+ return await run_db(detached_payload, job_id, resumed=True)
+ def _job_stats(db):
+ stats = JobService(db).stats(job_id)
+ return stats
+ stats = await run_in_session(_job_stats)
+ return {
+ "ok": done.get("status") == "completed",
+ "job_id": job_id,
+ "status": done.get("status"),
+ "stats": stats,
+ }
+
+ return {"ok": False, "error": f"未知 action: {act}"}
+
+ return run_job
diff --git a/src/backend/core/kb/kb_assets.py b/src/backend/core/kb/kb_assets.py
deleted file mode 100644
index 3f186ceb..00000000
--- a/src/backend/core/kb/kb_assets.py
+++ /dev/null
@@ -1,567 +0,0 @@
-"""Knowledge-base media assets: persistence, 图生文, and their retrieval rows.
-
-The text pipeline is unchanged — this module is the "路径 A" layer on top of it:
-
- parse → parents + RawAsset[] (core/kb/kb_parser.py)
- → persist_assets() bytes to object storage, rows to kb_assets,
- placeholders in the parent text rewritten to
- real URLs, each asset linked to its parent chunk
- → generate_captions() 图像理解走 core/vision 视觉桥(与对话同一条通路)
- → index_assets() 一条 ``row_type="image"``
- row per asset in the existing Milvus collection
-
-Because the image rows carry ``parent_chunk_id`` of the chunk that contains them, every
-downstream behaviour — dedup by parent, parent content fetched from PostgreSQL, RRF,
-reranking, per-document deletion — keeps working with no changes.
-
-Two extension seams are deliberate:
-
-* **路径 B(真多模态向量)** — a CLIP-family embedding cannot share this collection: the
- dimension differs and ``get_or_create_collection`` drops the collection on a dimension
- mismatch. It belongs in its own collection, recorded per asset under
- ``KBAsset.vector_state["visual"]``, and fused with these results in Python.
-* **音视频** — ``KBAsset.kind`` and ``RawAsset.kind`` already widen to audio/video. The
- medium-specific work is producing ``text_content`` (ASR transcript) and ``caption``;
- everything from :func:`index_assets` onwards is medium-agnostic.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import uuid
-from typing import Iterable, Optional, Sequence
-
-from core.kb.kb_parser import RawAsset, image_extension_for_mime
-
-logger = logging.getLogger(__name__)
-
-# Milvus ``row_type`` for a media asset row (alongside "chunk" / "question").
-ROW_TYPE_IMAGE = "image"
-
-# Milvus primary keys must not collide with chunk ids; asset rows are namespaced.
-_ASSET_ROW_PREFIX = "a_"
-
-# Images below this many bytes are almost always icons / rules / bullet glyphs that the
-# layout engine cut out. Describing them wastes a vision call per item and pollutes
-# retrieval with near-empty descriptions.
-_MIN_ASSET_BYTES = int(os.getenv("KB_ASSET_MIN_BYTES", "4096"))
-# 单文档图像理解张数上限,防一篇图海文档独占视觉桥。体积上限、单次调用超时都归视觉桥
-# 自己管(``VISION_MAX_IMAGE_BYTES`` / ``VISION_CALL_TIMEOUT_SECONDS``),这里不再重复设。
-_MAX_CAPTIONS_PER_DOC = int(os.getenv("KB_ASSET_CAPTION_MAX_PER_DOC", "200"))
-# Minimum pixel edge; only enforced when Pillow is importable (it arrives transitively
-# via python-pptx, so it is treated as optional).
-_MIN_ASSET_EDGE_PX = int(os.getenv("KB_ASSET_MIN_EDGE_PX", "64"))
-
-
-
-def multimodal_indexing_enabled(indexing_config: Optional[dict]) -> bool:
- """Whether to extract and index media for this document.
-
- Per-KB opt-out via ``indexing_config.multimodal_indexing``; the global default
- lives in ``KB_MULTIMODAL_INDEXING`` so an operator can turn the extra parse cost
- off fleet-wide without touching each knowledge base.
- """
- if isinstance(indexing_config, dict) and indexing_config.get("multimodal_indexing") is not None:
- return bool(indexing_config.get("multimodal_indexing"))
- return (os.getenv("KB_MULTIMODAL_INDEXING", "true") or "").strip().lower() not in (
- "false",
- "0",
- "no",
- )
-
-
-def asset_public_url(asset_id: str) -> str:
- """Frontend-resolvable URL for an asset. Matches the route in ``api/routes/v1/kb.py``.
-
- The ``/api`` prefix mirrors the frontend's ``VITE_API_BASE_URL`` default (nginx
- strips it before the backend sees the path). It is written into stored chunk text,
- so a deployment that serves the API elsewhere overrides ``KB_ASSET_URL_PREFIX``
- before indexing rather than rewriting rows afterwards.
- """
- prefix = (os.getenv("KB_ASSET_URL_PREFIX", "/api") or "").rstrip("/")
- return f"{prefix}/v1/catalog/kb/assets/{asset_id}"
-
-
-# ── Persistence ────────────────────────────────────────────────────────────────
-
-
-def _is_worth_indexing(asset: RawAsset) -> bool:
- """Filter out layout noise (rules, bullets, logos) before it costs a vision call."""
- if len(asset.data) < _MIN_ASSET_BYTES:
- return False
- try:
- import io
-
- from PIL import Image # optional dependency (transitive via python-pptx)
-
- with Image.open(io.BytesIO(asset.data)) as im:
- width, height = im.size
- if min(width, height) < _MIN_ASSET_EDGE_PX:
- return False
- except Exception:
- # Pillow missing or the bytes are not a decodable image — the byte-size floor
- # above is the fallback filter; never drop an asset because probing failed.
- pass
- return True
-
-
-def _locate_parent_chunk(parents: Sequence, placeholder: str) -> Optional[object]:
- """Find the parent chunk whose text references this asset."""
- for parent in parents:
- if placeholder and placeholder in (parent.content or ""):
- return parent
- return None
-
-
-def _rewrite_placeholder(parents: Sequence, placeholder: str, url: str) -> None:
- """Point the markdown image link at the stored asset.
-
- Before this, ```` in a chunk was a dead link — the layout
- service's temp path, with the bytes discarded. Rewriting in both the parent text
- and its children keeps the stored chunk, the embedded text, and what the UI renders
- consistent.
- """
- needle = f"]({placeholder})"
- replacement = f"]({url})"
- for parent in parents:
- if needle in (parent.content or ""):
- parent.content = parent.content.replace(needle, replacement)
- for child in getattr(parent, "children", []) or []:
- if needle in (child.content or ""):
- child.content = child.content.replace(needle, replacement)
-
-
-def persist_assets(
- db,
- *,
- kb_id: str,
- document_id: str,
- user_id: str,
- assets: Sequence[RawAsset],
- parents: Sequence,
-) -> list:
- """Store asset bytes, create ``kb_assets`` rows, and link them to parent chunks.
-
- Must run **before** the parent chunks are embedded and written: it rewrites the
- placeholder links inside ``parents`` in place, and the rewritten text is what gets
- stored in PostgreSQL and vectorised.
-
- Failures are per-asset — a storage error on one figure must not fail the document,
- whose text is already parsed and perfectly usable.
- """
- from core.db.models import KBAsset
- from core.storage import generate_storage_key, get_storage
-
- if not assets:
- return []
-
- env = os.getenv("ENVIRONMENT", "dev")
- storage = get_storage()
- created: list = []
-
- for asset in assets:
- if not _is_worth_indexing(asset):
- continue
-
- asset_id = uuid.uuid4().hex[:32]
- suffix = image_extension_for_mime(asset.mime_type) if asset.kind == "image" else ".bin"
- storage_key = generate_storage_key(
- env=env,
- user_id=user_id,
- category="kb_assets",
- filename=f"{asset_id}{suffix}",
- )
- try:
- storage.upload_bytes(asset.data, storage_key)
- except Exception as exc:
- logger.warning("资产 %s 落盘失败,跳过: %s", asset.placeholder, exc)
- continue
-
- parent = _locate_parent_chunk(parents, asset.placeholder)
- row = KBAsset(
- asset_id=asset_id,
- kb_id=kb_id,
- document_id=document_id,
- chunk_id=getattr(parent, "parent_id", None),
- kind=asset.kind,
- mime_type=asset.mime_type,
- storage_key=storage_key,
- size_bytes=len(asset.data),
- asset_index=asset.index,
- locator=asset.locator or {},
- text_content=asset.text_content or "",
- caption_status="pending",
- vector_state={},
- extra_data={"placeholder": asset.placeholder},
- )
- db.add(row)
- created.append(row)
- _rewrite_placeholder(parents, asset.placeholder, asset_public_url(asset_id))
-
- if created:
- db.commit()
- for row in created:
- db.refresh(row)
- logger.info("文档 %s 抽出并落盘 %d 个资产", document_id, len(created))
- return created
-
-
-def purge_document_assets(db, document_id: str) -> int:
- """Delete a document's assets (storage objects + rows). Used on re-index."""
- from core.db.models import KBAsset
-
- rows = db.query(KBAsset).filter(KBAsset.document_id == document_id).all()
- if not rows:
- return 0
-
- try:
- from core.storage import get_storage
-
- storage = get_storage()
- for row in rows:
- try:
- storage.delete(row.storage_key)
- except Exception as exc:
- logger.warning("资产 %s 存储清理失败: %s", row.asset_id, exc)
- except Exception as exc:
- logger.warning("存储后端不可用,仅清理资产表: %s", exc)
-
- count = db.query(KBAsset).filter(KBAsset.document_id == document_id).delete()
- db.commit()
- return count
-
-
-# ── 图生文:走视觉桥 ───────────────────────────────────────────────────────────
-
-# 单批提交给视觉桥的图片数。视觉桥的并发信号量是**进程级**的
-# (``VISION_MAX_CONCURRENCY``,默认 4),一篇上百张图的文档如果一次性 gather 过去,
-# 会把信号量占满,对话里的看图请求全被排在后面——批量离线任务不该抢交互路径的道。
-# 按小批提交保证 KB 任何时刻最多占用其中几格,其余留给对话。
-_CAPTION_BATCH = max(1, int(os.getenv("KB_ASSET_CAPTION_BATCH", "2")))
-# 落进 ``text_content`` 的转写上限:它要参与向量化,一张满是文字的扫描件能顶出几万字。
-_MAX_TEXT_CONTENT_CHARS = max(200, int(os.getenv("KB_ASSET_TEXT_MAX_CHARS", "4000")))
-
-# 给视觉桥的 focus:同一套证据契约,但把注意力挪到"文档插图"这个场景上。
-_CAPTION_FOCUS = (
- "这是文档中的插图,识别结果会被向量化用于知识库检索。请重点覆盖:"
- "图的类型(照片/流程图/架构图/折线图/柱状图/表格截图等)、"
- "图中出现的全部文字与数字(坐标轴、图例、标签、单位)、"
- "以及图所反映的关键结论或趋势。读不出来的一律写进 uncertainty,不要猜。"
-)
-
-
-def vision_available() -> bool:
- """当前是否有可用的图像理解模型(视觉桥)。"""
- try:
- from core.vision import is_available
-
- return is_available()
- except Exception as exc: # noqa: BLE001 — 视觉桥不可用不该让索引失败
- logger.debug("视觉桥可用性判定失败,按不可用处理: %s", exc)
- return False
-
-
-def _describe_batch(images: list, loop=None) -> list:
- """把一批图交给视觉桥识别。返回与输入等长的结果列表(失败位为 None)。
-
- 视觉桥是异步的,而索引是同步函数、跑在没有运行中事件循环的工作线程里。按可靠性
- 从高到低三级取宿主循环——只有跑在**宿主循环**上,视觉桥的证据缓存与并发信号量才
- 真的生效:
-
- 1. ``loop``:常驻索引 worker 显式传进来的调度循环。这是当前的生产路径——索引跑在
- worker 自己的 ``ThreadPoolExecutor`` 里(见 ``core/kb/index_worker.py``),那不是
- anyio 的工作线程,第 2 级在这里拿不到东西。
- 2. ``anyio.from_thread``:仍经 ``BackgroundTasks`` 进来的调用方(如我的空间同步产物
- 入库)跑在 Starlette 的 anyio 线程池里,天然带着回宿主循环的通路。
- 3. 临时循环 + **关掉证据缓存**:CLI / 脚本 / 测试。``core/infra/redis.py`` 的连接池是
- 模块级全局、绑定首次使用它的循环,让一个转瞬即逝的临时循环抢先创建它,会把整个
- 进程的 Redis 绑死在已销毁的循环上——那比"这次没吃到缓存"严重得多。
- """
- import asyncio
- import functools
-
- from core.vision import get_vision_bridge
-
- bridge = get_vision_bridge()
-
- if loop is not None and not loop.is_closed():
- future = asyncio.run_coroutine_threadsafe(
- bridge.describe_many(images, focus=_CAPTION_FOCUS), loop
- )
- return future.result()
-
- try:
- import anyio.from_thread
-
- return anyio.from_thread.run(
- functools.partial(bridge.describe_many, images, focus=_CAPTION_FOCUS)
- )
- except RuntimeError as exc:
- logger.debug("视觉桥回宿主循环不可用,改用临时循环且不走缓存: %s", exc)
-
- async def _uncached() -> list:
- tasks = [
- bridge.describe(data, mime, focus=_CAPTION_FOCUS, use_cache=False)
- for data, mime in images
- ]
- return list(await asyncio.gather(*tasks))
-
- return asyncio.run(_uncached())
-
-
-def _merge_text_content(existing: str, ocr_text: str) -> str:
- """图注 + OCR 转写合成检索用文本,去重后截断。
-
- 两者都留:图注是文档作者写的、往往点明了图的用途;OCR 是图里实际的字。
- """
- parts: list[str] = []
- for part in ((existing or "").strip(), (ocr_text or "").strip()):
- if part and part not in parts:
- parts.append(part)
- merged = "\n".join(parts)
- return merged[:_MAX_TEXT_CONTENT_CHARS]
-
-
-def _evidence_digest(result) -> dict:
- """留档进 ``extra_data`` 的证据摘要。
-
- 刻意不存 ``layout.regions`` 与 ``ocr.lines``:整段文字已经进了 ``text_content``,
- 再存一份逐行结构,一篇几百张图的文档会把 JSONB 撑得很难看。留下的是之后真会用到
- 的部分——实体/关系(可喂本体与 Wiki 层)、不确定项(人工复核时看)、模型元数据
- (换模型后判断哪些资产该重跑)。
- """
- evidence = result.evidence
- return {
- "summary": evidence.summary,
- "scene": evidence.semantics.scene,
- "intent": evidence.semantics.intent,
- "entities": [e.model_dump(exclude_none=True) for e in evidence.semantics.entities],
- "relations": [r.model_dump(exclude_none=True) for r in evidence.semantics.relations],
- "uncertainty": list(evidence.uncertainty),
- "meta": result.to_meta(),
- }
-
-
-def generate_captions(db, assets: Sequence, *, loop=None) -> int:
- """给资产补上图像理解结果:``caption`` 写概述,``text_content`` 并入 OCR 转写。
-
- 走的是 ``core/vision`` 视觉桥——和对话里"纯文本主模型读图"完全同一条通路、同一个
- ``vision`` 模型角色,因此知识库这边白得三样东西:按图片内容哈希的证据缓存(重新
- 索引同一篇文档不用再付一次钱)、三协议与结构化输出降级、以及"该角色只能指派多模态
- 模型"的硬约束。
-
- 没有可用的视觉模型是正常状态,不是错误:资产保留文档自带的图注并照常入索引,之后
- 配好 ``vision`` 角色再回填即可,**不用重新解析文档**。
- """
- if not assets:
- return 0
-
- if not vision_available():
- logger.info("视觉桥不可用(未指派 vision 角色且主模型不支持读图),跳过图生文;图注仍会入索引")
- for row in assets:
- row.caption_status = "skipped"
- db.commit()
- return 0
-
- from core.storage import get_storage
-
- storage = get_storage()
- images = [a for a in assets if a.kind == "image"]
- budget, overflow = images[:_MAX_CAPTIONS_PER_DOC], images[_MAX_CAPTIONS_PER_DOC:]
- # 没轮到的显式标注,让"上限生效"在数据里看得见,而不是像视觉模型什么都没返回。
- for row in overflow + [a for a in assets if a.kind != "image"]:
- row.caption_status = "skipped"
- if overflow:
- logger.warning(
- "文档图片数 %d 超过单文档图生文上限 %d,超出部分仅用图注索引",
- len(images),
- _MAX_CAPTIONS_PER_DOC,
- )
-
- done = 0
- for offset in range(0, len(budget), _CAPTION_BATCH):
- chunk = budget[offset : offset + _CAPTION_BATCH]
- payloads: list = []
- pending: list = []
- for row in chunk:
- try:
- data = storage.download_bytes(row.storage_key)
- except Exception as exc: # noqa: BLE001 — 单张读不出来不该拖垮整篇
- logger.warning("资产 %s 读取失败,跳过图生文: %s", row.asset_id, exc)
- row.caption_status = "failed"
- continue
- payloads.append((data, row.mime_type))
- pending.append(row)
-
- if not payloads:
- continue
-
- try:
- results = _describe_batch(payloads, loop)
- except Exception as exc: # noqa: BLE001 — 整批失败也只降级,正文索引已经好了
- logger.warning("视觉桥调用失败,该批 %d 张仅用图注索引: %s", len(pending), exc)
- for row in pending:
- row.caption_status = "failed"
- continue
-
- for row, result in zip(pending, results):
- if result is None:
- row.caption_status = "failed"
- continue
- evidence = result.evidence
- summary = (evidence.summary or "").strip()
- row.caption = summary
- row.text_content = _merge_text_content(row.text_content, evidence.ocr.full_text)
- meta = dict(row.extra_data or {})
- meta["vision"] = _evidence_digest(result)
- row.extra_data = meta
- row.caption_status = "completed" if summary else "failed"
- if summary:
- done += 1
-
- # 这里给 ``extra_data`` 赋的是**新字典**(不是原地改键),SQLAlchemy 的属性事件能
- # 直接捕获,不需要 flag_modified —— 那个只有原地改 JSON 才必须。
- db.commit()
- logger.info("图生文完成 %d/%d", done, len(budget))
- return done
-
-
-# ── Vector rows ────────────────────────────────────────────────────────────────
-
-
-def asset_index_text(asset, *, title: str = "") -> str:
- """The text an asset is retrieved by: 视觉桥概述 + 图注/OCR 转写 + 文档标题。
-
- Audio/video reuse this unchanged — ``text_content`` becomes the ASR transcript.
- """
- parts = [
- (asset.caption or "").strip(),
- (asset.text_content or "").strip(),
- (title or "").strip(),
- ]
- return "\n".join(p for p in parts if p)
-
-
-def index_assets(
- assets: Sequence, *, user_id: str, kb_id: str, document_id: str, title: str
-) -> int:
- """Write one ``row_type="image"`` row per asset into the existing collection.
-
- ``parent_chunk_id`` points at the chunk that contains the asset, so a hit here
- resolves to that chunk's full text through the existing parent-fetch path. Assets
- with no owning chunk (a standalone upload whose placeholder was lost) point at
- themselves and fall back to the caption as their content.
- """
- if not assets:
- return 0
-
- from core.kb.kb_vector import (
- TAGS_FIELD_MAX_BYTES,
- TITLE_FIELD_MAX_BYTES,
- embed_batch,
- text_to_sparse,
- truncate_utf8,
- upsert_rows,
- )
-
- indexable = [(a, asset_index_text(a, title=title)) for a in assets]
- indexable = [(a, t) for a, t in indexable if t.strip()]
- if not indexable:
- return 0
-
- try:
- vectors = embed_batch([t for _, t in indexable])
- except Exception as exc:
- logger.error("资产向量化失败,文档 %s 的图片不可检索: %s", document_id, exc)
- return 0
-
- rows = []
- for (asset, text), vector in zip(indexable, vectors):
- rows.append(
- {
- "chunk_id": f"{_ASSET_ROW_PREFIX}{asset.asset_id}",
- "parent_chunk_id": asset.chunk_id or f"{_ASSET_ROW_PREFIX}{asset.asset_id}",
- "row_type": ROW_TYPE_IMAGE,
- "user_id": user_id,
- "kb_id": kb_id,
- "document_id": document_id,
- "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES),
- "content": truncate_utf8(text),
- "tags_text": truncate_utf8(asset.text_content or "", TAGS_FIELD_MAX_BYTES),
- "chunk_index": asset.asset_index or 0,
- "dense_embedding": vector,
- "sparse_embedding": text_to_sparse(text),
- }
- )
-
- BATCH = 100
- for i in range(0, len(rows), BATCH):
- upsert_rows(rows[i : i + BATCH])
- logger.info("文档 %s 写入 %d 条资产检索行", document_id, len(rows))
- return len(rows)
-
-
-def mark_vector_state(db, assets: Sequence, space: str, state: str) -> None:
- """Record that these assets are indexed into a vector space.
-
- ``space`` is ``"text"`` for the caption/OCR rows written by :func:`index_assets`.
- 路径 B 落地时用 ``"visual"``,与文本行互不覆盖。
- """
- for row in assets:
- current = dict(row.vector_state or {})
- current[space] = state
- # 同上:赋新字典而不是原地改键,属性事件自然捕获得到。
- row.vector_state = current
- db.commit()
-
-
-# ── Retrieval helper ───────────────────────────────────────────────────────────
-
-
-def fetch_assets_for_chunks(chunk_ids: Iterable[str], *, max_per_chunk: int = 4) -> dict:
- """Map parent-chunk id → its assets, for attaching to retrieval results.
-
- Also keyed by the synthetic ``a_`` parent used when an asset has no
- owning chunk, so a hit on such a row still resolves to its own image.
- """
- ids = [c for c in dict.fromkeys(chunk_ids) if c]
- if not ids:
- return {}
-
- direct = [c for c in ids if not c.startswith(_ASSET_ROW_PREFIX)]
- orphan_asset_ids = [c[len(_ASSET_ROW_PREFIX) :] for c in ids if c.startswith(_ASSET_ROW_PREFIX)]
-
- try:
- from core.db.engine import SessionLocal
- from core.db.models import KBAsset
- from sqlalchemy import or_
-
- with SessionLocal() as db:
- clauses = []
- if direct:
- clauses.append(KBAsset.chunk_id.in_(direct))
- if orphan_asset_ids:
- clauses.append(KBAsset.asset_id.in_(orphan_asset_ids))
- if not clauses:
- return {}
- rows = db.query(KBAsset).filter(or_(*clauses)).order_by(KBAsset.asset_index).all()
- grouped: dict[str, list[dict]] = {}
- for row in rows:
- key = row.chunk_id or f"{_ASSET_ROW_PREFIX}{row.asset_id}"
- bucket = grouped.setdefault(key, [])
- if len(bucket) >= max_per_chunk:
- continue
- bucket.append(
- {
- "asset_id": row.asset_id,
- "kind": row.kind,
- "url": asset_public_url(row.asset_id),
- "caption": row.caption or row.text_content or "",
- "locator": row.locator or {},
- }
- )
- return grouped
- except Exception as exc:
- logger.warning("检索结果附图失败(不影响文本结果): %s", exc)
- return {}
diff --git a/src/backend/core/kb/kb_parser.py b/src/backend/core/kb/kb_parser.py
deleted file mode 100644
index 60525109..00000000
--- a/src/backend/core/kb/kb_parser.py
+++ /dev/null
@@ -1,933 +0,0 @@
-"""Document parser and parent-child chunker for private knowledge base.
-
-Supported formats: PDF, DOCX, TXT, MD, XLSX, 图片
-Output: list of ParentChunk, each containing child chunks for vector indexing,
-plus the non-text assets (图片;后续音视频) cut out of the document.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import math
-import re
-import uuid
-from dataclasses import dataclass, field
-from typing import Callable, Optional
-
-logger = logging.getLogger(__name__)
-
-# ── Data structures ────────────────────────────────────────────────────────────
-
-@dataclass
-class ChildChunk:
- child_id: str
- content: str
- index: int # position within parent
-
-
-@dataclass
-class ParentChunk:
- parent_id: str
- content: str # full parent text, returned to LLM on retrieval
- children: list[ChildChunk]
- char_start: Optional[int] = None
- char_end: Optional[int] = None
-
-
-@dataclass
-class RawAsset:
- """A non-text medium extracted from the source document, before persistence.
-
- ``placeholder`` is the exact token this medium is referenced by inside the
- paragraph text (``images/.jpg`` for a layout-extracted figure, a synthetic
- ``asset://`` for a standalone upload). It is what links the medium back to
- the parent chunk that contains it, and what gets rewritten to a real URL once the
- asset has an id.
-
- Medium-agnostic on purpose: ``kind`` widens to audio/video, and ``locator`` holds
- ``{page_idx, bbox}`` for an image where a transcript segment would hold
- ``{start_ms, end_ms}``.
- """
-
- data: bytes
- placeholder: str
- kind: str = "image"
- mime_type: str = "application/octet-stream"
- text_content: str = "" # 图注 / OCR(音视频接入后:ASR 转写)
- locator: dict = field(default_factory=dict)
- index: int = 0
-
-
-# ── Token counting ─────────────────────────────────────────────────────────────
-
-_tiktoken_enc = None
-
-
-def _get_tiktoken_enc():
- """Return cached tiktoken cl100k_base encoder."""
- global _tiktoken_enc
- if _tiktoken_enc is None:
- try:
- import tiktoken
- _tiktoken_enc = tiktoken.get_encoding("cl100k_base")
- except Exception:
- pass
- return _tiktoken_enc
-
-
-def _count_tokens(text: str) -> int:
- """Approximate token count using tiktoken cl100k_base."""
- enc = _get_tiktoken_enc()
- if enc is not None:
- return len(enc.encode(text))
- # Fallback: rough Chinese/English estimate
- return max(1, len(text) // 2)
-
-
-# ── MIME → filename suffix mapping ─────────────────────────────────────────────
-
-_MIME_TO_SUFFIX: dict[str, str] = {
- "application/pdf": ".pdf",
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
- "application/msword": ".doc",
- "text/plain": ".txt",
- "text/markdown": ".txt",
- "text/x-markdown": ".txt",
- "text/csv": ".csv",
- "application/json": ".txt",
-}
-
-
-# ── Text extraction ────────────────────────────────────────────────────────────
-
-def extract_text(file_bytes: bytes, mime_type: str) -> list[dict]:
- """Extract structured paragraphs from a document.
-
- Delegates to core/file_parser.py for PDF, DOCX, DOC, TXT (uses the
- external file-parser API for PDF, pandoc for DOCX, etc.).
- Falls back to built-in xlsx/plain-text extraction for unsupported types.
-
- Returns a list of paragraph dicts:
- {"text": str, "heading_level": int|None, "element_type": str}
- """
- paragraphs, _ = extract_text_with_assets(file_bytes, mime_type, with_assets=False)
- return paragraphs
-
-
-def extract_text_with_assets(
- file_bytes: bytes,
- mime_type: str,
- *,
- with_assets: bool = True,
-) -> tuple[list[dict], list[RawAsset]]:
- """Extract paragraphs and, when ``with_assets``, the media the document carries.
-
- Asset extraction is opt-in because it costs a heavier parse call: the layout
- service inlines every figure as base64 in the JSON response, which the text-only
- callers have no use for. When the rich call fails for any reason we fall back to
- the plain text path — a parse service that doesn't support the media params must
- degrade to today's behaviour, never fail the document.
- """
- mt = (mime_type or "").lower()
-
- # Standalone image upload. Without this branch an image falls through to
- # ``_extract_plain_text``, which latin-1 decodes the binary into garbage
- # paragraphs and indexes them — the upload allowlist accepts png/jpg/webp/gif
- # (``core/content/file_validation.py``) so this path is reachable in production.
- if mt.startswith("image/"):
- return _extract_standalone_image(file_bytes, mt, with_assets=with_assets)
-
- # XLSX — keep built-in handler (file_parser doesn't support spreadsheets)
- if mt in (
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- "application/vnd.ms-excel",
- ):
- return _extract_xlsx(file_bytes), []
-
- if with_assets and mt == "application/pdf":
- rich = _extract_pdf_rich(file_bytes)
- if rich is not None:
- return rich
-
- # Try core/file_parser for all other known types
- suffix = _MIME_TO_SUFFIX.get(mt)
- if suffix:
- try:
- from core.content.file_parser import parse_file
- filename = f"upload{suffix}"
- content = parse_file(file_bytes, filename)
- if content:
- return _markdown_to_paragraphs(content), []
- except RuntimeError:
- # file_parser failed (e.g. service down) — fall through to built-in
- pass
-
- # Fallback: treat as plain text
- return _extract_plain_text(file_bytes), []
-
-
-# Media placeholders the layout service emits: ````.
-_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)\)")
-
-_IMAGE_MIME_TO_EXT = {
- "image/png": ".png",
- "image/jpeg": ".jpg",
- "image/webp": ".webp",
- "image/gif": ".gif",
-}
-
-
-def _extract_standalone_image(
- file_bytes: bytes,
- mime_type: str,
- *,
- with_assets: bool,
-) -> tuple[list[dict], list[RawAsset]]:
- """An uploaded image is one asset with one synthetic paragraph holding its ref.
-
- The paragraph exists so the normal chunker still produces a parent chunk: retrieval
- fetches parent content from PostgreSQL, and an image-only document with no chunk
- would be unreachable. The caption/description is attached to the asset, not
- inlined here — it is not known until the VLM pass has run.
- """
- placeholder = f"asset://{uuid.uuid4().hex[:16]}"
- paragraphs = [{"text": f"", "heading_level": None, "element_type": "image"}]
- if not with_assets:
- return paragraphs, []
- asset = RawAsset(
- data=file_bytes,
- placeholder=placeholder,
- kind="image",
- mime_type=mime_type,
- index=0,
- )
- return paragraphs, [asset]
-
-
-def _extract_pdf_rich(file_bytes: bytes) -> Optional[tuple[list[dict], list[RawAsset]]]:
- """Parse a PDF keeping its figures. Returns None to signal "fall back to text-only"."""
- try:
- from core.content.file_parser import parse_pdf_rich
-
- result = parse_pdf_rich(file_bytes, "upload.pdf")
- except Exception as exc:
- logger.warning("PDF 富解析失败,退回纯文本解析: %s", exc)
- return None
-
- paragraphs = _markdown_to_paragraphs(result.markdown)
- if not result.images:
- return paragraphs, []
-
- # Layout metadata (caption / bbox / page) keyed by the same path the markdown
- # references, so each figure's asset carries where it came from.
- meta_by_path: dict[str, dict] = {}
- for block in result.blocks:
- if block.get("type") != "image":
- continue
- path = str(block.get("img_path") or "")
- if path:
- meta_by_path[path] = block
-
- assets: list[RawAsset] = []
- seen: set[str] = set()
- # Iterate in document order: the markdown's image links are the source of truth for
- # ordering, and only images actually referenced by the text get an asset.
- for idx, path in enumerate(_MD_IMAGE_RE.findall(result.markdown)):
- if path in seen:
- continue
- data = result.images.get(path)
- if not data:
- continue
- seen.add(path)
- meta = meta_by_path.get(path, {})
- caption_parts = [
- str(x).strip()
- for x in (meta.get("image_caption") or []) + (meta.get("image_footnote") or [])
- if str(x).strip()
- ]
- locator = {}
- if meta.get("page_idx") is not None:
- locator["page_idx"] = meta.get("page_idx")
- if meta.get("bbox"):
- locator["bbox"] = meta.get("bbox")
- assets.append(
- RawAsset(
- data=data,
- placeholder=path,
- kind="image",
- mime_type=_guess_image_mime(path),
- text_content=" ".join(caption_parts),
- locator=locator,
- index=idx,
- )
- )
-
- return paragraphs, assets
-
-
-def _guess_image_mime(path: str) -> str:
- lowered = path.lower()
- for mime, ext in _IMAGE_MIME_TO_EXT.items():
- if lowered.endswith(ext):
- return mime
- if lowered.endswith(".jpeg"):
- return "image/jpeg"
- return "image/jpeg"
-
-
-def image_extension_for_mime(mime_type: str) -> str:
- """Filename suffix for an image mime, used when building the storage key."""
- return _IMAGE_MIME_TO_EXT.get((mime_type or "").lower(), ".jpg")
-
-
-def _markdown_to_paragraphs(text: str) -> list[dict]:
- """Parse markdown text (from file_parser) into structured paragraph dicts."""
- paragraphs: list[dict] = []
- table_buffer: list[str] = []
-
- def _flush_table():
- if table_buffer:
- paragraphs.append({
- "text": "\n".join(table_buffer),
- "heading_level": None,
- "element_type": "table",
- })
- table_buffer.clear()
-
- for line in text.splitlines():
- stripped = line.strip()
-
- # Empty line — flush table buffer, skip
- if not stripped:
- _flush_table()
- continue
-
- # Markdown table row
- if stripped.startswith("|") and stripped.endswith("|"):
- table_buffer.append(stripped)
- continue
-
- # Not a table row — flush any pending table
- _flush_table()
-
- # Markdown heading
- md_match = re.match(r'^(#{1,6})\s+(.*)', stripped)
- if md_match:
- level = len(md_match.group(1))
- paragraphs.append({
- "text": stripped,
- "heading_level": level,
- "element_type": "heading",
- })
- continue
-
- # Chinese legal/report heading patterns
- heading_level = _detect_heading_level(stripped)
- if heading_level:
- paragraphs.append({
- "text": stripped,
- "heading_level": heading_level,
- "element_type": "heading",
- })
- continue
-
- # Regular paragraph
- paragraphs.append({
- "text": stripped,
- "heading_level": None,
- "element_type": "paragraph",
- })
-
- _flush_table()
- return paragraphs
-
-
-def _extract_xlsx(file_bytes: bytes) -> list[dict]:
- import io
- try:
- import openpyxl
- except ImportError:
- raise RuntimeError("openpyxl is required: pip install openpyxl")
-
- wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
- paragraphs = []
- for sheet in wb.worksheets:
- rows = list(sheet.iter_rows(values_only=True))
- if not rows:
- continue
- header = [str(c or "") for c in rows[0]]
- paragraphs.append({
- "text": f"## Sheet: {sheet.title}",
- "heading_level": 2,
- "element_type": "heading",
- })
- md_lines = ["| " + " | ".join(header) + " |"]
- md_lines.append("|" + "|".join("---" for _ in header) + "|")
- for row in rows[1:]:
- if all(c is None for c in row):
- continue
- md_lines.append("| " + " | ".join(str(c or "") for c in row) + " |")
- paragraphs.append({
- "text": "\n".join(md_lines),
- "heading_level": None,
- "element_type": "table",
- })
- return paragraphs
-
-
-def _extract_plain_text(file_bytes: bytes) -> list[dict]:
- """Fallback: decode as text and split into paragraphs."""
- # Try common encodings
- for enc in ("utf-8", "gbk", "latin-1"):
- try:
- text = file_bytes.decode(enc)
- break
- except UnicodeDecodeError:
- continue
- else:
- text = file_bytes.decode("utf-8", errors="replace")
-
- return _markdown_to_paragraphs(text)
-
-
-def _detect_heading_level(line: str) -> Optional[int]:
- """Detect Chinese legal/report heading patterns."""
- patterns = [
- (r'^第[一二三四五六七八九十百]+章', 1),
- (r'^第[一二三四五六七八九十百]+节', 2),
- (r'^第[一二三四五六七八九十百\d]+条', 3),
- (r'^[一二三四五六七八九十]+[、..]', 2),
- (r'^\d+[、..]\s*\S', 3),
- ]
- for pattern, level in patterns:
- if re.match(pattern, line):
- return level
- return None
-
-
-# ── Chunking ───────────────────────────────────────────────────────────────────
-
-def build_parent_child_chunks(
- paragraphs: list[dict],
- parent_size: int = 1024,
- child_size: int = 128,
- overlap: int = 20,
- chunk_method: str = "semantic",
- embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
- separators: Optional[list[str]] = None,
- child_separators: Optional[list[str]] = None,
-) -> list[ParentChunk]:
- """Build parent-child chunk pairs from extracted paragraphs.
-
- Parent chunks: large semantic units (~parent_size tokens), stored in PostgreSQL,
- returned to LLM as full context.
- Child chunks: small slices (~child_size tokens), vectorised in Milvus for retrieval.
-
- ``separators`` (parent-chunk separators): custom separator hierarchy that decides
- parent-chunk boundaries; only effective on paths that go through recursive
- splitting (the recursive method itself, and the recursive fallback of semantic
- chunking for oversized parents); when empty, falls back to the default hierarchy.
- ``child_separators`` (child-chunk separators): when a parent is split into
- children, split on these separators (then pack by child_size); when empty,
- children use the default fixed-length sliding-window split. The two are
- independent and do not affect each other.
- """
- seps = normalize_separators(separators)
- csep = normalize_separators(child_separators)
- if chunk_method == "qa":
- parents = _build_qa_chunks(paragraphs)
- elif chunk_method == "laws":
- parents = _build_law_chunks(paragraphs, parent_size, child_size, overlap)
- elif chunk_method == "recursive":
- parents = _build_recursive_chunks(paragraphs, parent_size, child_size, overlap, separators=seps)
- elif chunk_method == "embedding_semantic":
- parents = _build_embedding_semantic_chunks(
- paragraphs, parent_size, child_size, overlap, embed_fn=embed_fn, separators=seps,
- )
- else:
- # "semantic" / "structured" / default
- parents = _build_structured_chunks(paragraphs, parent_size, child_size, overlap)
-
- # Child-chunk separators: re-split children uniformly after parents are built
- # (overriding each builder's default fixed-length children), so we don't have to
- # thread the same parameter into every builder — one post-processing step covers
- # all chunking methods.
- if csep:
- _apply_child_separators(parents, child_size, csep)
- return parents
-
-
-def _apply_child_separators(parents: list[ParentChunk], child_size: int, separators: list[str]) -> None:
- """Re-split each parent's children by child_separators (replaces ``pc.children`` in place).
-
- Splits the parent body with the recursive separators and packs by child_size,
- aligning child boundaries to the user-specified separators.
- """
- for pc in parents:
- texts = [s.strip() for s in _recursive_split(pc.content, separators, child_size) if s.strip()]
- pc.children = [
- ChildChunk(child_id=f"{pc.parent_id}_{i}", content=c, index=i)
- for i, c in enumerate(texts or [pc.content])
- ]
-
-
-def _new_id() -> str:
- return uuid.uuid4().hex[:16]
-
-
-def _split_into_children(parent_text: str, child_size: int, overlap: int) -> list[str]:
- """Split parent text into overlapping child chunks by token count."""
- words = parent_text.split()
- if not words:
- return [parent_text]
-
- # Use character-based approximation when tiktoken unavailable
- chars_per_token = 2 # conservative for Chinese
- child_chars = child_size * chars_per_token
- overlap_chars = overlap * chars_per_token
-
- children = []
- start = 0
- while start < len(parent_text):
- end = start + child_chars
- chunk = parent_text[start:end].strip()
- if chunk:
- children.append(chunk)
- if end >= len(parent_text):
- break
- start = end - overlap_chars
- return children if children else [parent_text]
-
-
-def _make_parent_chunk(text: str, children_texts: list[str]) -> ParentChunk:
- pid = _new_id()
- children = [
- ChildChunk(child_id=f"{pid}_{i}", content=c, index=i)
- for i, c in enumerate(children_texts)
- ]
- return ParentChunk(parent_id=pid, content=text, children=children)
-
-
-# In structure-aware chunking, a heading is a "preferred break point" that only takes
-# effect once the buffer has accumulated to this fraction of parent_size.
-# 0.5 was chosen: subsections keep merging into the parent chunk until about half a
-# parent_size, then we break before the next heading — preserving structure awareness
-# while letting parent_size truly determine parent-chunk size (avoids heading-dense
-# documents being shredded).
-_HEADING_SOFT_RATIO = 0.5
-
-
-def _build_structured_chunks(
- paragraphs: list[dict],
- parent_size: int,
- child_size: int,
- overlap: int,
-) -> list[ParentChunk]:
- """Structure-aware chunking.
-
- A heading is a "preferred break point", not a "forced break point": we only break
- before the next heading once the buffer has accumulated to a certain fraction of
- parent_size (``_HEADING_SOFT_RATIO``); otherwise consecutive subsections are
- merged into the same parent chunk until it approaches parent_size. This keeps
- the heading/paragraph-aligned structure awareness while making ``parent_size``
- actually effective — heading-dense documents are no longer cut into lots of tiny
- pieces, and parents can yield enough children.
- """
- result = []
- buffer: list[str] = []
- buffer_tokens = 0
- soft_threshold = parent_size * _HEADING_SOFT_RATIO
-
- def _flush() -> None:
- nonlocal buffer, buffer_tokens
- parent_text = "\n".join(buffer)
- result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap)))
- buffer = []
- buffer_tokens = 0
-
- for para in paragraphs:
- text = para["text"]
- tokens = _count_tokens(text)
-
- # Hard cap: adding this paragraph would exceed parent_size, so break off the
- # accumulated content first
- if buffer and buffer_tokens + tokens > parent_size:
- _flush()
- # Soft boundary: break before a heading only when enough content has
- # accumulated (>= a fraction of parent_size); otherwise merge this
- # subsection into the current parent so parent_size is the real determinant
- # of parent-chunk size
- elif buffer and para.get("heading_level") and buffer_tokens >= soft_threshold:
- _flush()
-
- buffer.append(text)
- buffer_tokens += tokens
-
- if buffer:
- _flush()
-
- return result
-
-
-def _build_law_chunks(
- paragraphs: list[dict],
- parent_size: int,
- child_size: int,
- overlap: int,
-) -> list[ParentChunk]:
- """Split at article/chapter/section boundaries for legal documents."""
- result = []
- buffer: list[str] = []
-
- article_pattern = re.compile(r'^第[一二三四五六七八九十百\d]+[条章节]')
-
- for para in paragraphs:
- text = para["text"]
- is_boundary = bool(article_pattern.match(text)) or para.get("heading_level") in (1, 2)
-
- if is_boundary and buffer:
- parent_text = "\n".join(buffer)
- result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap)))
- buffer = []
-
- buffer.append(text)
-
- if buffer:
- parent_text = "\n".join(buffer)
- result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap)))
-
- return result
-
-
-def _build_qa_chunks(paragraphs: list[dict]) -> list[ParentChunk]:
- """Each Q&A pair becomes one parent chunk with one child."""
- result = []
- for para in paragraphs:
- text = para["text"]
- pid = _new_id()
- child = ChildChunk(child_id=f"{pid}_0", content=text, index=0)
- result.append(ParentChunk(parent_id=pid, content=text, children=[child]))
- return result
-
-
-# ── Recursive chunking ─────────────────────────────────────────────────────────
-
-_RECURSIVE_SEPARATORS = ["\n\n", "\n", "。", ".", ";", ";", ",", ",", " "]
-
-# Users may write escape sequences (e.g. \n, \t) in custom separators; restore them
-# to real characters here in one place
-_SEPARATOR_ESCAPES = {"\\n": "\n", "\\t": "\t", "\\r": "\r", "\\\\": "\\"}
-
-
-def normalize_separators(seps: Optional[list[str]]) -> Optional[list[str]]:
- """Normalize user-supplied custom separators into a usable hierarchy list.
-
- - Restore escapes like ``\\n`` / ``\\t``;
- - Drop empty strings, de-duplicate while preserving order;
- - Return None when all empty (callers fall back to the ``_RECURSIVE_SEPARATORS`` default hierarchy).
- """
- if not seps:
- return None
- out: list[str] = []
- seen: set[str] = set()
- for raw in seps:
- if raw is None:
- continue
- s = str(raw)
- for token, real in _SEPARATOR_ESCAPES.items():
- s = s.replace(token, real)
- if s == "" or s in seen:
- continue
- seen.add(s)
- out.append(s)
- return out or None
-
-
-def parse_separators_form(raw: Optional[str]) -> Optional[list[str]]:
- """Parse custom separators passed as a JSON-array string in a multipart form into a list.
-
- The preview endpoint takes Form params and cannot receive a list directly, so by
- convention a JSON string is passed (e.g. ``["\\n\\n", "。"]``). On parse failure
- or non-list input, silently return None (fall back to the default hierarchy)
- instead of 500-ing over tuning input.
- """
- if not raw or not raw.strip():
- return None
- try:
- data = json.loads(raw)
- except (ValueError, TypeError):
- return None
- if not isinstance(data, list):
- return None
- return normalize_separators([str(x) for x in data])
-
-
-def _recursive_split(text: str, separators: list[str], max_tokens: int) -> list[str]:
- """Recursively split text using a hierarchy of separators.
-
- Tries to split on the first separator that produces segments.
- Merges adjacent small segments up to max_tokens.
- Over-sized segments recurse with the next separator level.
- Final fallback: character-based truncation.
- """
- if _count_tokens(text) <= max_tokens:
- return [text]
-
- for i, sep in enumerate(separators):
- parts = text.split(sep)
- if len(parts) <= 1:
- continue
-
- # Merge adjacent small parts up to max_tokens
- result: list[str] = []
- current = ""
- current_tokens = 0
-
- for part in parts:
- candidate = (current + sep + part) if current else part
- candidate_tokens = _count_tokens(candidate)
-
- if candidate_tokens <= max_tokens:
- current = candidate
- current_tokens = candidate_tokens
- else:
- if current:
- result.append(current)
- # If this single part exceeds max_tokens, recurse deeper
- part_tokens = _count_tokens(part)
- if part_tokens > max_tokens:
- deeper = _recursive_split(part, separators[i + 1:], max_tokens)
- result.extend(deeper)
- current = ""
- current_tokens = 0
- else:
- current = part
- current_tokens = part_tokens
-
- if current:
- result.append(current)
-
- if result:
- return result
-
- # Final fallback: character-based truncation
- chars_per_token = 2
- chunk_chars = max_tokens * chars_per_token
- result = []
- start = 0
- while start < len(text):
- end = start + chunk_chars
- chunk = text[start:end].strip()
- if chunk:
- result.append(chunk)
- start = end
- return result if result else [text]
-
-
-def _build_recursive_chunks(
- paragraphs: list[dict],
- parent_size: int,
- child_size: int,
- overlap: int,
- separators: Optional[list[str]] = None,
-) -> list[ParentChunk]:
- """Recursive multi-level separator chunking."""
- # Join all paragraph text
- full_text = "\n".join(p["text"] for p in paragraphs)
- segments = _recursive_split(full_text, separators or _RECURSIVE_SEPARATORS, parent_size)
-
- result = []
- for seg in segments:
- seg = seg.strip()
- if not seg:
- continue
- result.append(_make_parent_chunk(seg, _split_into_children(seg, child_size, overlap)))
- return result
-
-
-# ── Embedding-based semantic chunking ──────────────────────────────────────────
-
-def _cosine_similarity(a: list[float], b: list[float]) -> float:
- """Compute cosine similarity between two vectors (pure Python)."""
- dot = sum(x * y for x, y in zip(a, b))
- norm_a = math.sqrt(sum(x * x for x in a))
- norm_b = math.sqrt(sum(x * x for x in b))
- if norm_a == 0 or norm_b == 0:
- return 0.0
- return dot / (norm_a * norm_b)
-
-
-def _build_embedding_semantic_chunks(
- paragraphs: list[dict],
- parent_size: int,
- child_size: int,
- overlap: int,
- embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
- separators: Optional[list[str]] = None,
-) -> list[ParentChunk]:
- """Semantic chunking based on embedding similarity between adjacent blocks.
-
- Algorithm:
- 1. Merge paragraphs into minimum ~50-token blocks
- 2. Batch-embed all blocks
- 3. Compute cosine similarity between adjacent blocks
- 4. Set breakpoints where similarity drops below P25 or is a local minimum
- 5. Merge blocks between breakpoints into parent chunks
- 6. Fallback to recursive splitting for oversized parents
- """
- if embed_fn is None:
- # Fallback to structured chunking when no embed function available
- return _build_structured_chunks(paragraphs, parent_size, child_size, overlap)
-
- # Step 1: Merge paragraphs into minimum-size blocks (~50 tokens)
- MIN_BLOCK_TOKENS = 50
- blocks: list[str] = []
- current_block = ""
- current_tokens = 0
-
- for para in paragraphs:
- text = para["text"]
- tokens = _count_tokens(text)
- if current_block and current_tokens + tokens > MIN_BLOCK_TOKENS:
- blocks.append(current_block)
- current_block = text
- current_tokens = tokens
- else:
- current_block = (current_block + "\n" + text) if current_block else text
- current_tokens += tokens
-
- if current_block:
- blocks.append(current_block)
-
- if len(blocks) <= 1:
- # Single block — just use structured
- return _build_structured_chunks(paragraphs, parent_size, child_size, overlap)
-
- # Step 2: Batch embed
- embeddings = embed_fn(blocks)
-
- # Step 3: Compute adjacent cosine similarities
- similarities: list[float] = []
- for i in range(len(embeddings) - 1):
- sim = _cosine_similarity(embeddings[i], embeddings[i + 1])
- similarities.append(sim)
-
- if not similarities:
- return _build_structured_chunks(paragraphs, parent_size, child_size, overlap)
-
- # Step 4: Find breakpoints
- sorted_sims = sorted(similarities)
- p25_idx = max(0, len(sorted_sims) // 4 - 1)
- p25_threshold = sorted_sims[p25_idx]
-
- breakpoints: set[int] = set()
- for i, sim in enumerate(similarities):
- # Below P25 threshold
- if sim <= p25_threshold:
- breakpoints.add(i)
- continue
- # Local minimum: lower than both neighbors
- is_local_min = True
- if i > 0 and sim >= similarities[i - 1]:
- is_local_min = False
- if i < len(similarities) - 1 and sim >= similarities[i + 1]:
- is_local_min = False
- if is_local_min and sim < p25_threshold * 1.2:
- breakpoints.add(i)
-
- # Step 5: Merge blocks between breakpoints into parent chunks
- result: list[ParentChunk] = []
- segment_start = 0
-
- sorted_breaks = sorted(breakpoints)
- # Add an artificial breakpoint at the end
- sorted_breaks.append(len(blocks) - 1)
-
- for bp in sorted_breaks:
- segment_blocks = blocks[segment_start:bp + 1]
- parent_text = "\n".join(segment_blocks).strip()
- segment_start = bp + 1
-
- if not parent_text:
- continue
-
- # Step 6: If parent is too large, use recursive split as fallback
- if _count_tokens(parent_text) > parent_size * 1.5:
- sub_segments = _recursive_split(parent_text, separators or _RECURSIVE_SEPARATORS, parent_size)
- for sub in sub_segments:
- sub = sub.strip()
- if sub:
- result.append(_make_parent_chunk(sub, _split_into_children(sub, child_size, overlap)))
- else:
- result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap)))
-
- # Handle remaining blocks
- if segment_start < len(blocks):
- remaining = "\n".join(blocks[segment_start:]).strip()
- if remaining:
- result.append(_make_parent_chunk(remaining, _split_into_children(remaining, child_size, overlap)))
-
- return result
-
-
-# ── Pipeline entry point ───────────────────────────────────────────────────────
-
-def parse_and_chunk(
- file_bytes: bytes,
- mime_type: str,
- chunk_method: str = "semantic",
- parent_size: int = 1024,
- child_size: int = 128,
- overlap: int = 20,
- embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
- separators: Optional[list[str]] = None,
- child_separators: Optional[list[str]] = None,
-) -> list[ParentChunk]:
- """Full pipeline: extract text then build parent-child chunks."""
- parents, _ = parse_and_chunk_rich(
- file_bytes,
- mime_type,
- chunk_method=chunk_method,
- parent_size=parent_size,
- child_size=child_size,
- overlap=overlap,
- embed_fn=embed_fn,
- separators=separators,
- child_separators=child_separators,
- with_assets=False,
- )
- return parents
-
-
-def parse_and_chunk_rich(
- file_bytes: bytes,
- mime_type: str,
- chunk_method: str = "semantic",
- parent_size: int = 1024,
- child_size: int = 128,
- overlap: int = 20,
- embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
- separators: Optional[list[str]] = None,
- child_separators: Optional[list[str]] = None,
- with_assets: bool = True,
-) -> tuple[list[ParentChunk], list[RawAsset]]:
- """Same pipeline as :func:`parse_and_chunk`, also returning the document's media.
-
- The assets are returned unpersisted; the caller links them to parent chunks and
- stores the bytes (see ``core/kb/kb_assets.py``).
- """
- paragraphs, assets = extract_text_with_assets(
- file_bytes, mime_type, with_assets=with_assets
- )
- parents = build_parent_child_chunks(
- paragraphs,
- parent_size=parent_size,
- child_size=child_size,
- overlap=overlap,
- chunk_method=chunk_method,
- embed_fn=embed_fn,
- separators=separators,
- child_separators=child_separators,
- )
- return parents, assets
diff --git a/src/backend/core/kb/kb_vector.py b/src/backend/core/kb/kb_vector.py
deleted file mode 100644
index 0582a462..00000000
--- a/src/backend/core/kb/kb_vector.py
+++ /dev/null
@@ -1,755 +0,0 @@
-"""Milvus vector store for private knowledge base.
-
-Collection: hugagent_kb_private
-- Stores both chunk rows (row_type='chunk') and question rows (row_type='question')
-- Child chunk vectors used for retrieval; parent chunk content fetched from PostgreSQL
-- User isolation enforced via user_id field in every query expression
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import os
-from typing import Any, Optional
-
-import requests
-
-logger = logging.getLogger(__name__)
-
-COLLECTION_NAME = "hugagent_kb_private"
-
-# Milvus VARCHAR field capacity (**bytes**, not characters) —— the schema definition and the write truncation share the same
-# set of constants, to avoid changing max_length but forgetting the truncation and hitting "length exceeds max length" again.
-# The content column is only a display fallback for retrieval results (dense/sparse vectors are computed from the full text,
-# and returned content is fetched full from PostgreSQL), so safe byte-wise truncation does not affect retrieval quality.
-TITLE_FIELD_MAX_BYTES = 500
-CONTENT_FIELD_MAX_BYTES = 4096
-TAGS_FIELD_MAX_BYTES = 1000
-
-
-def truncate_utf8(text: str, max_bytes: int = CONTENT_FIELD_MAX_BYTES) -> str:
- """Safe truncation by UTF-8 bytes (Chinese is 3 bytes/char); never cuts in the middle of a multi-byte character.
-
- Historical bug: using ``text[:4096]`` truncated by "character"; 4096 Chinese chars ≈ 12KB bytes, still exceeding Milvus's
- 4096 **byte** limit → ``length exceeds max length`` error.
- """
- if not text:
- return ""
- encoded = text.encode("utf-8")
- if len(encoded) <= max_bytes:
- return text
- return encoded[:max_bytes].decode("utf-8", "ignore")
-
-
-# Fallback dimension: used only when the real dimension can't be probed from the online embedding endpoint (not configured/unreachable).
-# In normal operation the dimension is auto-determined by detect_embed_dim() from the actual output of the chosen embedding model, no manual filling needed.
-_EMBED_DIMS = int(os.getenv("MEM0_EMBED_DIMS", "1024"))
-
-# Cache of probed dimensions: key=(embed_url, embed_model) → dim, to avoid hitting /embeddings on every index.
-_DETECTED_DIM_CACHE: dict[tuple[str, str], int] = {}
-# Collection dimension already confirmed in this process (avoids a describe on every get_or_create).
-_VERIFIED_COLLECTION_DIM: Optional[int] = None
-
-# Sparse vector dimension space (hash modulo)
-_SPARSE_DIM_SPACE = 100_000
-
-
-def _resolve_embed_config() -> tuple[str, str, str]:
- """Resolve embedding config from DB, with env fallback."""
- try:
- from core.services.model_config import ModelConfigService
-
- cfg = ModelConfigService.get_instance().resolve("embedding")
- if cfg:
- return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key
- except Exception:
- pass
- return (
- os.getenv("MEM0_EMBED_URL", "").rstrip("/"),
- os.getenv("MEM0_EMBED_MODEL", ""),
- os.getenv("MEM0_EMBED_API_KEY", ""),
- )
-
-
-def _resolve_reranker_config() -> tuple[str, str, str]:
- """Resolve reranker config from DB, with env fallback."""
- try:
- from core.services.model_config import ModelConfigService
-
- cfg = ModelConfigService.get_instance().resolve("reranker")
- if cfg:
- return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key
- except Exception:
- pass
- return (
- os.getenv("RERANKER_URL", "").rstrip("/"),
- os.getenv("RERANKER_MODEL", ""),
- os.getenv("RERANKER_API_KEY", ""),
- )
-
-
-# ── Embedding ──────────────────────────────────────────────────────────────────
-
-# Max number of texts packed into a single embedding request. A large document is split into hundreds or thousands of chunks;
-# sending the whole batch at once to a slow embedding model (e.g. 8B, ~0.45s/item serially) will inevitably hit the single-request
-# read timeout; splitting into sub-batches at this granularity keeps each request's latency under the timeout line.
-# Overridable via KB_EMBED_BATCH_SIZE.
-_EMBED_BATCH_SIZE = max(1, int(os.getenv("KB_EMBED_BATCH_SIZE", "32")))
-# Read timeout (seconds) for a single embedding request. Raise it to tolerate a slow model's per-batch latency; the real timeout
-# prevention is sub-batching, the two work together —— a sub-batch always has an upper bound, and a larger timeout is only a fallback.
-# Overridable via KB_EMBED_TIMEOUT.
-_EMBED_TIMEOUT = max(1, int(os.getenv("KB_EMBED_TIMEOUT", "120")))
-
-
-def _embed_request(
- url: str,
- headers: dict,
- inputs: list[str],
- embed_model: str,
- *,
- timeout: float | None = None,
-) -> list[list[float]]:
- """Post a single embedding request and return vectors ordered by input index."""
- payload = {"input": inputs, "model": embed_model}
- resp = requests.post(
- url,
- headers=headers,
- json=payload,
- timeout=timeout or _EMBED_TIMEOUT,
- )
- resp.raise_for_status()
- data = resp.json()
- return [item["embedding"] for item in sorted(data["data"], key=lambda x: x["index"])]
-
-
-def embed_text(text: str, *, timeout: float | None = None) -> list[float]:
- """Call the configured embedding service and return a dense vector."""
- embed_url, embed_model, api_key = _resolve_embed_config()
-
- if not embed_url:
- raise RuntimeError("Embedding model is not configured")
-
- url = f"{embed_url}/embeddings"
- from core.services.desktop_model_credentials import prepare_request_headers
- headers = prepare_request_headers(api_key, embed_url)
- vector = _embed_request(url, headers, [text], embed_model, timeout=timeout)[0]
- if vector:
- _DETECTED_DIM_CACHE[(embed_url, embed_model)] = len(vector)
- return vector
-
-
-def embed_batch(texts: list[str]) -> list[list[float]]:
- """Batch embed multiple texts, splitting into sub-batches with one-by-one fallback.
-
- A large document is split into hundreds or thousands of chunks. Sending the whole batch at once to a slow embedding
- model (e.g. 8B, ~0.45s/item serially) would time out the single request and fail the whole document's vectorization. So we
- split into sub-batches of _EMBED_BATCH_SIZE and request them one sub-batch at a time; if a sub-batch fails, we downgrade to
- embedding one item at a time within that sub-batch, preserving as many other chunks as possible so one timeout doesn't drag
- down the whole document. Only if the one-by-one attempt still fails do we re-raise (treated as a true failure).
- """
- embed_url, embed_model, api_key = _resolve_embed_config()
-
- if not embed_url:
- raise RuntimeError("Embedding model is not configured")
-
- url = f"{embed_url}/embeddings"
- from core.services.desktop_model_credentials import prepare_request_headers
- headers = prepare_request_headers(api_key, embed_url)
-
- results: list[list[float]] = []
- for start in range(0, len(texts), _EMBED_BATCH_SIZE):
- sub = texts[start : start + _EMBED_BATCH_SIZE]
- try:
- results.extend(_embed_request(url, headers, sub, embed_model))
- except Exception as exc:
- logger.warning(
- "Sub-batch embed failed for [%d:%d] (%s); falling back to one-by-one",
- start,
- start + len(sub),
- exc,
- )
- for one in sub:
- results.extend(_embed_request(url, headers, [one], embed_model))
- return results
-
-
-def detect_embed_dim() -> int:
- """Probe the real output dimension of the currently chosen embedding model (one /embeddings call, take the vector length).
-
- The result is cached by (embed_url, embed_model) to avoid repeated probing on every index; on probe failure (endpoint not
- configured/unreachable) it falls back to MEM0_EMBED_DIMS. This way the table dimension auto-follows after switching the
- embedding model, no manual filling needed.
- """
- embed_url, embed_model, _ = _resolve_embed_config()
- key = (embed_url, embed_model)
- cached = _DETECTED_DIM_CACHE.get(key)
- if cached:
- return cached
- try:
- dim = len(embed_text("dimension probe"))
- if dim > 0:
- _DETECTED_DIM_CACHE[key] = dim
- logger.info("Auto-detected embedding dim=%d (model=%s)", dim, embed_model or "?")
- return dim
- except Exception as exc:
- logger.warning(
- "Embedding dim auto-detect failed (%s); falling back to MEM0_EMBED_DIMS=%d",
- exc,
- _EMBED_DIMS,
- )
- return _EMBED_DIMS
-
-
-# ── Reranker ──────────────────────────────────────────────────────────────────
-
-
-def is_reranker_configured() -> bool:
- """Check if a reranker model endpoint is configured."""
- reranker_url, reranker_model, _ = _resolve_reranker_config()
- return bool(reranker_url and reranker_model)
-
-
-def rerank(
- query: str,
- documents: list[str],
- top_n: int | None = None,
- *,
- timeout: float = 30,
-) -> list[dict]:
- """Call the configured reranker endpoint (OpenAI-compatible /rerank).
-
- Returns a list of {"index": int, "relevance_score": float} sorted by score descending.
- """
- reranker_url, reranker_model, reranker_key = _resolve_reranker_config()
-
- if not reranker_url or not reranker_model:
- raise RuntimeError("Reranker is not configured")
-
- url = f"{reranker_url}/rerank"
- from core.services.desktop_model_credentials import prepare_request_headers
- headers = prepare_request_headers(reranker_key, reranker_url)
- payload: dict = {
- "model": reranker_model,
- "query": query,
- "documents": documents,
- }
- if top_n is not None:
- payload["top_n"] = top_n
-
- resp = requests.post(url, headers=headers, json=payload, timeout=timeout)
- resp.raise_for_status()
- data = resp.json()
-
- # Response format: {"results": [{"index": 0, "relevance_score": 0.95}, ...]}
- results = data.get("results", [])
- return sorted(results, key=lambda x: x.get("relevance_score", 0), reverse=True)
-
-
-# ── Milvus helpers ─────────────────────────────────────────────────────────────
-
-
-def _milvus_uri() -> str:
- from core.config.settings import settings
-
- return os.getenv("MILVUS_URL") or settings.memory.milvus_url
-
-
-def _is_lite() -> bool:
- """True when the vector backend is embedded Milvus Lite (a local file uri).
-
- Milvus Lite (no-Docker local profile) does **not** support SPARSE_FLOAT_VECTOR
- / SPARSE_INVERTED_INDEX / hybrid search / IVF / scalar INVERTED indexes, so the
- KB degrades to **dense-only** on it. A server uri (``http(s)://`` / ``tcp://``)
- is a full Milvus and keeps the dense+sparse hybrid path byte-for-byte unchanged.
- """
- return not _milvus_uri().startswith(("http://", "https://", "tcp://", "unix:"))
-
-
-def _get_client():
- """Return a MilvusClient connected to the configured Milvus instance."""
- from pymilvus import MilvusClient
-
- url = _milvus_uri()
- token = os.getenv("MILVUS_TOKEN", "")
- if token:
- return MilvusClient(uri=url, token=token)
- return MilvusClient(uri=url)
-
-
-def _upsert(client, data: list[dict[str, Any]]) -> None:
- """Upsert helper that drops the ``sparse_embedding`` field on Milvus Lite.
-
- On Lite the collection has no sparse field (see ``get_or_create_collection``),
- and dynamic fields are disabled, so an extra ``sparse_embedding`` key would be
- rejected. Stripping it here keeps every caller's row-building code identical
- across both backends.
- """
- if _is_lite():
- data = [{k: v for k, v in row.items() if k != "sparse_embedding"} for row in data]
- client.upsert(collection_name=COLLECTION_NAME, data=data)
-
-
-def _collection_dense_dim(client, *, timeout: float | None = None) -> Optional[int]:
- """Read the dense_embedding dimension of the existing collection.
-
- Connection and schema-read errors deliberately propagate to the caller. Treating
- an unavailable schema as a dimension mismatch used to trigger a destructive
- collection rebuild.
- """
- desc = client.describe_collection(COLLECTION_NAME, timeout=timeout)
- for f in desc.get("fields", []):
- if f.get("name") == "dense_embedding":
- dim = f.get("params", {}).get("dim")
- return int(dim) if dim is not None else None
- return None
-
-
-def get_or_create_collection(*, timeout: float | None = None) -> None:
- """Idempotently create hugagent_kb_private (hybrid retrieval schema) at the current embedding model's real dimension.
-
- New collections use the current embedding model's detected dimension. Existing
- collections are never dropped implicitly: an unreadable schema or dimension
- mismatch preserves all vectors and aborts the operation until an operator can
- verify the embedding configuration and explicitly re-index the collection.
- """
- global _VERIFIED_COLLECTION_DIM
- from pymilvus import DataType
-
- client = _get_client()
- target_dim = detect_embed_dim()
-
- if client.has_collection(COLLECTION_NAME, timeout=timeout):
- if _VERIFIED_COLLECTION_DIM == target_dim:
- return
- try:
- existing_dim = _collection_dense_dim(client, timeout=timeout)
- except Exception as exc:
- message = (
- "无法确认现有知识库集合的向量维度,已保留集合并中止本次操作;"
- "请检查 Milvus 连接后重试。"
- )
- logger.error("%s 原因:%s", message, exc, exc_info=True)
- raise RuntimeError(message) from exc
- if existing_dim is None:
- message = (
- "无法确认现有知识库集合的向量维度,已保留集合并中止本次操作;"
- "dense_embedding 字段或 dim 参数缺失。"
- )
- logger.error(message)
- raise RuntimeError(message)
- if existing_dim == target_dim:
- _VERIFIED_COLLECTION_DIM = target_dim
- return
- message = (
- f"KB collection 维度不匹配(已存在={existing_dim},当前模型={target_dim}),"
- "已保留集合并中止本次操作;请核实 embedding 配置并通过显式重建流程重新索引。"
- )
- logger.error(message)
- raise RuntimeError(message)
-
- lite = _is_lite()
-
- schema = client.create_schema(auto_id=False, enable_dynamic_field=False)
- schema.add_field("chunk_id", DataType.VARCHAR, max_length=64, is_primary=True)
- schema.add_field("parent_chunk_id", DataType.VARCHAR, max_length=64)
- schema.add_field("row_type", DataType.VARCHAR, max_length=16) # "chunk" | "question"
- schema.add_field("user_id", DataType.VARCHAR, max_length=64)
- schema.add_field("kb_id", DataType.VARCHAR, max_length=64)
- schema.add_field("document_id", DataType.VARCHAR, max_length=64)
- schema.add_field("title", DataType.VARCHAR, max_length=TITLE_FIELD_MAX_BYTES)
- schema.add_field("content", DataType.VARCHAR, max_length=CONTENT_FIELD_MAX_BYTES)
- schema.add_field(
- "tags_text", DataType.VARCHAR, max_length=TAGS_FIELD_MAX_BYTES
- ) # BM25 augmentation
- schema.add_field("chunk_index", DataType.INT64)
- schema.add_field("dense_embedding", DataType.FLOAT_VECTOR, dim=target_dim)
- if not lite:
- schema.add_field("sparse_embedding", DataType.SPARSE_FLOAT_VECTOR)
-
- index_params = client.prepare_index_params()
- if lite:
- # Milvus Lite: only a dense AUTOINDEX (FLAT under the hood). No sparse, no
- # scalar INVERTED indexes — filtering falls back to brute force, fine at
- # single-machine scale.
- index_params.add_index(
- field_name="dense_embedding",
- index_type="AUTOINDEX",
- metric_type="IP",
- )
- else:
- index_params.add_index(
- field_name="dense_embedding",
- index_type="IVF_FLAT",
- metric_type="IP",
- params={"nlist": 128},
- )
- index_params.add_index(
- field_name="sparse_embedding",
- index_type="SPARSE_INVERTED_INDEX",
- metric_type="IP",
- )
- # Scalar indexes for filtering acceleration
- for field in ("user_id", "kb_id", "document_id", "row_type"):
- index_params.add_index(field_name=field, index_type="INVERTED")
-
- client.create_collection(
- collection_name=COLLECTION_NAME,
- schema=schema,
- index_params=index_params,
- timeout=timeout,
- )
- _VERIFIED_COLLECTION_DIM = target_dim
- logger.info(
- "Created Milvus collection: %s (dim=%d, mode=%s)",
- COLLECTION_NAME,
- target_dim,
- "lite/dense-only" if lite else "hybrid",
- )
-
-
-# ── Write ──────────────────────────────────────────────────────────────────────
-
-
-def upsert_rows(rows: list[dict[str, Any]]) -> None:
- """Upsert a batch of rows into hugagent_kb_private.
-
- Each row must include all schema fields. dense_embedding and sparse_embedding
- must be pre-computed by the caller (use embed_text / build_sparse_text).
- """
- if not rows:
- return
- get_or_create_collection()
- client = _get_client()
- _upsert(client, rows)
- logger.debug("Upserted %d rows into %s", len(rows), COLLECTION_NAME)
-
-
-def delete_by_document(document_id: str, user_id: str) -> None:
- """Delete all Milvus rows (chunk + question) for a document."""
- get_or_create_collection()
- client = _get_client()
- expr = f'document_id == "{document_id}" and user_id == "{user_id}"'
- client.delete(collection_name=COLLECTION_NAME, filter=expr)
- logger.info("Deleted Milvus rows for document_id=%s user_id=%s", document_id, user_id)
-
-
-def delete_by_kb(kb_id: str, user_id: str) -> None:
- """Delete all Milvus rows for an entire KB space."""
- get_or_create_collection()
- client = _get_client()
- expr = f'kb_id == "{kb_id}" and user_id == "{user_id}"'
- client.delete(collection_name=COLLECTION_NAME, filter=expr)
- logger.info("Deleted Milvus rows for kb_id=%s user_id=%s", kb_id, user_id)
-
-
-def get_children_for_parent(parent_chunk_id: str) -> list[dict]:
- """Return the child-chunk rows of a given parent chunk in Milvus (for the "view chunks" parent→child display).
-
- Child rows have ``chunk_id`` shaped like ``{pid}_{i}`` with ``parent_chunk_id == pid``; in flat mode a parent has only one
- representative row (``chunk_id == parent_chunk_id``), which is filtered out —— i.e. flat mode returns an empty list.
- Returns an empty list when Milvus is unavailable / the query fails, without affecting the parent-chunk list display.
- """
- try:
- get_or_create_collection()
- client = _get_client()
- rows = client.query(
- collection_name=COLLECTION_NAME,
- # In flat mode a parent has only one representative row (chunk_id == parent_chunk_id); exclude it right in the query
- filter=(
- f'parent_chunk_id == "{parent_chunk_id}" and row_type == "chunk" '
- f'and chunk_id != "{parent_chunk_id}"'
- ),
- output_fields=["chunk_id", "parent_chunk_id", "chunk_index", "content"],
- limit=4096,
- )
- except Exception as exc:
- logger.warning("get_children_for_parent(%s) failed: %s", parent_chunk_id, exc)
- return []
- children = [
- {
- "chunk_id": r.get("chunk_id"),
- "chunk_index": r.get("chunk_index", 0),
- "content": r.get("content", ""),
- }
- for r in rows
- ]
- children.sort(key=lambda x: x["chunk_index"])
- return children
-
-
-def delete_by_chunk(parent_chunk_id: str) -> None:
- """Delete all Milvus rows (chunk + question) for a single parent chunk."""
- get_or_create_collection()
- client = _get_client()
- expr = f'parent_chunk_id == "{parent_chunk_id}"'
- client.delete(collection_name=COLLECTION_NAME, filter=expr)
- logger.info("Deleted Milvus rows for parent_chunk_id=%s", parent_chunk_id)
-
-
-# ── Search ─────────────────────────────────────────────────────────────────────
-
-
-def _hit_to_dict(ent, score: float) -> dict:
- """Build the shared KB search-result row. ``ent`` is any entity accessor with
- ``.get`` (a dict from Milvus Lite, or a Hit entity from hybrid search)."""
- return {
- "chunk_id": ent.get("chunk_id"),
- "parent_chunk_id": ent.get("parent_chunk_id") or ent.get("chunk_id"),
- "row_type": ent.get("row_type"),
- "kb_id": ent.get("kb_id"),
- "document_id": ent.get("document_id"),
- "title": ent.get("title"),
- "content": ent.get("content"),
- "chunk_index": ent.get("chunk_index"),
- "score": score,
- }
-
-
-def hybrid_search(
- user_id: str,
- kb_ids: list[str],
- query: str,
- query_vec: list[float],
- top_k: int = 10,
- *,
- public_kb_ids: list[str] | None = None,
- timeout: float | None = None,
-) -> list[dict[str, Any]]:
- """Hybrid search over dense + sparse vectors with RRF fusion.
-
- Searches both chunk rows and question rows simultaneously.
- Results are sorted by fused score; dedup by parent_chunk_id is done by the caller.
-
- ``kb_ids`` are private spaces and are filtered by ``user_id`` (owner isolation).
- ``public_kb_ids`` are admin-managed public spaces visible to everyone — they are
- filtered by ``kb_id`` only (no owner check), since public KBs are global and
- ``kb_id`` is globally unique.
- """
- from pymilvus import AnnSearchRequest, MilvusClient, RRFRanker
-
- get_or_create_collection(timeout=timeout)
- client = _get_client()
-
- clauses: list[str] = []
- if kb_ids:
- clauses.append(f'(user_id == "{user_id}" and kb_id in {json.dumps(kb_ids)})')
- if public_kb_ids:
- clauses.append(f"kb_id in {json.dumps(public_kb_ids)}")
- if not clauses:
- return []
- expr = " or ".join(clauses)
-
- output_fields = [
- "chunk_id",
- "parent_chunk_id",
- "row_type",
- "kb_id",
- "document_id",
- "title",
- "content",
- "chunk_index",
- ]
-
- # Milvus Lite (no-Docker local): no sparse field / hybrid search — fall back to
- # a plain dense ANN search over the same collection. Same result shape.
- if _is_lite():
- lite_res = client.search(
- collection_name=COLLECTION_NAME,
- data=[query_vec],
- anns_field="dense_embedding",
- search_params={"metric_type": "IP"},
- limit=top_k * 2,
- filter=expr,
- output_fields=output_fields,
- timeout=timeout,
- )
- return [
- _hit_to_dict(hit.get("entity", hit), hit.get("distance", 0.0)) for hit in lite_res[0]
- ]
-
- # Dense vector search request
- dense_req = AnnSearchRequest(
- data=[query_vec],
- anns_field="dense_embedding",
- param={"metric_type": "IP", "params": {"nprobe": 10}},
- limit=top_k * 3,
- expr=expr,
- )
-
- # Sparse search request — bag-of-words vector
- sparse_vec = text_to_sparse(query)
- sparse_req = AnnSearchRequest(
- data=[sparse_vec],
- anns_field="sparse_embedding",
- param={"metric_type": "IP"},
- limit=top_k * 3,
- expr=expr,
- )
-
- results = client.hybrid_search(
- collection_name=COLLECTION_NAME,
- reqs=[dense_req, sparse_req],
- ranker=RRFRanker(k=60),
- limit=top_k * 2,
- output_fields=output_fields,
- timeout=timeout,
- )
-
- return [_hit_to_dict(hit.entity, hit.score) for hit in results[0]]
-
-
-# ── Tag/question re-index ──────────────────────────────────────────────────────
-
-
-def build_sparse_text(content: str, tags: list[str]) -> str:
- """Build combined text for sparse vectorisation."""
- if not tags:
- return content
- tag_str = " ".join(f"[{t}]" for t in tags)
- return f"{content} {tag_str}"
-
-
-def text_to_sparse(text: str) -> dict[int, float]:
- """Convert text to a bag-of-words sparse vector using term-frequency + hashing.
-
- Compatible with Milvus SPARSE_FLOAT_VECTOR field (v2.4+).
- Each unique token is hashed to a dimension index; value = TF.
- """
- import hashlib
- import re
- from collections import Counter
-
- # Simple tokenisation: split on non-alphanumeric (works for CJK + Latin)
- tokens = re.findall(r"[\w\u4e00-\u9fff]+", text.lower())
- if not tokens:
- return {0: 1.0}
-
- counts = Counter(tokens)
- total = sum(counts.values())
-
- sparse: dict[int, float] = {}
- for token, cnt in counts.items():
- dim = int(hashlib.md5(token.encode()).hexdigest()[:8], 16) % _SPARSE_DIM_SPACE
- tf = cnt / total
- sparse[dim] = sparse.get(dim, 0.0) + tf
-
- return sparse if sparse else {0: 1.0}
-
-
-def reindex_chunk_tags(chunk_id: str, content: str, tags: list[str]) -> None:
- """Re-compute and upsert sparse_embedding for a chunk row after tag changes."""
- # Fetch the existing row to preserve all other fields
- get_or_create_collection()
- client = _get_client()
- rows = client.query(
- collection_name=COLLECTION_NAME,
- filter=f'chunk_id == "{chunk_id}"',
- output_fields=[
- "chunk_id",
- "parent_chunk_id",
- "row_type",
- "user_id",
- "kb_id",
- "document_id",
- "title",
- "chunk_index",
- "dense_embedding",
- ],
- )
- if not rows:
- logger.warning("reindex_chunk_tags: chunk_id %s not found in Milvus", chunk_id)
- return
-
- row = rows[0]
- sparse_text = build_sparse_text(content, tags)
- row["content"] = truncate_utf8(content)
- row["tags_text"] = truncate_utf8(" ".join(tags), TAGS_FIELD_MAX_BYTES)
- row["sparse_embedding"] = text_to_sparse(sparse_text)
- _upsert(client, [row])
-
-
-def reindex_chunk_content(
- chunk_id: str,
- content: str,
- tags: list[str],
- user_id: str,
- kb_id: str,
- document_id: str,
- title: str,
- chunk_index: int,
-) -> None:
- """Re-embed a parent chunk after a manual content edit.
-
- Removes the parent's existing chunk rows (child or flat) and upserts a single
- fresh chunk row with recomputed dense + sparse embeddings. Question rows are
- managed separately by :func:`upsert_question_rows`. Retrieval returns the parent
- content from PostgreSQL, so this single representative row is sufficient for the
- edited chunk to be matched and surfaced.
- """
- get_or_create_collection()
- client = _get_client()
- # Drop existing chunk rows for this parent (keep questions; they are re-upserted separately)
- client.delete(
- collection_name=COLLECTION_NAME,
- filter=f'parent_chunk_id == "{chunk_id}" and row_type == "chunk"',
- )
- sparse_text = build_sparse_text(content, tags or [])
- row = {
- "chunk_id": chunk_id,
- "parent_chunk_id": chunk_id,
- "row_type": "chunk",
- "user_id": user_id,
- "kb_id": kb_id,
- "document_id": document_id,
- "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES),
- "content": truncate_utf8(content),
- "tags_text": truncate_utf8(" ".join(tags or []), TAGS_FIELD_MAX_BYTES),
- "chunk_index": chunk_index,
- "dense_embedding": embed_text(content),
- "sparse_embedding": text_to_sparse(sparse_text),
- }
- _upsert(client, [row])
-
-
-def upsert_question_rows(
- parent_chunk_id: str,
- questions: list[str],
- user_id: str,
- kb_id: str,
- document_id: str,
- title: str,
- chunk_index: int,
-) -> None:
- """Delete existing question rows for a chunk and insert fresh ones."""
- get_or_create_collection()
- client = _get_client()
-
- # Delete old question rows
- expr = f'parent_chunk_id == "{parent_chunk_id}" and row_type == "question"'
- client.delete(collection_name=COLLECTION_NAME, filter=expr)
-
- if not questions:
- return
-
- # Embed all questions in one batch call
- vecs = embed_batch(questions)
- rows = []
- for i, (q, vec) in enumerate(zip(questions, vecs)):
- rows.append(
- {
- "chunk_id": f"q_{parent_chunk_id}_{i}",
- "parent_chunk_id": parent_chunk_id,
- "row_type": "question",
- "user_id": user_id,
- "kb_id": kb_id,
- "document_id": document_id,
- "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES),
- "content": truncate_utf8(q),
- "tags_text": "",
- "chunk_index": chunk_index,
- "dense_embedding": vec,
- "sparse_embedding": text_to_sparse(q),
- }
- )
- _upsert(client, rows)
diff --git a/src/backend/core/kb/wiki/local_provider.py b/src/backend/core/kb/wiki/local_provider.py
deleted file mode 100644
index 70a11701..00000000
--- a/src/backend/core/kb/wiki/local_provider.py
+++ /dev/null
@@ -1,523 +0,0 @@
-"""自建知识库 Wiki 的读取面。
-
-本模块的函数签名与返回结构,与外接知识库的 Wiki 模块**逐字段同构**——这是有意
-为之:路由层与 MCP 工具层只需要按 kb_id 选到不同的模块,调用代码一行都不用改,
-前端组件更是完全无感。
-
-相比外接透传,这里有一处实质优势:页面与 ``kb_chunks`` 同库,回溯原文是按
-chunk_id 直接查,而不是"拉整篇文档的全部分块再本地过滤"。
-"""
-
-from __future__ import annotations
-
-import logging
-import re
-from typing import Any, Dict, List, Optional, Sequence
-
-from core.db.engine import SessionLocal
-from core.db.models import KBChunk, KBDocument, KBWikiPage
-from core.db.models.kb_wiki import (
- WIKI_KNOWLEDGE_PAGE_TYPES,
- WIKI_PAGE_TYPE_INDEX,
- WIKI_PAGE_TYPE_LABELS,
- WIKI_PAGE_TYPE_SUMMARY,
-)
-from core.kb.wiki.config import MAX_GRAPH_NODES
-from core.kb.wiki.finalize import INDEX_SLUG
-from core.kb.wiki.slug import readable_from_slug
-from core.kb.wiki.store import (
- children_index,
- clean_category_path,
- descendant_folder_ids,
- get_page,
- load_folder_tree,
- page_counts_by_folder,
- subtree_counts,
- walk_folder_path,
-)
-from sqlalchemy import func, or_
-from sqlalchemy.orm import Session
-
-logger = logging.getLogger(__name__)
-
-PAGE_TYPE_LABELS = WIKI_PAGE_TYPE_LABELS
-KNOWLEDGE_PAGE_TYPES = WIKI_KNOWLEDGE_PAGE_TYPES
-
-_MAX_GRAPH_NODES = MAX_GRAPH_NODES
-_MAX_SEARCH_LIMIT = 50
-
-
-def _split_types(raw: str) -> List[str]:
- return [t.strip() for t in str(raw or "").split(",") if t.strip()]
-
-
-def _slim_page(page: KBWikiPage, *, with_content: bool = False) -> Dict[str, Any]:
- """页面 → 精简结构。
-
- 定位阶段只要标题/摘要/链接/血缘,正文按需带:一页动辄数千字,无差别塞进模型
- 上下文会把「先定位再取回」的成本优势整个吃掉。
- """
- item: Dict[str, Any] = {
- "slug": page.slug,
- "title": page.title,
- "page_type": page.page_type,
- "type_label": PAGE_TYPE_LABELS.get(page.page_type or "", ""),
- "summary": page.summary or "",
- "aliases": list(page.aliases or []),
- "category_path": list(page.category_path or []),
- "wiki_path": page.wiki_path or "",
- "out_links": list(page.out_links or []),
- "in_links": list(page.in_links or []),
- "source_refs": list(page.source_refs or []),
- "chunk_refs": list(page.chunk_refs or []),
- "updated_at": page.updated_at.isoformat() if page.updated_at else None,
- }
- if with_content:
- item["content"] = page.content or ""
- return item
-
-
-def _base_query(db: Session, kb_id: str):
- return db.query(KBWikiPage).filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None))
-
-
-# ── 统计 ────────────────────────────────────────────────────────────────────
-
-
-def wiki_stats(kb_id: str) -> Dict[str, Any]:
- """规模统计:总页数、分类型页数、链接数、孤儿页数。
-
- 只投影统计需要的三列——生成期间前端每 10 秒轮一次,把每页几 KB 的 ``content``
- 拉出来纯属浪费。
- """
- with SessionLocal() as db:
- rows = (
- db.query(KBWikiPage.page_type, KBWikiPage.in_links, KBWikiPage.out_links)
- .filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None))
- .all()
- )
- by_type: Dict[str, int] = {}
- total_links = 0
- orphan = 0
- for page_type, in_links, out_links in rows:
- by_type[page_type] = by_type.get(page_type, 0) + 1
- total_links += len(out_links or [])
- if not (in_links or []) and not (out_links or []):
- orphan += 1
- return {
- "total_pages": len(rows),
- "pages_by_type": by_type,
- "total_links": total_links,
- "orphan_count": orphan,
- }
-
-
-# ── 页面列表 ────────────────────────────────────────────────────────────────
-
-
-def wiki_list_pages(
- kb_id: str,
- page: int = 1,
- page_size: int = 50,
- page_type: str = "",
- category_path: str = "",
- category_depth: int = 0,
-) -> Dict[str, Any]:
- """分页列出页面。
-
- ``page_type`` 支持逗号分隔多选(「知识」视图把实体/概念/综述/对比合成一次
- 请求,避免按类型扇出);``category_path`` 把范围收窄到某个目录子树。
- """
- page = max(1, int(page or 1))
- page_size = max(1, int(page_size or 50))
- with SessionLocal() as db:
- query = _base_query(db, kb_id)
- types = _split_types(page_type)
- if types:
- query = query.filter(KBWikiPage.page_type.in_(types))
-
- labels = clean_category_path([category_path] if category_path else [])
- if labels:
- folder_id, _, _ = walk_folder_path(db, kb_id, labels, create=False)
- if folder_id is None:
- return {"pages": [], "total": 0, "page": page, "page_size": page_size}
- query = query.filter(
- KBWikiPage.folder_id.in_(descendant_folder_ids(db, kb_id, folder_id))
- )
-
- total = query.count()
- rows = (
- query.order_by(KBWikiPage.sort_order, KBWikiPage.title)
- .offset((page - 1) * page_size)
- .limit(page_size)
- .all()
- )
- return {
- "pages": [_slim_page(row) for row in rows],
- "total": total,
- "page": page,
- "page_size": page_size,
- }
-
-
-# ── 目录 ────────────────────────────────────────────────────────────────────
-
-
-def wiki_folders(kb_id: str, parent_id: str = "", page_types: str = "") -> Dict[str, Any]:
- """某一层的子目录,带**递归**页面计数,供前端逐层懒加载。
-
- 两次查询搞定:整棵目录树 + 一次按目录分组的页数聚合,其余在内存里算。原来是
- 每个子目录一次子树查询加一次计数,用户逐层展开时会反复付这个代价。
- """
- types = _split_types(page_types)
- with SessionLocal() as db:
- folders = load_folder_tree(db, kb_id)
- if not folders:
- return {"parent_id": parent_id or "", "folders": []}
- totals = subtree_counts(folders, page_counts_by_folder(db, kb_id, types))
- children = children_index(folders)
- items = [
- {
- "id": folder.folder_id,
- "name": folder.name,
- "parent_id": folder.parent_id or "",
- "page_count": totals.get(folder.folder_id, 0),
- "has_children": bool(children.get(folder.folder_id)),
- }
- for folder in children.get(parent_id or "", [])
- ]
- return {"parent_id": parent_id or "", "folders": items}
-
-
-# ── 索引总览 ────────────────────────────────────────────────────────────────
-
-
-def wiki_index(kb_id: str, limit: int = 20, types: str = "") -> Dict[str, Any]:
- """索引总览:一段总述 + 按类型分组的条目清单。"""
- limit = max(1, int(limit or 20))
- wanted = _split_types(types)
- with SessionLocal() as db:
- index_page = get_page(db, kb_id, INDEX_SLUG)
- intro = ""
- if index_page:
- intro = str((index_page.page_metadata or {}).get("intro") or "")
-
- groups = []
- for page_type in [*KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_SUMMARY]:
- if wanted and page_type not in wanted:
- continue
- query = _base_query(db, kb_id).filter(KBWikiPage.page_type == page_type)
- total = query.count()
- if not total:
- continue
- rows = query.order_by(KBWikiPage.title).limit(limit).all()
- groups.append(
- {
- "type": page_type,
- "type_label": PAGE_TYPE_LABELS.get(page_type, ""),
- "total": total,
- "items": [
- {
- "slug": row.slug,
- "title": row.title,
- "summary": row.summary or "",
- "wiki_path": row.wiki_path or "",
- }
- for row in rows
- ],
- }
- )
- return {
- "intro": intro,
- "version": index_page.version if index_page else None,
- "groups": groups,
- }
-
-
-# ── 检索 ────────────────────────────────────────────────────────────────────
-
-
-def wiki_search(kb_id: str, query: str, limit: int = 10) -> Dict[str, Any]:
- """按标题/别名/摘要/正文命中页面。
-
- 支持 ``资质|牌照`` 这样的交替写法——口语说法常常匹配不上书面术语,一次给多个
- 说法能显著提高命中率。交替在 Python 侧拆成多个 LIKE,避免依赖数据库正则。
- """
- limit = max(1, min(int(limit or 10), _MAX_SEARCH_LIMIT))
- terms = [t.strip() for t in re.split(r"[||]", str(query or "")) if t.strip()]
- if not terms:
- return {"pages": [], "total": 0}
-
- with SessionLocal() as db:
- conditions = []
- for term in terms[:8]:
- pattern = f"%{term.lower()}%"
- conditions.append(func.lower(KBWikiPage.title).like(pattern))
- conditions.append(func.lower(KBWikiPage.summary).like(pattern))
- conditions.append(func.lower(KBWikiPage.content).like(pattern))
- rows = (
- _base_query(db, kb_id)
- .filter(KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX)
- .filter(or_(*conditions))
- .order_by(KBWikiPage.page_type, KBWikiPage.title)
- .limit(limit * 3)
- .all()
- )
-
- # 别名命中在 SQL 里跨方言不好写,回到 Python 侧补一遍;同时按"命中位置"
- # 排序——标题命中远比正文里出现一次更相关
- lowered_terms = [t.lower() for t in terms]
-
- def _rank(page: KBWikiPage) -> int:
- title = (page.title or "").lower()
- aliases = " ".join(page.aliases or []).lower()
- summary = (page.summary or "").lower()
- if any(t == title for t in lowered_terms):
- return 0
- if any(t in title for t in lowered_terms):
- return 1
- if any(t in aliases for t in lowered_terms):
- return 2
- if any(t in summary for t in lowered_terms):
- return 3
- return 4
-
- ranked = sorted(rows, key=lambda p: (_rank(p), p.title or ""))[:limit]
- return {"pages": [_slim_page(p) for p in ranked], "total": len(ranked)}
-
-
-# ── 单页 ────────────────────────────────────────────────────────────────────
-
-
-def wiki_read_page(kb_id: str, slug: str, *, resolve_links: bool = True) -> Dict[str, Any]:
- """读取整页(含正文、双链、血缘坐标)。
-
- ``out_links`` 只有 slug,直接展示就是一串拼音,人和模型都读不出是什么。这里
- 顺手把邻居的标题查回来贴上——同库一次 IN 查询就够。
- """
- with SessionLocal() as db:
- page = get_page(db, kb_id, slug)
- if page is None:
- return {}
- item = _slim_page(page, with_content=True)
- item["related_pages"] = _resolve_link_titles(
- db, kb_id, item["out_links"], resolve=resolve_links
- )
- return item
-
-
-def _resolve_link_titles(
- db: Session, kb_id: str, slugs: Sequence[str], *, resolve: bool
-) -> List[Dict[str, str]]:
- if not slugs:
- return []
- titles: Dict[str, KBWikiPage] = {}
- if resolve:
- rows = _base_query(db, kb_id).filter(KBWikiPage.slug.in_(list(slugs))).all()
- titles = {row.slug: row for row in rows}
- resolved = []
- for slug in slugs:
- row = titles.get(slug)
- resolved.append(
- {
- "slug": slug,
- "title": (row.title if row else "") or readable_from_slug(slug),
- "page_type": (row.page_type if row else "") or slug.split("/", 1)[0],
- }
- )
- return resolved
-
-
-# ── 图谱 ────────────────────────────────────────────────────────────────────
-
-
-def wiki_graph(
- kb_id: str,
- mode: str = "overview",
- center: str = "",
- depth: int = 1,
- limit: int = 60,
- types: str = "",
-) -> Dict[str, Any]:
- """概念图谱。
-
- ``overview`` 取全局连接度最高的节点,``ego`` 以 center 为中心展开 depth 层。
- 大库有数千节点上万条边,前端必须靠这两种模式分批取,不能一次拉全量。
- """
- limit = max(1, min(int(limit or 60), _MAX_GRAPH_NODES))
- depth = max(1, min(int(depth or 1), 3))
- wanted = _split_types(types)
-
- with SessionLocal() as db:
- # 图谱只用到这四列;页面正文对邻接表毫无用处,却是最大的一列
- query = db.query(
- KBWikiPage.slug,
- KBWikiPage.title,
- KBWikiPage.page_type,
- KBWikiPage.out_links,
- ).filter(
- KBWikiPage.kb_id == kb_id,
- KBWikiPage.deleted_at.is_(None),
- KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX,
- )
- if wanted:
- query = query.filter(KBWikiPage.page_type.in_(wanted))
- rows = query.all()
-
- by_slug = {row.slug: row for row in rows}
- adjacency: Dict[str, List[str]] = {
- row.slug: [s for s in (row.out_links or []) if s in by_slug] for row in rows
- }
- # 边是无向展开用的,两个方向都要能走
- undirected: Dict[str, set] = {slug: set(links) for slug, links in adjacency.items()}
- for slug, links in adjacency.items():
- for target in links:
- undirected.setdefault(target, set()).add(slug)
-
- if mode == "ego":
- selected = _ego_nodes(undirected, center, depth, limit)
- if not selected:
- return {
- "nodes": [],
- "edges": [],
- "meta": {
- "mode": "ego",
- "center": center,
- "depth": depth,
- "truncated": False,
- "total": len(rows),
- "returned": 0,
- "total_pages": len(rows),
- },
- }
- else:
- ranked = sorted(rows, key=lambda r: len(undirected.get(r.slug, ())), reverse=True)
- selected = {r.slug for r in ranked[:limit]}
-
- nodes = [
- {
- "slug": slug,
- "title": by_slug[slug].title or readable_from_slug(slug),
- "page_type": by_slug[slug].page_type,
- "link_count": len(undirected.get(slug, ())),
- }
- for slug in selected
- if slug in by_slug
- ]
- edges = []
- seen_edges = set()
- for slug in selected:
- for target in adjacency.get(slug, []):
- if target not in selected:
- continue
- key = tuple(sorted((slug, target)))
- if key in seen_edges:
- continue
- seen_edges.add(key)
- edges.append({"source": slug, "target": target})
-
- return {
- "nodes": nodes,
- "edges": edges,
- "meta": {
- "mode": mode,
- "center": center,
- "depth": depth if mode == "ego" else None,
- "truncated": len(rows) > len(selected),
- # 前端类型声明读 total/returned;total_pages 是旧键,保留兼容
- "total": len(rows),
- "returned": len(nodes),
- "total_pages": len(rows),
- },
- }
-
-
-def _ego_nodes(adjacency: Dict[str, set], center: str, depth: int, limit: int) -> set:
- """从 center 出发做 BFS,取 depth 层内的节点。"""
- if not center or center not in adjacency:
- return set()
- selected = {center}
- frontier = {center}
- for _ in range(depth):
- nxt = set()
- for slug in frontier:
- for neighbour in adjacency.get(slug, ()):
- if neighbour not in selected:
- nxt.add(neighbour)
- if not nxt:
- break
- for slug in sorted(nxt, key=lambda s: len(adjacency.get(s, ())), reverse=True):
- if len(selected) >= limit:
- break
- selected.add(slug)
- frontier = nxt
- if len(selected) >= limit:
- break
- return selected
-
-
-# ── 顺血缘取回原文 ──────────────────────────────────────────────────────────
-
-
-def wiki_source_chunks(kb_id: str, slug: str, max_chunks: int = 6) -> Dict[str, Any]:
- """按页面的 ``chunk_refs`` 直取原文分块。
-
- 这是「地图 → 定位 → 顺坐标取回原文」这条链的最后一跳:**按 ID 直取**,不是
- 再检索一次。页面与分块同库,一次 IN 查询就够。
- """
- max_chunks = max(1, min(int(max_chunks or 6), 20))
- with SessionLocal() as db:
- page = get_page(db, kb_id, slug)
- if page is None:
- return {"page": {}, "chunks": []}
-
- item = _slim_page(page, with_content=True)
- item["related_pages"] = _resolve_link_titles(db, kb_id, item["out_links"], resolve=True)
-
- chunk_ids = list(page.chunk_refs or [])
- rows: List[KBChunk] = []
- if chunk_ids:
- rows = (
- db.query(KBChunk)
- .filter(KBChunk.kb_id == kb_id, KBChunk.chunk_id.in_(chunk_ids))
- .all()
- )
- # 保持页面记录的引用顺序:那是证据的呈现次序,不是数据库的返回次序
- order = {cid: i for i, cid in enumerate(chunk_ids)}
- rows.sort(key=lambda r: order.get(r.chunk_id, 10**6))
-
- # chunk_refs 对不上(页面被改写过、或文档重新索引换了分块)时,退回该页
- # 主要来源文档的前几块正文——给不出精确证据,至少不要空手而归
- if not rows and (page.source_refs or []):
- rows = (
- db.query(KBChunk)
- .filter(KBChunk.kb_id == kb_id, KBChunk.document_id == page.source_refs[0])
- .order_by(KBChunk.chunk_index)
- .limit(max_chunks)
- .all()
- )
-
- titles = _document_titles(db, kb_id, [r.document_id for r in rows])
- chunks = [
- {
- "chunk_id": row.chunk_id,
- "document_id": row.document_id,
- "document_title": titles.get(row.document_id, ""),
- "content": row.content or "",
- }
- for row in rows[:max_chunks]
- if (row.content or "").strip()
- ]
- return {"page": item, "chunks": chunks}
-
-
-def _document_titles(db: Session, kb_id: str, document_ids: Sequence[str]) -> Dict[str, str]:
- ids = [d for d in dict.fromkeys(document_ids) if d]
- if not ids:
- return {}
- rows = (
- db.query(KBDocument.document_id, KBDocument.title, KBDocument.filename)
- .filter(KBDocument.kb_id == kb_id, KBDocument.document_id.in_(ids))
- .all()
- )
- return {row.document_id: (row.title or row.filename or row.document_id) for row in rows}
diff --git a/src/backend/core/llm/__init__.py b/src/backend/core/llm/__init__.py
index 4d9fbb7d..0546c2bc 100644
--- a/src/backend/core/llm/__init__.py
+++ b/src/backend/core/llm/__init__.py
@@ -1,17 +1,5 @@
"""LLM package.
Keep package import framework-neutral: pure context/manifest modules must be
-usable in tests and offline tooling without importing AgentScope. Legacy
-attribute exports remain available lazily.
+usable in tests and offline tooling without importing AgentScope.
"""
-
-
-def __getattr__(name): # noqa: ANN001, ANN202
- if name in {"make_chat_model", "get_default_model"}:
- from core.llm import chat_models
-
- return getattr(chat_models, name)
- raise AttributeError(name)
-
-
-__all__ = ["get_default_model", "make_chat_model"]
diff --git a/src/backend/core/llm/_distill_shared.py b/src/backend/core/llm/_distill_shared.py
deleted file mode 100644
index 96569922..00000000
--- a/src/backend/core/llm/_distill_shared.py
+++ /dev/null
@@ -1,156 +0,0 @@
-"""Shared helpers for distillation LLM callers (skill_distiller / persona_distiller).
-
-Extracted from skill_distiller.py so the per-chat auto distiller and the
-persona-level (colleague / personal) distiller share one implementation of:
-- model resolution (role 'skill_distiller' → fallback 'main_agent')
-- tolerant strict-JSON parsing ( blocks, ``` fences, surrounding prose)
-- rough cost estimation
-- skill dict → SKILL.md rendering
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-from typing import Any, Dict, Optional
-
-from core.services.model_config import ModelConfigService, ResolvedModelConfig
-
-logger = logging.getLogger(__name__)
-
-
-# OpenAI-style rough price table per model. Override via env / admin panel later.
-# USD per 1M tokens. Default to a conservative flat rate when model unknown.
-PRICE_PER_MTOK: Dict[str, Dict[str, float]] = {
- "default": {"input": 3.0, "output": 15.0},
- "deepseek": {"input": 0.27, "output": 1.1},
- "qwen": {"input": 0.5, "output": 1.5},
-}
-
-
-def estimate_cost_usd(model: str, prompt_tokens: int, completion_tokens: int) -> float:
- price = PRICE_PER_MTOK["default"]
- for key, v in PRICE_PER_MTOK.items():
- if key in (model or "").lower():
- price = v
- break
- return (prompt_tokens * price["input"] + completion_tokens * price["output"]) / 1_000_000.0
-
-
-def resolve_distiller_model() -> ResolvedModelConfig:
- svc = ModelConfigService.get_instance()
- cfg = svc.resolve("skill_distiller") or svc.resolve("main_agent")
- if cfg is None:
- raise RuntimeError("skill_distiller: no model configured for 'skill_distiller' or 'main_agent'")
- return cfg
-
-
-_THINK_BLOCK_RE = re.compile(r".*? ", re.DOTALL | re.IGNORECASE)
-_THINK_CLOSE_RE = re.compile(r" ", re.IGNORECASE)
-_THINK_OPEN_RE = re.compile(r"", re.IGNORECASE)
-
-
-def strip_think_blocks(text: str) -> str:
- """Strip a reasoning model's think blocks and return the body.
-
- Covers three shapes:
- - closed ``... `` blocks (possibly multiple);
- - only `` `` (some serving stacks swallow the opening tag into the chat template, with the body following the closing tag)
- → take the content after the last closing tag;
- - only ```` (output was truncated mid-thought by max_tokens, with no body after)
- → discard everything from the opening tag onward.
- """
- t = _THINK_BLOCK_RE.sub("", text)
- m = None
- for m in _THINK_CLOSE_RE.finditer(t):
- pass
- if m is not None:
- t = t[m.end():]
- m = _THINK_OPEN_RE.search(t)
- if m is not None:
- t = t[: m.start()]
- return t.strip()
-
-
-def extract_first_json_object(text: str) -> Optional[str]:
- start = text.find("{")
- if start < 0:
- return None
- depth = 0
- in_str = False
- escape = False
- for i, ch in enumerate(text[start:], start=start):
- if in_str:
- if escape:
- escape = False
- elif ch == "\\":
- escape = True
- elif ch == '"':
- in_str = False
- continue
- if ch == '"':
- in_str = True
- elif ch == "{":
- depth += 1
- elif ch == "}":
- depth -= 1
- if depth == 0:
- return text[start:i + 1]
- return None
-
-
-def parse_strict_json(text: str) -> Optional[Dict[str, Any]]:
- """Parse JSON tolerantly: strips blocks, ``` fences, and surrounding prose."""
- if not text:
- return None
- # If the body is already valid JSON, return it directly to avoid corruption from stripping a think tag that happens to appear inside a string value
- try:
- parsed = json.loads(text.strip())
- if isinstance(parsed, dict):
- return parsed
- except Exception:
- pass
- t = strip_think_blocks(text.strip())
-
- if t.startswith("```"):
- lines = [l for l in t.splitlines() if not l.strip().startswith("```")]
- t = "\n".join(lines).strip()
-
- try:
- return json.loads(t)
- except Exception:
- pass
-
- candidate = extract_first_json_object(t)
- if candidate:
- try:
- return json.loads(candidate)
- except Exception:
- return None
- return None
-
-
-def skill_to_markdown(skill: Dict[str, Any]) -> str:
- """Render a skill dict back to SKILL.md text with YAML frontmatter."""
- try:
- import yaml # type: ignore
- except ImportError:
- yaml = None # type: ignore
-
- fm = skill.get("frontmatter") or {}
- body = skill.get("instructions_md") or ""
-
- if yaml is not None:
- fm_text = yaml.safe_dump(fm, allow_unicode=True, sort_keys=False).strip()
- else:
- # Fallback: basic key: value formatting (only primitive types)
- parts = []
- for k, v in fm.items():
- if isinstance(v, (list, dict)):
- parts.append(f"{k}: {json.dumps(v, ensure_ascii=False)}")
- else:
- parts.append(f"{k}: {v}")
- fm_text = "\n".join(parts)
-
- return f"---\n{fm_text}\n---\n\n{body}\n"
diff --git a/src/backend/core/llm/api_runtime/__init__.py b/src/backend/core/llm/api_runtime/__init__.py
new file mode 100644
index 00000000..14f23927
--- /dev/null
+++ b/src/backend/core/llm/api_runtime/__init__.py
@@ -0,0 +1 @@
+"""Agent runtime for authenticated API-key requests."""
diff --git a/src/backend/core/llm/agent_api_runtime.py b/src/backend/core/llm/api_runtime/agent.py
similarity index 97%
rename from src/backend/core/llm/agent_api_runtime.py
rename to src/backend/core/llm/api_runtime/agent.py
index 5d2b0f5f..46afdd5e 100644
--- a/src/backend/core/llm/agent_api_runtime.py
+++ b/src/backend/core/llm/api_runtime/agent.py
@@ -106,7 +106,7 @@ def scope_mcp_servers(servers: dict, scope: AgentApiExecutionScope) -> dict:
existing = next((key for key in headers if key.lower() == header.lower()), header)
if not str(headers.get(existing) or "").strip():
headers[existing] = "__agent_api_no_kb__"
- from core.llm.mcp_invocation import for_url
+ from core.llm.mcp.invocation import for_url
headers.update(for_url(item["url"], scope.sandbox_user_id, scope.chat_id))
item["headers"] = headers
out[name] = item
@@ -119,7 +119,7 @@ def artifact_in_scope(file_id: str, scope: AgentApiExecutionScope) -> bool:
return False
from core.db.engine import SessionLocal
from core.db.models import Artifact, ChatSession
- from core.services.agent_api_service import session_matches_scope
+ from core.auth.agent_api import session_matches_scope
with SessionLocal() as db:
session = db.get(ChatSession, scope.chat_id)
if not session or not session_matches_scope(session, {"version": 1, **{key: value for key, value in asdict(scope).items() if value is not None}}):
@@ -133,7 +133,7 @@ def artifact_in_scope(file_id: str, scope: AgentApiExecutionScope) -> bool:
)
# An output is in the file index before pinning/finalization persists it.
# Read only index metadata here; get_artifact resolves host project paths.
- from core.artifacts.store import _read_record
+ from core.files.artifacts.store import _read_record
item = _read_record(file_id) if file_id else None
metadata = (item or {}).get("metadata") or {}
return (
diff --git a/src/backend/core/llm/agent_api_skills.py b/src/backend/core/llm/api_runtime/agent_skills.py
similarity index 96%
rename from src/backend/core/llm/agent_api_skills.py
rename to src/backend/core/llm/api_runtime/agent_skills.py
index 110a38a1..e6b8beb1 100644
--- a/src/backend/core/llm/agent_api_skills.py
+++ b/src/backend/core/llm/api_runtime/agent_skills.py
@@ -5,7 +5,7 @@
import re
from pathlib import Path
-from core.llm.agent_api_runtime import AgentApiExecutionScope
+from core.llm.api_runtime.agent import AgentApiExecutionScope
def _skill_files(skill_dirs: dict[str, str]):
diff --git a/src/backend/core/llm/agent_api_tools.py b/src/backend/core/llm/api_runtime/agent_tools.py
similarity index 90%
rename from src/backend/core/llm/agent_api_tools.py
rename to src/backend/core/llm/api_runtime/agent_tools.py
index 96c41212..7149bf29 100644
--- a/src/backend/core/llm/agent_api_tools.py
+++ b/src/backend/core/llm/api_runtime/agent_tools.py
@@ -1,5 +1,6 @@
"""Native tools for an API agent: private account data is never mounted."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import base64
@@ -9,8 +10,8 @@
import tempfile
from pathlib import Path
-from core.llm.agent_api_runtime import AgentApiExecutionScope, artifact_in_scope
-from core.llm.tools._tool_helpers import _resp_json
+from core.llm.api_runtime.agent import AgentApiExecutionScope, artifact_in_scope
+from core.llm.tools._tool_helpers import resp_json as _resp_json
def _provider():
@@ -38,7 +39,7 @@ def _workspace_path(value: str) -> str:
def register_agent_api_tools(toolkit, scope: AgentApiExecutionScope, *, read_only=False, skill_dirs=None):
"""Register a fixed scoped surface; do not wrap account-private tools."""
from core.sandbox import ProcessRequest, SandboxError
- from core.llm.agent_api_skills import ApiSkillStager
+ from core.llm.api_runtime.agent_skills import ApiSkillStager
skills = ApiSkillStager(skill_dirs)
async def Bash(command: str, timeout: int | None = None, yield_time_ms: int = 60000):
@@ -77,11 +78,11 @@ async def read_artifact(
"""读取当前 API 会话中生成的文件,不能读取账号其他文件。"""
if not await asyncio.to_thread(artifact_in_scope, file_id, scope):
return _resp_json({"error": "文件不属于当前 API 会话或已删除"})
- from core.llm.tool_collector import ToolCollector
+ from core.llm.factory.tools.definitions import ToolAssembly
from core.llm.tools.read_artifact_tool import register_read_artifact
- collector = ToolCollector()
+ collector = ToolAssembly()
register_read_artifact(collector, user_id=scope.owner_user_id)
- return await collector.get_tool("read_artifact")._func(
+ return await next(t for t in collector if t.name == "read_artifact")._func(
file_id=file_id, offset=offset, limit=limit,
sheet_name=sheet_name, slide_index=slide_index,
)
@@ -93,11 +94,11 @@ async def pin_to_workspace(file_ids: list[str]):
for file_id in file_ids:
if not isinstance(file_id, str) or not await asyncio.to_thread(artifact_in_scope, file_id, scope):
return _resp_json({"error": "文件不属于当前 API 会话或已删除"})
- from core.artifacts.store import get_artifact
+ from core.files.artifacts.store import get_artifact
from core.db.engine import SessionLocal
from core.db.models import Artifact
- from core.llm import workspace
- from core.services.artifact_service import persist_artifacts
+ from core.llm.runtime import workspace
+ from core.files.artifacts.service import persist_artifacts
def persist():
refs = []
@@ -142,7 +143,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str):
path = _workspace_path(dest_path)
if not await asyncio.to_thread(artifact_in_scope, artifact_id, scope):
raise ValueError("文件不属于当前 API 会话或已删除")
- from core.llm.tools._tool_helpers import _resolve_artifact_files
+ from core.files.artifacts.generated import resolve_artifact_files as _resolve_artifact_files
files, error = await asyncio.to_thread(
_resolve_artifact_files, {path: artifact_id}, scope.owner_user_id,
)
@@ -159,7 +160,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str):
async def sandbox_get_artifact(src_path: str, name: str = ""):
"""保存当前 API 沙箱中的产物并返回文件 ID;不读取个人空间。"""
from core.config.settings import settings
- from core.llm.tools._tool_helpers import _store_generated_file_path
+ from core.files.artifacts.generated import store_generated_file_path as _store_generated_file_path
target = None
try:
provider = _provider()
@@ -192,7 +193,7 @@ async def sandbox_get_artifact(src_path: str, name: str = ""):
target.unlink(missing_ok=True)
for tool in (read_artifact, sandbox_get_artifact, pin_to_workspace):
- toolkit.register_tool_function(tool, namesake_strategy="override")
+ toolkit.append(function_tool(tool))
if not read_only:
for tool in (Bash, write_stdin, sandbox_put_artifact):
- toolkit.register_tool_function(tool, namesake_strategy="override")
+ toolkit.append(function_tool(tool))
diff --git a/src/backend/core/llm/project_api_runtime.py b/src/backend/core/llm/api_runtime/project.py
similarity index 94%
rename from src/backend/core/llm/project_api_runtime.py
rename to src/backend/core/llm/api_runtime/project.py
index 1a7c638b..2bb36c2e 100644
--- a/src/backend/core/llm/project_api_runtime.py
+++ b/src/backend/core/llm/api_runtime/project.py
@@ -23,7 +23,7 @@ async def bind_project_api_runtime(request) -> None:
def authorize():
from core.db.engine import SessionLocal
- from core.services.api_key_runs import require_admission_key
+ from core.auth.api_key_runs import require_admission_key
with SessionLocal() as db:
require_admission_key(
diff --git a/src/backend/core/llm/capability_tools.py b/src/backend/core/llm/capability_tools.py
deleted file mode 100644
index 5e1ed5ff..00000000
--- a/src/backend/core/llm/capability_tools.py
+++ /dev/null
@@ -1,106 +0,0 @@
-"""Per-request connector guards; pooled clients remain account-neutral."""
-
-import asyncio
-from agentscope.tool import ToolBase, ToolChunk
-from agentscope.message import TextBlock, ToolResultState
-from core.capabilities import runtime
-from core.capabilities.errors import CapabilityError
-from core.capabilities.paths import LOCAL_PROFILE
-
-
-def connector_available(run, server_id):
- runtime.validate(run)
- latest = runtime.get(run.run_id, scope_id=run.scope_id) or run
- if "mcp:" + server_id in latest.unavailable:
- return False
- binding = run.mcp_bindings.get(server_id)
- if binding is not None:
- from core.capabilities import registry
-
- relationships = latest.dependency_report.get("connector_parents", {})
- parents = {
- *relationships.get(server_id, []),
- *relationships.get(binding["install_id"].split(":", 2)[2], []),
- }
- for parent_id in parents:
- parent = registry.get(parent_id)
- if parent is None or not parent.enabled or parent.state == "removed":
- return False
- owner = parent.payload.get("owner_user_id")
- if owner and str(owner) != run.user_id:
- return False
- if binding is None or not binding.get("authorization_checked"):
- return True
- _, profile, key = binding["install_id"].split(":", 2)
- if profile == "local-json":
- from core.capabilities.mcp_json import local_server_configs
-
- return key in local_server_configs()
- if profile == LOCAL_PROFILE:
- from core.services.mcp_service import McpServerConfigService
-
- service = McpServerConfigService.get_instance()
- return key in {
- **service.get_all_servers(enabled_only=True),
- **service.get_owned_servers(run.user_id, enabled_only=True),
- }
- from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs
-
- return server_id in cloud_gateway_mcp_configs([server_id])
-
-
-class AvailableMCPTool(ToolBase):
- def __init__(self, tool, run, server_id):
- self.tool, self.run, self.server_id = tool, run, server_id
- for name in (
- "name",
- "description",
- "input_schema",
- "is_concurrency_safe",
- "is_read_only",
- "is_external_tool",
- "is_state_injected",
- "is_mcp",
- "mcp_name",
- ):
- setattr(self, name, getattr(tool, name))
-
- def __getattr__(self, name):
- return getattr(self.tool, name)
-
- async def check_permissions(self, *args, **kwargs):
- return await self.tool.check_permissions(*args, **kwargs)
-
- async def __call__(self, **kwargs):
- try:
- available = await asyncio.to_thread(connector_available, self.run, self.server_id)
- except (CapabilityError, OSError):
- available = False
- if not available:
- return ToolChunk(
- state=ToolResultState.ERROR,
- content=[
- TextBlock(
- text="该连接器已停用或当前不可用,本次调用未执行。请说明情况并继续处理可完成的部分。"
- )
- ],
- )
- # Do not catch an uncertain write outcome or retry a side effect.
- return await self.tool(**kwargs)
-
-
-class AvailableMCPClient:
- def __init__(self, client, run):
- self.client, self.run = client, run
- self.name = client.name
-
- def __getattr__(self, name):
- return getattr(self.client, name)
-
- async def list_tools(self):
- return [
- AvailableMCPTool(tool, self.run, self.name) for tool in await self.client.list_tools()
- ]
-
- async def get_tool(self, name):
- return AvailableMCPTool(await self.client.get_tool(name), self.run, self.name)
diff --git a/src/backend/core/llm/citations/__init__.py b/src/backend/core/llm/citations/__init__.py
new file mode 100644
index 00000000..242360b5
--- /dev/null
+++ b/src/backend/core/llm/citations/__init__.py
@@ -0,0 +1 @@
+"""Number tool-result sources as citation anchors the model can cite."""
diff --git a/src/backend/core/llm/citations/allocator.py b/src/backend/core/llm/citations/allocator.py
new file mode 100644
index 00000000..b481a98d
--- /dev/null
+++ b/src/backend/core/llm/citations/allocator.py
@@ -0,0 +1,115 @@
+"""Per-chat citation anchor allocator and registry."""
+from __future__ import annotations
+from contextvars import ContextVar
+from typing import Any, Dict, List, Optional, Sequence
+import logging
+from core.llm.citations import anchor_state
+from core.llm.citations import types
+
+logger = logging.getLogger(__name__)
+
+class AnchorAllocator:
+ """会话内单调锚点计数器 + 按 tool_id 的引用注册表。
+
+ 并发安全性:分配是同步代码段(无 await 点),asyncio 并行工具任务
+ 通过 ContextVar 共享同一实例,不会重号。
+ """
+
+ def __init__(self, start: int = 1) -> None:
+ self._next = max(1, int(start or 1))
+ self._by_tool_id: Dict[str, List[types.AnchorCitation]] = {}
+ self.all: List[types.AnchorCitation] = []
+
+ def next_id(self) -> str:
+ aid = f"e{self._next}"
+ self._next += 1
+ return aid
+
+ def register(self, tool_id: Optional[str], items: Sequence[types.AnchorCitation]) -> None:
+ if not items:
+ return
+ self.all.extend(items)
+ if tool_id:
+ self._by_tool_id.setdefault(str(tool_id), []).extend(items)
+
+ def citations_for(self, tool_id: Optional[str]) -> List[types.AnchorCitation]:
+ if not tool_id:
+ return []
+ return list(self._by_tool_id.get(str(tool_id), []))
+
+
+def attach_allocator(agent: Any, allocator: AnchorAllocator) -> AnchorAllocator:
+ """run 入口调用:把发号器绑定到 agent(并写 ContextVar 供子链路兜底)。"""
+ try:
+ setattr(agent, ALLOC_ATTR, allocator)
+ except Exception as exc: # noqa: BLE001 - pydantic 模型等禁止动态属性时降级到 ContextVar
+ logger.debug("[citation-anchor] attach to agent failed: %s", exc)
+ CITATION_ALLOCATOR.set(allocator)
+ return allocator
+
+
+def resolve_allocator(agent: Any) -> AnchorAllocator:
+ """中间件调用:取本 run 的发号器;缺失时就地建一个并绑定,保证始终可用。"""
+ allocator = getattr(agent, ALLOC_ATTR, None)
+ if isinstance(allocator, AnchorAllocator):
+ return allocator
+ allocator = CITATION_ALLOCATOR.get()
+ if not isinstance(allocator, AnchorAllocator):
+ allocator = AnchorAllocator()
+ return attach_allocator(agent, allocator)
+
+
+def collect_citation_dicts(
+ tool_id: Optional[str],
+ allocator: Optional[AnchorAllocator] = None,
+) -> List[Dict[str, Any]]:
+ """编排层统一取引用入口(dict 形态,直接可进 SSE / trace / 落库)。
+
+ 发号器是**唯一**编号方:中间件在结果回给模型前已完成注号并按 tool_id 登记,
+ 这里只做精确取回,不做二次提取。``allocator`` 由调用方显式传入(run 入口绑定
+ 在 agent 上的那一个);不传时退到 ContextVar(子智能体链路)。
+ """
+ if allocator is None:
+ allocator = CITATION_ALLOCATOR.get()
+ if allocator is None:
+ return []
+ return [item.to_dict() for item in allocator.citations_for(tool_id)]
+
+
+def anchor_start_for_chat(chat_id: Optional[str]) -> int:
+ """跨轮续号:扫描该会话已落库消息的 citations,取最大锚点 + 1。
+
+ 任何异常都回退 1(引用功能降级,不影响对话)。
+ """
+ if not chat_id:
+ return 1
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.models import ChatMessage
+
+ max_seen = 0
+ with SessionLocal() as db:
+ rows = (
+ db.query(ChatMessage.extra_data)
+ .filter(ChatMessage.chat_id == chat_id)
+ .order_by(ChatMessage.chat_seq.desc())
+ .limit(200)
+ .all()
+ )
+ for (extra,) in rows:
+ for cit in (extra or {}).get("citations") or []:
+ m = anchor_state.ANCHOR_RE.match(str(cit.get("id", "")) if isinstance(cit, dict) else "")
+ if m:
+ max_seen = max(max_seen, int(m.group(1)))
+ return max_seen + 1
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[citation-anchor] start scan failed for chat %s: %s", chat_id, exc)
+ return 1
+
+
+CITATION_ALLOCATOR: ContextVar[Optional[AnchorAllocator]] = ContextVar(
+ "jx_citation_allocator", default=None
+)
+
+
+ALLOC_ATTR = "_jx_citation_allocator"
diff --git a/src/backend/core/llm/citations/anchor.py b/src/backend/core/llm/citations/anchor.py
new file mode 100644
index 00000000..2843cafd
--- /dev/null
+++ b/src/backend/core/llm/citations/anchor.py
@@ -0,0 +1,63 @@
+"""Annotate tool results with citation anchors."""
+from __future__ import annotations
+import json
+from typing import List, Optional, Tuple
+import logging
+from core.llm.citations.allocator import AnchorAllocator
+from core.llm.citations import anchor_state
+from core.llm.citations import extraction
+from core.llm.citations import types
+
+logger = logging.getLogger(__name__)
+
+def annotate_tool_result(
+ tool_name: str,
+ tool_id: Optional[str],
+ text: str,
+ allocator: AnchorAllocator,
+) -> Tuple[str, List[types.AnchorCitation]]:
+ """给一份工具结果文本注号。
+
+ 返回 (注号后的文本, 登记的引用条目)。不可注号(跳过名单 / 空结果 /
+ 解析失败且判定无引用价值)时原样返回 (text, [])。**绝不抛错**。
+ """
+ if not text or tool_name in anchor_state.SKIP_TOOLS:
+ return text, []
+ try:
+ parsed = json.loads(text)
+ except (json.JSONDecodeError, TypeError):
+ parsed = None
+
+ try:
+ if isinstance(parsed, dict):
+ items = extraction.extract_and_inject(tool_name, tool_id, parsed, allocator)
+ if not items:
+ return text, []
+ return json.dumps(parsed, ensure_ascii=False), items
+ if isinstance(parsed, list):
+ # 裸数组结果:包壳后按启发式处理,回注仍输出裸数组语义(保持兼容 → 整份 1 条)
+ parsed = {"items": parsed}
+ items = extraction.extract_and_inject(tool_name, tool_id, parsed, allocator)
+ if not items:
+ return text, []
+ return json.dumps(parsed["items"], ensure_ascii=False), items
+ # 纯文本结果:整份 1 条,文末追加锚点行
+ spec = anchor_state.TOOL_SPECS.get(tool_name)
+ if spec and spec.mode == "list":
+ # 声明为列表型但拿到纯文本 → 形态异常,放行不注号
+ return text, []
+ aid = allocator.next_id()
+ item = types.AnchorCitation(
+ id=aid,
+ tool_name=tool_name,
+ tool_id=tool_id,
+ title=extraction.display_title(tool_name, spec),
+ url="",
+ snippet=str(text)[:anchor_state.SNIPPET_MAX],
+ source_type=(spec.source_type if spec else "unknown"),
+ item_index=-1,
+ )
+ return f"{text}\n\n[cite_id: {aid}]", [item]
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[citation-anchor] annotate failed tool=%s: %s", tool_name, exc)
+ return text, []
diff --git a/src/backend/core/llm/citations/anchor_state.py b/src/backend/core/llm/citations/anchor_state.py
new file mode 100644
index 00000000..a5e99dd3
--- /dev/null
+++ b/src/backend/core/llm/citations/anchor_state.py
@@ -0,0 +1,101 @@
+"""Anchor patterns, length limits and tool lists for citations."""
+from __future__ import annotations
+import re
+from typing import Dict, Tuple
+from core.llm.citations import types
+
+ANCHOR_RE = re.compile(r"^e(\d+)$")
+
+
+LIST_FIELD_ALIASES: Tuple[str, ...] = (
+ "items",
+ "results",
+ "events",
+ "pages",
+ "records",
+ "entries",
+ "data",
+ "list",
+ "rows",
+ "docs",
+)
+
+
+TITLE_MAX = 120
+
+
+SNIPPET_MAX = 3000
+
+
+WHOLE_SNIPPET_MAX = 500
+
+
+SKIP_TOOLS = frozenset(
+ {
+ # 自研文件/沙盒操作
+ "Write",
+ "Edit",
+ "Bash",
+ "write_stdin",
+ "sandbox_put_artifact",
+ "sandbox_get_artifact",
+ "pin_to_workspace",
+ "channel_read_attachment",
+ "choose_design",
+ "ask_user_question",
+ "view_text_file",
+ "get_data_context",
+ "load_skill",
+ "Glob",
+ "Grep",
+ # 产物/发布/批量
+ "publish_site",
+ # 定时任务写操作
+ "create_scheduled_task",
+ "update_scheduled_task",
+ "pause_scheduled_task",
+ "resume_scheduled_task",
+ "delete_scheduled_task",
+ "list_scheduled_tasks",
+ "list_channel_conversations",
+ "get_scheduled_task",
+ # 技能管理
+ "search_marketplace",
+ "list_my_skills",
+ "install_from_marketplace",
+ "register_skill",
+ "submit_to_marketplace",
+ "delete_skill",
+ "edit_skill",
+ # 能力/资产发现
+ "list_datasets",
+ # 子智能体的引用由其内部工具的嵌套事件承载
+ "call_subagent",
+ }
+)
+
+
+TOOL_SPECS: Dict[str, types.CitationSpec] = {
+ # 检索/知识类(逐条)
+ "internet_search": types.CitationSpec(
+ mode="list",
+ items_paths=(("result", "results"), ("results",)),
+ source_type="internet",
+ ),
+ "retrieve_dataset_content": types.CitationSpec(
+ mode="list", items_paths=(("items",),), source_type="knowledge_base"
+ ),
+ "retrieve_local_kb": types.CitationSpec(
+ mode="list", items_paths=(("items",),), source_type="knowledge_base"
+ ),
+ "wiki_fetch_source": types.CitationSpec(
+ mode="list", items_paths=(("items",),), source_type="knowledge_base"
+ ),
+ "wiki_locate": types.CitationSpec(
+ mode="list", items_paths=(("pages",),), source_type="knowledge_base"
+ ),
+ "wiki_read_page": types.CitationSpec(mode="whole", title="Wiki 页面", source_type="knowledge_base"),
+ "web_fetch": types.CitationSpec(mode="whole", title="网页内容", source_type="internet"),
+ # 数据/报告类(整份)
+ "query_database": types.CitationSpec(mode="whole", title="数据库查询结果", source_type="database"),
+}
diff --git a/src/backend/core/llm/citations/extraction.py b/src/backend/core/llm/citations/extraction.py
new file mode 100644
index 00000000..f0abbc17
--- /dev/null
+++ b/src/backend/core/llm/citations/extraction.py
@@ -0,0 +1,208 @@
+"""Extract citation items from tool results and inject their ids."""
+from __future__ import annotations
+import json
+import re
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+from core.llm.citations.allocator import AnchorAllocator
+from core.llm.citations import anchor_state
+from core.llm.citations import types
+
+def first_str(item: Dict[str, Any], keys: Sequence[str]) -> str:
+ for key in keys:
+ val = item.get(key)
+ if isinstance(val, (str, int, float)) and str(val).strip():
+ return str(val)
+ return ""
+
+
+def normalize_citation_url(raw: str) -> str:
+ """把工具结果里的来源链接规整成「浏览器点了不会跑偏」的绝对地址。
+
+ 检索工具返回的 url 千奇百怪,直接透传会有两类坏结果:
+
+ - **缺协议**(``www.example.gov.cn/a.html``):前端 ```` 会把它当相对
+ 路径,按当前站点 origin 解析,于是点开变成 ``http://<本平台>/www.example…``,
+ 落到我们自己的 nginx 上返回 404 —— 看起来像「引用地址错误」。
+ - **纯相对路径**(``/module/download/x.doc``):没有站点根就无从还原,同样会
+ 指到本平台。这种宁可不给链接,也好过给一个必然打不开的。
+
+ 另外只放行 http/https:javascript:、data: 之类既没有意义也不该出现在来源里。
+ """
+ url = (raw or "").strip()
+ if not url:
+ return ""
+ lowered = url.lower()
+ if lowered.startswith(("http://", "https://")):
+ return url
+ # 协议相对://example.com/a → https://example.com/a
+ if url.startswith("//"):
+ return f"https:{url}"
+ # 明确的其它协议(javascript:/data:/ftp: …)一律丢弃
+ if re.match(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:", url):
+ return ""
+ # 相对路径:没有站点根,还原不出来
+ if url.startswith("/"):
+ return ""
+ # 剩下的形如 example.gov.cn/path —— 看着像域名就补上协议
+ host = url.split("/", 1)[0]
+ if "." in host and " " not in host:
+ return f"https://{url}"
+ return ""
+
+
+def dig(data: Dict[str, Any], path: Sequence[str]) -> Any:
+ cur: Any = data
+ for key in path:
+ if not isinstance(cur, dict):
+ return None
+ cur = cur.get(key)
+ return cur
+
+
+def display_title(tool_name: str, spec: Optional[types.CitationSpec]) -> str:
+ if spec and spec.title:
+ return spec.title
+ return f"{tool_name} 结果"
+
+
+def item_citation(
+ aid: str,
+ tool_name: str,
+ tool_id: Optional[str],
+ spec: types.CitationSpec,
+ item: Dict[str, Any],
+ index: int,
+) -> types.AnchorCitation:
+ title = first_str(item, spec.title_keys)[:anchor_state.TITLE_MAX] or display_title(tool_name, spec)
+ if spec.snippet_join:
+ parts = [first_str(item, (k,)) for k in spec.snippet_join]
+ snippet = " · ".join(p for p in parts if p)
+ else:
+ snippet = first_str(item, spec.snippet_keys)
+ return types.AnchorCitation(
+ id=aid,
+ tool_name=tool_name,
+ tool_id=tool_id,
+ title=title,
+ url=normalize_citation_url(first_str(item, spec.url_keys)),
+ snippet=snippet[:anchor_state.SNIPPET_MAX],
+ source_type=spec.source_type,
+ item_index=index,
+ )
+
+
+def looks_like_error(data: Dict[str, Any]) -> bool:
+ if not data.get("error"):
+ return False
+ # 列表字段仍有内容时不算整体失败(部分工具 error+items 并存)
+ for key in anchor_state.LIST_FIELD_ALIASES:
+ val = data.get(key)
+ if isinstance(val, list) and val:
+ return False
+ return True
+
+
+def heuristic_list_field(data: Dict[str, Any]) -> Optional[Tuple[Tuple[str, ...], List[Any]]]:
+ """启发式找列表字段:别名优先;否则要求「唯一」的字典数组字段。"""
+ scopes: List[Tuple[Tuple[str, ...], Dict[str, Any]]] = [((), data)]
+ inner = data.get("result")
+ if isinstance(inner, dict):
+ scopes.append((("result",), inner))
+ for prefix, scope in scopes:
+ for alias in anchor_state.LIST_FIELD_ALIASES:
+ val = scope.get(alias)
+ if isinstance(val, list) and val and all(isinstance(x, dict) for x in val):
+ return prefix + (alias,), val
+ for prefix, scope in scopes:
+ candidates = [
+ (key, val)
+ for key, val in scope.items()
+ if isinstance(val, list) and val and all(isinstance(x, dict) for x in val)
+ ]
+ if len(candidates) == 1:
+ key, val = candidates[0]
+ return prefix + (key,), val
+ return None
+
+
+def extract_and_inject(
+ tool_name: str,
+ tool_id: Optional[str],
+ data: Dict[str, Any],
+ allocator: AnchorAllocator,
+) -> List[types.AnchorCitation]:
+ """在 data 上就地注入 cite_id,返回登记的引用条目。"""
+ spec = anchor_state.TOOL_SPECS.get(tool_name)
+
+ # L1:工具自声明 __citations__(优先级最高,条目顺序即 item_index)
+ declared = data.get("__citations__")
+ if isinstance(declared, list) and declared and all(isinstance(x, dict) for x in declared):
+ base = spec or types.CitationSpec(source_type="unknown")
+ out: List[types.AnchorCitation] = []
+ for idx, entry in enumerate(declared):
+ aid = allocator.next_id()
+ entry["cite_id"] = aid
+ out.append(
+ types.AnchorCitation(
+ id=aid,
+ tool_name=tool_name,
+ tool_id=tool_id,
+ title=str(entry.get("title") or display_title(tool_name, spec))[:anchor_state.TITLE_MAX],
+ url=normalize_citation_url(str(entry.get("url") or "")),
+ snippet=str(entry.get("snippet") or "")[:anchor_state.SNIPPET_MAX],
+ source_type=str(entry.get("source_type") or base.source_type),
+ item_index=idx,
+ )
+ )
+ return out
+
+ if looks_like_error(data):
+ return []
+
+ # L2:配置映射(列表型)
+ if spec and spec.mode == "list":
+ out = []
+ for path in spec.items_paths:
+ items = dig(data, path)
+ if not isinstance(items, list):
+ continue
+ for item in items:
+ if not isinstance(item, dict) or item.get("error"):
+ continue
+ aid = allocator.next_id()
+ item["cite_id"] = aid
+ out.append(item_citation(aid, tool_name, tool_id, spec, item, len(out)))
+ return out
+
+ # L3:无 spec → 启发式找唯一字典数组
+ if spec is None:
+ found = heuristic_list_field(data)
+ if found is not None:
+ _path, items = found
+ generic = types.CitationSpec(mode="list", source_type="unknown")
+ out = []
+ for item in items:
+ if item.get("error"):
+ continue
+ aid = allocator.next_id()
+ item["cite_id"] = aid
+ out.append(item_citation(aid, tool_name, tool_id, generic, item, len(out)))
+ if out:
+ return out
+
+ # L4:整份 1 条(显式 whole spec 或彻底认不出)
+ aid = allocator.next_id()
+ data["cite_id"] = aid
+ snippet = json.dumps(data, ensure_ascii=False)[:anchor_state.WHOLE_SNIPPET_MAX]
+ return [
+ types.AnchorCitation(
+ id=aid,
+ tool_name=tool_name,
+ tool_id=tool_id,
+ title=display_title(tool_name, spec),
+ url=normalize_citation_url(first_str(data, (spec or types.CitationSpec()).url_keys)),
+ snippet=snippet,
+ source_type=(spec.source_type if spec else "unknown"),
+ item_index=-1,
+ )
+ ]
diff --git a/src/backend/core/llm/citations/hook.py b/src/backend/core/llm/citations/hook.py
new file mode 100644
index 00000000..2dc9133e
--- /dev/null
+++ b/src/backend/core/llm/citations/hook.py
@@ -0,0 +1,38 @@
+"""Annotate a buffered neutral tool result before model and SSE exposure."""
+from collections.abc import Mapping
+from core.harness.events import thaw_value
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage
+from core.llm.citations.allocator import CITATION_ALLOCATOR
+from core.llm.citations.anchor import annotate_tool_result
+from core.llm.citations.anchor_state import SKIP_TOOLS
+
+
+def citation_candidate(name):
+ return bool(name and name not in SKIP_TOOLS)
+
+
+def annotate_result(invocation):
+ if not citation_candidate(invocation.operation_name):
+ return Decision()
+ result = thaw_value(invocation.data.get("tool_result"))
+ if not result or result.get("state") != "success":
+ return Decision()
+ blocks = result.get("content") or []
+ if not blocks or any(not isinstance(block, Mapping) or block.get("type") != "text" for block in blocks):
+ return Decision()
+ allocator = CITATION_ALLOCATOR.get()
+ if allocator is None:
+ return Decision()
+ tool_id = str(invocation.data.get("metadata", {}).get("tool_call_id") or "")
+ text = "".join(block.get("text") or "" for block in blocks)
+ annotated, items = annotate_tool_result(invocation.operation_name, tool_id, text, allocator)
+ if not items:
+ return Decision()
+ allocator.register(tool_id, items)
+ result["content"] = [{"type": "text", "text": annotated}]
+ return Decision.modify({"tool_result": result})
+
+
+def citation_hook():
+ return HookSpec("citation_anchor", HookStage.AFTER_TOOL, annotate_result, priority=200,
+ failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"tool_result"}))
diff --git a/src/backend/core/llm/citations/types.py b/src/backend/core/llm/citations/types.py
new file mode 100644
index 00000000..b1809f0d
--- /dev/null
+++ b/src/backend/core/llm/citations/types.py
@@ -0,0 +1,63 @@
+"""Citation item and per-tool extraction spec types."""
+from __future__ import annotations
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, Optional, Tuple
+
+@dataclass
+class AnchorCitation:
+ """带全局锚点的引用条目(CitationItem 的超集,dict 形态向后兼容)。"""
+
+ id: str # 全局锚点,如 "e7"
+ tool_name: str
+ tool_id: Optional[str]
+ title: str
+ url: str
+ snippet: str
+ source_type: str
+ item_index: int = -1 # 条目在该次工具结果列表中的 0-based 下标;整体型为 -1
+
+ def to_dict(self) -> Dict[str, Any]:
+ return asdict(self)
+
+
+DEFAULT_TITLE_KEYS: Tuple[str, ...] = (
+ "title",
+ "标题",
+ "文件名称",
+ "企业名称",
+ "产品名称",
+ "name",
+ "名称",
+ "document_name",
+)
+
+
+DEFAULT_URL_KEYS: Tuple[str, ...] = ("url", "链接", "link", "href")
+
+
+DEFAULT_SNIPPET_KEYS: Tuple[str, ...] = (
+ "content",
+ "文件内容",
+ "snippet",
+ "摘要",
+ "summary",
+ "description",
+ "abstract",
+ "text",
+)
+
+
+@dataclass(frozen=True)
+class CitationSpec:
+ """单个工具的引用提取规格。mode: list(逐条)/ whole(整份 1 条)。"""
+
+ mode: str = "whole"
+ # 列表字段路径(支持多层如 ("result", "results");支持并列多列表)
+ items_paths: Tuple[Tuple[str, ...], ...] = ()
+ title_keys: Tuple[str, ...] = DEFAULT_TITLE_KEYS
+ url_keys: Tuple[str, ...] = DEFAULT_URL_KEYS
+ snippet_keys: Tuple[str, ...] = DEFAULT_SNIPPET_KEYS
+ # 摘要拼接键(如企业搜索的 法定代表人·注册资金·企业状态);命中则优先于 snippet_keys
+ snippet_join: Tuple[str, ...] = ()
+ source_type: str = "unknown"
+ title: str = "" # whole 模式的显示标题;空则用工具名
diff --git a/src/backend/core/llm/command_middleware.py b/src/backend/core/llm/command_middleware.py
deleted file mode 100644
index 48b55717..00000000
--- a/src/backend/core/llm/command_middleware.py
+++ /dev/null
@@ -1,39 +0,0 @@
-"""Insert terminal command facts only between complete ReAct steps."""
-import asyncio
-from agentscope.middleware import MiddlewareBase
-from core.llm.context_adapter import append_context_text
-from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER
-from core.services.commands import delivery
-from core.services.commands import store
-
-class CommandCompletionMiddleware(MiddlewareBase):
- async def on_reasoning(self, agent, input_kwargs, next_handler):
- state = agent.state
- run_id = str(state.run_id or "")
- rows = []
- if (run_id and state.chat_id and state.user_id
- and await asyncio.to_thread(store.has_pending, state.chat_id, state.user_id)):
- rows = await asyncio.to_thread(delivery.reserve_boundary,
- state.chat_id, state.user_id, run_id)
- if rows:
- append_context_text(agent, delivery.notice_text(rows), kind=KIND_REMINDER,
- origin="harness:command_completion", trust="system", priority=950,
- token_budget=4000, truncation_policy=POLICY_NEVER)
- async for item in next_handler(**input_kwargs):
- yield item
-
- async def on_reply(self, agent, input_kwargs, next_handler):
- if not (agent.state.run_id and agent.state.chat_id and agent.state.user_id):
- async for item in next_handler(**input_kwargs):
- yield item
- return
- current = input_kwargs
- while True:
- async for item in next_handler(**current):
- yield item
- # Close the final-response race: completions arriving during the last
- # output receive a new reasoning step in the same active run.
- rows = await asyncio.to_thread(store.pending_notices, agent.state.chat_id)
- if not any(r["notice_status"] == "pending" and r["user_id"] == agent.state.user_id for r in rows):
- return
- current = {**input_kwargs, "inputs": None}
diff --git a/src/backend/core/llm/context/__init__.py b/src/backend/core/llm/context/__init__.py
new file mode 100644
index 00000000..454203f7
--- /dev/null
+++ b/src/backend/core/llm/context/__init__.py
@@ -0,0 +1 @@
+"""Framework-neutral context IR: items, assembly and conversion to provider messages."""
diff --git a/src/backend/core/llm/context/adapter.py b/src/backend/core/llm/context/adapter.py
new file mode 100644
index 00000000..a5da94ad
--- /dev/null
+++ b/src/backend/core/llm/context/adapter.py
@@ -0,0 +1,362 @@
+"""Translate between the context IR and AgentScope messages."""
+from __future__ import annotations
+from core.llm.context import provider_conversion
+from core.llm.context import reference_conversion
+from core.llm.context import message_defaults
+from dataclasses import replace
+from typing import Any, Iterable, Mapping, Optional, Sequence
+from agentscope.message import Msg, TextBlock
+from core.infra.immutable import thaw_json
+from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, KIND_COMPACTION, KIND_USER_INPUT, POLICY_DROP, POLICY_HEAD_TAIL, POLICY_NEVER, SESSION_CONTEXT_META_KEY, STEP_KINDS, VISIBILITY_MODEL
+from core.llm.context.items import ContextItem
+
+
+def _block_payload(block: Any) -> Any:
+ if isinstance(block, Mapping):
+ return dict(block)
+ if hasattr(block, "model_dump"):
+ return block.model_dump(mode="json")
+ return block
+
+
+def _block_type(block: Any) -> str:
+ if isinstance(block, Mapping):
+ return str(block.get("type") or "")
+ return str(getattr(block, "type", "") or "")
+
+
+def _block_text(block: Any) -> str:
+ if isinstance(block, Mapping):
+ return str(block.get("text") or "")
+ return str(getattr(block, "text", "") or "")
+
+
+CONTEXT_META_KEY = "harness_context_items"
+
+
+class AgentScopeContextAdapter:
+ """Translate the IR while preserving current model-facing roles/blocks."""
+
+ @staticmethod
+ def wrap_content(content: Any) -> list[Any]:
+ """Construct AgentScope-compatible blocks inside the sole SDK seam."""
+ if isinstance(content, str):
+ return [TextBlock(type="text", text=content)] if content else []
+ if isinstance(content, list):
+ return content
+ return [content] if content else []
+
+ @staticmethod
+ def _defaults(role: str, block_type: str) -> dict[str, Any]:
+ return message_defaults.defaults(role, block_type)
+
+ @staticmethod
+ def _metadata_entries(message: Msg) -> list[Mapping[str, Any]]:
+ metadata = getattr(message, "metadata", None) or {}
+ raw = metadata.get(CONTEXT_META_KEY) if isinstance(metadata, Mapping) else None
+ if not isinstance(raw, list):
+ return []
+ return [entry for entry in raw if isinstance(entry, Mapping)]
+
+ def items_from_messages(
+ self,
+ messages: Sequence[Msg],
+ *,
+ summary_text: Any = None,
+ promote_latest_user: bool = True,
+ ) -> list[ContextItem]:
+ items: list[ContextItem] = []
+ summary = str(summary_text or "")
+ tool_occurrences: dict[str, int] = {}
+ pending_tool_pairs: dict[str, list[str]] = {}
+ orphan_results: dict[str, int] = {}
+ # One ReAct step = what the model produced in one response plus the
+ # results of the tool calls in it. A step opens on the first thinking,
+ # text or tool_call block after a tool result (or after any other
+ # message) and its results join it through the pending call ids. This
+ # is the same boundary the reasoning-echo formatter flushes on.
+ step_unit: Optional[str] = None
+ step_closed = False
+ pair_units: dict[str, str] = {}
+ for message_index, message in enumerate(messages):
+ role = str(getattr(message, "role", "user") or "user")
+ name = str(getattr(message, "name", role) or role)
+ blocks = list(getattr(message, "content", None) or [])
+ entries = self._metadata_entries(message)
+ for block_index, block in enumerate(blocks):
+ block_type = _block_type(block)
+ payload = _block_text(block) if block_type == "text" else _block_payload(block)
+ defaults = self._defaults(role, block_type)
+ explicit = entries[block_index] if block_index < len(entries) else {}
+ if (
+ not explicit
+ and summary
+ and role == "user"
+ and isinstance(payload, str)
+ and payload == summary
+ ):
+ defaults.update(
+ kind=KIND_COMPACTION,
+ origin="harness:compaction",
+ trust="system",
+ priority=850,
+ truncation_policy=POLICY_HEAD_TAIL,
+ render_role="user",
+ )
+ raw_pair_id = str(
+ payload.get("id")
+ if block_type in {"tool_call", "tool_result"} and isinstance(payload, Mapping)
+ else ""
+ )
+ pair_id = str(explicit.get("pair_id") or "")
+ if block_type == "tool_call" and raw_pair_id:
+ occurrence = tool_occurrences.get(raw_pair_id, 0) + 1
+ tool_occurrences[raw_pair_id] = occurrence
+ pair_id = f"{raw_pair_id}#{occurrence}"
+ pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id)
+ elif block_type == "tool_result" and raw_pair_id:
+ pending = pending_tool_pairs.get(raw_pair_id) or []
+ if pending:
+ pair_id = pending.pop(0)
+ else:
+ occurrence = orphan_results.get(raw_pair_id, 0) + 1
+ orphan_results[raw_pair_id] = occurrence
+ pair_id = f"{raw_pair_id}#orphan-result-{occurrence}"
+ item_id = str(
+ (
+ f"message:{message_index}:block:{block_index}:{block_type}:{pair_id}"
+ if block_type in {"tool_call", "tool_result"}
+ else explicit.get("item_id")
+ or f"message:{message_index}:block:{block_index}"
+ )
+ )
+ kind = str(explicit.get("kind") or defaults["kind"])
+ if kind in STEP_KINDS and block_type != "tool_result":
+ if step_unit is None or step_closed:
+ step_unit = f"step:{message_index}:{block_index}"
+ step_closed = False
+ unit_id = step_unit
+ if block_type == "tool_call":
+ pair_units[pair_id] = unit_id
+ elif block_type == "tool_result":
+ unit_id = pair_units.pop(pair_id, f"orphan:{item_id}")
+ step_closed = True
+ else:
+ unit_id = f"item:{item_id}"
+ step_unit = None
+ step_closed = False
+ pair_units.clear()
+ created_seq = int(
+ explicit.get("created_seq")
+ if explicit.get("created_seq") is not None
+ else message_index * CONTEXT_SEQUENCE_STRIDE + block_index
+ )
+ item = ContextItem.create(
+ item_id=item_id,
+ kind=kind,
+ origin=str(explicit.get("origin") or defaults["origin"]),
+ trust=str(explicit.get("trust") or defaults["trust"]),
+ visibility=str(explicit.get("visibility") or VISIBILITY_MODEL),
+ priority=int(
+ explicit["priority"]
+ if explicit.get("priority") is not None
+ else defaults["priority"]
+ ),
+ token_budget=int(
+ explicit["token_budget"]
+ if explicit.get("token_budget") is not None
+ else defaults["token_budget"]
+ ),
+ truncation_policy=str(
+ explicit.get("truncation_policy") or defaults["truncation_policy"]
+ ),
+ content=payload,
+ cache_class=str(explicit.get("cache_class") or "dynamic"),
+ created_seq=created_seq,
+ render_role=str(explicit.get("render_role") or defaults["render_role"]),
+ # The outer message carries the actual transport name.
+ # Persisted manifests contain only its hash.
+ render_name=name,
+ pair_id=pair_id,
+ message_group=str(explicit.get("message_group") or f"message:{message_index}"),
+ unit_id=unit_id,
+ content_ref=(
+ str(explicit["content_ref"]) if explicit.get("content_ref") else None
+ ),
+ content_hash=(
+ str(explicit["content_hash"]) if explicit.get("content_hash") else None
+ ),
+ token_estimate=(
+ int(explicit["token_estimate"])
+ if explicit.get("token_estimate") is not None
+ else None
+ ),
+ metadata={},
+ )
+ items.append(item)
+
+ # The latest real user instruction is a hard inclusion. Memory and
+ # reminders may render as user for compatibility but keep distinct kind.
+ user_indices = [i for i, item in enumerate(items) if item.kind == KIND_USER_INPUT]
+ if promote_latest_user and user_indices:
+ idx = user_indices[-1]
+ items[idx] = replace(
+ items[idx],
+ priority=max(900, items[idx].priority),
+ truncation_policy=POLICY_NEVER,
+ )
+ return items
+
+ def items_from_session_dict(
+ self,
+ message: Mapping[str, Any],
+ *,
+ created_seq: int,
+ ) -> list[ContextItem]:
+ role = str(message.get("role") or "user")
+ role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role)
+ content = message.get("content")
+ blocks = (
+ content
+ if isinstance(content, list)
+ else ([{"type": "text", "text": content}] if content else [])
+ )
+ explicit = message.get(SESSION_CONTEXT_META_KEY)
+ metadata_entries = []
+ if isinstance(explicit, Mapping):
+ metadata_entries = [dict(explicit) for _ in blocks]
+ msg = Msg(
+ name=str(message.get("name") or role),
+ role=role if role in {"system", "assistant", "user"} else "user",
+ content=blocks,
+ metadata={CONTEXT_META_KEY: metadata_entries} if metadata_entries else {},
+ )
+ # A persisted row is converted in isolation, so it cannot know whether
+ # it is the current user instruction. Promotion happens once, later,
+ # when ManifestBoundAgent sees the complete final request.
+ items = self.items_from_messages([msg], promote_latest_user=False)
+ has_explicit_seq = isinstance(explicit, Mapping) and explicit.get("created_seq") is not None
+ return [
+ replace(
+ item,
+ item_id=(
+ item.item_id
+ if isinstance(explicit, Mapping) and explicit.get("item_id")
+ else f"session:{created_seq}:block:{index}"
+ ),
+ created_seq=(
+ item.created_seq
+ if has_explicit_seq
+ else created_seq * CONTEXT_SEQUENCE_STRIDE + index
+ ),
+ message_group=f"session:{created_seq}",
+ unit_id=f"session:{created_seq}:{item.unit_id}",
+ )
+ for index, item in enumerate(items)
+ ]
+
+ def message_from_session_dict(
+ self,
+ message: Mapping[str, Any],
+ *,
+ created_seq: int,
+ ) -> Msg:
+ """Render one persisted history record through the canonical adapter."""
+ items = self.items_from_session_dict(message, created_seq=created_seq)
+ rendered = self.messages_from_items(items)
+ if rendered:
+ return rendered[0]
+ role = str(message.get("role") or "user")
+ role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role)
+ if role not in {"system", "assistant", "user"}:
+ role = "user"
+ return Msg(
+ name=str(message.get("name") or role),
+ role=role,
+ content=[],
+ metadata={CONTEXT_META_KEY: []},
+ )
+
+ def messages_from_items(self, items: Iterable[ContextItem]) -> list[Msg]:
+ visible = [item for item in items if item.visibility == VISIBILITY_MODEL]
+ messages: list[Msg] = []
+ current_group: Optional[tuple[str, str]] = None
+ current_items: list[ContextItem] = []
+
+ def flush() -> None:
+ nonlocal current_group, current_items
+ if not current_items or current_group is None:
+ return
+ blocks = []
+ manifests = []
+ for item in current_items:
+ content = thaw_json(item.content)
+ if isinstance(content, str):
+ blocks.append(TextBlock(type="text", text=content))
+ manifests.append(item.to_manifest())
+ elif isinstance(content, Mapping):
+ blocks.append(dict(content))
+ manifests.append(item.to_manifest())
+ elif isinstance(content, list):
+ for block_index, block in enumerate(content):
+ blocks.append(block)
+ normalized = (
+ _block_text(block)
+ if _block_type(block) == "text"
+ else _block_payload(block)
+ )
+ derived = replace(
+ item,
+ item_id=f"{item.item_id}:block:{block_index}",
+ truncation_policy=(
+ POLICY_DROP
+ if _block_type(block) == "data"
+ else item.truncation_policy
+ ),
+ ).with_content(normalized)
+ manifests.append(derived.to_manifest())
+ elif content is not None:
+ blocks.append(TextBlock(type="text", text=str(content)))
+ manifests.append(item.with_content(str(content)).to_manifest())
+ if blocks:
+ name = str(current_items[0].render_name or current_group[1])
+ messages.append(
+ Msg(
+ name=name,
+ role=current_group[1],
+ content=blocks,
+ metadata={CONTEXT_META_KEY: manifests},
+ )
+ )
+ current_group = None
+ current_items = []
+
+ for item in visible:
+ group = (item.message_group, item.render_role)
+ if current_group is not None and group != current_group:
+ flush()
+ current_group = group
+ current_items.append(item)
+ flush()
+ return messages
+
+ def annotate_provider_messages(
+ self,
+ source_messages: Sequence[Msg],
+ formatted_messages: Sequence[Mapping[str, Any]],
+ row_counts: Sequence[int],
+ ) -> list[dict[str, Any]]:
+ return provider_conversion.annotate_provider_messages(self, source_messages, formatted_messages, row_counts)
+
+ def items_from_provider_messages(
+ self,
+ messages: Sequence[Mapping[str, Any]],
+ ) -> list[ContextItem]:
+ return provider_conversion.items_from_provider_messages(self, messages)
+
+ @staticmethod
+ def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]:
+ return provider_conversion.provider_messages_from_items(items)
+
+ def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]:
+ return reference_conversion.reference_items_from_execution_manifest(self, manifest)
diff --git a/src/backend/core/llm/context/attachment_hook.py b/src/backend/core/llm/context/attachment_hook.py
new file mode 100644
index 00000000..b9c71d19
--- /dev/null
+++ b/src/backend/core/llm/context/attachment_hook.py
@@ -0,0 +1,81 @@
+"""Attachment context policy using immutable hook inputs and neutral content."""
+import asyncio
+from core.files import artifact_bytes
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage, Invocation
+from core.llm.runtime import file_context
+from core.models.vision import resolve_vision_mode
+
+
+def supports_native_images(provider_id: str) -> bool:
+ return resolve_vision_mode(provider_id or "") == "native"
+
+
+def native_image_content(image_files: list, user_id):
+ blocks, names = [], []
+ for file in image_files:
+ result = file_context.fetch_image_base64(file, user_id=user_id)
+ if result:
+ encoded, mime = result
+ blocks.append({"type": "data", "source": {
+ "type": "base64", "media_type": mime, "data": encoded,
+ }})
+ names.append(file.get("name", "图片"))
+ if not blocks:
+ return None
+ prefix = {"type": "text", "text": f"[用户上传了 {len(blocks)} 张图片:{'、'.join(names)}]"}
+ return [prefix, *blocks]
+
+
+async def vision_evidence(image_files: list, user_id, precomputed: str = "") -> str:
+ if precomputed.strip():
+ return precomputed.strip()
+ from core.models.vision.attachments import transcribe_attachments
+ result = await transcribe_attachments(image_files, user_id=user_id,
+ download_bytes=artifact_bytes.download_artifact_bytes)
+ return result.text if result is not None and result.text else ""
+
+
+def payload(content, *, kind, origin, trust, token_budget=8000):
+ return dict(content=content, kind=kind, origin=origin, trust=trust,
+ priority=650, token_budget=token_budget)
+
+
+class AttachmentContextHook:
+ def __init__(self):
+ self.injected = False
+
+ async def __call__(self, invocation: Invocation) -> Decision:
+ if invocation.operation_name != "agent_reply":
+ return Decision()
+ runtime = invocation.data.get("runtime", {})
+ uploaded = list(runtime.get("uploaded_files") or [])
+ historical = list(runtime.get("historical_files") or [])
+ if self.injected or not (uploaded or historical):
+ return Decision()
+ self.injected = True
+ user_id = runtime.get("user_id") or None
+ items = list(invocation.data.get("context_items") or [])
+ if historical:
+ text = file_context.build_historical_files_context(historical)
+ if text:
+ items.append(payload(text, kind="project_material", origin="workspace:historical_files", trust="workspace"))
+ text = await asyncio.to_thread(file_context.build_file_context, uploaded, user_id=user_id) if uploaded else ""
+ if text:
+ items.append(payload(text, kind="attachment", origin="user:uploaded_files", trust="user"))
+ images = [file for file in uploaded if artifact_bytes.is_image(file)]
+ if images:
+ if supports_native_images(str(runtime.get("model_provider_id") or "")):
+ content = await asyncio.to_thread(native_image_content, images, user_id)
+ if content:
+ items.append(payload(content, kind="attachment", origin="user:uploaded_images", trust="user", token_budget=16000))
+ else:
+ text = await vision_evidence(images, user_id, str(runtime.get("vision_evidence_text") or ""))
+ if text:
+ items.append(payload(text, kind="attachment", origin="vision:transcription", trust="tool"))
+ return Decision.modify({"context_items": items})
+
+
+def attachment_context_hook() -> HookSpec:
+ return HookSpec("attachment_context", HookStage.TRANSFORM_CONTEXT, AttachmentContextHook(),
+ priority=100, timeout_ms=180000, failure_mode=FailureMode.OPEN,
+ mutable_fields=frozenset({"context_items"}))
diff --git a/src/backend/core/services/compaction_budget.py b/src/backend/core/llm/context/budget.py
similarity index 98%
rename from src/backend/core/services/compaction_budget.py
rename to src/backend/core/llm/context/budget.py
index 9fb16d3e..4ddf2687 100644
--- a/src/backend/core/services/compaction_budget.py
+++ b/src/backend/core/llm/context/budget.py
@@ -5,7 +5,7 @@
import json
from typing import Any, Dict, List, Optional
-from core.llm import compaction as C
+from core.llm.context import compaction as C
def estimate_history_tokens(history: List[Dict[str, Any]]) -> int:
diff --git a/src/backend/core/llm/compacting_agent.py b/src/backend/core/llm/context/compacting_agent.py
similarity index 96%
rename from src/backend/core/llm/compacting_agent.py
rename to src/backend/core/llm/context/compacting_agent.py
index 0822defb..679e2748 100644
--- a/src/backend/core/llm/compacting_agent.py
+++ b/src/backend/core/llm/context/compacting_agent.py
@@ -15,9 +15,9 @@
from agentscope.message import DataBlock, TextBlock, ToolResultBlock
from agentscope.model import ChatUsage
from agentscope.workspace import Offloader
-from core.llm.context_safety import enforce_context_budget
-from core.llm.execution_manifest import stable_hash
-from core.llm.manifest_agent import ManifestBoundAgent
+from core.llm.context.safety import enforce_context_budget
+from core.harness.execution_manifest import stable_hash
+from core.llm.runtime.manifest_agent import ManifestBoundAgent
logger = logging.getLogger(__name__)
@@ -247,7 +247,7 @@ async def _measure_context_tokens(self) -> int:
trailing = None
self._jx_observation = None
if trailing is not None:
- from core.services.compaction_budget import estimate_history_tokens
+ from core.llm.context.budget import estimate_history_tokens
return obs.tokens + (estimate_history_tokens(trailing) if trailing else 0)
return self._estimate_prompt_tokens()
@@ -260,7 +260,7 @@ def _estimate_prompt_tokens(self) -> int:
units as the observed one above — both reserve media instead of billing
its transport bytes.
"""
- from core.services.compaction_budget import estimate_context_budget
+ from core.llm.context.budget import estimate_context_budget
system_prompt, tool_schema = frozen_prompt_surface(self)
return estimate_context_budget(
@@ -271,7 +271,7 @@ def _estimate_prompt_tokens(self) -> int:
# ── MidTurn compaction ───────────────────────────────────────────────────
- async def compress_context(self, context_config: ContextConfig | None = None) -> None:
+ async def compress_context(self, context_config: ContextConfig | None = None, instructions=None) -> None:
"""Compact at a ReAct step boundary without replacing failed work with a summary."""
del context_config
await enforce_context_budget(self)
@@ -317,8 +317,8 @@ async def compress_context(self, context_config: ContextConfig | None = None) ->
history = self._history_for_summary()
try:
- from core.llm.compaction import CompactionPhase
- from core.services.compaction_service import run_compaction, run_mid_turn_compaction
+ from core.llm.context.compaction import CompactionPhase
+ from core.harness.registry.compaction import run_compaction, run_mid_turn_compaction
if chat_id:
system_prompt, tool_schema = frozen_prompt_surface(self)
@@ -365,7 +365,7 @@ async def _context_offload_notice(self, dropped: list[Any]) -> str:
async def _should_compact(self) -> tuple[Optional[int], int]:
"""Return ``(limit, measured_tokens)``; ``limit`` is None when undeterminable."""
- from core.services.compaction_service import resolve_token_limit
+ from core.harness.registry.compaction import resolve_token_limit
window = int(getattr(self.model, "context_size", 0) or 0)
limit = resolve_token_limit(window, ratio=self._jx_trigger_ratio)
@@ -389,7 +389,7 @@ async def _apply_replacement(self, replacement: List[Dict[str, Any]]) -> None:
session that owns that path need not outlive this turn, and a checkpoint
that points at a vanished file is worse than one that never mentions it.
"""
- from core.llm.message_compat import session_to_msgs
+ from core.llm.runtime.message_compat import session_to_msgs
dropped = list(self.state.context)
replacement = [dict(m) for m in replacement]
diff --git a/src/backend/core/llm/compaction.py b/src/backend/core/llm/context/compaction.py
similarity index 98%
rename from src/backend/core/llm/compaction.py
rename to src/backend/core/llm/context/compaction.py
index 8f3dd6c5..7fb835b2 100644
--- a/src/backend/core/llm/compaction.py
+++ b/src/backend/core/llm/context/compaction.py
@@ -6,13 +6,10 @@
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple
-from core.llm.context_ir import (
- KIND_COMPACTION,
- SESSION_CONTEXT_META_KEY,
- make_text_context_item,
- image_token_reserve,
-)
-from core.llm.model_steps import explode_history_rows
+from core.llm.context.ir_constants import KIND_COMPACTION, SESSION_CONTEXT_META_KEY
+from core.llm.context.items import make_text_context_item
+from core.llm.context.content import image_token_reserve
+from core.llm.runtime.model_steps import explode_history_rows
# ── Constants ────────────────────────────────────────────────────────────────
diff --git a/src/backend/core/llm/context/content.py b/src/backend/core/llm/context/content.py
new file mode 100644
index 00000000..e3ea04a3
--- /dev/null
+++ b/src/backend/core/llm/context/content.py
@@ -0,0 +1,212 @@
+"""Normalise, measure and truncate context content."""
+from __future__ import annotations
+from core.models.constants import IMAGE_TOKEN_RESERVE
+from typing import Any, Mapping
+from core.infra.immutable import thaw_json
+from core.harness.execution_manifest import canonical_json
+from core.llm.context import ir_constants
+
+def normalize_context_content(content: Any) -> Any:
+ """Convert SDK block objects into stable, framework-neutral JSON values."""
+ if hasattr(content, "model_dump"):
+ return normalize_context_content(content.model_dump(mode="json"))
+ if isinstance(content, Mapping):
+ return {
+ str(key): normalize_context_content(value)
+ for key, value in content.items()
+ }
+ if isinstance(content, (list, tuple)):
+ return [normalize_context_content(value) for value in content]
+ return content
+
+
+def tool_result_images(content: Any) -> list[Any]:
+ """Image payloads are atomic media, not part of a tool's text allowance."""
+ if not isinstance(content, Mapping) or content.get("type") != "tool_result":
+ return []
+ output = content.get("output")
+ if not isinstance(output, (list, tuple)):
+ return []
+ return [
+ block
+ for block in output
+ if isinstance(block, Mapping)
+ and block.get("type") == "data"
+ and isinstance(block.get("source"), Mapping)
+ and str(block["source"].get("media_type") or "").startswith("image/")
+ ]
+
+
+def image_token_reserve(content: Any) -> int:
+ """Reserve media occupancy without measuring image transport bytes."""
+ if isinstance(content, (list, tuple)):
+ return sum(image_token_reserve(block) for block in content)
+ if isinstance(content, Mapping):
+ source = content.get("source")
+ if (
+ content.get("type") == "data"
+ and isinstance(source, Mapping)
+ and str(source.get("media_type") or "").startswith("image/")
+ ):
+ return IMAGE_TOKEN_RESERVE
+ if content.get("type") == "tool_result":
+ return image_token_reserve(content.get("output", content.get("content")))
+ return 0
+
+
+def estimate_context_tokens(content: Any) -> int:
+ """Deterministic conservative estimate without provider-specific imports."""
+ content = normalize_context_content(content)
+ if content is None:
+ return 0
+ if isinstance(content, str):
+ return max(1, (len(content.encode("utf-8")) + 3) // 4) if content else 0
+ if isinstance(content, Mapping) and content.get("type") == "data":
+ # Provider image accounting is not proportional to base64 bytes. Keep a
+ # conservative fixed reserve without letting transport encoding evict
+ # the actual attachment before the provider can count it precisely.
+ return IMAGE_TOKEN_RESERVE
+ if tool_result_images(content):
+ # Count image reserves recursively instead of tokenizing their base64
+ # inside the enclosing tool-result JSON.
+ return estimate_context_tokens({**content, "output": []}) + sum(
+ estimate_context_tokens(block) for block in content["output"]
+ )
+ if isinstance(content, list):
+ return sum(estimate_context_tokens(item) for item in content)
+ return max(1, (len(canonical_json(content).encode("utf-8")) + 3) // 4)
+
+
+def truncate_text(text: str, max_tokens: int, *, tail_only: bool = False) -> str:
+ if not text or max_tokens <= 0:
+ return ""
+ max_bytes = max(4, max_tokens * 4)
+ if len(text.encode("utf-8")) <= max_bytes:
+ return text
+ # Character slicing is deterministic and safe for Unicode. Iterate down if
+ # multi-byte text still exceeds the byte reserve.
+ max_chars = max(1, max_bytes)
+ head = 0
+ tail = 0
+ marker = ""
+ if tail_only:
+ tail = max_chars
+ result = text[-tail:]
+ elif max_chars < 40:
+ head = max_chars
+ result = text[:head]
+ else:
+ marker = "\n[… omitted …]\n"
+ body = max(2, max_chars - len(marker))
+ head = max(1, int(body * 0.6))
+ tail = max(1, body - head)
+ result = text[:head] + marker + text[-tail:]
+ while len(result.encode("utf-8")) > max_bytes and len(result) > 1:
+ if tail_only:
+ tail = max(1, tail - 1)
+ result = text[-tail:]
+ elif marker:
+ # Preserve both semantic ends; shrink the larger slice first.
+ if head >= tail and head > 1:
+ head -= 1
+ elif tail > 1:
+ tail -= 1
+ elif head > 1:
+ head -= 1
+ else:
+ break
+ result = text[:head] + marker + text[-tail:]
+ else:
+ head = max(1, head - 1)
+ result = text[:head]
+ return result
+
+
+def truncate_content(content: Any, max_tokens: int, policy: str) -> Any:
+ """Shorten prunable text only; every structured block keeps its shape.
+
+ Plain text and a tool result's output are the only things a budget may
+ shorten. Any other block — thinking, tool call, data, or a shape this
+ module does not know — is returned untouched so the caller keeps or drops
+ it whole. Rewriting such a block into a JSON string would hand the model
+ its own reasoning back as an answer.
+ """
+ if max_tokens <= 0:
+ return ""
+ if isinstance(content, str):
+ return truncate_text(content, max_tokens, tail_only=policy == ir_constants.POLICY_TAIL)
+ if isinstance(content, Mapping):
+ mutable = thaw_json(content)
+ block_type = str(mutable.get("type") or "")
+ if block_type not in ir_constants.TRUNCATABLE_BLOCK_TYPES:
+ return mutable
+ for key in ("output", "content", "text"):
+ value = mutable.get(key)
+ if isinstance(value, str):
+ overhead = estimate_context_tokens({**mutable, key: ""})
+ mutable[key] = truncate_text(
+ value,
+ max(1, max_tokens - overhead),
+ tail_only=policy == ir_constants.POLICY_TAIL,
+ )
+ return mutable
+ if key == "output" and isinstance(value, (list, Mapping)):
+ images = tool_result_images(mutable)
+ if images:
+ text = "\n".join(
+ (
+ str(block["text"])
+ if isinstance(block, Mapping) and block.get("text") is not None
+ else block if isinstance(block, str) else canonical_json(block)
+ )
+ for block in value
+ if block not in images
+ )
+ output = [{"type": "text", "text": ""}, *images]
+ overhead = estimate_context_tokens({**mutable, key: output})
+ output[0]["text"] = truncate_text(
+ text, max(1, max_tokens - overhead), tail_only=policy == ir_constants.POLICY_TAIL
+ )
+ mutable[key] = output
+ return mutable
+ if isinstance(value, list):
+ parts = []
+ for block in value:
+ if isinstance(block, Mapping) and block.get("text") is not None:
+ parts.append(str(block["text"]))
+ elif isinstance(block, str):
+ parts.append(block)
+ else:
+ parts.append(canonical_json(block))
+ flattened = "\n".join(parts)
+ else:
+ flattened = canonical_json(value)
+ overhead = estimate_context_tokens({**mutable, key: ""})
+ mutable[key] = truncate_text(
+ flattened,
+ max(1, max_tokens - overhead),
+ tail_only=policy == ir_constants.POLICY_TAIL,
+ )
+ return mutable
+ return mutable
+ return thaw_json(content)
+
+
+def prune_tool_output(content: Any, original_tokens: int) -> Any:
+ """Replace a tool result's output with an explicit placeholder."""
+ mutable = thaw_json(content)
+ if not isinstance(mutable, dict) or str(mutable.get("type") or "") != "tool_result":
+ raise ValueError("only tool_result content can be pruned")
+ images = tool_result_images(mutable)
+ if images:
+ mutable["output"] = [
+ {"type": "text", "text": ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)},
+ *images,
+ ]
+ return mutable
+ for key in ("output", "content", "text"):
+ if key in mutable:
+ mutable[key] = ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)
+ return mutable
+ mutable["output"] = ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)
+ return mutable
diff --git a/src/backend/core/llm/context/ir.py b/src/backend/core/llm/context/ir.py
new file mode 100644
index 00000000..bb1f8a84
--- /dev/null
+++ b/src/backend/core/llm/context/ir.py
@@ -0,0 +1,327 @@
+"""Assemble context items into the window sent to the model."""
+from __future__ import annotations
+from dataclasses import replace
+from typing import Any, Iterable, Mapping, Optional, Sequence
+from core.harness.execution_manifest import stable_hash
+from core.llm.context import content
+from core.llm.context import ir_constants
+from core.llm.context.items import ContextAssembly, ContextItem
+
+class ContextAssembler:
+ """Explicit, unit-level selection in context order.
+
+ The unit of selection is a whole ReAct step — the reasoning, text and tool
+ calls the model produced together with the results those calls returned —
+ never a single block. A step is kept or dropped as one; the only thing a
+ budget may shorten inside it is a tool result's output.
+
+ Ordering is **not** the assembler's job: items arrive in the order the agent
+ context already holds them, and they leave in that same order. The live
+ user instruction carries a sequence 64 strides above the context tail (see
+ ``next_request_sequence``), so sorting by sequence would move it behind the
+ turn it had just started.
+
+ Budget pressure is resolved in a fixed order: per-item caps first, then
+ tool outputs of older steps are pruned (oldest first), and only then are
+ whole steps cut from the oldest end of the conversation. The steps of the
+ turn in progress — everything after the current user instruction — are
+ protected: they are never pruned, cut or split. When they alone exceed the
+ budget the assembly reports ``over_budget`` instead of quietly removing
+ anything; compaction owns that case.
+ """
+
+ def __init__(
+ self,
+ *,
+ total_budget: int,
+ budget_details: Optional[Mapping[str, int]] = None,
+ ) -> None:
+ self.total_budget = max(0, int(total_budget))
+ self.budget_details = {
+ str(key): max(0, int(value))
+ for key, value in sorted((budget_details or {}).items())
+ }
+
+ @staticmethod
+ def _mandatory(item: ContextItem) -> bool:
+ if item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY:
+ return True
+ if item.kind in ir_constants.STEP_KINDS:
+ return False
+ return item.truncation_policy == ir_constants.POLICY_NEVER
+
+ @staticmethod
+ def _unit_key(items: Sequence[ContextItem]) -> tuple[Any, ...]:
+ return (
+ -max(item.priority for item in items),
+ -max(item.created_seq for item in items),
+ tuple(sorted(item.item_id for item in items)),
+ )
+
+ @staticmethod
+ def _tokens(item: ContextItem) -> int:
+ return 0 if item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY else item.token_estimate
+
+ def _cap_item(
+ self,
+ item: ContextItem,
+ records: dict[str, dict[str, Any]],
+ ) -> Optional[ContextItem]:
+ original = item.token_estimate
+ # Media still contributes to whole-context occupancy, but cannot spend
+ # or be cut by the per-item text allowance.
+ budget = item.token_budget + sum(
+ content.estimate_context_tokens(block) for block in content.tool_result_images(item.content)
+ )
+ if (
+ item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY
+ or item.truncation_policy == ir_constants.POLICY_NEVER
+ or original <= budget
+ ):
+ records[item.item_id] = {
+ "action": "included",
+ "original_tokens": original,
+ "final_tokens": original,
+ }
+ return item
+ if item.truncation_policy in {ir_constants.POLICY_HEAD_TAIL, ir_constants.POLICY_TAIL} and item.token_budget > 0:
+ target = budget
+ capped = item.with_content(
+ content.truncate_content(item.content, target, item.truncation_policy)
+ )
+ # JSON escaping of newlines/quotes can exceed the initial text
+ # allowance. Refit using the measured cost, never discard an image
+ # result just because its accompanying text needs a smaller slice.
+ while capped.token_estimate > budget and target > 1:
+ target = max(1, target - (capped.token_estimate - budget))
+ capped = item.with_content(
+ content.truncate_content(item.content, target, item.truncation_policy)
+ )
+ if capped.token_estimate <= budget:
+ records[item.item_id] = {
+ "action": "truncated",
+ "original_tokens": original,
+ "final_tokens": capped.token_estimate,
+ }
+ return capped
+ records[item.item_id] = {
+ "action": "excluded",
+ "reason": "item_budget",
+ "original_tokens": original,
+ "final_tokens": 0,
+ }
+ return None
+
+ @staticmethod
+ def _tool_pairs_balanced(unit: Sequence[ContextItem]) -> bool:
+ pairs: dict[str, list[ContextItem]] = {}
+ for item in unit:
+ if item.kind in ir_constants.TOOL_KINDS:
+ if not item.pair_id:
+ return False
+ pairs.setdefault(item.pair_id, []).append(item)
+ for pair_items in pairs.values():
+ calls = [item for item in pair_items if item.kind == ir_constants.KIND_TOOL_CALL]
+ results = [item for item in pair_items if item.kind == ir_constants.KIND_TOOL_RESULT]
+ if not calls or len(calls) != len(results):
+ return False
+ return True
+
+ def assemble(self, items: Iterable[ContextItem]) -> ContextAssembly:
+ raw_items = tuple(items)
+ item_ids = [item.item_id for item in raw_items]
+ duplicates = sorted(
+ item_id for item_id in set(item_ids) if item_ids.count(item_id) > 1
+ )
+ if duplicates:
+ raise ValueError(f"context item_id values must be unique: {duplicates}")
+ original_items = list(raw_items)
+ arrival = {item.item_id: index for index, item in enumerate(original_items)}
+ records: dict[str, dict[str, Any]] = {}
+ excluded: list[ContextItem] = []
+
+ capped: list[ContextItem] = []
+ for item in original_items:
+ candidate = self._cap_item(item, records)
+ if candidate is None:
+ excluded.append(item)
+ continue
+ capped.append(candidate)
+
+ # A tool result belongs to the step that issued its call, whatever unit
+ # the caller stamped on it: the pair id is the structural link.
+ call_units: dict[str, str] = {}
+ ambiguous_pairs: set[str] = set()
+ for item in capped:
+ if item.kind == ir_constants.KIND_TOOL_CALL and item.pair_id:
+ # One pair id may name a provider parallel batch (several calls
+ # in one unit); the same id across two units is unresolvable.
+ if call_units.get(item.pair_id, item.unit_id) != item.unit_id:
+ ambiguous_pairs.add(item.pair_id)
+ call_units[item.pair_id] = item.unit_id
+ units: dict[str, list[ContextItem]] = {}
+ for candidate in capped:
+ if candidate.kind in ir_constants.TOOL_KINDS and candidate.pair_id in ambiguous_pairs:
+ records[candidate.item_id].update(
+ action="excluded", reason="malformed_tool_pair", final_tokens=0
+ )
+ excluded.append(candidate)
+ continue
+ if candidate.kind == ir_constants.KIND_TOOL_RESULT and candidate.pair_id in call_units:
+ candidate = replace(candidate, unit_id=call_units[candidate.pair_id])
+ units.setdefault(candidate.unit_id, []).append(candidate)
+
+ # The request that opened the turn in progress. Everything after it is
+ # the model's own live work and is protected as a whole.
+ request_index = -1
+ for item in original_items:
+ if (
+ item.kind == ir_constants.KIND_USER_INPUT
+ and item.truncation_policy == ir_constants.POLICY_NEVER
+ and item.visibility == ir_constants.VISIBILITY_MODEL
+ ):
+ request_index = arrival[item.item_id]
+
+ def unit_start(unit: Sequence[ContextItem]) -> int:
+ return min(arrival[item.item_id] for item in unit)
+
+ mandatory: list[list[ContextItem]] = []
+ protected: list[list[ContextItem]] = []
+ aside: list[list[ContextItem]] = []
+ conversation: list[list[ContextItem]] = []
+ for unit in units.values():
+ is_step = any(item.kind in ir_constants.STEP_KINDS for item in unit)
+ if any(self._mandatory(item) for item in unit):
+ mandatory.append(unit)
+ elif is_step and request_index >= 0 and unit_start(unit) > request_index:
+ protected.append(unit)
+ elif not self._tool_pairs_balanced(unit):
+ for item in unit:
+ records[item.item_id].update(
+ action="excluded", reason="malformed_tool_pair", final_tokens=0
+ )
+ excluded.append(item)
+ elif all(item.kind in ir_constants.CONVERSATION_KINDS for item in unit):
+ conversation.append(unit)
+ else:
+ aside.append(unit)
+
+ mandatory.sort(key=unit_start)
+ protected.sort(key=unit_start)
+ conversation.sort(key=unit_start)
+ # Selection order only for non-conversation material; it never reaches
+ # the output, which is re-sorted into context order.
+ aside.sort(key=self._unit_key)
+
+ def unit_tokens(unit: Sequence[ContextItem]) -> int:
+ return sum(self._tokens(item) for item in unit)
+
+ included: list[ContextItem] = []
+ used = 0
+ for unit in mandatory + protected:
+ included.extend(unit)
+ used += unit_tokens(unit)
+ over_budget = used > self.total_budget
+
+ for unit in aside:
+ tokens = unit_tokens(unit)
+ remaining = max(0, self.total_budget - used)
+ if tokens <= remaining:
+ included.extend(unit)
+ used += tokens
+ continue
+ item = unit[0]
+ if (
+ len(unit) == 1
+ and remaining > 0
+ and item.truncation_policy in {ir_constants.POLICY_HEAD_TAIL, ir_constants.POLICY_TAIL}
+ ):
+ shrunk = item.with_content(
+ content.truncate_content(item.content, remaining, item.truncation_policy)
+ )
+ if self._tokens(shrunk) < self._tokens(item) and self._tokens(shrunk) <= remaining:
+ records[item.item_id].update(
+ action="truncated", final_tokens=shrunk.token_estimate
+ )
+ included.append(shrunk)
+ used += self._tokens(shrunk)
+ continue
+ for item in unit:
+ records[item.item_id].update(action="excluded", reason="budget", final_tokens=0)
+ excluded.append(item)
+
+ # Conversation history: prune old tool outputs first, then cut whole
+ # steps from the oldest end. Never a hole in the middle.
+ pruned_items = 0
+ conversation_tokens = sum(unit_tokens(unit) for unit in conversation)
+ if used + conversation_tokens > self.total_budget:
+ for unit in conversation:
+ if used + conversation_tokens <= self.total_budget:
+ break
+ for index, item in enumerate(unit):
+ if used + conversation_tokens <= self.total_budget:
+ break
+ if item.kind != ir_constants.KIND_TOOL_RESULT or item.truncation_policy == ir_constants.POLICY_NEVER:
+ continue
+ pruned = item.with_content(content.prune_tool_output(item.content, item.token_estimate))
+ if self._tokens(pruned) >= self._tokens(item):
+ continue
+ conversation_tokens -= self._tokens(item) - self._tokens(pruned)
+ unit[index] = pruned
+ records[item.item_id].update(action="pruned", final_tokens=pruned.token_estimate)
+ pruned_items += 1
+ cut_units = 0
+ while conversation and used + conversation_tokens > self.total_budget:
+ dropped = conversation.pop(0)
+ conversation_tokens -= unit_tokens(dropped)
+ cut_units += 1
+ for item in dropped:
+ records[item.item_id].update(action="excluded", reason="budget_cut", final_tokens=0)
+ excluded.append(item)
+ for unit in conversation:
+ included.extend(unit)
+ used += unit_tokens(unit)
+
+ included.sort(key=lambda item: arrival[item.item_id])
+ excluded_by_id = {item.item_id: item for item in excluded}
+ excluded = sorted(excluded_by_id.values(), key=lambda item: arrival[item.item_id])
+
+ included_manifest = []
+ excluded_manifest = []
+ included_ids = {item.item_id for item in included}
+ final_by_id = {item.item_id: item for item in included}
+ for original in original_items:
+ record = dict(records.get(original.item_id) or {})
+ final = final_by_id.get(original.item_id, original)
+ entry = final.to_manifest()
+ entry.update(record)
+ if original.item_id in included_ids:
+ included_manifest.append(entry)
+ else:
+ excluded_manifest.append(entry)
+
+ payload = {
+ "schema_version": ir_constants.CONTEXT_SCHEMA_VERSION,
+ "total_budget": self.total_budget,
+ "used_tokens": used,
+ "over_budget": over_budget,
+ "protected_units": len(protected),
+ "cut_units": cut_units,
+ "pruned_items": pruned_items,
+ "included": included_manifest,
+ "excluded": excluded_manifest,
+ }
+ if self.budget_details:
+ payload["budget_details"] = dict(self.budget_details)
+ manifest_hash = stable_hash(payload)
+ return ContextAssembly(
+ included=tuple(included),
+ excluded=tuple(excluded),
+ _manifest=payload,
+ manifest_hash=manifest_hash,
+ used_tokens=used,
+ total_budget=self.total_budget,
+ over_budget=over_budget,
+ cut_units=cut_units,
+ pruned_items=pruned_items,
+ )
diff --git a/src/backend/core/llm/context/ir_constants.py b/src/backend/core/llm/context/ir_constants.py
new file mode 100644
index 00000000..931faf2b
--- /dev/null
+++ b/src/backend/core/llm/context/ir_constants.py
@@ -0,0 +1,86 @@
+"""Context item kinds, policies and metadata keys."""
+from __future__ import annotations
+
+
+CONTEXT_SCHEMA_VERSION = "harness.context.v2"
+
+
+SESSION_CONTEXT_META_KEY = "_context_item"
+
+
+CONTEXT_SEQUENCE_STRIDE = 1_000
+
+
+KIND_SYSTEM_RULE = "system_rule"
+
+
+KIND_USER_INPUT = "user_input"
+
+
+KIND_ASSISTANT = "assistant_history"
+
+
+KIND_MEMORY = "memory"
+
+
+KIND_IDENTITY = "identity"
+
+
+KIND_PROJECT = "project_material"
+
+
+KIND_THINKING = "thinking"
+
+
+KIND_TOOL_CALL = "tool_call"
+
+
+KIND_TOOL_RESULT = "tool_result"
+
+
+KIND_REMINDER = "reminder"
+
+
+KIND_COMPACTION = "compaction_summary"
+
+
+KIND_ATTACHMENT = "attachment"
+
+
+KIND_STEER = "steer"
+
+
+KIND_REFERENCE = "reference"
+
+
+POLICY_NEVER = "never"
+
+
+POLICY_DROP = "drop"
+
+
+POLICY_HEAD_TAIL = "head_tail"
+
+
+POLICY_TAIL = "tail"
+
+
+VISIBILITY_MODEL = "model"
+
+
+VISIBILITY_MANIFEST_ONLY = "manifest_only"
+
+
+TOOL_KINDS = {KIND_TOOL_CALL, KIND_TOOL_RESULT}
+
+
+STEP_KINDS = frozenset({KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT})
+
+
+CONVERSATION_KINDS = STEP_KINDS | {KIND_USER_INPUT}
+
+
+TRUNCATABLE_BLOCK_TYPES = {"tool_result"}
+
+
+PRUNED_OUTPUT_TEMPLATE = "[tool output pruned to fit the context budget: {tokens} tokens omitted]"
diff --git a/src/backend/core/llm/context/items.py b/src/backend/core/llm/context/items.py
new file mode 100644
index 00000000..22618c10
--- /dev/null
+++ b/src/backend/core/llm/context/items.py
@@ -0,0 +1,215 @@
+"""Context items and their provenance metadata."""
+from __future__ import annotations
+from dataclasses import dataclass, field, replace
+from typing import Any, Mapping, Optional
+from core.infra.immutable import FrozenDict, freeze_json, thaw_json
+from core.harness.execution_manifest import stable_hash
+from core.llm.context.content import estimate_context_tokens, normalize_context_content
+from core.llm.context import ir_constants
+
+@dataclass(frozen=True)
+class ContextItem:
+ item_id: str
+ kind: str
+ origin: str
+ trust: str
+ visibility: str
+ priority: int
+ token_budget: int
+ truncation_policy: str
+ content_ref: str
+ content_hash: str
+ cache_class: str
+ created_seq: int
+ token_estimate: int
+ render_role: str = "user"
+ render_name: str = ""
+ pair_id: str = ""
+ message_group: str = ""
+ unit_id: str = ""
+ content: Any = field(default=None, repr=False, compare=False)
+ metadata: Mapping[str, Any] = field(
+ default_factory=FrozenDict, repr=False, compare=False
+ )
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "content", freeze_json(self.content))
+ object.__setattr__(self, "metadata", freeze_json(self.metadata or {}))
+
+ @classmethod
+ def create(
+ cls,
+ *,
+ item_id: str,
+ kind: str,
+ origin: str,
+ trust: str,
+ visibility: str,
+ priority: int,
+ token_budget: int,
+ truncation_policy: str,
+ content: Any,
+ cache_class: str,
+ created_seq: int,
+ render_role: str = "user",
+ pair_id: str = "",
+ message_group: str = "",
+ unit_id: str = "",
+ content_ref: Optional[str] = None,
+ content_hash: Optional[str] = None,
+ token_estimate: Optional[int] = None,
+ render_name: str = "",
+ metadata: Optional[Mapping[str, Any]] = None,
+ ) -> "ContextItem":
+ normalized_content = normalize_context_content(content)
+ if (
+ isinstance(normalized_content, Mapping)
+ and str(normalized_content.get("type") or "") == "data"
+ ):
+ truncation_policy = ir_constants.POLICY_DROP
+ computed_hash = stable_hash(normalized_content)
+ supplied_hash = str(content_hash or "")
+ external_reference = normalized_content is None and bool(supplied_hash)
+ digest = supplied_hash if external_reference else computed_hash
+ estimate = (
+ max(0, int(token_estimate or 0))
+ if external_reference
+ else estimate_context_tokens(normalized_content)
+ )
+ reference = (
+ str(content_ref)
+ if content_ref is not None
+ and (external_reference or supplied_hash == computed_hash)
+ else f"sha256:{digest}"
+ )
+ return cls(
+ item_id=str(item_id),
+ kind=str(kind),
+ origin=str(origin),
+ trust=str(trust),
+ visibility=str(visibility),
+ priority=int(priority),
+ token_budget=max(0, int(token_budget)),
+ truncation_policy=str(truncation_policy),
+ content_ref=reference,
+ content_hash=digest,
+ cache_class=str(cache_class),
+ created_seq=int(created_seq),
+ token_estimate=estimate,
+ render_role=str(render_role),
+ render_name=str(render_name or ""),
+ pair_id=str(pair_id or ""),
+ message_group=str(message_group or f"item:{item_id}"),
+ unit_id=str(unit_id or f"item:{item_id}"),
+ content=normalized_content,
+ metadata=metadata or {},
+ )
+
+ def with_content(self, content: Any) -> "ContextItem":
+ normalized_content = normalize_context_content(content)
+ digest = stable_hash(normalized_content)
+ return replace(
+ self,
+ content=freeze_json(normalized_content),
+ content_hash=digest,
+ content_ref=f"sha256:{digest}",
+ token_estimate=estimate_context_tokens(normalized_content),
+ )
+
+ def to_manifest(self) -> dict[str, Any]:
+ return {
+ "item_id": self.item_id,
+ "kind": self.kind,
+ "origin": self.origin,
+ "trust": self.trust,
+ "visibility": self.visibility,
+ "priority": self.priority,
+ "token_budget": self.token_budget,
+ "truncation_policy": self.truncation_policy,
+ "content_ref": self.content_ref,
+ "content_hash": self.content_hash,
+ "cache_class": self.cache_class,
+ "created_seq": self.created_seq,
+ "token_estimate": self.token_estimate,
+ "render_role": self.render_role,
+ "render_name_hash": stable_hash(self.render_name),
+ "pair_id": self.pair_id or None,
+ "message_group": self.message_group,
+ "unit_id": self.unit_id,
+ }
+
+
+def make_text_context_item(
+ text: str,
+ *,
+ item_id: str,
+ kind: str,
+ origin: str,
+ trust: str,
+ created_seq: int,
+ priority: int = 700,
+ token_budget: int = 4_000,
+ truncation_policy: str = ir_constants.POLICY_HEAD_TAIL,
+ render_role: str = "user",
+ cache_class: str = "dynamic",
+) -> ContextItem:
+ """Create a framework-neutral, explicitly-provenanced text item."""
+ return ContextItem.create(
+ item_id=item_id,
+ kind=kind,
+ origin=origin,
+ trust=trust,
+ visibility=ir_constants.VISIBILITY_MODEL,
+ priority=priority,
+ token_budget=token_budget,
+ truncation_policy=truncation_policy,
+ content=str(text or ""),
+ cache_class=cache_class,
+ created_seq=created_seq,
+ render_role=render_role,
+ render_name=render_role,
+ message_group=item_id,
+ )
+
+
+def session_context_metadata(item: ContextItem) -> dict[str, Any]:
+ """Carry provenance on a positional session row without stale identity/sequence."""
+ payload = item.to_manifest()
+ for key in (
+ "item_id",
+ "created_seq",
+ "message_group",
+ "unit_id",
+ "content_ref",
+ "content_hash",
+ "token_estimate",
+ "render_name_hash",
+ ):
+ payload.pop(key, None)
+ return payload
+
+
+@dataclass(frozen=True)
+class ContextAssembly:
+ included: tuple[ContextItem, ...]
+ excluded: tuple[ContextItem, ...]
+ _manifest: Mapping[str, Any] = field(repr=False)
+ manifest_hash: str
+ used_tokens: int
+ total_budget: int
+ over_budget: bool = False
+ cut_units: int = 0
+ pruned_items: int = 0
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "included", tuple(self.included))
+ object.__setattr__(self, "excluded", tuple(self.excluded))
+ object.__setattr__(self, "_manifest", freeze_json(self._manifest))
+
+ @property
+ def manifest(self) -> dict[str, Any]:
+ """Return a detached JSON-serializable copy of the audit manifest."""
+ manifest = thaw_json(self._manifest)
+ if not isinstance(manifest, dict): # pragma: no cover - frozen in __post_init__
+ raise TypeError("context manifest must thaw to a dictionary")
+ return manifest
diff --git a/src/backend/core/llm/context/message_defaults.py b/src/backend/core/llm/context/message_defaults.py
new file mode 100644
index 00000000..ef5ce116
--- /dev/null
+++ b/src/backend/core/llm/context/message_defaults.py
@@ -0,0 +1,78 @@
+"""Default kind and truncation policy for each message role and block type."""
+from __future__ import annotations
+from typing import Any
+from core.llm.context.ir_constants import KIND_ASSISTANT, KIND_ATTACHMENT, KIND_SYSTEM_RULE, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT, POLICY_DROP, POLICY_HEAD_TAIL, POLICY_NEVER
+
+def defaults(role: str, block_type: str) -> dict[str, Any]:
+ if block_type == "thinking":
+ # Reasoning is handed back to the model verbatim. It is counted on its
+ # own so the usage panel can show what it costs, but it is never
+ # shortened or dropped apart from the step that produced it.
+ return {
+ "kind": KIND_THINKING,
+ "origin": "assistant:reasoning",
+ "trust": "assistant",
+ "priority": 400,
+ "token_budget": 0,
+ "truncation_policy": POLICY_NEVER,
+ "render_role": "assistant",
+ }
+ if block_type == "tool_call":
+ return {
+ "kind": KIND_TOOL_CALL,
+ "origin": "agent:tool_call",
+ "trust": "assistant",
+ "priority": 700,
+ "token_budget": 0,
+ "truncation_policy": POLICY_NEVER,
+ "render_role": "assistant",
+ }
+ if block_type == "tool_result":
+ return {
+ "kind": KIND_TOOL_RESULT,
+ "origin": "tool:result",
+ "trust": "tool",
+ "priority": 700,
+ "token_budget": 20_000,
+ "truncation_policy": POLICY_HEAD_TAIL,
+ "render_role": "assistant",
+ }
+ if block_type == "data":
+ return {
+ "kind": KIND_ATTACHMENT,
+ "origin": "user:attachment",
+ "trust": "user",
+ "priority": 650,
+ "token_budget": 4_000,
+ "truncation_policy": POLICY_DROP,
+ "render_role": "user",
+ }
+ if role == "system":
+ return {
+ "kind": KIND_SYSTEM_RULE,
+ "origin": "platform:system_prompt",
+ "trust": "platform",
+ "priority": 1_000,
+ "token_budget": 100_000,
+ "truncation_policy": POLICY_NEVER,
+ "render_role": "system",
+ }
+ if role == "assistant":
+ return {
+ "kind": KIND_ASSISTANT,
+ "origin": "assistant:history",
+ "trust": "assistant",
+ "priority": 400,
+ "token_budget": 0,
+ "truncation_policy": POLICY_NEVER,
+ "render_role": "assistant",
+ }
+ return {
+ "kind": KIND_USER_INPUT,
+ "origin": "user:chat",
+ "trust": "user",
+ "priority": 500,
+ "token_budget": 100_000,
+ "truncation_policy": POLICY_HEAD_TAIL,
+ "render_role": "user",
+ }
diff --git a/src/backend/core/llm/offloader.py b/src/backend/core/llm/context/offloader.py
similarity index 96%
rename from src/backend/core/llm/offloader.py
rename to src/backend/core/llm/context/offloader.py
index 75e5d1cb..886e086e 100644
--- a/src/backend/core/llm/offloader.py
+++ b/src/backend/core/llm/context/offloader.py
@@ -14,7 +14,7 @@
import uuid
from typing import Any, Optional
-from core.llm.message_compat import flatten_tool_output
+from core.llm.runtime.message_compat import flatten_tool_output
logger = logging.getLogger(__name__)
_WRITE_TIMEOUT_SEC = 30
@@ -43,7 +43,7 @@ def __init__(
self._user_id = user_id
def _path(self, prefix: str) -> str:
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
if not self._sess:
raise OffloadError("A persistent tool workspace is required")
diff --git a/src/backend/core/llm/context/provider_conversion.py b/src/backend/core/llm/context/provider_conversion.py
new file mode 100644
index 00000000..901d19a6
--- /dev/null
+++ b/src/backend/core/llm/context/provider_conversion.py
@@ -0,0 +1,239 @@
+"""Carry context provenance through provider-formatted messages."""
+from __future__ import annotations
+from core.models.constants import PROVIDER_CONTEXT_META_KEY
+from typing import Any, Iterable, Mapping, Sequence
+from agentscope.message import Msg
+from core.infra.immutable import thaw_json
+from core.harness.execution_manifest import stable_hash
+from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, KIND_ASSISTANT, KIND_ATTACHMENT, KIND_SYSTEM_RULE, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT, POLICY_NEVER, VISIBILITY_MODEL
+from core.llm.context.items import ContextItem
+from core.llm.context.content import estimate_context_tokens
+
+def annotate_provider_messages(
+ self,
+ source_messages: Sequence[Msg],
+ formatted_messages: Sequence[Mapping[str, Any]],
+ row_counts: Sequence[int],
+) -> list[dict[str, Any]]:
+ """Carry canonical provenance across an SDK formatter boundary.
+
+ AgentScope's OpenAI formatter intentionally drops arbitrary message
+ metadata and may expand one tool-result message into a tool row plus a
+ synthetic media row. ``row_counts`` is obtained by formatting each
+ source message in isolation after a provider has rejected media. This
+ lets the retry inherit the original IR provenance without guessing it
+ from compatibility roles or XML-like text.
+ """
+ annotated = [dict(message) for message in formatted_messages]
+ cursor = 0
+ for source, count in zip(source_messages, row_counts):
+ entries = self._metadata_entries(source)
+ source_rows = annotated[cursor : cursor + max(0, int(count))]
+ cursor += max(0, int(count))
+ if not entries:
+ continue
+
+ def pick(kind: str) -> Mapping[str, Any]:
+ match = next((entry for entry in entries if entry.get("kind") == kind), None)
+ if match is not None:
+ return match
+ user = next(
+ (entry for entry in entries if entry.get("kind") == KIND_USER_INPUT),
+ None,
+ )
+ return user or entries[0]
+
+ for row in source_rows:
+ role = str(row.get("role") or "user")
+ name = str(row.get("name") or role)
+ if name == "system-reminder":
+ base = pick(KIND_TOOL_RESULT)
+ metadata = {
+ "kind": KIND_ATTACHMENT,
+ "origin": f"{base.get('origin') or 'tool:result'}:media",
+ "trust": str(base.get("trust") or "tool"),
+ "priority": int(base.get("priority") or 700),
+ }
+ elif role == "tool":
+ base = pick(KIND_TOOL_RESULT)
+ metadata = {
+ "kind": KIND_TOOL_RESULT,
+ "origin": str(base.get("origin") or "tool:result"),
+ "trust": str(base.get("trust") or "tool"),
+ "priority": int(base.get("priority") or 700),
+ }
+ elif role == "assistant" and row.get("tool_calls"):
+ base = pick(KIND_TOOL_CALL)
+ metadata = {
+ "kind": KIND_TOOL_CALL,
+ "origin": str(base.get("origin") or "agent:tool_call"),
+ "trust": str(base.get("trust") or "assistant"),
+ "priority": int(base.get("priority") or 700),
+ }
+ else:
+ base = pick("")
+ metadata = {
+ key: base[key]
+ for key in ("kind", "origin", "trust", "priority")
+ if base.get(key) is not None
+ }
+ row[PROVIDER_CONTEXT_META_KEY] = metadata
+ if cursor != len(annotated):
+ # A formatter shape change must fail visibly rather than silently
+ # assigning later rows the wrong trust boundary.
+ raise ValueError(
+ "provider formatter row mapping mismatch: "
+ f"mapped={cursor} formatted={len(annotated)}"
+ )
+ return annotated
+
+
+def items_from_provider_messages(
+ self,
+ messages: Sequence[Mapping[str, Any]],
+) -> list[ContextItem]:
+ """Reify an already-formatted provider retry as exact canonical items.
+
+ A provider may reject media after the normal AgentScope request has
+ already been assembled. The retry is therefore a new execution
+ surface. Each complete wire message is kept atomically so its hash
+ covers role, name, tool identifiers and content together; transient
+ provenance metadata is removed before hashing and before transport.
+ """
+ items: list[ContextItem] = []
+ pending_tool_pairs: dict[str, list[str]] = {}
+ orphan_results: dict[str, int] = {}
+ for index, raw_message in enumerate(messages):
+ message = dict(raw_message)
+ explicit = message.pop(PROVIDER_CONTEXT_META_KEY, None)
+ explicit = explicit if isinstance(explicit, Mapping) else {}
+ role = str(message.get("role") or "user")
+ name = str(message.get("name") or role)
+ tool_calls = message.get("tool_calls")
+ if role == "assistant" and isinstance(tool_calls, list) and tool_calls:
+ pair_id = f"provider-tool-batch:{index}"
+ message_hash = stable_hash(message)
+ message_tokens = estimate_context_tokens(message)
+ for call_index, call in enumerate(tool_calls):
+ call = call if isinstance(call, Mapping) else {}
+ raw_pair_id = str(call.get("id") or "")
+ pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id)
+ items.append(
+ ContextItem.create(
+ item_id=f"provider-retry:{index}:tool-call:{call_index}",
+ kind=str(explicit.get("kind") or KIND_TOOL_CALL),
+ origin=str(explicit.get("origin") or "assistant:provider_tool_call"),
+ trust=str(explicit.get("trust") or "assistant"),
+ visibility=VISIBILITY_MODEL,
+ priority=int(explicit.get("priority") or 900),
+ token_budget=message_tokens if call_index == 0 else 0,
+ truncation_policy=POLICY_NEVER,
+ content=message if call_index == 0 else None,
+ content_ref=f"provider-message:{index}",
+ content_hash=message_hash,
+ token_estimate=message_tokens if call_index == 0 else 0,
+ cache_class="dynamic",
+ created_seq=index * CONTEXT_SEQUENCE_STRIDE + call_index,
+ render_role=role,
+ render_name=name,
+ pair_id=pair_id,
+ message_group=f"provider-retry:{index}",
+ unit_id=pair_id,
+ )
+ )
+ continue
+ if role == "tool":
+ raw_pair_id = str(message.get("tool_call_id") or "")
+ pending = pending_tool_pairs.get(raw_pair_id) or []
+ if pending:
+ pair_id = pending.pop(0)
+ else:
+ occurrence = orphan_results.get(raw_pair_id, 0) + 1
+ orphan_results[raw_pair_id] = occurrence
+ pair_id = f"{raw_pair_id}#orphan-result-{occurrence}"
+ defaults = {
+ "kind": KIND_TOOL_RESULT,
+ "origin": "tool:provider_result",
+ "trust": "tool",
+ "priority": 900,
+ "pair_id": pair_id,
+ }
+ elif role == "system":
+ defaults = {
+ "kind": KIND_SYSTEM_RULE,
+ "origin": "platform:provider_request",
+ "trust": "platform",
+ "priority": 1_000,
+ }
+ elif role == "assistant":
+ defaults = {
+ "kind": KIND_ASSISTANT,
+ "origin": "assistant:provider_history",
+ "trust": "assistant",
+ "priority": 600,
+ }
+ else:
+ defaults = {
+ "kind": KIND_USER_INPUT,
+ "origin": "user:provider_request",
+ "trust": "user",
+ "priority": 900,
+ }
+ items.append(
+ ContextItem.create(
+ item_id=str(explicit.get("item_id") or f"provider-retry:{index}"),
+ kind=str(explicit.get("kind") or defaults["kind"]),
+ origin=str(explicit.get("origin") or defaults["origin"]),
+ trust=str(explicit.get("trust") or defaults["trust"]),
+ visibility=VISIBILITY_MODEL,
+ priority=int(explicit.get("priority") or defaults["priority"]),
+ # These rows have already survived the canonical budget
+ # pass. Media recovery only removes data or replaces it
+ # with a smaller textual description, so the retry must
+ # preserve every resulting row exactly.
+ token_budget=max(1, estimate_context_tokens(message)),
+ truncation_policy=POLICY_NEVER,
+ content=message,
+ cache_class="dynamic",
+ created_seq=int(
+ explicit.get("created_seq")
+ if explicit.get("created_seq") is not None
+ else index * CONTEXT_SEQUENCE_STRIDE
+ ),
+ render_role=role,
+ render_name=name,
+ pair_id=str(defaults.get("pair_id") or ""),
+ message_group=f"provider-retry:{index}",
+ unit_id=str(defaults.get("pair_id") or f"provider-retry:{index}"),
+ )
+ )
+ return items
+
+
+def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]:
+ """Render exact provider rows while stripping harness-only metadata."""
+ rendered: list[dict[str, Any]] = []
+ by_group: dict[str, dict[str, Any]] = {}
+ deferred_groups: set[str] = set()
+ for item in items:
+ if item.visibility != VISIBILITY_MODEL:
+ continue
+ value = thaw_json(item.content)
+ if value is None:
+ deferred_groups.add(item.message_group)
+ continue
+ if not isinstance(value, Mapping):
+ raise ValueError("provider retry context item must contain one message mapping")
+ row = dict(value)
+ row.pop(PROVIDER_CONTEXT_META_KEY, None)
+ previous = by_group.get(item.message_group)
+ if previous is not None:
+ if previous != row:
+ raise ValueError("provider retry group contains conflicting wire messages")
+ continue
+ by_group[item.message_group] = row
+ rendered.append(row)
+ missing = deferred_groups - set(by_group)
+ if missing:
+ raise ValueError(f"provider retry groups lack a wire row: {sorted(missing)}")
+ return rendered
diff --git a/src/backend/core/llm/context/reasoning_hooks.py b/src/backend/core/llm/context/reasoning_hooks.py
new file mode 100644
index 00000000..68cbf7c6
--- /dev/null
+++ b/src/backend/core/llm/context/reasoning_hooks.py
@@ -0,0 +1,49 @@
+"""Reasoning policies consume plain snapshots and emit bounded context additions."""
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage
+
+
+def reminder(text, origin):
+ return dict(content=f"\n{text}\n ",
+ kind="reminder", origin=origin, trust="system",
+ priority=850, token_budget=4000)
+
+
+class IterationBudgetHook:
+ def __init__(self, threshold=2):
+ from core.config.settings import env_bool as _bool, env_value as _env
+ self.threshold = threshold
+ self.force_text = _bool(_env("CHAT_FINAL_ITER_FORCE_TEXT", "true"))
+ self.last_key = None
+
+ def __call__(self, invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ runtime = invocation.data["runtime"]
+ maximum = int(runtime.get("max_iters") or 0)
+ current = int(runtime.get("cur_iter") or 0)
+ remaining = maximum - current
+ if maximum <= self.threshold + 1:
+ return Decision()
+ patch = {}
+ if remaining <= 1 and self.force_text and invocation.data.get("tool_choice") is None:
+ patch["tool_choice"] = {"mode": "none"}
+ key = (runtime.get("reply_id") or "", current)
+ if remaining <= self.threshold and key != self.last_key:
+ self.last_key = key
+ if remaining <= 1:
+ text = ("这是本次回复的最后一轮推理:不要再调用任何工具(再调用会被系统强制"
+ "中断,已完成的工作将无法交付)。请直接基于已获得的信息输出最终回复;"
+ "若任务未全部完成,如实汇报已产出的成果、当前进展和未完成的部分。")
+ else:
+ text = (f"注意:推理-工具调用轮次即将用尽,包含本轮在内最多还剩 {remaining} 轮。"
+ "请停止新的探索,不要再重试已反复失败的操作,立即整合已获得的信息进行"
+ "收尾;若任务无法在剩余轮次内全部完成,优先输出已有成果与进展说明。")
+ patch["context_items"] = [*invocation.data.get("context_items", ()),
+ reminder(text, "harness:iteration_budget")]
+ return Decision.modify(patch) if patch else Decision()
+
+
+def iteration_budget_hook():
+ return HookSpec("iteration_budget", HookStage.TRANSFORM_CONTEXT, IterationBudgetHook(),
+ priority=300, failure_mode=FailureMode.OPEN,
+ mutable_fields=frozenset({"context_items", "tool_choice"}))
diff --git a/src/backend/core/llm/context/reference_conversion.py b/src/backend/core/llm/context/reference_conversion.py
new file mode 100644
index 00000000..426832eb
--- /dev/null
+++ b/src/backend/core/llm/context/reference_conversion.py
@@ -0,0 +1,79 @@
+"""Context items for the references listed in an execution manifest."""
+from __future__ import annotations
+from typing import Any, Mapping
+from core.llm.context.ir_constants import KIND_PROJECT, KIND_SYSTEM_RULE, POLICY_HEAD_TAIL, POLICY_NEVER, VISIBILITY_MANIFEST_ONLY, VISIBILITY_MODEL
+from core.llm.context.items import ContextItem
+
+def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]:
+ if hasattr(manifest, "to_dict"):
+ payload = manifest.to_dict()
+ elif isinstance(manifest, Mapping):
+ payload = dict(manifest)
+ else:
+ return []
+ sections = (payload.get("prompt_manifest") or {}).get("sections") or []
+ items: list[ContextItem] = []
+ for index, section in enumerate(sections):
+ if not isinstance(section, Mapping):
+ continue
+ section_id = str(section.get("id") or "")
+ content_hash = str(section.get("content_hash") or "")
+ if not section_id or not content_hash:
+ continue
+ is_project = section_id == "runtime/project"
+ live_content = ""
+ content_reader = getattr(manifest, "prompt_section_content", None)
+ if callable(content_reader):
+ live_content = str(content_reader(index) or "")
+ if is_project and live_content:
+ items.append(
+ ContextItem.create(
+ item_id=f"prompt:{index}:{section_id}",
+ kind=KIND_PROJECT,
+ origin=str(section.get("origin") or "workspace:project"),
+ trust=str(section.get("trust") or "workspace"),
+ visibility=VISIBILITY_MODEL,
+ priority=int(section.get("priority") or 900),
+ token_budget=max(
+ 1,
+ int(section.get("budget") or section.get("token_estimate") or 1),
+ ),
+ truncation_policy=POLICY_HEAD_TAIL,
+ content=live_content,
+ content_ref=(
+ str(section["reference"]) if section.get("reference") else None
+ ),
+ content_hash=content_hash,
+ token_estimate=int(section.get("token_estimate") or 0),
+ cache_class=str(section.get("cache_class") or "workspace"),
+ created_seq=1,
+ render_role="system",
+ render_name="system",
+ message_group=f"prompt:{index}",
+ )
+ )
+ continue
+ items.append(
+ ContextItem.create(
+ item_id=f"prompt:{index}:{section_id}",
+ kind=KIND_PROJECT if is_project else KIND_SYSTEM_RULE,
+ origin=str(section.get("origin") or "prompt:unknown"),
+ trust=str(section.get("trust") or "platform"),
+ visibility=VISIBILITY_MANIFEST_ONLY,
+ priority=int(section.get("priority") or 0),
+ token_budget=int(section.get("budget") or 0),
+ truncation_policy=POLICY_NEVER,
+ content=None,
+ content_ref=str(
+ section.get("reference")
+ or f"prompt:{section_id}@{section.get('version') or '1'}"
+ ),
+ content_hash=content_hash,
+ token_estimate=int(section.get("token_estimate") or 0),
+ cache_class=str(section.get("cache_class") or "stable"),
+ created_seq=-10_000 + index,
+ render_role="system",
+ message_group=f"prompt:{index}",
+ )
+ )
+ return items
diff --git a/src/backend/core/llm/context/rendering.py b/src/backend/core/llm/context/rendering.py
new file mode 100644
index 00000000..9eff0361
--- /dev/null
+++ b/src/backend/core/llm/context/rendering.py
@@ -0,0 +1,114 @@
+"""Render context items into AgentScope state and allocate their sequence numbers."""
+from __future__ import annotations
+from core.llm.context import adapter
+from dataclasses import replace
+from typing import Any, Mapping, Sequence
+from agentscope.message import Msg, TextBlock
+from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, POLICY_HEAD_TAIL
+from core.llm.context.items import ContextItem, make_text_context_item
+
+def append_state_context_item(state: Any, item: ContextItem) -> Msg:
+ """Render and append one explicitly-provenanced item to AgentScope state."""
+ messages = adapter.AgentScopeContextAdapter().messages_from_items([item])
+ if len(messages) != 1:
+ raise ValueError("context item did not render to exactly one message")
+ message = messages[0]
+ state.context.append(message)
+ return message
+
+
+def render_text_block(text: str) -> TextBlock:
+ """Construct an AgentScope text block inside the SDK adapter boundary."""
+ return TextBlock(type="text", text=str(text or ""))
+
+
+def next_context_sequence(messages: Sequence[Any]) -> int:
+ """Allocate one monotonic sequence across replay and live injections."""
+ max_sequence = -CONTEXT_SEQUENCE_STRIDE
+ for message_index, message in enumerate(messages):
+ entries = adapter.AgentScopeContextAdapter._metadata_entries(message)
+ blocks = list(getattr(message, "content", None) or [])
+ for block_index, _block in enumerate(blocks or [None]):
+ explicit = entries[block_index] if block_index < len(entries) else {}
+ candidate = explicit.get("created_seq")
+ if candidate is None:
+ candidate = message_index * CONTEXT_SEQUENCE_STRIDE + block_index
+ try:
+ max_sequence = max(max_sequence, int(candidate))
+ except (TypeError, ValueError):
+ continue
+ return ((max_sequence // CONTEXT_SEQUENCE_STRIDE) + 1) * CONTEXT_SEQUENCE_STRIDE
+
+
+def next_request_sequence(messages: Sequence[Any]) -> int:
+ """Reserve a pre-reply lane above the context tail for the user instruction.
+
+ AgentScope runs ``on_reply`` middleware before it appends ``inputs`` to
+ state. File/project attachment middleware can therefore add a few explicit
+ context items first without colliding with the request's sequence. The
+ sequence no longer decides message order — the assembler emits items in
+ context order — it only keeps identifiers distinct and marks recency for
+ budget eviction.
+ """
+ pre_reply_slots = 64
+ return next_context_sequence(messages) + pre_reply_slots * CONTEXT_SEQUENCE_STRIDE
+
+
+def render_context_item(item: ContextItem) -> Msg:
+ """Render one IR item for ``Agent.reply`` without mutating agent state."""
+ messages = adapter.AgentScopeContextAdapter().messages_from_items([item])
+ if len(messages) != 1:
+ raise ValueError("context item did not render to exactly one message")
+ return messages[0]
+
+
+def render_session_input(message: Mapping[str, Any], *, created_seq: int) -> Msg:
+ """Render one pending positional row at an explicit live request sequence."""
+ context_adapter = adapter.AgentScopeContextAdapter()
+ items = context_adapter.items_from_session_dict(message, created_seq=0)
+ positioned = [
+ replace(
+ item,
+ created_seq=created_seq + index,
+ message_group=f"request:{created_seq}",
+ )
+ for index, item in enumerate(items)
+ ]
+ rendered = context_adapter.messages_from_items(positioned)
+ if len(rendered) != 1:
+ raise ValueError("session input did not render to exactly one message")
+ return rendered[0]
+
+
+def append_context_item(agent: Any, item: ContextItem) -> Msg:
+ """Append one explicitly-provenanced item through the adapter seam."""
+ return append_state_context_item(agent.state, item)
+
+
+def append_context_text(
+ agent: Any,
+ text: str,
+ *,
+ kind: str,
+ origin: str,
+ trust: str,
+ priority: int = 700,
+ token_budget: int = 4_000,
+ truncation_policy: str = POLICY_HEAD_TAIL,
+ render_role: str = "user",
+) -> Msg:
+ """Append provenanced text without creating AgentScope messages elsewhere."""
+ created_seq = next_context_sequence(agent.state.context)
+ item = make_text_context_item(
+ text,
+ item_id=f"{origin}:{created_seq}",
+ kind=kind,
+ origin=origin,
+ trust=trust,
+ created_seq=created_seq,
+ priority=priority,
+ token_budget=token_budget,
+ truncation_policy=truncation_policy,
+ render_role=render_role,
+ )
+ return append_context_item(agent, item)
diff --git a/src/backend/core/llm/context_safety.py b/src/backend/core/llm/context/safety.py
similarity index 87%
rename from src/backend/core/llm/context_safety.py
rename to src/backend/core/llm/context/safety.py
index a1836da8..94f0ea76 100644
--- a/src/backend/core/llm/context_safety.py
+++ b/src/backend/core/llm/context/safety.py
@@ -8,7 +8,7 @@
from typing import Any
from core.harness.hooks import HookPaused
-from core.llm.context_manager import usable_context_window
+from core.models.context_window import usable_context_window
class ContextBudgetExceeded(HookPaused):
@@ -40,9 +40,11 @@ def check(self) -> None:
@contextmanager
def request_admission_scope(*, context=None, checkpoint=None):
admission = RequestAdmission(context=context, checkpoint=checkpoint)
+ from core.models.admission import request_admission_callback
token = _admission.set(admission)
try:
- yield admission
+ with request_admission_callback(admission.check):
+ yield admission
finally:
_admission.reset(token)
@@ -77,16 +79,10 @@ async def enforce_context_budget(agent: Any, assembly: Any = None) -> None:
def remember_context_output(*, message_id: str, steps: list, content: str) -> None:
admission = _admission.get()
if admission is not None:
- from core.services.context_budget_checkpoint import ContextStopCheckpoint
+ from core.harness.registry.compaction import create_context_checkpoint
- admission.checkpoint = ContextStopCheckpoint(
+ admission.checkpoint = create_context_checkpoint(
message_id=message_id,
steps=list(steps),
content=content,
)
-
-
-def check_request_admission() -> None:
- admission = _admission.get()
- if admission is not None:
- admission.check()
diff --git a/src/backend/core/llm/system_reminder.py b/src/backend/core/llm/context/system_reminder.py
similarity index 94%
rename from src/backend/core/llm/system_reminder.py
rename to src/backend/core/llm/context/system_reminder.py
index a4c10113..543dd68a 100644
--- a/src/backend/core/llm/system_reminder.py
+++ b/src/backend/core/llm/context/system_reminder.py
@@ -16,8 +16,8 @@
import logging
from typing import Any
-from core.llm.context_adapter import append_context_text
-from core.llm.context_ir import KIND_REMINDER
+from core.llm.context.rendering import append_context_text
+from core.llm.context.ir_constants import KIND_REMINDER
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/llm/context_usage.py b/src/backend/core/llm/context/usage.py
similarity index 99%
rename from src/backend/core/llm/context_usage.py
rename to src/backend/core/llm/context/usage.py
index 6d596cb4..c32537c3 100644
--- a/src/backend/core/llm/context_usage.py
+++ b/src/backend/core/llm/context/usage.py
@@ -9,7 +9,7 @@
import math
import time
-from typing import Any, Dict, Mapping, Optional
+from typing import Any, Dict, Mapping
SCHEMA_VERSION = "context-usage.v2"
BREAKDOWN_KEYS = ("messages", "tools", "thinking", "files", "system", "input")
diff --git a/src/backend/core/llm/context/workspace_hooks.py b/src/backend/core/llm/context/workspace_hooks.py
new file mode 100644
index 00000000..dc1d604c
--- /dev/null
+++ b/src/backend/core/llm/context/workspace_hooks.py
@@ -0,0 +1,125 @@
+"""Workspace and plan reminders operate on plain context snapshots."""
+import json
+from collections.abc import Mapping
+from core.harness.events import thaw_value
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage
+from core.llm.context.reasoning_hooks import reminder
+from core.llm.runtime.hooks_constants import FILE_ID_RE, PIN_HINT_SKIP_TOOLS
+from core.llm.runtime.message_compat import flatten_tool_output
+from core.llm.tools.plan_update import parse_plan_update_args
+
+
+def content_blocks(context, kind):
+ for message in context:
+ if not isinstance(message, Mapping):
+ continue
+ for block in message.get("content") or ():
+ if isinstance(block, Mapping) and block.get("type") == kind:
+ yield block
+
+
+def append_reminder(invocation, text, origin):
+ return Decision.modify({"context_items": [*invocation.data.get("context_items", ()),
+ reminder(text, origin)]})
+
+
+class WorkspacePinHook:
+ def __init__(self):
+ self.seen = set()
+ self.last_signature = ""
+
+ def reset(self, invocation):
+ self.seen.clear()
+ self.last_signature = ""
+ return Decision()
+
+ def finish(self, invocation):
+ runtime = invocation.data.get("runtime", {})
+ if runtime.get("finish_pins_completed"):
+ return Decision()
+ remaining = sorted(self.seen - set(runtime.get("pinned_file_ids") or ()))
+ return Decision.modify({"pin_file_ids": remaining}) if remaining else Decision()
+
+ def __call__(self, invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ context = list(invocation.data.get("context") or ())
+ for block in content_blocks(context[-12:], "tool_result"):
+ if block.get("name", "") in PIN_HINT_SKIP_TOOLS:
+ continue
+ text = flatten_tool_output(block.get("output"))
+ if '"file_id"' in text:
+ self.seen.update(FILE_ID_RE.findall(text))
+ unpinned = self.seen - set(invocation.data["runtime"].get("pinned_file_ids") or ())
+ signature = ",".join(sorted(unpinned))
+ if not unpinned or signature == self.last_signature:
+ return Decision()
+ preview = sorted(unpinned)[:6]
+ rendered = ", ".join(preview)
+ if len(unpinned) > len(preview):
+ rendered += f", …(+{len(unpinned) - len(preview)} 个)"
+ self.last_signature = signature
+ return append_reminder(invocation,
+ f"沙盒里有 {len(unpinned)} 个 file_id 还没 pin:[{rendered}]。"
+ "若是给用户的最终产物,必须调 `pin_to_workspace(file_ids=[...])` 才能交付。",
+ "harness:workspace_pin")
+
+
+class PlanStaleHook:
+ def __init__(self, interval=10):
+ self.interval = max(1, int(interval))
+ self.since_last = 0
+ self.fingerprint = None
+
+ def __call__(self, invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ blocks = list(content_blocks(invocation.data.get("context") or (), "tool_call"))
+ plan = None
+ for block in reversed(blocks):
+ if block.get("name") != "update_plan":
+ continue
+ raw = block.get("input")
+ if isinstance(raw, str):
+ try:
+ raw = json.loads(raw)
+ except (ValueError, TypeError):
+ raw = None
+ plan = parse_plan_update_args(thaw_value(raw))
+ fingerprint = str(block.get("id") or "") or json.dumps(plan or {}, ensure_ascii=False, sort_keys=True)
+ break
+ if not plan:
+ return Decision()
+ if fingerprint != self.fingerprint:
+ self.fingerprint, self.since_last = fingerprint, 0
+ return Decision()
+ steps = plan.get("steps") or []
+ if not any(step.get("status") in ("pending", "in_progress") for step in steps):
+ return Decision()
+ self.since_last += 1
+ if self.since_last < self.interval:
+ return Decision()
+ done = sum(step.get("status") == "completed" for step in steps)
+ checklist = "\n".join(f" {i}. [{step.get('status', 'pending')}] {step.get('title', '')}"
+ for i, step in enumerate(steps, 1))
+ text = PLAN_REMINDER.format(done=done, total=len(steps), rounds=self.since_last, checklist=checklist)
+ self.since_last = 0
+ return append_reminder(invocation, text, "harness:plan_stale")
+
+
+def workspace_pin_hooks(*, allow_finish=False):
+ policy = WorkspacePinHook()
+ specs = (HookSpec("workspace_pin_reset", HookStage.BEFORE_RUN, policy.reset, failure_mode=FailureMode.OPEN),
+ HookSpec("workspace_pin", HookStage.TRANSFORM_CONTEXT, policy, priority=200,
+ failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"})))
+ if allow_finish:
+ specs += (HookSpec("finish_pin_guard", HookStage.BEFORE_FINISH, policy.finish,
+ failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"pin_file_ids"})),)
+ return specs
+
+
+def plan_stale_hook():
+ return HookSpec("plan_stale", HookStage.TRANSFORM_CONTEXT, PlanStaleHook(), priority=600,
+ failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"}))
+
+PLAN_REMINDER = '`update_plan` 已经 {rounds} 轮没有用过了。当前计划栏停在 {done}/{total}:\n{checklist}\n\n如果其中有步骤其实已经做完,可以调用一次 `update_plan` 把状态补上(传全量列表);如果这份\n清单已经不符合你正在做的事,重写它比让它停在原地更好。仅在确实相关时才使用——这只是一条\n温和的提醒,不适用就忽略,不要为了回应它而打断手上的工作。不要向用户提起这条提醒。'
diff --git a/src/backend/core/llm/context_adapter.py b/src/backend/core/llm/context_adapter.py
deleted file mode 100644
index 297f5a10..00000000
--- a/src/backend/core/llm/context_adapter.py
+++ /dev/null
@@ -1,867 +0,0 @@
-"""The sole Canonical Context IR ↔ AgentScope message adapter.
-
-Compatibility roles are a rendering detail here. Provenance and trust travel
-in structured message metadata and never have to be inferred from XML labels.
-"""
-
-from __future__ import annotations
-
-from dataclasses import replace
-from typing import Any, Iterable, Mapping, Optional, Sequence
-
-from agentscope.message import Msg, TextBlock
-
-from core.immutable import thaw_json
-from core.llm.execution_manifest import stable_hash
-from core.llm.context_ir import (
- CONTEXT_SEQUENCE_STRIDE,
- ContextItem,
- KIND_ASSISTANT,
- KIND_ATTACHMENT,
- KIND_COMPACTION,
- KIND_MEMORY,
- KIND_PROJECT,
- KIND_REFERENCE,
- KIND_REMINDER,
- KIND_SYSTEM_RULE,
- KIND_THINKING,
- KIND_TOOL_CALL,
- KIND_TOOL_RESULT,
- KIND_USER_INPUT,
- POLICY_DROP,
- POLICY_HEAD_TAIL,
- POLICY_NEVER,
- SESSION_CONTEXT_META_KEY,
- STEP_KINDS,
- VISIBILITY_MANIFEST_ONLY,
- VISIBILITY_MODEL,
- estimate_context_tokens,
- make_text_context_item,
-)
-
-CONTEXT_META_KEY = "harness_context_items"
-PROVIDER_CONTEXT_META_KEY = "_harness_context_item"
-
-
-def _block_payload(block: Any) -> Any:
- if isinstance(block, Mapping):
- return dict(block)
- if hasattr(block, "model_dump"):
- return block.model_dump(mode="json")
- return block
-
-
-def _block_type(block: Any) -> str:
- if isinstance(block, Mapping):
- return str(block.get("type") or "")
- return str(getattr(block, "type", "") or "")
-
-
-def _block_text(block: Any) -> str:
- if isinstance(block, Mapping):
- return str(block.get("text") or "")
- return str(getattr(block, "text", "") or "")
-
-
-class AgentScopeContextAdapter:
- """Translate the IR while preserving current model-facing roles/blocks."""
-
- @staticmethod
- def wrap_content(content: Any) -> list[Any]:
- """Construct AgentScope-compatible blocks inside the sole SDK seam."""
- if isinstance(content, str):
- return [TextBlock(type="text", text=content)] if content else []
- if isinstance(content, list):
- return content
- return [content] if content else []
-
- @staticmethod
- def _defaults(role: str, block_type: str) -> dict[str, Any]:
- if block_type == "thinking":
- # Reasoning is handed back to the model verbatim. It is counted on its
- # own so the usage panel can show what it costs, but it is never
- # shortened or dropped apart from the step that produced it.
- return {
- "kind": KIND_THINKING,
- "origin": "assistant:reasoning",
- "trust": "assistant",
- "priority": 400,
- "token_budget": 0,
- "truncation_policy": POLICY_NEVER,
- "render_role": "assistant",
- }
- if block_type == "tool_call":
- return {
- "kind": KIND_TOOL_CALL,
- "origin": "agent:tool_call",
- "trust": "assistant",
- "priority": 700,
- "token_budget": 0,
- "truncation_policy": POLICY_NEVER,
- "render_role": "assistant",
- }
- if block_type == "tool_result":
- return {
- "kind": KIND_TOOL_RESULT,
- "origin": "tool:result",
- "trust": "tool",
- "priority": 700,
- "token_budget": 20_000,
- "truncation_policy": POLICY_HEAD_TAIL,
- "render_role": "assistant",
- }
- if block_type == "data":
- return {
- "kind": KIND_ATTACHMENT,
- "origin": "user:attachment",
- "trust": "user",
- "priority": 650,
- "token_budget": 4_000,
- "truncation_policy": POLICY_DROP,
- "render_role": "user",
- }
- if role == "system":
- return {
- "kind": KIND_SYSTEM_RULE,
- "origin": "platform:system_prompt",
- "trust": "platform",
- "priority": 1_000,
- "token_budget": 100_000,
- "truncation_policy": POLICY_NEVER,
- "render_role": "system",
- }
- if role == "assistant":
- return {
- "kind": KIND_ASSISTANT,
- "origin": "assistant:history",
- "trust": "assistant",
- "priority": 400,
- "token_budget": 0,
- "truncation_policy": POLICY_NEVER,
- "render_role": "assistant",
- }
- return {
- "kind": KIND_USER_INPUT,
- "origin": "user:chat",
- "trust": "user",
- "priority": 500,
- "token_budget": 100_000,
- "truncation_policy": POLICY_HEAD_TAIL,
- "render_role": "user",
- }
-
- @staticmethod
- def _metadata_entries(message: Msg) -> list[Mapping[str, Any]]:
- metadata = getattr(message, "metadata", None) or {}
- raw = metadata.get(CONTEXT_META_KEY) if isinstance(metadata, Mapping) else None
- if not isinstance(raw, list):
- return []
- return [entry for entry in raw if isinstance(entry, Mapping)]
-
- def items_from_messages(
- self,
- messages: Sequence[Msg],
- *,
- summary_text: Any = None,
- promote_latest_user: bool = True,
- ) -> list[ContextItem]:
- items: list[ContextItem] = []
- summary = str(summary_text or "")
- tool_occurrences: dict[str, int] = {}
- pending_tool_pairs: dict[str, list[str]] = {}
- orphan_results: dict[str, int] = {}
- # One ReAct step = what the model produced in one response plus the
- # results of the tool calls in it. A step opens on the first thinking,
- # text or tool_call block after a tool result (or after any other
- # message) and its results join it through the pending call ids. This
- # is the same boundary the reasoning-echo formatter flushes on.
- step_unit: Optional[str] = None
- step_closed = False
- pair_units: dict[str, str] = {}
- for message_index, message in enumerate(messages):
- role = str(getattr(message, "role", "user") or "user")
- name = str(getattr(message, "name", role) or role)
- blocks = list(getattr(message, "content", None) or [])
- entries = self._metadata_entries(message)
- for block_index, block in enumerate(blocks):
- block_type = _block_type(block)
- payload = _block_text(block) if block_type == "text" else _block_payload(block)
- defaults = self._defaults(role, block_type)
- explicit = entries[block_index] if block_index < len(entries) else {}
- if (
- not explicit
- and summary
- and role == "user"
- and isinstance(payload, str)
- and payload == summary
- ):
- defaults.update(
- kind=KIND_COMPACTION,
- origin="harness:compaction",
- trust="system",
- priority=850,
- truncation_policy=POLICY_HEAD_TAIL,
- render_role="user",
- )
- raw_pair_id = str(
- payload.get("id")
- if block_type in {"tool_call", "tool_result"} and isinstance(payload, Mapping)
- else ""
- )
- pair_id = str(explicit.get("pair_id") or "")
- if block_type == "tool_call" and raw_pair_id:
- occurrence = tool_occurrences.get(raw_pair_id, 0) + 1
- tool_occurrences[raw_pair_id] = occurrence
- pair_id = f"{raw_pair_id}#{occurrence}"
- pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id)
- elif block_type == "tool_result" and raw_pair_id:
- pending = pending_tool_pairs.get(raw_pair_id) or []
- if pending:
- pair_id = pending.pop(0)
- else:
- occurrence = orphan_results.get(raw_pair_id, 0) + 1
- orphan_results[raw_pair_id] = occurrence
- pair_id = f"{raw_pair_id}#orphan-result-{occurrence}"
- item_id = str(
- (
- f"message:{message_index}:block:{block_index}:{block_type}:{pair_id}"
- if block_type in {"tool_call", "tool_result"}
- else explicit.get("item_id")
- or f"message:{message_index}:block:{block_index}"
- )
- )
- kind = str(explicit.get("kind") or defaults["kind"])
- if kind in STEP_KINDS and block_type != "tool_result":
- if step_unit is None or step_closed:
- step_unit = f"step:{message_index}:{block_index}"
- step_closed = False
- unit_id = step_unit
- if block_type == "tool_call":
- pair_units[pair_id] = unit_id
- elif block_type == "tool_result":
- unit_id = pair_units.pop(pair_id, f"orphan:{item_id}")
- step_closed = True
- else:
- unit_id = f"item:{item_id}"
- step_unit = None
- step_closed = False
- pair_units.clear()
- created_seq = int(
- explicit.get("created_seq")
- if explicit.get("created_seq") is not None
- else message_index * CONTEXT_SEQUENCE_STRIDE + block_index
- )
- item = ContextItem.create(
- item_id=item_id,
- kind=kind,
- origin=str(explicit.get("origin") or defaults["origin"]),
- trust=str(explicit.get("trust") or defaults["trust"]),
- visibility=str(explicit.get("visibility") or VISIBILITY_MODEL),
- priority=int(
- explicit["priority"]
- if explicit.get("priority") is not None
- else defaults["priority"]
- ),
- token_budget=int(
- explicit["token_budget"]
- if explicit.get("token_budget") is not None
- else defaults["token_budget"]
- ),
- truncation_policy=str(
- explicit.get("truncation_policy") or defaults["truncation_policy"]
- ),
- content=payload,
- cache_class=str(explicit.get("cache_class") or "dynamic"),
- created_seq=created_seq,
- render_role=str(explicit.get("render_role") or defaults["render_role"]),
- # The outer message carries the actual transport name.
- # Persisted manifests contain only its hash.
- render_name=name,
- pair_id=pair_id,
- message_group=str(explicit.get("message_group") or f"message:{message_index}"),
- unit_id=unit_id,
- content_ref=(
- str(explicit["content_ref"]) if explicit.get("content_ref") else None
- ),
- content_hash=(
- str(explicit["content_hash"]) if explicit.get("content_hash") else None
- ),
- token_estimate=(
- int(explicit["token_estimate"])
- if explicit.get("token_estimate") is not None
- else None
- ),
- metadata={},
- )
- items.append(item)
-
- # The latest real user instruction is a hard inclusion. Memory and
- # reminders may render as user for compatibility but keep distinct kind.
- user_indices = [i for i, item in enumerate(items) if item.kind == KIND_USER_INPUT]
- if promote_latest_user and user_indices:
- idx = user_indices[-1]
- items[idx] = replace(
- items[idx],
- priority=max(900, items[idx].priority),
- truncation_policy=POLICY_NEVER,
- )
- return items
-
- def items_from_session_dict(
- self,
- message: Mapping[str, Any],
- *,
- created_seq: int,
- ) -> list[ContextItem]:
- role = str(message.get("role") or "user")
- role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role)
- content = message.get("content")
- blocks = (
- content
- if isinstance(content, list)
- else ([{"type": "text", "text": content}] if content else [])
- )
- explicit = message.get(SESSION_CONTEXT_META_KEY)
- metadata_entries = []
- if isinstance(explicit, Mapping):
- metadata_entries = [dict(explicit) for _ in blocks]
- msg = Msg(
- name=str(message.get("name") or role),
- role=role if role in {"system", "assistant", "user"} else "user",
- content=blocks,
- metadata={CONTEXT_META_KEY: metadata_entries} if metadata_entries else {},
- )
- # A persisted row is converted in isolation, so it cannot know whether
- # it is the current user instruction. Promotion happens once, later,
- # when ManifestBoundAgent sees the complete final request.
- items = self.items_from_messages([msg], promote_latest_user=False)
- has_explicit_seq = isinstance(explicit, Mapping) and explicit.get("created_seq") is not None
- return [
- replace(
- item,
- item_id=(
- item.item_id
- if isinstance(explicit, Mapping) and explicit.get("item_id")
- else f"session:{created_seq}:block:{index}"
- ),
- created_seq=(
- item.created_seq
- if has_explicit_seq
- else created_seq * CONTEXT_SEQUENCE_STRIDE + index
- ),
- message_group=f"session:{created_seq}",
- unit_id=f"session:{created_seq}:{item.unit_id}",
- )
- for index, item in enumerate(items)
- ]
-
- def message_from_session_dict(
- self,
- message: Mapping[str, Any],
- *,
- created_seq: int,
- ) -> Msg:
- """Render one persisted history record through the canonical adapter."""
- items = self.items_from_session_dict(message, created_seq=created_seq)
- rendered = self.messages_from_items(items)
- if rendered:
- return rendered[0]
- role = str(message.get("role") or "user")
- role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role)
- if role not in {"system", "assistant", "user"}:
- role = "user"
- return Msg(
- name=str(message.get("name") or role),
- role=role,
- content=[],
- metadata={CONTEXT_META_KEY: []},
- )
-
- def messages_from_items(self, items: Iterable[ContextItem]) -> list[Msg]:
- visible = [item for item in items if item.visibility == VISIBILITY_MODEL]
- messages: list[Msg] = []
- current_group: Optional[tuple[str, str]] = None
- current_items: list[ContextItem] = []
-
- def flush() -> None:
- nonlocal current_group, current_items
- if not current_items or current_group is None:
- return
- blocks = []
- manifests = []
- for item in current_items:
- content = thaw_json(item.content)
- if isinstance(content, str):
- blocks.append(TextBlock(type="text", text=content))
- manifests.append(item.to_manifest())
- elif isinstance(content, Mapping):
- blocks.append(dict(content))
- manifests.append(item.to_manifest())
- elif isinstance(content, list):
- for block_index, block in enumerate(content):
- blocks.append(block)
- normalized = (
- _block_text(block)
- if _block_type(block) == "text"
- else _block_payload(block)
- )
- derived = replace(
- item,
- item_id=f"{item.item_id}:block:{block_index}",
- truncation_policy=(
- POLICY_DROP
- if _block_type(block) == "data"
- else item.truncation_policy
- ),
- ).with_content(normalized)
- manifests.append(derived.to_manifest())
- elif content is not None:
- blocks.append(TextBlock(type="text", text=str(content)))
- manifests.append(item.with_content(str(content)).to_manifest())
- if blocks:
- name = str(current_items[0].render_name or current_group[1])
- messages.append(
- Msg(
- name=name,
- role=current_group[1],
- content=blocks,
- metadata={CONTEXT_META_KEY: manifests},
- )
- )
- current_group = None
- current_items = []
-
- for item in visible:
- group = (item.message_group, item.render_role)
- if current_group is not None and group != current_group:
- flush()
- current_group = group
- current_items.append(item)
- flush()
- return messages
-
- def annotate_provider_messages(
- self,
- source_messages: Sequence[Msg],
- formatted_messages: Sequence[Mapping[str, Any]],
- row_counts: Sequence[int],
- ) -> list[dict[str, Any]]:
- """Carry canonical provenance across an SDK formatter boundary.
-
- AgentScope's OpenAI formatter intentionally drops arbitrary message
- metadata and may expand one tool-result message into a tool row plus a
- synthetic media row. ``row_counts`` is obtained by formatting each
- source message in isolation after a provider has rejected media. This
- lets the retry inherit the original IR provenance without guessing it
- from compatibility roles or XML-like text.
- """
- annotated = [dict(message) for message in formatted_messages]
- cursor = 0
- for source, count in zip(source_messages, row_counts):
- entries = self._metadata_entries(source)
- source_rows = annotated[cursor : cursor + max(0, int(count))]
- cursor += max(0, int(count))
- if not entries:
- continue
-
- def pick(kind: str) -> Mapping[str, Any]:
- match = next((entry for entry in entries if entry.get("kind") == kind), None)
- if match is not None:
- return match
- user = next(
- (entry for entry in entries if entry.get("kind") == KIND_USER_INPUT),
- None,
- )
- return user or entries[0]
-
- for row in source_rows:
- role = str(row.get("role") or "user")
- name = str(row.get("name") or role)
- if name == "system-reminder":
- base = pick(KIND_TOOL_RESULT)
- metadata = {
- "kind": KIND_ATTACHMENT,
- "origin": f"{base.get('origin') or 'tool:result'}:media",
- "trust": str(base.get("trust") or "tool"),
- "priority": int(base.get("priority") or 700),
- }
- elif role == "tool":
- base = pick(KIND_TOOL_RESULT)
- metadata = {
- "kind": KIND_TOOL_RESULT,
- "origin": str(base.get("origin") or "tool:result"),
- "trust": str(base.get("trust") or "tool"),
- "priority": int(base.get("priority") or 700),
- }
- elif role == "assistant" and row.get("tool_calls"):
- base = pick(KIND_TOOL_CALL)
- metadata = {
- "kind": KIND_TOOL_CALL,
- "origin": str(base.get("origin") or "agent:tool_call"),
- "trust": str(base.get("trust") or "assistant"),
- "priority": int(base.get("priority") or 700),
- }
- else:
- base = pick("")
- metadata = {
- key: base[key]
- for key in ("kind", "origin", "trust", "priority")
- if base.get(key) is not None
- }
- row[PROVIDER_CONTEXT_META_KEY] = metadata
- if cursor != len(annotated):
- # A formatter shape change must fail visibly rather than silently
- # assigning later rows the wrong trust boundary.
- raise ValueError(
- "provider formatter row mapping mismatch: "
- f"mapped={cursor} formatted={len(annotated)}"
- )
- return annotated
-
- def items_from_provider_messages(
- self,
- messages: Sequence[Mapping[str, Any]],
- ) -> list[ContextItem]:
- """Reify an already-formatted provider retry as exact canonical items.
-
- A provider may reject media after the normal AgentScope request has
- already been assembled. The retry is therefore a new execution
- surface. Each complete wire message is kept atomically so its hash
- covers role, name, tool identifiers and content together; transient
- provenance metadata is removed before hashing and before transport.
- """
- items: list[ContextItem] = []
- pending_tool_pairs: dict[str, list[str]] = {}
- orphan_results: dict[str, int] = {}
- for index, raw_message in enumerate(messages):
- message = dict(raw_message)
- explicit = message.pop(PROVIDER_CONTEXT_META_KEY, None)
- explicit = explicit if isinstance(explicit, Mapping) else {}
- role = str(message.get("role") or "user")
- name = str(message.get("name") or role)
- tool_calls = message.get("tool_calls")
- if role == "assistant" and isinstance(tool_calls, list) and tool_calls:
- pair_id = f"provider-tool-batch:{index}"
- message_hash = stable_hash(message)
- message_tokens = estimate_context_tokens(message)
- for call_index, call in enumerate(tool_calls):
- call = call if isinstance(call, Mapping) else {}
- raw_pair_id = str(call.get("id") or "")
- pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id)
- items.append(
- ContextItem.create(
- item_id=f"provider-retry:{index}:tool-call:{call_index}",
- kind=str(explicit.get("kind") or KIND_TOOL_CALL),
- origin=str(explicit.get("origin") or "assistant:provider_tool_call"),
- trust=str(explicit.get("trust") or "assistant"),
- visibility=VISIBILITY_MODEL,
- priority=int(explicit.get("priority") or 900),
- token_budget=message_tokens if call_index == 0 else 0,
- truncation_policy=POLICY_NEVER,
- content=message if call_index == 0 else None,
- content_ref=f"provider-message:{index}",
- content_hash=message_hash,
- token_estimate=message_tokens if call_index == 0 else 0,
- cache_class="dynamic",
- created_seq=index * CONTEXT_SEQUENCE_STRIDE + call_index,
- render_role=role,
- render_name=name,
- pair_id=pair_id,
- message_group=f"provider-retry:{index}",
- unit_id=pair_id,
- )
- )
- continue
- if role == "tool":
- raw_pair_id = str(message.get("tool_call_id") or "")
- pending = pending_tool_pairs.get(raw_pair_id) or []
- if pending:
- pair_id = pending.pop(0)
- else:
- occurrence = orphan_results.get(raw_pair_id, 0) + 1
- orphan_results[raw_pair_id] = occurrence
- pair_id = f"{raw_pair_id}#orphan-result-{occurrence}"
- defaults = {
- "kind": KIND_TOOL_RESULT,
- "origin": "tool:provider_result",
- "trust": "tool",
- "priority": 900,
- "pair_id": pair_id,
- }
- elif role == "system":
- defaults = {
- "kind": KIND_SYSTEM_RULE,
- "origin": "platform:provider_request",
- "trust": "platform",
- "priority": 1_000,
- }
- elif role == "assistant":
- defaults = {
- "kind": KIND_ASSISTANT,
- "origin": "assistant:provider_history",
- "trust": "assistant",
- "priority": 600,
- }
- else:
- defaults = {
- "kind": KIND_USER_INPUT,
- "origin": "user:provider_request",
- "trust": "user",
- "priority": 900,
- }
- items.append(
- ContextItem.create(
- item_id=str(explicit.get("item_id") or f"provider-retry:{index}"),
- kind=str(explicit.get("kind") or defaults["kind"]),
- origin=str(explicit.get("origin") or defaults["origin"]),
- trust=str(explicit.get("trust") or defaults["trust"]),
- visibility=VISIBILITY_MODEL,
- priority=int(explicit.get("priority") or defaults["priority"]),
- # These rows have already survived the canonical budget
- # pass. Media recovery only removes data or replaces it
- # with a smaller textual description, so the retry must
- # preserve every resulting row exactly.
- token_budget=max(1, estimate_context_tokens(message)),
- truncation_policy=POLICY_NEVER,
- content=message,
- cache_class="dynamic",
- created_seq=int(
- explicit.get("created_seq")
- if explicit.get("created_seq") is not None
- else index * CONTEXT_SEQUENCE_STRIDE
- ),
- render_role=role,
- render_name=name,
- pair_id=str(defaults.get("pair_id") or ""),
- message_group=f"provider-retry:{index}",
- unit_id=str(defaults.get("pair_id") or f"provider-retry:{index}"),
- )
- )
- return items
-
- @staticmethod
- def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]:
- """Render exact provider rows while stripping harness-only metadata."""
- rendered: list[dict[str, Any]] = []
- by_group: dict[str, dict[str, Any]] = {}
- deferred_groups: set[str] = set()
- for item in items:
- if item.visibility != VISIBILITY_MODEL:
- continue
- value = thaw_json(item.content)
- if value is None:
- deferred_groups.add(item.message_group)
- continue
- if not isinstance(value, Mapping):
- raise ValueError("provider retry context item must contain one message mapping")
- row = dict(value)
- row.pop(PROVIDER_CONTEXT_META_KEY, None)
- previous = by_group.get(item.message_group)
- if previous is not None:
- if previous != row:
- raise ValueError("provider retry group contains conflicting wire messages")
- continue
- by_group[item.message_group] = row
- rendered.append(row)
- missing = deferred_groups - set(by_group)
- if missing:
- raise ValueError(f"provider retry groups lack a wire row: {sorted(missing)}")
- return rendered
-
- def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]:
- if hasattr(manifest, "to_dict"):
- payload = manifest.to_dict()
- elif isinstance(manifest, Mapping):
- payload = dict(manifest)
- else:
- return []
- sections = (payload.get("prompt_manifest") or {}).get("sections") or []
- items: list[ContextItem] = []
- for index, section in enumerate(sections):
- if not isinstance(section, Mapping):
- continue
- section_id = str(section.get("id") or "")
- content_hash = str(section.get("content_hash") or "")
- if not section_id or not content_hash:
- continue
- is_project = section_id == "runtime/project"
- live_content = ""
- content_reader = getattr(manifest, "prompt_section_content", None)
- if callable(content_reader):
- live_content = str(content_reader(index) or "")
- if is_project and live_content:
- items.append(
- ContextItem.create(
- item_id=f"prompt:{index}:{section_id}",
- kind=KIND_PROJECT,
- origin=str(section.get("origin") or "workspace:project"),
- trust=str(section.get("trust") or "workspace"),
- visibility=VISIBILITY_MODEL,
- priority=int(section.get("priority") or 900),
- token_budget=max(
- 1,
- int(section.get("budget") or section.get("token_estimate") or 1),
- ),
- truncation_policy=POLICY_HEAD_TAIL,
- content=live_content,
- content_ref=(
- str(section["reference"]) if section.get("reference") else None
- ),
- content_hash=content_hash,
- token_estimate=int(section.get("token_estimate") or 0),
- cache_class=str(section.get("cache_class") or "workspace"),
- created_seq=1,
- render_role="system",
- render_name="system",
- message_group=f"prompt:{index}",
- )
- )
- continue
- items.append(
- ContextItem.create(
- item_id=f"prompt:{index}:{section_id}",
- kind=KIND_PROJECT if is_project else KIND_SYSTEM_RULE,
- origin=str(section.get("origin") or "prompt:unknown"),
- trust=str(section.get("trust") or "platform"),
- visibility=VISIBILITY_MANIFEST_ONLY,
- priority=int(section.get("priority") or 0),
- token_budget=int(section.get("budget") or 0),
- truncation_policy=POLICY_NEVER,
- content=None,
- content_ref=str(
- section.get("reference")
- or f"prompt:{section_id}@{section.get('version') or '1'}"
- ),
- content_hash=content_hash,
- token_estimate=int(section.get("token_estimate") or 0),
- cache_class=str(section.get("cache_class") or "stable"),
- created_seq=-10_000 + index,
- render_role="system",
- message_group=f"prompt:{index}",
- )
- )
- return items
-
-
-def append_state_context_item(state: Any, item: ContextItem) -> Msg:
- """Render and append one explicitly-provenanced item to AgentScope state."""
- messages = AgentScopeContextAdapter().messages_from_items([item])
- if len(messages) != 1:
- raise ValueError("context item did not render to exactly one message")
- message = messages[0]
- state.context.append(message)
- return message
-
-
-def render_text_block(text: str) -> TextBlock:
- """Construct an AgentScope text block inside the SDK adapter boundary."""
- return TextBlock(type="text", text=str(text or ""))
-
-
-def next_context_sequence(messages: Sequence[Any]) -> int:
- """Allocate one monotonic sequence across replay and live injections."""
- max_sequence = -CONTEXT_SEQUENCE_STRIDE
- for message_index, message in enumerate(messages):
- entries = AgentScopeContextAdapter._metadata_entries(message)
- blocks = list(getattr(message, "content", None) or [])
- for block_index, _block in enumerate(blocks or [None]):
- explicit = entries[block_index] if block_index < len(entries) else {}
- candidate = explicit.get("created_seq")
- if candidate is None:
- candidate = message_index * CONTEXT_SEQUENCE_STRIDE + block_index
- try:
- max_sequence = max(max_sequence, int(candidate))
- except (TypeError, ValueError):
- continue
- return ((max_sequence // CONTEXT_SEQUENCE_STRIDE) + 1) * CONTEXT_SEQUENCE_STRIDE
-
-
-def next_request_sequence(messages: Sequence[Any]) -> int:
- """Reserve a pre-reply lane above the context tail for the user instruction.
-
- AgentScope runs ``on_reply`` middleware before it appends ``inputs`` to
- state. File/project attachment middleware can therefore add a few explicit
- context items first without colliding with the request's sequence. The
- sequence no longer decides message order — the assembler emits items in
- context order — it only keeps identifiers distinct and marks recency for
- budget eviction.
- """
- pre_reply_slots = 64
- return next_context_sequence(messages) + pre_reply_slots * CONTEXT_SEQUENCE_STRIDE
-
-
-def render_context_item(item: ContextItem) -> Msg:
- """Render one IR item for ``Agent.reply`` without mutating agent state."""
- messages = AgentScopeContextAdapter().messages_from_items([item])
- if len(messages) != 1:
- raise ValueError("context item did not render to exactly one message")
- return messages[0]
-
-
-def render_session_input(message: Mapping[str, Any], *, created_seq: int) -> Msg:
- """Render one pending positional row at an explicit live request sequence."""
- adapter = AgentScopeContextAdapter()
- items = adapter.items_from_session_dict(message, created_seq=0)
- positioned = [
- replace(
- item,
- created_seq=created_seq + index,
- message_group=f"request:{created_seq}",
- )
- for index, item in enumerate(items)
- ]
- rendered = adapter.messages_from_items(positioned)
- if len(rendered) != 1:
- raise ValueError("session input did not render to exactly one message")
- return rendered[0]
-
-
-def append_context_item(agent: Any, item: ContextItem) -> Msg:
- """Append one explicitly-provenanced item through the adapter seam."""
- return append_state_context_item(agent.state, item)
-
-
-def append_context_text(
- agent: Any,
- text: str,
- *,
- kind: str,
- origin: str,
- trust: str,
- priority: int = 700,
- token_budget: int = 4_000,
- truncation_policy: str = POLICY_HEAD_TAIL,
- render_role: str = "user",
-) -> Msg:
- """Append provenanced text without creating AgentScope messages elsewhere."""
- created_seq = next_context_sequence(agent.state.context)
- item = make_text_context_item(
- text,
- item_id=f"{origin}:{created_seq}",
- kind=kind,
- origin=origin,
- trust=trust,
- created_seq=created_seq,
- priority=priority,
- token_budget=token_budget,
- truncation_policy=truncation_policy,
- render_role=render_role,
- )
- return append_context_item(agent, item)
-
-
-__all__ = [
- "AgentScopeContextAdapter",
- "CONTEXT_META_KEY",
- "PROVIDER_CONTEXT_META_KEY",
- "SESSION_CONTEXT_META_KEY",
- "append_context_item",
- "append_state_context_item",
- "append_context_text",
- "make_text_context_item",
- "next_context_sequence",
- "next_request_sequence",
- "render_context_item",
- "render_session_input",
- "render_text_block",
-]
diff --git a/src/backend/core/llm/context_ir.py b/src/backend/core/llm/context_ir.py
deleted file mode 100644
index 5f68c755..00000000
--- a/src/backend/core/llm/context_ir.py
+++ /dev/null
@@ -1,855 +0,0 @@
-"""Framework-neutral canonical context IR and deterministic budget assembler.
-
-This module deliberately has no AgentScope dependency. It owns provenance,
-trust, selection, truncation and the sanitized inclusion/exclusion manifest;
-``context_adapter`` is the only layer that translates these items to model SDK
-messages.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field, replace
-from typing import Any, Iterable, Mapping, Optional, Protocol, Sequence
-
-from core.immutable import FrozenDict, freeze_json, thaw_json
-from core.llm.execution_manifest import canonical_json, stable_hash
-
-CONTEXT_SCHEMA_VERSION = "harness.context.v2"
-SESSION_CONTEXT_META_KEY = "_context_item"
-CONTEXT_SEQUENCE_STRIDE = 1_000
-# Matches openai/codex's per-image estimate (RESIZED_IMAGE_BYTES_ESTIMATE = 7373
-# bytes over its 4-bytes/token heuristic). One price per image, never its
-# transport bytes; the provider's reported usage supersedes it once a reply lands.
-IMAGE_TOKEN_RESERVE = 1_844
-
-KIND_SYSTEM_RULE = "system_rule"
-KIND_USER_INPUT = "user_input"
-KIND_ASSISTANT = "assistant_history"
-KIND_MEMORY = "memory"
-KIND_IDENTITY = "identity"
-KIND_PROJECT = "project_material"
-KIND_THINKING = "thinking"
-KIND_TOOL_CALL = "tool_call"
-KIND_TOOL_RESULT = "tool_result"
-KIND_REMINDER = "reminder"
-KIND_COMPACTION = "compaction_summary"
-KIND_ATTACHMENT = "attachment"
-KIND_STEER = "steer"
-KIND_REFERENCE = "reference"
-
-POLICY_NEVER = "never"
-POLICY_DROP = "drop"
-POLICY_HEAD_TAIL = "head_tail"
-POLICY_TAIL = "tail"
-
-VISIBILITY_MODEL = "model"
-VISIBILITY_MANIFEST_ONLY = "manifest_only"
-
-_TOOL_KINDS = {KIND_TOOL_CALL, KIND_TOOL_RESULT}
-# One ReAct step is the atomic unit of history: what the model said (thinking,
-# text, tool calls) together with the results those calls produced. These
-# kinds are never budgeted block by block.
-STEP_KINDS = frozenset({KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT})
-# Conversation history is selected as a contiguous tail; everything else
-# (memory, project material, reminders, attachments, the compaction summary)
-# keeps priority-based selection because it is not ordered by time.
-CONVERSATION_KINDS = STEP_KINDS | {KIND_USER_INPUT}
-_TRUNCATABLE_BLOCK_TYPES = {"tool_result"}
-_PRUNED_OUTPUT_TEMPLATE = "[tool output pruned to fit the context budget: {tokens} tokens omitted]"
-
-
-class ContextAdapterProtocol(Protocol):
- """Framework-neutral seam for turning transport rows into canonical IR."""
-
- def items_from_messages(
- self,
- messages: Sequence[Any],
- *,
- summary_text: Any = None,
- promote_latest_user: bool = True,
- ) -> list["ContextItem"]: ...
-
- def messages_from_items(self, items: Iterable["ContextItem"]) -> list[Any]: ...
-
- def reference_items_from_execution_manifest(
- self, manifest: Any
- ) -> list["ContextItem"]: ...
-
- def items_from_provider_messages(
- self,
- messages: Sequence[Mapping[str, Any]],
- ) -> list["ContextItem"]: ...
-
- def provider_messages_from_items(
- self,
- items: Iterable["ContextItem"],
- ) -> list[Mapping[str, Any]]: ...
-
-
-def _normalize_context_content(content: Any) -> Any:
- """Convert SDK block objects into stable, framework-neutral JSON values."""
- if hasattr(content, "model_dump"):
- return _normalize_context_content(content.model_dump(mode="json"))
- if isinstance(content, Mapping):
- return {
- str(key): _normalize_context_content(value)
- for key, value in content.items()
- }
- if isinstance(content, (list, tuple)):
- return [_normalize_context_content(value) for value in content]
- return content
-
-
-def _tool_result_images(content: Any) -> list[Any]:
- """Image payloads are atomic media, not part of a tool's text allowance."""
- if not isinstance(content, Mapping) or content.get("type") != "tool_result":
- return []
- output = content.get("output")
- if not isinstance(output, (list, tuple)):
- return []
- return [
- block
- for block in output
- if isinstance(block, Mapping)
- and block.get("type") == "data"
- and isinstance(block.get("source"), Mapping)
- and str(block["source"].get("media_type") or "").startswith("image/")
- ]
-
-
-def image_token_reserve(content: Any) -> int:
- """Reserve media occupancy without measuring image transport bytes."""
- if isinstance(content, (list, tuple)):
- return sum(image_token_reserve(block) for block in content)
- if isinstance(content, Mapping):
- source = content.get("source")
- if (
- content.get("type") == "data"
- and isinstance(source, Mapping)
- and str(source.get("media_type") or "").startswith("image/")
- ):
- return IMAGE_TOKEN_RESERVE
- if content.get("type") == "tool_result":
- return image_token_reserve(content.get("output", content.get("content")))
- return 0
-
-
-def estimate_context_tokens(content: Any) -> int:
- """Deterministic conservative estimate without provider-specific imports."""
- content = _normalize_context_content(content)
- if content is None:
- return 0
- if isinstance(content, str):
- return max(1, (len(content.encode("utf-8")) + 3) // 4) if content else 0
- if isinstance(content, Mapping) and content.get("type") == "data":
- # Provider image accounting is not proportional to base64 bytes. Keep a
- # conservative fixed reserve without letting transport encoding evict
- # the actual attachment before the provider can count it precisely.
- return IMAGE_TOKEN_RESERVE
- if _tool_result_images(content):
- # Count image reserves recursively instead of tokenizing their base64
- # inside the enclosing tool-result JSON.
- return estimate_context_tokens({**content, "output": []}) + sum(
- estimate_context_tokens(block) for block in content["output"]
- )
- if isinstance(content, list):
- return sum(estimate_context_tokens(item) for item in content)
- return max(1, (len(canonical_json(content).encode("utf-8")) + 3) // 4)
-
-
-def _truncate_text(text: str, max_tokens: int, *, tail_only: bool = False) -> str:
- if not text or max_tokens <= 0:
- return ""
- max_bytes = max(4, max_tokens * 4)
- if len(text.encode("utf-8")) <= max_bytes:
- return text
- # Character slicing is deterministic and safe for Unicode. Iterate down if
- # multi-byte text still exceeds the byte reserve.
- max_chars = max(1, max_bytes)
- head = 0
- tail = 0
- marker = ""
- if tail_only:
- tail = max_chars
- result = text[-tail:]
- elif max_chars < 40:
- head = max_chars
- result = text[:head]
- else:
- marker = "\n[… omitted …]\n"
- body = max(2, max_chars - len(marker))
- head = max(1, int(body * 0.6))
- tail = max(1, body - head)
- result = text[:head] + marker + text[-tail:]
- while len(result.encode("utf-8")) > max_bytes and len(result) > 1:
- if tail_only:
- tail = max(1, tail - 1)
- result = text[-tail:]
- elif marker:
- # Preserve both semantic ends; shrink the larger slice first.
- if head >= tail and head > 1:
- head -= 1
- elif tail > 1:
- tail -= 1
- elif head > 1:
- head -= 1
- else:
- break
- result = text[:head] + marker + text[-tail:]
- else:
- head = max(1, head - 1)
- result = text[:head]
- return result
-
-
-def _truncate_content(content: Any, max_tokens: int, policy: str) -> Any:
- """Shorten prunable text only; every structured block keeps its shape.
-
- Plain text and a tool result's output are the only things a budget may
- shorten. Any other block — thinking, tool call, data, or a shape this
- module does not know — is returned untouched so the caller keeps or drops
- it whole. Rewriting such a block into a JSON string would hand the model
- its own reasoning back as an answer.
- """
- if max_tokens <= 0:
- return ""
- if isinstance(content, str):
- return _truncate_text(content, max_tokens, tail_only=policy == POLICY_TAIL)
- if isinstance(content, Mapping):
- mutable = thaw_json(content)
- block_type = str(mutable.get("type") or "")
- if block_type not in _TRUNCATABLE_BLOCK_TYPES:
- return mutable
- for key in ("output", "content", "text"):
- value = mutable.get(key)
- if isinstance(value, str):
- overhead = estimate_context_tokens({**mutable, key: ""})
- mutable[key] = _truncate_text(
- value,
- max(1, max_tokens - overhead),
- tail_only=policy == POLICY_TAIL,
- )
- return mutable
- if key == "output" and isinstance(value, (list, Mapping)):
- images = _tool_result_images(mutable)
- if images:
- text = "\n".join(
- (
- str(block["text"])
- if isinstance(block, Mapping) and block.get("text") is not None
- else block if isinstance(block, str) else canonical_json(block)
- )
- for block in value
- if block not in images
- )
- output = [{"type": "text", "text": ""}, *images]
- overhead = estimate_context_tokens({**mutable, key: output})
- output[0]["text"] = _truncate_text(
- text, max(1, max_tokens - overhead), tail_only=policy == POLICY_TAIL
- )
- mutable[key] = output
- return mutable
- if isinstance(value, list):
- parts = []
- for block in value:
- if isinstance(block, Mapping) and block.get("text") is not None:
- parts.append(str(block["text"]))
- elif isinstance(block, str):
- parts.append(block)
- else:
- parts.append(canonical_json(block))
- flattened = "\n".join(parts)
- else:
- flattened = canonical_json(value)
- overhead = estimate_context_tokens({**mutable, key: ""})
- mutable[key] = _truncate_text(
- flattened,
- max(1, max_tokens - overhead),
- tail_only=policy == POLICY_TAIL,
- )
- return mutable
- return mutable
- return thaw_json(content)
-
-
-def _prune_tool_output(content: Any, original_tokens: int) -> Any:
- """Replace a tool result's output with an explicit placeholder."""
- mutable = thaw_json(content)
- if not isinstance(mutable, dict) or str(mutable.get("type") or "") != "tool_result":
- raise ValueError("only tool_result content can be pruned")
- images = _tool_result_images(mutable)
- if images:
- mutable["output"] = [
- {"type": "text", "text": _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)},
- *images,
- ]
- return mutable
- for key in ("output", "content", "text"):
- if key in mutable:
- mutable[key] = _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)
- return mutable
- mutable["output"] = _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)
- return mutable
-
-
-@dataclass(frozen=True)
-class ContextItem:
- item_id: str
- kind: str
- origin: str
- trust: str
- visibility: str
- priority: int
- token_budget: int
- truncation_policy: str
- content_ref: str
- content_hash: str
- cache_class: str
- created_seq: int
- token_estimate: int
- render_role: str = "user"
- render_name: str = ""
- pair_id: str = ""
- message_group: str = ""
- unit_id: str = ""
- content: Any = field(default=None, repr=False, compare=False)
- metadata: Mapping[str, Any] = field(
- default_factory=FrozenDict, repr=False, compare=False
- )
-
- def __post_init__(self) -> None:
- object.__setattr__(self, "content", freeze_json(self.content))
- object.__setattr__(self, "metadata", freeze_json(self.metadata or {}))
-
- @classmethod
- def create(
- cls,
- *,
- item_id: str,
- kind: str,
- origin: str,
- trust: str,
- visibility: str,
- priority: int,
- token_budget: int,
- truncation_policy: str,
- content: Any,
- cache_class: str,
- created_seq: int,
- render_role: str = "user",
- pair_id: str = "",
- message_group: str = "",
- unit_id: str = "",
- content_ref: Optional[str] = None,
- content_hash: Optional[str] = None,
- token_estimate: Optional[int] = None,
- render_name: str = "",
- metadata: Optional[Mapping[str, Any]] = None,
- ) -> "ContextItem":
- normalized_content = _normalize_context_content(content)
- if (
- isinstance(normalized_content, Mapping)
- and str(normalized_content.get("type") or "") == "data"
- ):
- truncation_policy = POLICY_DROP
- computed_hash = stable_hash(normalized_content)
- supplied_hash = str(content_hash or "")
- external_reference = normalized_content is None and bool(supplied_hash)
- digest = supplied_hash if external_reference else computed_hash
- estimate = (
- max(0, int(token_estimate or 0))
- if external_reference
- else estimate_context_tokens(normalized_content)
- )
- reference = (
- str(content_ref)
- if content_ref is not None
- and (external_reference or supplied_hash == computed_hash)
- else f"sha256:{digest}"
- )
- return cls(
- item_id=str(item_id),
- kind=str(kind),
- origin=str(origin),
- trust=str(trust),
- visibility=str(visibility),
- priority=int(priority),
- token_budget=max(0, int(token_budget)),
- truncation_policy=str(truncation_policy),
- content_ref=reference,
- content_hash=digest,
- cache_class=str(cache_class),
- created_seq=int(created_seq),
- token_estimate=estimate,
- render_role=str(render_role),
- render_name=str(render_name or ""),
- pair_id=str(pair_id or ""),
- message_group=str(message_group or f"item:{item_id}"),
- unit_id=str(unit_id or f"item:{item_id}"),
- content=normalized_content,
- metadata=metadata or {},
- )
-
- def with_content(self, content: Any) -> "ContextItem":
- normalized_content = _normalize_context_content(content)
- digest = stable_hash(normalized_content)
- return replace(
- self,
- content=freeze_json(normalized_content),
- content_hash=digest,
- content_ref=f"sha256:{digest}",
- token_estimate=estimate_context_tokens(normalized_content),
- )
-
- def to_manifest(self) -> dict[str, Any]:
- return {
- "item_id": self.item_id,
- "kind": self.kind,
- "origin": self.origin,
- "trust": self.trust,
- "visibility": self.visibility,
- "priority": self.priority,
- "token_budget": self.token_budget,
- "truncation_policy": self.truncation_policy,
- "content_ref": self.content_ref,
- "content_hash": self.content_hash,
- "cache_class": self.cache_class,
- "created_seq": self.created_seq,
- "token_estimate": self.token_estimate,
- "render_role": self.render_role,
- "render_name_hash": stable_hash(self.render_name),
- "pair_id": self.pair_id or None,
- "message_group": self.message_group,
- "unit_id": self.unit_id,
- }
-
-
-def make_text_context_item(
- text: str,
- *,
- item_id: str,
- kind: str,
- origin: str,
- trust: str,
- created_seq: int,
- priority: int = 700,
- token_budget: int = 4_000,
- truncation_policy: str = POLICY_HEAD_TAIL,
- render_role: str = "user",
- cache_class: str = "dynamic",
-) -> ContextItem:
- """Create a framework-neutral, explicitly-provenanced text item."""
- return ContextItem.create(
- item_id=item_id,
- kind=kind,
- origin=origin,
- trust=trust,
- visibility=VISIBILITY_MODEL,
- priority=priority,
- token_budget=token_budget,
- truncation_policy=truncation_policy,
- content=str(text or ""),
- cache_class=cache_class,
- created_seq=created_seq,
- render_role=render_role,
- render_name=render_role,
- message_group=item_id,
- )
-
-
-def session_context_metadata(item: ContextItem) -> dict[str, Any]:
- """Carry provenance on a positional session row without stale identity/sequence."""
- payload = item.to_manifest()
- for key in (
- "item_id",
- "created_seq",
- "message_group",
- "unit_id",
- "content_ref",
- "content_hash",
- "token_estimate",
- "render_name_hash",
- ):
- payload.pop(key, None)
- return payload
-
-
-@dataclass(frozen=True)
-class ContextAssembly:
- included: tuple[ContextItem, ...]
- excluded: tuple[ContextItem, ...]
- _manifest: Mapping[str, Any] = field(repr=False)
- manifest_hash: str
- used_tokens: int
- total_budget: int
- over_budget: bool = False
- cut_units: int = 0
- pruned_items: int = 0
-
- def __post_init__(self) -> None:
- object.__setattr__(self, "included", tuple(self.included))
- object.__setattr__(self, "excluded", tuple(self.excluded))
- object.__setattr__(self, "_manifest", freeze_json(self._manifest))
-
- @property
- def manifest(self) -> dict[str, Any]:
- """Return a detached JSON-serializable copy of the audit manifest."""
- manifest = thaw_json(self._manifest)
- if not isinstance(manifest, dict): # pragma: no cover - frozen in __post_init__
- raise TypeError("context manifest must thaw to a dictionary")
- return manifest
-
-
-class ContextAssembler:
- """Explicit, unit-level selection in context order.
-
- The unit of selection is a whole ReAct step — the reasoning, text and tool
- calls the model produced together with the results those calls returned —
- never a single block. A step is kept or dropped as one; the only thing a
- budget may shorten inside it is a tool result's output.
-
- Ordering is **not** the assembler's job: items arrive in the order the agent
- context already holds them, and they leave in that same order. The live
- user instruction carries a sequence 64 strides above the context tail (see
- ``next_request_sequence``), so sorting by sequence would move it behind the
- turn it had just started.
-
- Budget pressure is resolved in a fixed order: per-item caps first, then
- tool outputs of older steps are pruned (oldest first), and only then are
- whole steps cut from the oldest end of the conversation. The steps of the
- turn in progress — everything after the current user instruction — are
- protected: they are never pruned, cut or split. When they alone exceed the
- budget the assembly reports ``over_budget`` instead of quietly removing
- anything; compaction owns that case.
- """
-
- def __init__(
- self,
- *,
- total_budget: int,
- budget_details: Optional[Mapping[str, int]] = None,
- ) -> None:
- self.total_budget = max(0, int(total_budget))
- self.budget_details = {
- str(key): max(0, int(value))
- for key, value in sorted((budget_details or {}).items())
- }
-
- @staticmethod
- def _mandatory(item: ContextItem) -> bool:
- if item.visibility == VISIBILITY_MANIFEST_ONLY:
- return True
- if item.kind in STEP_KINDS:
- return False
- return item.truncation_policy == POLICY_NEVER
-
- @staticmethod
- def _unit_key(items: Sequence[ContextItem]) -> tuple[Any, ...]:
- return (
- -max(item.priority for item in items),
- -max(item.created_seq for item in items),
- tuple(sorted(item.item_id for item in items)),
- )
-
- @staticmethod
- def _tokens(item: ContextItem) -> int:
- return 0 if item.visibility == VISIBILITY_MANIFEST_ONLY else item.token_estimate
-
- def _cap_item(
- self,
- item: ContextItem,
- records: dict[str, dict[str, Any]],
- ) -> Optional[ContextItem]:
- original = item.token_estimate
- # Media still contributes to whole-context occupancy, but cannot spend
- # or be cut by the per-item text allowance.
- budget = item.token_budget + sum(
- estimate_context_tokens(block) for block in _tool_result_images(item.content)
- )
- if (
- item.visibility == VISIBILITY_MANIFEST_ONLY
- or item.truncation_policy == POLICY_NEVER
- or original <= budget
- ):
- records[item.item_id] = {
- "action": "included",
- "original_tokens": original,
- "final_tokens": original,
- }
- return item
- if item.truncation_policy in {POLICY_HEAD_TAIL, POLICY_TAIL} and item.token_budget > 0:
- target = budget
- capped = item.with_content(
- _truncate_content(item.content, target, item.truncation_policy)
- )
- # JSON escaping of newlines/quotes can exceed the initial text
- # allowance. Refit using the measured cost, never discard an image
- # result just because its accompanying text needs a smaller slice.
- while capped.token_estimate > budget and target > 1:
- target = max(1, target - (capped.token_estimate - budget))
- capped = item.with_content(
- _truncate_content(item.content, target, item.truncation_policy)
- )
- if capped.token_estimate <= budget:
- records[item.item_id] = {
- "action": "truncated",
- "original_tokens": original,
- "final_tokens": capped.token_estimate,
- }
- return capped
- records[item.item_id] = {
- "action": "excluded",
- "reason": "item_budget",
- "original_tokens": original,
- "final_tokens": 0,
- }
- return None
-
- @staticmethod
- def _tool_pairs_balanced(unit: Sequence[ContextItem]) -> bool:
- pairs: dict[str, list[ContextItem]] = {}
- for item in unit:
- if item.kind in _TOOL_KINDS:
- if not item.pair_id:
- return False
- pairs.setdefault(item.pair_id, []).append(item)
- for pair_items in pairs.values():
- calls = [item for item in pair_items if item.kind == KIND_TOOL_CALL]
- results = [item for item in pair_items if item.kind == KIND_TOOL_RESULT]
- if not calls or len(calls) != len(results):
- return False
- return True
-
- def assemble(self, items: Iterable[ContextItem]) -> ContextAssembly:
- raw_items = tuple(items)
- item_ids = [item.item_id for item in raw_items]
- duplicates = sorted(
- item_id for item_id in set(item_ids) if item_ids.count(item_id) > 1
- )
- if duplicates:
- raise ValueError(f"context item_id values must be unique: {duplicates}")
- original_items = list(raw_items)
- arrival = {item.item_id: index for index, item in enumerate(original_items)}
- records: dict[str, dict[str, Any]] = {}
- excluded: list[ContextItem] = []
-
- capped: list[ContextItem] = []
- for item in original_items:
- candidate = self._cap_item(item, records)
- if candidate is None:
- excluded.append(item)
- continue
- capped.append(candidate)
-
- # A tool result belongs to the step that issued its call, whatever unit
- # the caller stamped on it: the pair id is the structural link.
- call_units: dict[str, str] = {}
- ambiguous_pairs: set[str] = set()
- for item in capped:
- if item.kind == KIND_TOOL_CALL and item.pair_id:
- # One pair id may name a provider parallel batch (several calls
- # in one unit); the same id across two units is unresolvable.
- if call_units.get(item.pair_id, item.unit_id) != item.unit_id:
- ambiguous_pairs.add(item.pair_id)
- call_units[item.pair_id] = item.unit_id
- units: dict[str, list[ContextItem]] = {}
- for candidate in capped:
- if candidate.kind in _TOOL_KINDS and candidate.pair_id in ambiguous_pairs:
- records[candidate.item_id].update(
- action="excluded", reason="malformed_tool_pair", final_tokens=0
- )
- excluded.append(candidate)
- continue
- if candidate.kind == KIND_TOOL_RESULT and candidate.pair_id in call_units:
- candidate = replace(candidate, unit_id=call_units[candidate.pair_id])
- units.setdefault(candidate.unit_id, []).append(candidate)
-
- # The request that opened the turn in progress. Everything after it is
- # the model's own live work and is protected as a whole.
- request_index = -1
- for item in original_items:
- if (
- item.kind == KIND_USER_INPUT
- and item.truncation_policy == POLICY_NEVER
- and item.visibility == VISIBILITY_MODEL
- ):
- request_index = arrival[item.item_id]
-
- def unit_start(unit: Sequence[ContextItem]) -> int:
- return min(arrival[item.item_id] for item in unit)
-
- mandatory: list[list[ContextItem]] = []
- protected: list[list[ContextItem]] = []
- aside: list[list[ContextItem]] = []
- conversation: list[list[ContextItem]] = []
- for unit in units.values():
- is_step = any(item.kind in STEP_KINDS for item in unit)
- if any(self._mandatory(item) for item in unit):
- mandatory.append(unit)
- elif is_step and request_index >= 0 and unit_start(unit) > request_index:
- protected.append(unit)
- elif not self._tool_pairs_balanced(unit):
- for item in unit:
- records[item.item_id].update(
- action="excluded", reason="malformed_tool_pair", final_tokens=0
- )
- excluded.append(item)
- elif all(item.kind in CONVERSATION_KINDS for item in unit):
- conversation.append(unit)
- else:
- aside.append(unit)
-
- mandatory.sort(key=unit_start)
- protected.sort(key=unit_start)
- conversation.sort(key=unit_start)
- # Selection order only for non-conversation material; it never reaches
- # the output, which is re-sorted into context order.
- aside.sort(key=self._unit_key)
-
- def unit_tokens(unit: Sequence[ContextItem]) -> int:
- return sum(self._tokens(item) for item in unit)
-
- included: list[ContextItem] = []
- used = 0
- for unit in mandatory + protected:
- included.extend(unit)
- used += unit_tokens(unit)
- over_budget = used > self.total_budget
-
- for unit in aside:
- tokens = unit_tokens(unit)
- remaining = max(0, self.total_budget - used)
- if tokens <= remaining:
- included.extend(unit)
- used += tokens
- continue
- item = unit[0]
- if (
- len(unit) == 1
- and remaining > 0
- and item.truncation_policy in {POLICY_HEAD_TAIL, POLICY_TAIL}
- ):
- shrunk = item.with_content(
- _truncate_content(item.content, remaining, item.truncation_policy)
- )
- if self._tokens(shrunk) < self._tokens(item) and self._tokens(shrunk) <= remaining:
- records[item.item_id].update(
- action="truncated", final_tokens=shrunk.token_estimate
- )
- included.append(shrunk)
- used += self._tokens(shrunk)
- continue
- for item in unit:
- records[item.item_id].update(action="excluded", reason="budget", final_tokens=0)
- excluded.append(item)
-
- # Conversation history: prune old tool outputs first, then cut whole
- # steps from the oldest end. Never a hole in the middle.
- pruned_items = 0
- conversation_tokens = sum(unit_tokens(unit) for unit in conversation)
- if used + conversation_tokens > self.total_budget:
- for unit in conversation:
- if used + conversation_tokens <= self.total_budget:
- break
- for index, item in enumerate(unit):
- if used + conversation_tokens <= self.total_budget:
- break
- if item.kind != KIND_TOOL_RESULT or item.truncation_policy == POLICY_NEVER:
- continue
- pruned = item.with_content(_prune_tool_output(item.content, item.token_estimate))
- if self._tokens(pruned) >= self._tokens(item):
- continue
- conversation_tokens -= self._tokens(item) - self._tokens(pruned)
- unit[index] = pruned
- records[item.item_id].update(action="pruned", final_tokens=pruned.token_estimate)
- pruned_items += 1
- cut_units = 0
- while conversation and used + conversation_tokens > self.total_budget:
- dropped = conversation.pop(0)
- conversation_tokens -= unit_tokens(dropped)
- cut_units += 1
- for item in dropped:
- records[item.item_id].update(action="excluded", reason="budget_cut", final_tokens=0)
- excluded.append(item)
- for unit in conversation:
- included.extend(unit)
- used += unit_tokens(unit)
-
- included.sort(key=lambda item: arrival[item.item_id])
- excluded_by_id = {item.item_id: item for item in excluded}
- excluded = sorted(excluded_by_id.values(), key=lambda item: arrival[item.item_id])
-
- included_manifest = []
- excluded_manifest = []
- included_ids = {item.item_id for item in included}
- final_by_id = {item.item_id: item for item in included}
- for original in original_items:
- record = dict(records.get(original.item_id) or {})
- final = final_by_id.get(original.item_id, original)
- entry = final.to_manifest()
- entry.update(record)
- if original.item_id in included_ids:
- included_manifest.append(entry)
- else:
- excluded_manifest.append(entry)
-
- payload = {
- "schema_version": CONTEXT_SCHEMA_VERSION,
- "total_budget": self.total_budget,
- "used_tokens": used,
- "over_budget": over_budget,
- "protected_units": len(protected),
- "cut_units": cut_units,
- "pruned_items": pruned_items,
- "included": included_manifest,
- "excluded": excluded_manifest,
- }
- if self.budget_details:
- payload["budget_details"] = dict(self.budget_details)
- manifest_hash = stable_hash(payload)
- return ContextAssembly(
- included=tuple(included),
- excluded=tuple(excluded),
- _manifest=payload,
- manifest_hash=manifest_hash,
- used_tokens=used,
- total_budget=self.total_budget,
- over_budget=over_budget,
- cut_units=cut_units,
- pruned_items=pruned_items,
- )
-
-
-__all__ = [
- "CONTEXT_SCHEMA_VERSION",
- "CONTEXT_SEQUENCE_STRIDE",
- "CONVERSATION_KINDS",
- "STEP_KINDS",
- "SESSION_CONTEXT_META_KEY",
- "ContextAssembler",
- "ContextAdapterProtocol",
- "ContextAssembly",
- "ContextItem",
- "KIND_ASSISTANT",
- "KIND_ATTACHMENT",
- "KIND_COMPACTION",
- "KIND_MEMORY",
- "KIND_IDENTITY",
- "KIND_PROJECT",
- "KIND_REFERENCE",
- "KIND_REMINDER",
- "KIND_STEER",
- "KIND_SYSTEM_RULE",
- "KIND_TOOL_CALL",
- "KIND_TOOL_RESULT",
- "KIND_USER_INPUT",
- "POLICY_DROP",
- "POLICY_HEAD_TAIL",
- "POLICY_NEVER",
- "POLICY_TAIL",
- "VISIBILITY_MANIFEST_ONLY",
- "VISIBILITY_MODEL",
- "estimate_context_tokens",
- "make_text_context_item",
- "session_context_metadata",
-]
diff --git a/src/backend/core/llm/factory/README.md b/src/backend/core/llm/factory/README.md
index 1cd4e5c4..e9839f53 100644
--- a/src/backend/core/llm/factory/README.md
+++ b/src/backend/core/llm/factory/README.md
@@ -1,7 +1,7 @@
# Agent factory
Public entry: `from core.llm.factory import create_agent_executor`.
-`warmup_mcp_tools` is the startup entry for shared MCP discovery.
+`mcp_connections` discovers native MCP capabilities within the authorized run.
The initializer loads these entries lazily so importing selection helpers does
not initialize the model/tool stack. No legacy `agent_factory` module remains.
@@ -17,8 +17,8 @@ not initialize the model/tool stack. No legacy `agent_factory` module remains.
| `runtime/` | Select the model and budgets; attach middleware, manifests and listeners | Agent execution policy changes |
| `models.py`, `defaults.py` | Assembly-owned value types and defaults | Multiple phases share a domain value |
-The factory consumes `core.plugins.runtime`; it does not own plugin lifecycle
-or manifest ingestion. Plugin behavior belongs in `core/plugins/`.
+The factory consumes `core.capabilities.plugins.runtime`; it does not own plugin lifecycle
+or manifest ingestion. Plugin behavior belongs in `core/capabilities/plugins/`.
The orchestration layer calls the public entry; it should not coordinate the
private phases itself. Keep phase inputs explicit rather than introducing a
mutable catch-all context or arbitrary pipeline registry.
diff --git a/src/backend/core/llm/factory/__init__.py b/src/backend/core/llm/factory/__init__.py
index 2e2b49c5..a5f0a66a 100644
--- a/src/backend/core/llm/factory/__init__.py
+++ b/src/backend/core/llm/factory/__init__.py
@@ -6,11 +6,7 @@ def __getattr__(name):
from .build import create_agent_executor
return create_agent_executor
- if name == "warmup_mcp_tools":
- from .tools.mcp_config import warmup_mcp_tools
-
- return warmup_mcp_tools
raise AttributeError(name)
-__all__ = ["create_agent_executor", "warmup_mcp_tools"]
+__all__ = ["create_agent_executor"]
diff --git a/src/backend/core/llm/factory/assembly.py b/src/backend/core/llm/factory/assembly.py
index 6e0380e1..eb682a07 100644
--- a/src/backend/core/llm/factory/assembly.py
+++ b/src/backend/core/llm/factory/assembly.py
@@ -11,8 +11,8 @@
from core.llm.factory.runtime.manifest_binding import manifest_binding
from core.llm.factory.runtime.middleware_stack import middleware_stack
from core.llm.factory.runtime.model_selection import model_selection
-from core.llm.factory.selection.device_bindings import device_bindings
-from core.llm.factory.selection.device_preparation import device_preparation
+from core.llm.factory.runtime.capability_bindings import device_bindings
+from core.harness.registry.run_capabilities import prepare_run_capabilities
from core.llm.factory.selection.entry_policy import entry_policy
from core.llm.factory.selection.mode_bindings import mode_bindings
from core.llm.factory.selection.profile_selection import profile_selection
@@ -28,7 +28,7 @@
async def assemble_agent(request: AgentRequest):
- from core.llm.project_environment import bind_project_environment
+ from core.llm.runtime.project_environment import bind_project_environment
_caps_skill_ids = None
(
@@ -124,7 +124,7 @@ async def assemble_agent(request: AgentRequest):
_caps_skill_ids,
_desktop_progressive,
_prepared_capabilities,
- ) = await device_preparation(
+ ) = await prepare_run_capabilities(
request,
_api_scope=_api_scope,
_capability_run_key=_capability_run_key,
@@ -177,11 +177,9 @@ async def assemble_agent(request: AgentRequest):
_required_connector_tool_names,
_required_plugin_mcp_tool_names,
_tool_approval_available,
- http_clients,
+ mcp_resources,
loader,
- mcp_clients,
toolkit,
- transient_mcp_clients,
) = await mcp_connections(
request,
_api_scope=_api_scope,
@@ -261,9 +259,8 @@ async def assemble_agent(request: AgentRequest):
required=required,
_required_connector_tool_names=_required_connector_tool_names,
_required_plugin_mcp_tool_names=_required_plugin_mcp_tool_names,
- http_clients=http_clients,
+ mcp_resources=mcp_resources,
loader=loader,
- mcp_clients=mcp_clients,
skill_ids_to_register=skill_ids_to_register,
toolkit=toolkit,
)
@@ -400,10 +397,9 @@ async def assemble_agent(request: AgentRequest):
_manifest_for_surface=_manifest_for_surface,
_plugin_runtime=_plugin_runtime,
agent=agent,
- http_clients=http_clients,
+ mcp_resources=mcp_resources,
profile=profile,
skill_selection=skill_selection,
system_prompt=system_prompt,
toolkit=toolkit,
- transient_mcp_clients=transient_mcp_clients,
)
diff --git a/src/backend/core/llm/factory/build.py b/src/backend/core/llm/factory/build.py
index 2b68d9df..39c40de5 100644
--- a/src/backend/core/llm/factory/build.py
+++ b/src/backend/core/llm/factory/build.py
@@ -8,8 +8,8 @@
from agentscope.mcp import MCPClient
from core.llm.factory.assembly import assemble_agent
from core.llm.factory.request import AgentRequest
-from core.llm.mcp_manager import close_factory_clients_on_error
-from orchestration.registry import AgentSpec
+from core.llm.factory.specs import AgentSpec
+from core.llm.mcp.manager import close_factory_clients_on_error
@close_factory_clients_on_error
@@ -115,7 +115,7 @@ async def create_agent_executor(
required_plugin_name: Optional[str] = None,
required_plugin_skill_ids: Optional[List[str]] = None,
required_plugin_mcp_ids: Optional[List[str]] = None,
- # mode_spec: 对话模式的装配契约(core/services/chat_mode_service.ChatModeSpec)。
+ # mode_spec: 对话模式的装配契约(core.chat.mode_types.ChatModeSpec)。
# 「模式」把原来写死的极速模式泛化成一张表:工具面 / 技能 / 插件 / 提示词 kind /
# 迭代上限都由它给。turbo_mode 现在的含义是"这个模式要收窄工具面",收窄成什么
# 由 mode_spec 说了算。缺省(老调用方没传)时退回历史的 turbo.* 配置键读取,
diff --git a/src/backend/core/llm/factory/defaults.py b/src/backend/core/llm/factory/defaults.py
index 2aeabf55..d6705a88 100644
--- a/src/backend/core/llm/factory/defaults.py
+++ b/src/backend/core/llm/factory/defaults.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from dataclasses import dataclass, field
from typing import Dict
from core.config.settings import settings as _settings
@@ -90,9 +91,6 @@
"一旦写入,「哪些还没做」就不再可判定。\n"
)
-_HTTP_MCP_FAIL_AT: Dict[str, float] = {}
-
-_HTTP_MCP_FAIL_COOLDOWN_S = 60.0
KB_MCP_HTTP_URL = (
f"http://{_settings.server.mcp_host}:{_MCP_PORTS['retrieve_dataset_content']}/mcp/"
@@ -109,3 +107,15 @@
"# 可用技能(技能名、说明文件、适用场景):{% for skill in skills %}\n"
"- `{{ skill.name }}`:`{{ skill.dir }}/SKILL.md` — {{ skill.description }}{% endfor %}"
)
+
+
+@dataclass
+class DefaultsState:
+ HTTP_MCP_FAIL_AT: Dict[str, float] = field(default_factory=lambda: {})
+
+
+_STATE = DefaultsState()
+
+
+def process_state() -> DefaultsState:
+ return _STATE
diff --git a/src/backend/core/llm/factory/prompts/policy.py b/src/backend/core/llm/factory/prompts/policy.py
index eabe1708..cfeff1b8 100644
--- a/src/backend/core/llm/factory/prompts/policy.py
+++ b/src/backend/core/llm/factory/prompts/policy.py
@@ -4,9 +4,10 @@
import logging
-logger = logging.getLogger(__name__)
from core.llm.factory.defaults import DYNAMIC_BLOCK_HEADER
+logger = logging.getLogger(__name__)
+
def _render_turn_budget_hint(max_iters: int) -> str:
"""Tell an agent that *does* have a turn budget how to spend it.
diff --git a/src/backend/core/llm/factory/prompts/system.py b/src/backend/core/llm/factory/prompts/system.py
index aa76009d..7aec1737 100644
--- a/src/backend/core/llm/factory/prompts/system.py
+++ b/src/backend/core/llm/factory/prompts/system.py
@@ -1,15 +1,17 @@
"""Agent assembly phase: system prompt. """
from __future__ import annotations
+from core.llm.tools.plugin_activation import register_load_plugin
+from core.llm import prompt_rendering
import os
from typing import Any, Dict
from core.llm.factory.defaults import _WORKFLOW_MODE_HINT
from core.llm.factory.request import AgentRequest
-from core.llm.project_environment import validate_project_environment
-from core.ontology.validator import render_runtime_prompt
-from core.services.system_config import code_capability_enabled
+from core.llm.runtime.project_environment import validate_project_environment
+from core.ontology.runtime_pack import render_runtime_prompt
+from core.config.store.execution_options import code_capability_enabled
from prompts.prompt_runtime import build_subagent_system_prompt, build_system_prompt
@@ -37,7 +39,7 @@ async def system_prompt(
toolkit,
):
if request.profile:
- from core.extensions import agent_profile
+ from core.harness.registry import agent_profile
system_prompt = await agent_profile(request.profile).system_prompt(request)
return _agent_ref, _plan_tool_enabled, _plugin_runtime, system_prompt
@@ -93,7 +95,7 @@ async def system_prompt(
# Bound plugins deferred in the override block above; in-run activation
# only (persist=False — see the note there).
if _subagent_progressive is not None:
- from core.plugins import runtime as _plug
+ from core.capabilities.plugins import runtime as _plug
_plugin_dir_section = _plug.build_plugin_directory_section(
_subagent_progressive.directory
@@ -128,7 +130,7 @@ async def system_prompt(
"approval_available": _tool_approval_available,
"ontology_runtime": _ontology_runtime,
}
- _plug.register_load_plugin(
+ register_load_plugin(
toolkit, _subagent_progressive.deferred_by_slug(), _plugin_runtime
)
_log.info(
@@ -146,14 +148,13 @@ async def system_prompt(
# 能配自己的提示词。之前把它写在 turbo 分支里,非收窄模式配了也不生效。
_mode_prompt = ""
if request.mode_spec is not None:
- from core.services import prompt_version_service as _pvs_mode
_mode_prompt_text = getattr(request.mode_spec, "prompt_text", None)
_mode_prompt_kind = getattr(request.mode_spec, "prompt_kind", None)
if _mode_prompt_text:
_mode_prompt = str(_mode_prompt_text)
elif _mode_prompt_kind and _mode_prompt_kind != "turbo":
- _mode_prompt = _pvs_mode.render_kind_segment(_mode_prompt_kind, fs_fallback=False)
+ _mode_prompt = prompt_rendering.render_kind_segment(_mode_prompt_kind, fs_fallback=False)
if request.turbo_mode and not _turbo_code_exec:
# Turbo swaps in the standalone prompt (DB "turbo" active version →
@@ -162,10 +163,9 @@ async def system_prompt(
# (开了代码执行位的收窄模式不进这条:装配确实带沙箱/文件工具,
# turbo 正文"秒级检索、不执行代码"的叙事反而是错的——没配专属
# 提示词时走默认装配,工具段按实际 toolkit 动态生成。)
- from core.services import prompt_version_service as _pvs_turbo
# 收窄模式没配专属提示词时退回历史的 turbo 正文——极速模式绑的就是它。
- system_prompt = _mode_prompt or _pvs_turbo.render_turbo_system_prompt()
+ system_prompt = _mode_prompt or prompt_rendering.render_turbo_system_prompt()
from core.config.local_mode import local_mode_enabled
if local_mode_enabled():
@@ -275,9 +275,9 @@ async def system_prompt(
# Single source of truth render_kind_segment (same source as the Config console preview).
if code_capability_enabled() and (not request.turbo_mode or _turbo_code_exec):
try:
- from core.services import prompt_version_service as _pvs
+ from core.llm import prompt_versions as _pvs
- _code_exec_text = _pvs.render_kind_segment("code_exec")
+ _code_exec_text = prompt_rendering.render_kind_segment("code_exec")
except Exception:
_code_exec_text = ""
if _code_exec_text:
@@ -302,7 +302,7 @@ async def system_prompt(
# independent of activation state) so an activation never perturbs this
# part of the prefix; only the tools/skills it adds do, once.
if _progressive is not None:
- from core.plugins import runtime as _plug
+ from core.capabilities.plugins import runtime as _plug
_plugin_dir_section = _plug.build_plugin_directory_section(_progressive.directory)
if _plugin_dir_section:
@@ -334,7 +334,7 @@ async def system_prompt(
"approval_available": _tool_approval_available,
"ontology_runtime": _ontology_runtime,
}
- _plug.register_load_plugin(toolkit, _progressive.deferred_by_slug(), _plugin_runtime)
+ register_load_plugin(toolkit, _progressive.deferred_by_slug(), _plugin_runtime)
_log.info(
"[factory] +%s progressive plugins: %d in directory, %d deferred "
"(skills=%d, mcp=%d)",
@@ -361,8 +361,8 @@ async def system_prompt(
# ── Register call_subagent tool for main agent ──
if request.visible_subagents:
- from core.llm.builtin_subagents import refresh_builtin_subagents
- from core.llm.subagent_tool import build_subagent_prompt_section, register_subagent_tool
+ from core.capabilities.agents.builtin import refresh_builtin_subagents
+ from core.llm.subagents.tool import build_subagent_prompt_section, register_subagent_tool
# Refresh platform-default rows only after the parent toolset has
# completed catalog defaults, permission filtering, skill-bound MCP
@@ -443,13 +443,10 @@ async def system_prompt(
# derived/non-interactive constructions get nothing by default. The DB
# switch auto_plan_entry_enabled (which itself returns False on
# config-layer errors) can turn this off entirely.
- from core.services.system_config import auto_plan_entry_enabled
+ from core.config.store.execution_options import auto_plan_entry_enabled
if request.top_level_chat and auto_plan_entry_enabled():
- from core.llm.plan_update_tool import (
- build_plan_update_prompt_section,
- register_plan_update_tool,
- )
+ from core.llm.tools.plan_update import build_plan_update_prompt_section, register_plan_update_tool
# 计划栏的催更中间件挂在同一个正向开关上:工具没注册就绝不该有人催更新。
_plan_tool_enabled = True
diff --git a/src/backend/core/llm/factory/request.py b/src/backend/core/llm/factory/request.py
index 09ef6c5a..51af1a0c 100644
--- a/src/backend/core/llm/factory/request.py
+++ b/src/backend/core/llm/factory/request.py
@@ -5,12 +5,12 @@
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
-from orchestration.registry import AgentSpec
+from core.llm.factory.specs import AgentSpec
@dataclass
class AgentRequest:
- # A registered module agent profile (core.extensions.AgentProfile) and that
+ # A registered module agent profile (core.harness.registry.AgentProfile) and that
# module's own reference, e.g. the task the dedicated agent works on.
profile: Optional[str] = None
profile_ref: Optional[str] = None
diff --git a/src/backend/core/llm/factory/runtime/agent_construction.py b/src/backend/core/llm/factory/runtime/agent_construction.py
index 0b545539..8ca8118e 100644
--- a/src/backend/core/llm/factory/runtime/agent_construction.py
+++ b/src/backend/core/llm/factory/runtime/agent_construction.py
@@ -4,14 +4,15 @@
import logging
-logger = logging.getLogger(__name__)
import os
from agentscope.agent import ReActConfig
-from agentscope.agent._config import ModelConfig
-from core.llm.compacting_agent import CompactingAgent
+from agentscope.agent import ModelConfig, InjectionConfig
+from core.llm.context.compacting_agent import CompactingAgent
from core.llm.factory.request import AgentRequest
+logger = logging.getLogger(__name__)
+
async def agent_construction(
request: AgentRequest,
@@ -63,11 +64,11 @@ async def agent_construction(
_offloader = None
if not request.disable_tools and os.getenv("SANDBOX_TOOLS_ENABLED", "true").lower() == "true":
try:
- from core.llm.offloader import SandboxOffloader
+ from core.llm.context.offloader import SandboxOffloader
from core.sandbox.factory import get_sandbox_provider
if _api_scope is not None:
- from core.llm.agent_api_tools import _provider as api_sandbox_provider
+ from core.llm.api_runtime.agent_tools import _provider as api_sandbox_provider
_offloader = SandboxOffloader(
api_sandbox_provider(),
@@ -83,7 +84,7 @@ async def agent_construction(
# CompactingAgent, not the bare framework Agent: its compress_context
# override is what routes the ReAct step boundary into the one compaction
- # engine and persists the checkpoint (see core/llm/compacting_agent.py).
+ # engine and persists the checkpoint (see core/llm/context/compacting_agent.py).
agent = CompactingAgent(
name=_agent_name,
system_prompt=system_prompt,
@@ -93,7 +94,8 @@ async def agent_construction(
state=_state,
context_config=context_config,
model_config=ModelConfig(max_retries=3, fallback_model=None),
- react_config=ReActConfig(max_iters=_max_iters),
+ injection_config=InjectionConfig(inject_runtime_state=False),
+ react_config=ReActConfig(max_iters=_max_iters, interruption_raise_cancelled_error=True),
offloader=_offloader,
)
# Compaction trigger accounting reuses the exact already-frozen execution
@@ -107,8 +109,9 @@ async def agent_construction(
# and a seed write on a cold config cache, inside the ReAct loop).
try:
agent._jx_trigger_ratio = _trigger_ratio
- except Exception: # pragma: no cover - agent may reject attributes
- pass
+ except Exception as fallback_error: # pragma: no cover - agent may reject attributes
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.factory.runtime.agent_construction:agent_construction', fallback_error)
# Set the agent reference so the call_subagent closure can extract shared context
if _agent_ref is not None:
diff --git a/src/backend/core/llm/factory/selection/device_bindings.py b/src/backend/core/llm/factory/runtime/capability_bindings.py
similarity index 93%
rename from src/backend/core/llm/factory/selection/device_bindings.py
rename to src/backend/core/llm/factory/runtime/capability_bindings.py
index c6006bf7..768d7398 100644
--- a/src/backend/core/llm/factory/selection/device_bindings.py
+++ b/src/backend/core/llm/factory/runtime/capability_bindings.py
@@ -4,8 +4,7 @@
import asyncio
-from core.capabilities import runtime as capability_runtime
-from core.llm.agent_api_runtime import scope_mcp_servers
+from core.llm.api_runtime.agent import scope_mcp_servers
from core.llm.factory.models import RequiredCapabilities, StickyCapabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.tools import mcp_config as factory_mcp_config
@@ -58,8 +57,7 @@ async def device_bindings(
)
if _prepared_capabilities is not None:
enabled_servers = await asyncio.to_thread(
- capability_runtime.bind_mcp,
- _prepared_capabilities,
+ _prepared_capabilities.bind_mcp,
enabled_servers,
_capability_mcp_resolution[0] if _capability_mcp_resolution else None,
)
@@ -73,8 +71,7 @@ async def device_bindings(
if required.plugin_id:
_dependency_plugins.append(required.plugin_id)
_prepared_capabilities = await asyncio.to_thread(
- capability_runtime.preflight,
- _prepared_capabilities,
+ _prepared_capabilities.preflight,
plugin_ids=list(dict.fromkeys(_dependency_plugins)),
available_mcp=set(enabled_servers),
available_kb=set(request.enabled_kb_ids or []),
@@ -115,7 +112,7 @@ async def device_bindings(
else:
_progressive = _desktop_progressive
- enabled_servers = factory_mcp_config._inject_runtime_headers(
+ enabled_servers = factory_mcp_config.inject_runtime_headers(
enabled_servers,
current_user_id=request.current_user_id,
chat_id=request.chat_id,
diff --git a/src/backend/core/llm/factory/runtime/execution_budget.py b/src/backend/core/llm/factory/runtime/execution_budget.py
index 9d5f7d0f..9558944e 100644
--- a/src/backend/core/llm/factory/runtime/execution_budget.py
+++ b/src/backend/core/llm/factory/runtime/execution_budget.py
@@ -4,8 +4,8 @@
from agentscope.agent import ContextConfig
from core.config.settings import settings as _settings
-from core.llm.compaction import SUMMARIZATION_PROMPT
-from core.llm.context_manager import AUTO_COMPACT_MAX_RATIO
+from core.llm.context.compaction import SUMMARIZATION_PROMPT
+from core.models.context_window import AUTO_COMPACT_MAX_RATIO
from core.llm.factory.prompts import policy as factory_prompt_policy
from core.llm.factory.request import AgentRequest
@@ -24,7 +24,7 @@ async def execution_budget(
):
_ctx_window = int(getattr(default_model, "context_size", 0) or 0)
# Resolve the shared ratio once instead of reading config at each ReAct step.
- from core.services.compaction_service import resolve_token_limit, resolve_trigger_ratio
+ from core.harness.registry.compaction import resolve_token_limit, resolve_trigger_ratio
_trigger_ratio = resolve_trigger_ratio()
_log.info(
@@ -128,7 +128,7 @@ async def execution_budget(
# like all env config).
if profile.max_react_turns:
_max_iters = profile.max_react_turns
- from core.config.settings import _env, _int
+ from core.config.settings import env_value as _env, env_int as _int
_env_iters = _int(_env("CHAT_MAIN_MAX_ITERS"), 0)
if _env_iters > 0:
@@ -138,7 +138,7 @@ async def execution_budget(
# Hard cap for the quick-lookup contract: 1-2 retrieval rounds (each may
# fan out parallel calls) plus the final answer. Admin-tunable via
# 系统配置「极速模式」turbo.max_iters; wins over profile/env.
- from core.services.system_config import turbo_max_iters
+ from core.config.store.execution_options import turbo_max_iters
_mode_iters = getattr(request.mode_spec, "max_iters", None) if request.mode_spec else None
if _mode_iters:
diff --git a/src/backend/core/llm/factory/runtime/listeners.py b/src/backend/core/llm/factory/runtime/listeners.py
index 2b4738c0..5ace4861 100644
--- a/src/backend/core/llm/factory/runtime/listeners.py
+++ b/src/backend/core/llm/factory/runtime/listeners.py
@@ -15,17 +15,16 @@ async def runtime_listeners(
_manifest_for_surface,
_plugin_runtime,
agent,
- http_clients,
+ mcp_resources,
profile,
skill_selection,
system_prompt,
toolkit,
- transient_mcp_clients,
):
async def _publish_context_manifest(request_manifest): # noqa: ANN001, ANN202
"""Bind the exact post-budget context manifest used for this request."""
try:
- from core.evolution.runtime_binding import rebind_execution_manifest
+ from core.evolution.cycle.runtime_binding import rebind_execution_manifest
request_bundle = rebind_execution_manifest(
run_id=request.run_id,
@@ -46,7 +45,7 @@ async def _publish_context_manifest(request_manifest): # noqa: ANN001, ANN202
# treats a missing bundle as partial instead of falsely persisting
# the pre-context surface as complete request evidence.
agent.clear_request_evidence(request_manifest)
- from core.evolution.runtime_binding import clear_run_binding
+ from core.evolution.cycle.runtime_binding import clear_run_binding
clear_run_binding(request.run_id, capability_scope=request.capability_scope)
@@ -71,15 +70,17 @@ async def _publish_surface_generation(surface): # noqa: ANN001, ANN202
_log.warning("[manifest] surface generation refresh unavailable: %s", exc)
try:
agent.bind_execution_surface(None, bundle=_bind_manifest(None))
- except Exception: # pragma: no cover - last-resort availability path
- pass
+ except Exception as fallback_error: # pragma: no cover - last-resort availability path
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.factory.runtime.listeners:_publish_surface_generation', fallback_error)
toolkit.set_execution_surface_listener(_publish_surface_generation)
if skill_selection is not None:
try:
setattr(agent, "_jx_skill_selection", skill_selection)
- except Exception: # pragma: no cover
- pass
+ except Exception as fallback_error: # pragma: no cover
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.factory.runtime.listeners:runtime_listeners', fallback_error)
# Which assembly governed this run, for the turn card. Carried rather than
# re-resolved after the response: by then a profile may have been published
# or switched off, and the card would name one the run never used.
@@ -94,17 +95,21 @@ async def _publish_surface_generation(surface): # noqa: ANN001, ANN202
"narrowed_tools": profile.tool_allowlist is not None,
},
)
- except Exception: # pragma: no cover
- pass
+ except Exception as fallback_error: # pragma: no cover
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.factory.runtime.listeners:runtime_listeners', fallback_error)
_log.info("[factory] +%s agent created, TOTAL setup done", _elapsed())
- # Only transient (per-request) stdio clients + HTTP clients get closed;
- # pooled stable clients stay open for reuse (closing them would defeat the
- # pool and hand dead clients to the next request).
- all_transient = [*transient_mcp_clients, *http_clients]
+ for resource in mcp_resources:
+ if resource.interactions is not None:
+ resource.interactions.agent = agent
+ # Release each run's leases; chat-owned STDIO sessions expire independently.
+ all_transient = list(mcp_resources)
# Clients connected by a mid-run load_plugin activation are appended to this
# same list object, so the caller's close_clients() teardown covers them.
if _plugin_runtime is not None:
_plugin_runtime["close_list"] = all_transient
+ _plugin_runtime["agent"] = agent
+ _plugin_runtime["request"] = request
return agent, all_transient
diff --git a/src/backend/core/llm/factory/runtime/manifest_binding.py b/src/backend/core/llm/factory/runtime/manifest_binding.py
index 5cc32d85..6daa142e 100644
--- a/src/backend/core/llm/factory/runtime/manifest_binding.py
+++ b/src/backend/core/llm/factory/runtime/manifest_binding.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from core.llm.execution_manifest import tool_manifest_from_schemas
+from core.harness.execution_manifest import tool_manifest_from_schemas
from core.llm.factory.request import AgentRequest
@@ -46,7 +46,7 @@ def _manifest_for_surface(surface): # noqa: ANN001, ANN202
)
def _bind_manifest(manifest): # noqa: ANN001, ANN202
- from core.evolution.runtime_binding import bind_runtime_assets
+ from core.evolution.cycle.runtime_binding import bind_runtime_assets
return bind_runtime_assets(
run_id=request.run_id,
diff --git a/src/backend/core/llm/factory/runtime/middleware_stack.py b/src/backend/core/llm/factory/runtime/middleware_stack.py
index cc4d4edd..493ff855 100644
--- a/src/backend/core/llm/factory/runtime/middleware_stack.py
+++ b/src/backend/core/llm/factory/runtime/middleware_stack.py
@@ -2,27 +2,11 @@
from __future__ import annotations
-from core.llm.steer_middleware import SteerMiddleware
-from core.llm.command_middleware import CommandCompletionMiddleware
-from core.llm.agentscope_hook_adapter import AgentScopeHookAdapter
+from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter
from core.llm.factory.request import AgentRequest
from core.llm.factory.runtime import evidence as factory_evidence_helpers
-from core.llm.middlewares import (
- ActingToolCallIdMiddleware,
- AgentRuntimeState,
- CitationAnchorMiddleware,
- DynamicModelMiddleware,
- FileContextMiddleware,
- FinishPinGuardMiddleware,
- IterBudgetReminderMiddleware,
- JobLedgerReminderMiddleware,
- OntologyGateMiddleware,
- PlanStaleReminderMiddleware,
- StallInterventionMiddleware,
- ToolEffectMiddleware,
- WorkspacePinHintMiddleware,
-)
+from core.llm.runtime.state import AgentRuntimeState
async def middleware_stack(
@@ -43,14 +27,11 @@ async def middleware_stack(
tool_schemas,
toolkit,
):
- from core.llm.tool_permissions import (
- PermissionRuntime,
- ToolPermissionMiddleware,
- ToolPermissionRegistry,
- ToolPermissionService,
- )
+ from core.sandbox.permissions.dispatch_models import PermissionRuntime
+ from core.sandbox.permissions.tool_specs import ToolPermissionRegistry
+ from core.llm.tool_permissions import ToolPermissionService
- _builtin_tool_names = {ft.name for ft in toolkit.function_tools}
+ _builtin_tool_names = {ft.name for ft in list(toolkit)}
_all_tool_names = {
str(s.get("function", {}).get("name") or "")
for s in tool_schemas
@@ -84,7 +65,7 @@ async def middleware_stack(
)
# AgentScope's native permission engine remains a coarse first gate. Names
- # present in the registry additionally pass ToolPermissionMiddleware and
+ # present in the registry additionally pass the permission hook and
# the final Toolkit ticket guard below.
from agentscope.permission import PermissionContext
@@ -104,62 +85,51 @@ async def middleware_stack(
permission_context=PermissionContext(),
)
- _policy_middlewares: list = [
- DynamicModelMiddleware(), # on_reply: switch models by chat_mode
- FileContextMiddleware(), # on_reply: inject file context
- *([CommandCompletionMiddleware()] if _eval_scope is None and _api_scope is None else []),
- SteerMiddleware(), # on_acting/on_reasoning: inject queued user steer before tool I/O
- WorkspacePinHintMiddleware(), # on_reasoning: remind to pin
- IterBudgetReminderMiddleware(), # on_reasoning: inject a wrap-up reminder near max_iters
- # on_acting: the active profile's intervention rules, applied to *this*
- # loop. Previously they only reached the autonomous loop, which left the
- # orchestration profile with one field that governed nothing on the axis
- # almost all traffic takes.
- StallInterventionMiddleware(profile.intervention_rules),
- OntologyGateMiddleware(_ontology_runtime), # on_acting: zero-LLM L-a contract gate
- ToolPermissionMiddleware(_permission_service),
- CitationAnchorMiddleware(), # on_acting: 证据锚点——工具结果回给模型前发号回注 cite_id
- ActingToolCallIdMiddleware(), # on_acting: expose call_subagent's tool_call.id to tools (parent-child linkage)
- ToolEffectMiddleware(), # on_acting: durable Intent before every actual tool invocation
- ]
- if request.project_environment is not None:
- from core.llm.project_environment_middleware import ProjectEnvironmentMiddleware
-
- _policy_middlewares.insert(0, ProjectEnvironmentMiddleware(request))
+ from core.harness.registry import hook_specs
- # on_reasoning: 会话里有未收敛的批量作业时,每轮把台账数字回灌进上下文。
- # 进度是外部事实(job_items 表),不是模型的记忆——不主动回灌,隔十几轮之后就会
- # 退化成"边际收益递减,先交付吧"(568 行只补 66 行正是这么停的)。
- # 子作业内部不挂(isolated),避免嵌套噪声。
- if request.workflow_mode and not request.isolated and request.chat_id:
- _policy_middlewares.append(
- JobLedgerReminderMiddleware(chat_id=request.chat_id, user_id=request.current_user_id)
- )
- # on_reasoning: 计划栏停在半路时把当前清单回灌回去,催模型调 update_plan。
- # 只在真的注册了 update_plan 工具的那条路径上挂(同一个正向开关),派生/非交互
- # 的构造一律拿不到——催一个不存在的工具只会让模型编造调用。
- if _plan_tool_enabled:
- _policy_middlewares.append(PlanStaleReminderMiddleware())
- # The legacy guard directly pins ids found in arbitrary tool output. API
- # delivery must go through the scoped tool's ownership check and persistence.
- if _api_scope is None and _eval_scope is None:
- _policy_middlewares.append(FinishPinGuardMiddleware())
+ from core.llm.tool_permissions.hook import ToolPermissionHook
+ permission_policy = ToolPermissionHook(_permission_service)
+ adapter = AgentScopeHookAdapter(permission_policy=permission_policy, durable_tools=True)
+ from core.harness.registry.hook_factories import create_steering_policy
+ steering = None
if request.profile:
- from core.extensions import agent_profile
-
- for middleware in agent_profile(request.profile).middlewares(request):
- if isinstance(middleware, SteerMiddleware):
- # Profiles change the source, not its position before approvals
- # and durable tool effects.
- index = next(i for i, item in enumerate(_policy_middlewares)
- if isinstance(item, SteerMiddleware))
- _policy_middlewares[index] = middleware
- else:
- _policy_middlewares.append(middleware)
- # The Agent sees one framework adapter. Transitional AgentScope policies
- # execute inside its compatibility chain and can be deleted one by one as
- # their neutral HookSpec replacements reach parity.
- _middlewares: list = [AgentScopeHookAdapter(legacy_middlewares=tuple(_policy_middlewares))]
+ from core.harness.registry import agent_profile
+ selected_profile = agent_profile(request.profile)
+ steering = selected_profile.steering(request)
+ for spec in selected_profile.hooks(request):
+ adapter.bus.register(spec)
+ for spec in (steering or create_steering_policy()).specs():
+ adapter.bus.register(spec)
+ if request.project_environment is not None:
+ from core.llm.runtime.project_environment_hook import project_environment_hook
+ adapter.bus.register(project_environment_hook(request))
+ if _eval_scope is None and _api_scope is None:
+ from core.harness.registry.hook_factories import command_completion_hooks
+ for spec in command_completion_hooks():
+ adapter.bus.register(spec)
+ adapter.bus.register(permission_policy.spec())
+ from core.models.selection_hook import model_selection_hook
+ adapter.bus.register(model_selection_hook())
+ from core.llm.citations.hook import citation_hook
+ adapter.bus.register(citation_hook())
+ from core.ontology.tool_hook import ontology_tool_hooks
+ for spec in ontology_tool_hooks(_ontology_runtime):
+ adapter.bus.register(spec)
+ from core.evolution.profiles.stall_hook import stall_hooks
+ for spec in stall_hooks(profile.intervention_rules):
+ adapter.bus.register(spec)
+ from core.llm.context.attachment_hook import attachment_context_hook
+ adapter.bus.register(attachment_context_hook())
+ from core.llm.context.reasoning_hooks import iteration_budget_hook
+ adapter.bus.register(iteration_budget_hook())
+ from core.llm.context.workspace_hooks import workspace_pin_hooks, plan_stale_hook
+ for spec in workspace_pin_hooks(allow_finish=_api_scope is None and _eval_scope is None):
+ adapter.bus.register(spec)
+ if _plan_tool_enabled:
+ adapter.bus.register(plan_stale_hook())
+ for spec in hook_specs(request):
+ adapter.bus.register(spec)
+ _middlewares: list = [adapter]
# At this point the collector has gathered all tools (including any subagent tools) → construct the final Toolkit.
toolkit = _build_toolkit()
diff --git a/src/backend/core/llm/factory/runtime/model_selection.py b/src/backend/core/llm/factory/runtime/model_selection.py
index f5bcb2de..e92800ca 100644
--- a/src/backend/core/llm/factory/runtime/model_selection.py
+++ b/src/backend/core/llm/factory/runtime/model_selection.py
@@ -4,7 +4,7 @@
from dataclasses import replace
-from core.llm.chat_models import get_default_model, make_chat_model
+from core.models.chat_models import get_default_model, make_chat_model
from core.llm.factory.request import AgentRequest
@@ -15,15 +15,15 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba
_selected_provider_id = (request.model_provider_id or "").strip()
if _selected_provider_id:
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
_selected_provider_cfg = (
_shared_subagent_cfg
or ModelConfigService.get_instance().resolve_provider(_selected_provider_id)
)
if _selected_provider_cfg:
- from core.llm.chat_models import build_model_for_mode
- from core.llm.failover import with_failover
+ from core.models.chat_models import build_model_for_mode
+ from core.models.failover import with_failover
_mode = (request.chat_mode or "medium").lower()
default_model = with_failover(
@@ -41,11 +41,11 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba
_mode_role = request.model_role or ("plan_agent" if request.plan_mode else None)
if default_model is None and _mode_role:
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
_mode_cfg = ModelConfigService.get_instance().resolve(_mode_role)
if _mode_cfg:
- from core.llm.failover import with_failover
+ from core.models.failover import with_failover
default_model = with_failover(
make_chat_model(
@@ -90,8 +90,8 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba
if _user_provider_id or _user_temp is not None or _user_max_tokens or _user_timeout:
try:
- from core.llm.chat_models import build_model_for_mode
- from core.services.model_config import ModelConfigService
+ from core.models.chat_models import build_model_for_mode
+ from core.models.config import ModelConfigService
_model_service = ModelConfigService.get_instance()
_provider_cfg = (
@@ -111,7 +111,7 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba
max_tokens=_user_max_tokens or _base_cfg.max_tokens,
timeout=_user_timeout or _base_cfg.timeout,
)
- from core.llm.failover import with_failover
+ from core.models.failover import with_failover
_overrides = {
key: value
@@ -143,7 +143,7 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba
_log.warning("[factory] subagent config override failed: %s, using default", exc)
if request.run_id:
- from core.llm.model_usage import instrument_model_usage
+ from core.models.usage import instrument_model_usage
instrument_model_usage(default_model)
diff --git a/src/backend/core/llm/factory/selection/capabilities.py b/src/backend/core/llm/factory/selection/capabilities.py
index 390144da..f490a25d 100644
--- a/src/backend/core/llm/factory/selection/capabilities.py
+++ b/src/backend/core/llm/factory/selection/capabilities.py
@@ -1,11 +1,13 @@
"""Agent assembly: capabilities. """
from __future__ import annotations
+from core.capabilities.skills import loader_factory
+from core.harness.registry import execution_account as skill_account
+from core.harness.registry import run_capabilities as skills
from typing import Optional
-import core.agent_skills.loader as skill_loader
-import core.config.catalog as catalog
+import core.capabilities.catalog.api as catalog
def _device_available_skill_ids(
@@ -23,18 +25,17 @@ def _device_available_skill_ids(
import logging as _logging
_log = _logging.getLogger(__name__)
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
return skill_ids
- from core.capabilities import skills as _caps_skills
- from core.capabilities.preparation import ensure_cloud_ready
+ from core.harness.registry.run_capabilities import ensure_cloud_ready
try:
ensure_cloud_ready(user_id, skill_keys=skill_ids, plugin_keys=plugin_ids)
available = set(
- _caps_skills.filter_available_names(
- skill_ids, user_id=_caps_skills.current_local_user_id()
+ skills.filter_available_names(
+ skill_ids, user_id=skill_account.current_local_user_id()
)
)
except Exception as exc: # noqa: BLE001 — 解析失败时不放大成整轮失败,退回原始集合
@@ -46,7 +47,7 @@ def _device_available_skill_ids(
return [sid for sid in skill_ids if sid in available]
-def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> list[str]:
+def filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> list[str]:
"""Strip out skill ids this user must not load.
Two independent rules, applied at the one choke point every agent (main,
@@ -67,7 +68,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) ->
try:
from core.db.engine import SessionLocal
from core.db.models import AdminSkill
- from core.evolution.exposure import filter_skill_ids as filter_evolved
+ from core.evolution.evidence.exposure import filter_skill_ids as filter_evolved
with SessionLocal() as db:
owned = dict(
@@ -86,7 +87,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) ->
# read is withheld, because the unsafe direction here is loading an
# unvetted self-authored capability.
try:
- from core.evolution.exposure import is_evolved_skill_id
+ from core.evolution.evidence.exposure import is_evolved_skill_id
return [sid for sid in skill_ids if not is_evolved_skill_id(sid)]
except Exception:
@@ -96,7 +97,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) ->
def _filter_kb_ids_for_user(kb_ids: list[str], user_id: Optional[str]) -> list[str]:
"""Strip out KB ids the current user has no access to (local + external collections), preventing unauthorized ids passed in from the frontend.
- Single source of truth ``core.auth.kb_permissions``: public KBs are visible
+ Single source of truth ``features.knowledge.edition``: public KBs are visible
to everyone, private KBs to their owner, and scoped-visibility KBs per
grant. On failure, fall back to returning the input unchanged (this doesn't
escalate permissions — it only avoids hurting availability, and the
@@ -105,7 +106,7 @@ def _filter_kb_ids_for_user(kb_ids: list[str], user_id: Optional[str]) -> list[s
if not kb_ids or not user_id:
return kb_ids
try:
- from core.auth.kb_permissions import filter_accessible_kb_ids
+ from core.harness.registry.retrieval_context import filter_accessible_kb_ids
from core.db.engine import SessionLocal
with SessionLocal() as db:
@@ -137,16 +138,16 @@ def _expand_plugin_bindings(
)
for r in rows:
cids = r.component_ids or {}
- from core.plugins.packaging.sources import _component_keys
+ from core.capabilities.plugins.packaging.sources import component_keys as _component_keys
skills.extend(_component_keys(cids, "skills"))
mcp.extend(_component_keys(cids, "mcp"))
except Exception: # noqa: BLE001
skills, mcp = [], []
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities.plugins import cloud_binding_ids
+ if has_run_capabilities():
+ from core.harness.registry.run_capabilities import cloud_binding_ids
cloud_skills, cloud_mcp = cloud_binding_ids(plugin_ids, user_id=user_id)
skills.extend(cloud_skills)
@@ -154,16 +155,16 @@ def _expand_plugin_bindings(
return list(dict.fromkeys(skills)), list(dict.fromkeys(mcp))
-def _effective_main_available_skills() -> list[str]:
+def effective_main_available_skills() -> list[str]:
"""Resolve main-agent skills from currently enabled catalog skills."""
def available(names):
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities import skills
+ if has_run_capabilities():
+ from core.harness.registry import run_capabilities as skills
- return skills.filter_available_names(names, user_id=skills.current_local_user_id())
+ return skills.filter_available_names(names, user_id=skill_account.current_local_user_id())
return names
enabled_ids = [
@@ -174,18 +175,19 @@ def available(names):
# On the server, an empty default catalog means no ambient skills. The
# loader also knows installed-but-disabled skills for explicit selection.
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
return []
try:
- loader = skill_loader.get_skill_loader()
+ loader = loader_factory.get_skill_loader()
discovered = sorted(loader.load_all_metadata().keys())
if discovered:
return available(discovered)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.factory.selection.capabilities:_effective_main_available_skills', fallback_error)
return []
@@ -195,7 +197,7 @@ def _mcp_ids_bound_to_skills(skill_ids: list[str]) -> list[str]:
if not skill_ids:
return []
try:
- metadata = skill_loader.get_skill_loader().load_all_metadata()
+ metadata = loader_factory.get_skill_loader().load_all_metadata()
except Exception:
return []
result: list[str] = []
diff --git a/src/backend/core/llm/factory/selection/entry_policy.py b/src/backend/core/llm/factory/selection/entry_policy.py
index 11d84ae0..9d608ad2 100644
--- a/src/backend/core/llm/factory/selection/entry_policy.py
+++ b/src/backend/core/llm/factory/selection/entry_policy.py
@@ -10,16 +10,16 @@
async def entry_policy(request: AgentRequest):
"""Normalize execution scopes and initialize per-construction capability state."""
- from core.llm.project_api_runtime import bind_project_api_runtime
+ from core.llm.api_runtime.project import bind_project_api_runtime
await bind_project_api_runtime(request)
required = RequiredCapabilities()
sticky = StickyCapabilities()
import asyncio
- from core.llm.agent_api_runtime import parse_api_scope
- from core.llm.evaluation_runtime import evaluation_user_agent, parse_evaluation_scope
- from core.llm.middlewares import CURRENT_RUN_BINDING
+ from core.llm.api_runtime.agent import parse_api_scope
+ from core.harness.registry.evaluation import evaluation_user_agent, parse_evaluation_scope
+ from core.harness.tool_call_context import CURRENT_RUN_BINDING
_eval_scope = parse_evaluation_scope(
owner_user_id=request.current_user_id,
@@ -43,7 +43,7 @@ async def entry_policy(request: AgentRequest):
request.ontology_runtime = {}
request.workspace_id = request.sandbox_session_id = _eval_scope.session_id
request.user_agent = evaluation_user_agent(request.user_agent)
- from core.llm.builtin_subagents import get_builtin_subagent
+ from core.capabilities.agents.builtin import get_builtin_subagent
request.visible_subagents = [
item
@@ -73,7 +73,7 @@ async def entry_policy(request: AgentRequest):
if request.user_agent is not None and not (
_api_scope and _api_scope.public_chat_visitor and request.model_provider_id
):
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
_model_service = ModelConfigService.get_instance()
_role_cfg = await asyncio.to_thread(_model_service.resolve, "subagent")
@@ -90,13 +90,13 @@ async def entry_policy(request: AgentRequest):
request.run_id = request.run_id or inherited_run_binding[0]
request.journal_owner = request.journal_owner or inherited_run_binding[1]
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
_capability_run_key = request.run_id
- if capabilities_enabled():
+ if has_run_capabilities():
import uuid as _cap_uuid
- from core.capabilities import runtime as capability_runtime
+ from core.harness.registry import capability_runtime
_capability_run_key = request.run_id or "ephemeral:" + _cap_uuid.uuid4().hex
if request.user_agent is not None:
@@ -111,7 +111,7 @@ async def entry_policy(request: AgentRequest):
import logging
import time
- from core.llm.tool_permissions import resolve_approval_mode
+ from core.sandbox.permissions.approval import resolve_approval_mode
request.approval_mode = resolve_approval_mode(
request.approval_mode, user_id=request.current_user_id
diff --git a/src/backend/core/llm/factory/selection/mode_bindings.py b/src/backend/core/llm/factory/selection/mode_bindings.py
index 52d258df..f27f57d0 100644
--- a/src/backend/core/llm/factory/selection/mode_bindings.py
+++ b/src/backend/core/llm/factory/selection/mode_bindings.py
@@ -22,12 +22,7 @@ async def mode_bindings(request: AgentRequest, *, required: RequiredCapabilities
_mode_plugin_ids = list(getattr(request.mode_spec, "plugin_ids", ()) or ())
_turbo_code_exec = bool(getattr(request.mode_spec, "code_exec_enabled", False))
else:
- from core.services.system_config import (
- turbo_manual_invoke_enabled,
- turbo_mcp_server_ids,
- turbo_plugin_ids,
- turbo_skill_ids,
- )
+ from core.config.store.execution_options import turbo_manual_invoke_enabled, turbo_mcp_server_ids, turbo_plugin_ids, turbo_skill_ids
_mode_manual_invoke = turbo_manual_invoke_enabled()
_mode_mcp_ids = sorted(turbo_mcp_server_ids())
diff --git a/src/backend/core/llm/factory/selection/profile_selection.py b/src/backend/core/llm/factory/selection/profile_selection.py
index aa07f8c9..cb57c964 100644
--- a/src/backend/core/llm/factory/selection/profile_selection.py
+++ b/src/backend/core/llm/factory/selection/profile_selection.py
@@ -6,15 +6,17 @@
from core.llm.factory.models import RequiredCapabilities, StickyCapabilities
-logger = logging.getLogger(__name__)
import asyncio
-import core.config.catalog as catalog
+import core.capabilities.catalog.api as catalog
from core.auth.tenancy import tenant_of
-from core.evolution.agent_profile import builtin_profile, load_active_profile
-from core.llm.execution_manifest import PromptManifestBuilder
+from core.evolution.profiles.agent_profile import builtin_profile
+from core.evolution.profiles.profile_selection import load_active_profile
+from core.harness.execution_manifest import PromptManifestBuilder
from core.llm.factory.request import AgentRequest
+logger = logging.getLogger(__name__)
+
async def profile_selection(
request: AgentRequest,
@@ -88,7 +90,7 @@ async def profile_selection(
# what the decremental engine needs; narrowing what gets loaded is not.
skill_selection = None
try:
- from core.agent_skills.selection_record import build_selection
+ from core.capabilities.skills.selection_record import build_selection
skill_selection = build_selection(
all_candidate_ids=list(skill_ids_for_bindings or []),
diff --git a/src/backend/core/llm/factory/selection/progressive_bindings.py b/src/backend/core/llm/factory/selection/progressive_bindings.py
index 8f637fd9..87bbec9a 100644
--- a/src/backend/core/llm/factory/selection/progressive_bindings.py
+++ b/src/backend/core/llm/factory/selection/progressive_bindings.py
@@ -4,15 +4,15 @@
import asyncio
-import core.config.catalog as catalog
-from core.capabilities.paths import capabilities_enabled
+import core.capabilities.catalog.api as catalog
+from core.harness.registry.run_capabilities import has_run_capabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.selection import capabilities as factory_capabilities
async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
- if capabilities_enabled() and not request.disable_tools and request.enabled_skill_ids:
- from core.capabilities.preparation import ensure_cloud_ready
+ if has_run_capabilities() and not request.disable_tools and request.enabled_skill_ids:
+ from core.harness.registry.run_capabilities import ensure_cloud_ready
await asyncio.to_thread(
ensure_cloud_ready, request.current_user_id, skill_keys=request.enabled_skill_ids
@@ -20,7 +20,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
# Security: strip out other users' private skills, preventing unauthorized skill_ids passed in from the frontend
if request.enabled_skill_ids:
- request.enabled_skill_ids = factory_capabilities._filter_skill_ids_for_user(
+ request.enabled_skill_ids = factory_capabilities.filter_skill_ids_for_user(
request.enabled_skill_ids, request.current_user_id
)
@@ -31,7 +31,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
# its explicit lists are its whole surface and its tool filter would hide
# `load_plugin`, leaving deferred plugins unreachable. Here, installed plugins' components are
# removed from this run's enabled sets and replaced by a one-line directory
- # entry + a `load_plugin` activation tool (see core/plugins/runtime/__init__.py).
+ # entry + a `load_plugin` activation tool (see core/capabilities/plugins/runtime/__init__.py).
# Deferral happens BEFORE the skill-bound-MCP merge below so a deferred
# skill doesn't pull its bound MCP servers into the assembly either.
_progressive = None
@@ -43,16 +43,16 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
and request.current_user_id
and _eval_scope is None
):
- from core.plugins import runtime as _plug
+ from core.capabilities.plugins import runtime as _plug
- if not capabilities_enabled() and _plug.progressive_plugin_loading_enabled():
+ if not has_run_capabilities() and _plug.progressive_plugin_loading_enabled():
try:
# Normalize the None fallbacks to their concrete resolutions
# (identical sources to the later phases) so the subtraction
# below has explicit lists to operate on.
if request.enabled_skill_ids is None:
- request.enabled_skill_ids = factory_capabilities._filter_skill_ids_for_user(
- factory_capabilities._effective_main_available_skills(),
+ request.enabled_skill_ids = factory_capabilities.filter_skill_ids_for_user(
+ factory_capabilities.effective_main_available_skills(),
request.current_user_id,
)
if not isinstance(request.enabled_mcp_ids, list):
@@ -95,7 +95,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
skill_ids_for_bindings = (
request.enabled_skill_ids
if request.enabled_skill_ids is not None
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
skill_bound_mcp_ids = (
factory_capabilities._mcp_ids_bound_to_skills(skill_ids_for_bindings)
@@ -103,7 +103,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log):
else []
)
if request.profile:
- from core.extensions import agent_profile
+ from core.harness.registry import agent_profile
if not agent_profile(request.profile).inherit_default_mcps:
skill_bound_mcp_ids = [m for m in skill_bound_mcp_ids if m in (request.enabled_mcp_ids or [])]
if skill_bound_mcp_ids:
diff --git a/src/backend/core/llm/factory/selection/server_selection.py b/src/backend/core/llm/factory/selection/server_selection.py
index 542feab1..f61ed6c5 100644
--- a/src/backend/core/llm/factory/selection/server_selection.py
+++ b/src/backend/core/llm/factory/selection/server_selection.py
@@ -2,11 +2,11 @@
from __future__ import annotations
-from core.capabilities.paths import capabilities_enabled
+from core.harness.registry.run_capabilities import has_run_capabilities
from core.llm.factory.models import RequiredCapabilities, StickyCapabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.tools import mcp_config as factory_mcp_config
-from core.services.mcp_service import McpServerConfigService
+from core.capabilities.mcp.service import McpServerConfigService
async def server_selection(
@@ -54,7 +54,7 @@ async def server_selection(
_capability_mcp_resolution = []
_unavailable_connectors = {}
try:
- from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs
+ from core.harness.registry.execution_catalog import cloud_gateway_mcp_configs
bridge_mcp_servers = cloud_gateway_mcp_configs(
request.enabled_mcp_ids,
@@ -62,7 +62,7 @@ async def server_selection(
unavailable_out=_unavailable_connectors,
)
except Exception: # noqa: BLE001
- if capabilities_enabled():
+ if has_run_capabilities():
raise # a desktop binding failure must not fall back to another source
bridge_mcp_servers = {}
@@ -77,7 +77,7 @@ async def server_selection(
]
# Determine which MCP servers to connect
- enabled_mcp_keys = factory_mcp_config._effective_mcp_server_keys(
+ enabled_mcp_keys = factory_mcp_config.effective_mcp_server_keys(
cfg,
request.agent_spec,
enabled_mcp_ids=request.enabled_mcp_ids,
diff --git a/src/backend/core/llm/factory/selection/sticky_capabilities.py b/src/backend/core/llm/factory/selection/sticky_capabilities.py
index a0eed260..04520eb0 100644
--- a/src/backend/core/llm/factory/selection/sticky_capabilities.py
+++ b/src/backend/core/llm/factory/selection/sticky_capabilities.py
@@ -4,8 +4,8 @@
import asyncio
-import core.config.catalog as catalog
-from core.capabilities.paths import capabilities_enabled
+import core.capabilities.catalog.api as catalog
+from core.harness.registry.run_capabilities import has_run_capabilities
from core.llm.factory.models import RequiredCapabilities, StickyCapabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.selection import capabilities as factory_capabilities
@@ -23,7 +23,7 @@ async def sticky_capabilities(
sticky: StickyCapabilities,
):
# Dedicated profiles own their full binding list. Chat history must not widen it.
- from core.extensions import agent_profile
+ from core.harness.registry import agent_profile
explicit_profile = bool(request.profile and not agent_profile(request.profile).inherit_default_mcps)
inherited_skills = set(request.enabled_skill_ids or [])
inherited_mcps = set(request.enabled_mcp_ids or [])
@@ -31,18 +31,18 @@ async def sticky_capabilities(
request.capability_source_chat_id if request.derived_main else None
) or request.chat_id
if (
- capabilities_enabled()
+ has_run_capabilities()
and not request.disable_tools
and not request.turbo_mode
and request.user_agent is None
and request.enabled_skill_ids is None
):
- from core.plugins.runtime import prepare_desktop_plugin_skill_defaults
+ from core.harness.registry.run_capabilities import plugin_skill_defaults
request.enabled_skill_ids = await asyncio.to_thread(
- prepare_desktop_plugin_skill_defaults,
+ plugin_skill_defaults,
request.current_user_id,
- factory_capabilities._effective_main_available_skills(),
+ factory_capabilities.effective_main_available_skills(),
)
_log.info("[factory] +%s default capabilities prepared", _elapsed())
@@ -56,7 +56,7 @@ async def sticky_capabilities(
base_skill_ids = (
list(request.enabled_skill_ids)
if isinstance(request.enabled_skill_ids, list)
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
request.enabled_skill_ids = list(
dict.fromkeys([*base_skill_ids, *required.plugin_skill_ids])
@@ -72,7 +72,7 @@ async def sticky_capabilities(
base_skill_ids = (
list(request.enabled_skill_ids)
if isinstance(request.enabled_skill_ids, list)
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
request.enabled_skill_ids = list(dict.fromkeys([*base_skill_ids, required.skill_id]))
@@ -101,8 +101,8 @@ async def sticky_capabilities(
and _eval_scope is None
and (not request.turbo_mode or _mode_manual_invoke)
):
- from core.llm import session_capabilities as _sticky_direct
- from core.plugins import runtime as _sticky_plugins
+ from core.llm.runtime import session_capabilities as _sticky_direct
+ from core.capabilities.plugins import runtime as _sticky_plugins
try:
_sticky, _direct = await asyncio.gather(
@@ -132,7 +132,7 @@ async def sticky_capabilities(
base_skill_ids = (
list(request.enabled_skill_ids)
if isinstance(request.enabled_skill_ids, list)
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
request.enabled_skill_ids = list(
dict.fromkeys([*base_skill_ids, *sticky_skill_ids])
@@ -154,7 +154,7 @@ async def sticky_capabilities(
len(sticky_mcp_ids),
)
except Exception as exc: # noqa: BLE001
- if capabilities_enabled():
+ if has_run_capabilities():
raise # a saved source must not silently disappear or change account
_log.warning("[factory] sticky capability restore failed: %s", exc)
diff --git a/src/backend/core/llm/factory/selection/subagent_bindings.py b/src/backend/core/llm/factory/selection/subagent_bindings.py
index 81d25453..d9a6bfaf 100644
--- a/src/backend/core/llm/factory/selection/subagent_bindings.py
+++ b/src/backend/core/llm/factory/selection/subagent_bindings.py
@@ -6,7 +6,7 @@
from dataclasses import replace
import prompts.prompt_config as prompt_config
-from core.capabilities.paths import capabilities_enabled
+from core.harness.registry.run_capabilities import has_run_capabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.selection import capabilities as factory_capabilities
@@ -49,11 +49,11 @@ async def subagent_bindings(request: AgentRequest, *, _api_scope, _elapsed, _log
# Expand bound plugins into their component skills + MCPs (a plugin = a detachable capability bundle). Merge with the loose bindings, deduplicated.
plugin_ids = request.user_agent.plugin_ids or []
if plugin_ids:
- from core.plugins import runtime as _plug_sub
+ from core.capabilities.plugins import runtime as _plug_sub
if (
not request.disable_tools
- and not capabilities_enabled()
+ and not has_run_capabilities()
and _api_scope is None
and _plug_sub.progressive_plugin_loading_enabled()
):
@@ -64,7 +64,7 @@ def _resolve_bound():
# would have received via factory_capabilities._filter_skill_ids_for_user.
return _plug_sub.resolve_bound_progressive_plugins(
list(plugin_ids),
- skill_filter=lambda sids: factory_capabilities._filter_skill_ids_for_user(
+ skill_filter=lambda sids: factory_capabilities.filter_skill_ids_for_user(
sids, request.current_user_id
),
)
diff --git a/src/backend/orchestration/registry.py b/src/backend/core/llm/factory/specs.py
similarity index 99%
rename from src/backend/orchestration/registry.py
rename to src/backend/core/llm/factory/specs.py
index 3e8a7e58..e68c4f56 100755
--- a/src/backend/orchestration/registry.py
+++ b/src/backend/core/llm/factory/specs.py
@@ -9,7 +9,6 @@
from dataclasses import dataclass, field
from typing import Any, Callable, List, Optional
-
AgentFactory = Callable[[], Any]
@@ -48,4 +47,3 @@ class AgentSpec:
model_name: Optional[str] = None
temperature: Optional[float] = None
max_tokens: Optional[int] = None
-
diff --git a/src/backend/core/llm/factory/tools/definitions.py b/src/backend/core/llm/factory/tools/definitions.py
new file mode 100644
index 00000000..fddc89e0
--- /dev/null
+++ b/src/backend/core/llm/factory/tools/definitions.py
@@ -0,0 +1,35 @@
+"""Native tool values and their domain policy declarations during assembly."""
+from agentscope.tool import FunctionTool, ToolBase
+from agentscope.permission import PermissionBehavior, PermissionDecision
+from core.sandbox.permissions.tool_specs import builtin_tool_permission
+
+
+class ToolAssembly(list[ToolBase]):
+ def __init__(self):
+ super().__init__()
+ self.skill_loaders = []
+
+ @property
+ def permission_specs(self):
+ from extensions.mcp_servers.registry import SERVERS
+ specs = {}
+ for server in SERVERS.values():
+ specs.update(server.permission_specs())
+ for tool in self:
+ spec = getattr(tool, "dispatch_spec", None) or builtin_tool_permission(tool.name)
+ if spec is not None:
+ previous = specs.get(tool.name)
+ if previous is not None and previous.key != spec.key:
+ raise ValueError(f"Conflicting permission declaration for {tool.name!r}")
+ specs[tool.name] = spec
+ return specs
+
+
+def function_tool(func, *, description=None, dispatch_spec=None):
+ tool = FunctionTool(
+ func, description=description, is_concurrency_safe=False,
+ permission=PermissionDecision(behavior=PermissionBehavior.ALLOW,
+ message="Subject to the run's execution policy"),
+ )
+ tool.dispatch_spec = dispatch_spec
+ return tool
diff --git a/src/backend/core/llm/factory/tools/interactive.py b/src/backend/core/llm/factory/tools/interactive.py
index bb4d8b4c..d30fbe9a 100644
--- a/src/backend/core/llm/factory/tools/interactive.py
+++ b/src/backend/core/llm/factory/tools/interactive.py
@@ -24,7 +24,7 @@ async def interactive_tools(
):
_log.info("[factory] +%s skills registered", _elapsed())
- from core.services.project_scope import project_scope_from_context
+ from core.projects.edition import project_scope_from_context
_proj_scope = project_scope_from_context(request.project_ctx or {})
diff --git a/src/backend/core/llm/factory/tools/mcp_config.py b/src/backend/core/llm/factory/tools/mcp_config.py
index 7ca5bfd2..e5b2956d 100644
--- a/src/backend/core/llm/factory/tools/mcp_config.py
+++ b/src/backend/core/llm/factory/tools/mcp_config.py
@@ -4,14 +4,13 @@
from typing import Any, Dict, List, Optional, Set
-import core.config.catalog as catalog
-from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
-from core.llm.mcp_pool import MCPConnectionPool
-from core.services.mcp_service import McpServerConfigService
-from orchestration.registry import AgentSpec
+import core.capabilities.catalog.api as catalog
+from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
+from core.llm.factory.specs import AgentSpec
+from core.capabilities.mcp.service import McpServerConfigService
-def _effective_mcp_server_keys(
+def effective_mcp_server_keys(
cfg,
agent_spec: Optional[AgentSpec],
enabled_mcp_ids: Optional[list[str]] = None,
@@ -103,7 +102,7 @@ def _required_mcp_server_keys(
return resolved
-def _inject_runtime_headers(
+def inject_runtime_headers(
enabled_servers: dict,
*,
current_user_id: Optional[str] = None,
@@ -125,7 +124,7 @@ def _inject_runtime_headers(
if not enabled_servers:
return enabled_servers
- from core.kb.external_provider import runtime_request_context
+ from core.harness.registry.retrieval_context import runtime_request_context
normalized = [str(x).strip() for x in (enabled_kb_ids or []) if str(x).strip()]
local_ids = [x for x in normalized if x.startswith("kb_")]
@@ -163,7 +162,7 @@ def _inject_runtime_headers(
if is_http:
headers = dict(c.get("headers") or {})
headers.update(ctx_headers)
- from core.llm.mcp_invocation import PLUGIN_HEADER, for_url
+ from core.llm.mcp.invocation import PLUGIN_HEADER, for_url
plugin_id = ""
if c.get("origin") == "local_plugin" and c.get("source_plugin"):
plugin_id = c.get("plugin_install_id") or "plugin:local:" + c["source_plugin"]
@@ -176,41 +175,3 @@ def _inject_runtime_headers(
c["env"] = env_cfg
out[key] = c
return out
-
-
-async def warmup_mcp_tools() -> None:
- """Initialize the MCP connection pool at startup.
-
- Reads MCP server configs from DB (via McpServerConfigService) and
- connects to all stable servers. Per-request servers (e.g.
- retrieve_dataset_content) are spawned on demand.
- """
- import logging
- import time
-
- log = logging.getLogger(__name__)
-
- # DB overlays (model config, system config) are already applied inside
- # McpServerConfigService._build_env(), so no manual overlay needed here.
- svc = McpServerConfigService.get_instance()
- servers = svc.get_all_servers(enabled_only=True)
-
- if not servers:
- log.info("[warmup] No MCP servers configured – skipping warmup")
- return
-
- log.info("[warmup] Initializing MCP connection pool for %d server(s)…", len(servers))
- start = time.monotonic()
-
- try:
- pool = MCPConnectionPool.get_instance()
- await pool.initialize(servers)
- elapsed = time.monotonic() - start
- log.info(
- "[warmup] MCP pool initialized: %d stable connections in %.2fs",
- pool.stable_client_count,
- elapsed,
- )
- except Exception as exc:
- elapsed = time.monotonic() - start
- log.warning("[warmup] MCP pool initialization failed after %.2fs: %s", elapsed, exc)
diff --git a/src/backend/core/llm/factory/tools/mcp_connections.py b/src/backend/core/llm/factory/tools/mcp_connections.py
index 84f171f9..b09dc69c 100644
--- a/src/backend/core/llm/factory/tools/mcp_connections.py
+++ b/src/backend/core/llm/factory/tools/mcp_connections.py
@@ -1,24 +1,25 @@
"""Agent assembly phase: mcp connections. """
from __future__ import annotations
+from core.capabilities.skills import loader_factory
import asyncio
-import time
from typing import List
-import core.agent_skills.loader as skill_loader
-from agentscope.mcp import MCPClient
-from core.llm.factory.defaults import (
- _HTTP_MCP_FAIL_AT,
- _HTTP_MCP_FAIL_COOLDOWN_S,
- _SKILL_INSTRUCTION_TEMPLATE,
- KB_MCP_HTTP_URL,
-)
+from core.llm.factory.defaults import _SKILL_INSTRUCTION_TEMPLATE
from core.llm.factory.models import RequiredCapabilities
from core.llm.factory.request import AgentRequest
-from core.llm.mcp_manager import own_factory_client
-from core.llm.mcp_pool import MCPConnectionPool
-from core.llm.tool_collector import ToolCollector
+from core.llm.mcp.manager import own_factory_client
+from core.llm.factory.tools.definitions import ToolAssembly
+
+
+def _leaf_errors(exc):
+ """Task groups wrap the transport's real error; report the leaves, not the wrapper."""
+ if isinstance(exc, BaseExceptionGroup):
+ for inner in exc.exceptions:
+ yield from _leaf_errors(inner)
+ else:
+ yield exc
async def mcp_connections(
@@ -36,7 +37,7 @@ async def mcp_connections(
):
def _preload_skill_metadata():
"""Pre-warm skill metadata cache so registration is fast."""
- loader = skill_loader.get_skill_loader()
+ loader = loader_factory.get_skill_loader()
loader.load_all_metadata()
return loader
@@ -48,24 +49,8 @@ def _preload_skill_metadata():
loader = await asyncio.to_thread(_preload_skill_metadata)
_log.info("[factory] +%s skill metadata pre-loaded", _elapsed())
- # ── Phase 2: MCP toolkit (async, may spawn per-request subprocesses) ──
- mcp_clients: List[MCPClient] = []
- # Stable (pooled) clients must be reused and must never be closed at the end
- # of a request; only transient (per-request spawned) stdio clients go on the
- # close list. mcp_clients contains stable+transient (for Toolkit
- # construction); transient_mcp_clients is only for closing.
- transient_mcp_clients: List[MCPClient] = []
- http_clients: List[MCPClient] = []
- # AgentScope 2.0: the Toolkit is constructed once — there are no incremental
- # register_* calls. Use ToolCollector to duck-type-compatibly collect our
- # in-house tools/skills (the register_* functions barely change), then
- # construct the real Toolkit at the end.
- toolkit = ToolCollector()
- # Policies must exist before deferred plugins become visible mid-run.
- from extensions.mcp_servers.registry import SERVERS
- for server in SERVERS.values():
- for name, spec in server.permission_specs().items():
- toolkit.register_tool_permission(name, spec)
+ mcp_resources = []
+ toolkit = ToolAssembly()
# ⚠️ This is a Jinja2 template, rendered by toolkit.get_skill_instructions()
# with the ``skills`` variable. It MUST contain the
# ``{% for skill in skills %}`` loop to actually list the skills — otherwise
@@ -89,160 +74,41 @@ def _preload_skill_metadata():
)
if not request.disable_tools and enabled_servers:
- from core.llm.mcp_pool import HTTP_TRANSPORTS, make_client, uses_manifest_schema
-
- http_server_cfgs = {
- k: v for k, v in enabled_servers.items() if v.get("transport") in HTTP_TRANSPORTS
- }
- stdio_servers = {
- k: v for k, v in enabled_servers.items() if v.get("transport") not in HTTP_TRANSPORTS
- }
-
- # ``isolated`` callers run in their own event loop (subagent_tool
- # worker threads), so they MUST NOT touch the shared MCP pool — pool
- # clients are bound to the main loop's task scope and would crash
- # anyio on cross-loop teardown. Spawn fresh per-request stdio + HTTP
- # instead, and rely on close_clients() in the caller's loop.
- if request.isolated or _api_scope is not None:
- from core.llm.mcp_manager import connect_mcp_clients
-
- mcp_clients = await connect_mcp_clients(stdio_servers)
- transient_mcp_clients = mcp_clients # all freshly spawned → all closable
- per_request_http = http_server_cfgs
- else:
- pool = MCPConnectionPool.get_instance()
- pool_managed = pool.stable_server_ids if pool.is_initialized else frozenset()
- per_request_http = {k: v for k, v in http_server_cfgs.items() if k not in pool_managed}
- if pool.is_initialized:
- per_request_stdio = {
- k: v for k, v in stdio_servers.items() if k not in pool_managed
- }
- # 2.0: the pool returns a list of connected MCPClients
- # (stable+transient); the Toolkit(mcps=...) is constructed
- # uniformly below.
- mcp_clients, transient_mcp_clients = await pool.get_request_clients(
- enabled_keys=enabled_mcp_keys,
- per_request_servers_cfg=per_request_stdio,
- )
- else:
- from core.llm.mcp_manager import connect_mcp_clients
-
- mcp_clients = await connect_mcp_clients(stdio_servers)
- transient_mcp_clients = mcp_clients # pool off → all transient
+ from core.llm.mcp.lifetime import discover
+ from core.harness.registry.local_publication import ensure_native_servers
- # Per-request HTTP — pool can't carry per-request headers that some
- # servers (e.g. retrieve_dataset_content) require. BaseException is
- # caught because the mcp HTTP client's SSE task can propagate
- # CancelledError on transient failures.
- async def _connect_http(key: str, cfg: dict):
- start = time.monotonic()
- last_fail = _HTTP_MCP_FAIL_AT.get(key, 0.0)
- if last_fail and start - last_fail < _HTTP_MCP_FAIL_COOLDOWN_S:
- return None
- # ⚠️ 2.0 key point: HTTP MCP uses is_stateful=False (a new connection
- # per call), avoiding the stateful client's task-binding problem
- # (the connect task differs from the request task → cancel-scope
- # crash, so tool_result is never received). Stateless clients need
- # not and must not connect(); a single list_tools serves as the
- # liveness probe (lazy connect + enumerate, verifying reachability),
- # after which the Toolkit opens a fresh connection on every call.
- _http_cfg = {**cfg, "url": cfg.get("url", KB_MCP_HTTP_URL),
- "read_only": request.read_only}
- if _http_cfg.get("schema_source") == "cloud_manifest" and not uses_manifest_schema(
- _http_cfg
- ):
- _log.warning(
- "[factory] HTTP MCP '%s' ignored because its cloud manifest is invalid",
- key,
- )
- return None
- client = own_factory_client(make_client(key, _http_cfg, is_stateful=False))
- if uses_manifest_schema(_http_cfg):
- # The complete, authorized schema arrived in the dynamic cloud
- # manifest. Agent construction is network-free; only a model-
- # selected tool call reaches the JSON gateway.
- _HTTP_MCP_FAIL_AT.pop(key, None)
- _log.info(
- "[factory] HTTP MCP '%s' loaded from cloud manifest (%d tools)",
- key,
- len(_http_cfg.get("manifest_tools") or []),
- )
- return client
+ async def load(key, cfg):
+ cfg = {**cfg, "read_only": request.read_only}
try:
- from orchestration.local_plugin_sidecars import ensure_native_servers
-
- await ensure_native_servers({key: _http_cfg})
- await client.list_tools()
- _HTTP_MCP_FAIL_AT.pop(key, None)
- _log.info(
- "[factory] HTTP MCP '%s' (stateless) probed in %.0fms",
- key,
- (time.monotonic() - start) * 1000,
- )
- return client
- except BaseException as exc:
- _HTTP_MCP_FAIL_AT[key] = time.monotonic()
- _log.warning(
- "[factory] HTTP MCP '%s' connect failed (%s, cooldown %.0fs): %s",
- key,
- type(exc).__name__,
- _HTTP_MCP_FAIL_COOLDOWN_S,
- exc,
- )
- # Only propagate CancelledError when the *outer* task is itself
- # being cancelled (real user/system cancel). anyio's SSE-client
- # cleanup raises CancelledError as a scope-exit signal even when
- # nobody cancelled us — re-raising those was killing the whole
- # chat run whenever any single HTTP MCP (e.g. a freshly-removed
- # word_mcp / ppt_mcp / excel_mcp / pdf_mcp whose admin_mcp_servers row was still
- # ``is_enabled=true``) was unreachable.
- if isinstance(exc, asyncio.CancelledError):
- current = asyncio.current_task()
- if current is not None and getattr(current, "cancelling", lambda: 0)() > 0:
- raise
+ if cfg.get("schema_source") not in {"cloud_manifest", "plugin_manifest"}:
+ await ensure_native_servers({key: cfg})
+ from core.llm.mcp.callbacks import Interactions
+ interactions = Interactions(request, key, _tool_approval_available)
+ scope = ((request.current_user_id, request.chat_id, request.capability_scope)
+ if request.current_user_id and request.chat_id and not request.isolated else None)
+ resource = await discover(key, cfg, interactions=interactions, scope=scope)
+ return own_factory_client(resource)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ target = str(cfg.get("url") or cfg.get("command") or "").split("?", 1)[0]
+ _log.warning("MCP %s discovery failed (%s %s): %s", key, cfg.get("transport") or "stdio", target,
+ "; ".join(f"{type(e).__name__}: {e}"[:300] for e in _leaf_errors(exc)))
return None
- if per_request_http:
- results = await asyncio.gather(
- *(_connect_http(k, v) for k, v in per_request_http.items()),
- return_exceptions=False,
- )
- http_clients.extend(c for c in results if c is not None)
-
- _log.info(
- "[factory] +%s MCP tools loaded (transient_stdio=%d, http=%d)",
- _elapsed(),
- len(mcp_clients),
- len(http_clients),
- )
+ results = await asyncio.gather(*(load(k, c) for k, c in enabled_servers.items()))
+ mcp_resources = [resource for resource in results if resource is not None]
+ failed = [key for key, resource in zip(enabled_servers, results) if resource is None]
+ if failed:
+ _note_unavailable("以下 MCP 服务连接失败,本轮无法使用其工具:" + "、".join(failed))
- connected_clients = {
- str(getattr(client, "name", "") or ""): client for client in [*mcp_clients, *http_clients]
- }
+ connected_clients = {resource.name: resource for resource in mcp_resources}
async def _mcp_tool_names_for_servers(server_keys: List[str], *, log_prefix: str) -> List[str]:
- connected_keys = [key for key in server_keys if key in connected_clients]
- if not connected_keys:
- return []
- listed_tools = await asyncio.gather(
- *(connected_clients[key].list_tools() for key in connected_keys),
- return_exceptions=True,
- )
- names: List[str] = []
- for server_key, result in zip(connected_keys, listed_tools):
- if isinstance(result, BaseException):
- _log.warning(
- "[%s] list_tools failed for %s: %s",
- log_prefix,
- server_key,
- result,
- )
- continue
- for tool in result:
- tool_name = str(getattr(tool, "name", "") or "")
- if tool_name and tool_name not in names:
- names.append(tool_name)
- return names
+ return list(dict.fromkeys(
+ tool.name for key in server_keys if key in connected_clients
+ for tool in connected_clients[key].tools
+ ))
_required_connector_tool_names = await _mcp_tool_names_for_servers(
_required_connector_server_keys,
@@ -267,9 +133,7 @@ async def _mcp_tool_names_for_servers(server_keys: List[str], *, log_prefix: str
_required_connector_tool_names,
_required_plugin_mcp_tool_names,
_tool_approval_available,
- http_clients,
+ mcp_resources,
loader,
- mcp_clients,
toolkit,
- transient_mcp_clients,
)
diff --git a/src/backend/core/llm/factory/tools/native.py b/src/backend/core/llm/factory/tools/native.py
index 4675bb22..830e7185 100644
--- a/src/backend/core/llm/factory/tools/native.py
+++ b/src/backend/core/llm/factory/tools/native.py
@@ -2,15 +2,13 @@
from __future__ import annotations
-import asyncio
+from core.harness.registry import NativeToolContext, install_tools
from core.llm.factory.request import AgentRequest
from core.llm.tools import (
ReadStateTracker,
register_bash,
- register_channel_attachment,
register_edit,
- register_get_data_context,
register_glob,
register_grep,
register_pin_to_workspace,
@@ -22,7 +20,7 @@
register_sandboxed_view_text_file,
register_write,
)
-from core.services.system_config import code_capability_enabled
+from core.config.store.execution_options import code_capability_enabled
async def native_tools(
@@ -46,7 +44,7 @@ async def native_tools(
_progressive=None,
):
if request.profile:
- from core.extensions import agent_profile
+ from core.harness.registry import agent_profile
agent_profile(request.profile).register_tools(toolkit, request, loader=loader)
return
@@ -68,12 +66,11 @@ async def native_tools(
# Skill files reach the sandbox via the unified /workspace/skills bind
# mount (built-in synced at startup, DB skills materialized on demand —
# see agent_skills.config.get_sandbox_skills_dir), so Bash needs no
- # per-call sync. loader/loaded_skill_ids kept for backward compat.
+ # per-call sync.
if request.allow_bash:
register_bash(
toolkit,
loader=loader,
- loaded_skill_ids=loaded_skill_ids,
source_run_id=request.run_id,
chat_id=request.chat_id,
sandbox_session_id=_sbx_sess,
@@ -102,29 +99,11 @@ async def native_tools(
# longer registered here (see extensions/mcp_servers/site_publish_mcp +
# extensions/plugins/marketplace/sites).
- # Site-builder design pick (choose one of three): registered only for
- # sessions with the site-builder skill enabled (per-run conditional
- # registration, not in the catalog). Interactivity uses the second-tier
- # judgment _ui_reachable (≠ _interactive): write confirmation has the
- # allow_session out-of-band pre-authorization path on IM channels and
- # the automation confirmation panel in automation sessions, but the
- # suspended picker has no clickable UI in either place — so degrade to
- # non-interactive (the tool just lets the model pick its own design
- # instead of suspending for 2h).
- from core.llm.tools import design_picker_tool
-
- _ui_reachable = _interactive and not _is_channel_run and not request.automation_run
- if skill_ids_to_register and any(
- design_picker_tool.skill_uses_choose_design(str(sid)) for sid in skill_ids_to_register
- ):
- design_picker_tool.register_choose_design(
- toolkit,
- chat_id=request.chat_id,
- interactive=_ui_reachable,
- sandbox_session_id=_sbx_sess,
- user_id=request.current_user_id,
- scope=_proj_scope,
- )
+ contribution_context = NativeToolContext(
+ request, _interactive, _is_channel_run, _sbx_sess, _proj_scope,
+ skill_ids_to_register, enabled_mcp_keys, _log,
+ )
+ await install_tools("after_sandbox", toolkit, contribution_context)
# ── Phase 3.6: Register file-operation tools (Read/Edit/Write/Glob/
# Grep). These tools share a single
@@ -143,7 +122,7 @@ async def native_tools(
# Decided once per build from the model this run will actually use: a
# pinned subagent model wins over the request's provider (mirrors the
# sub-agent override below and DynamicModelMiddleware).
- from core.vision import resolve_vision_mode
+ from core.models.vision import resolve_vision_mode
_vision_mode = resolve_vision_mode(
(_shared_subagent_cfg.provider_id if _shared_subagent_cfg else None)
@@ -222,18 +201,11 @@ async def native_tools(
vision_mode=_vision_mode,
)
- # ── Phase 3.7b: channel_read_attachment (channel runs only) ──
- # Group listening records bystander attachments by key without downloading them;
- # this is how the agent pulls one in on demand. Gated on channel runs so the tool
- # never clutters the toolkit of web conversations, where it could never resolve.
- if _is_channel_run:
- register_channel_attachment(
- toolkit, user_id=request.current_user_id, chat_id=request.chat_id
- )
+ await install_tools("after_image", toolkit, contribution_context)
# ── Phase 3.8: Register pin_to_workspace ──
# Lets the agent gate which generated files reach the user-visible
- # assistant message. See core/llm/workspace.py for the per-run state.
+ # assistant message. See core/llm/runtime/workspace.py for the per-run state.
register_pin_to_workspace(toolkit, scope=_proj_scope, sandbox_session_id=_sbx_sess)
# ── Phase 3.85: run_job(工作流模式的作业编排面) ──
@@ -245,8 +217,8 @@ async def native_tools(
# 就自行收工)。run_job 把循环体交给沙箱脚本,模型调用由后端代持凭据派出——
# 脚本因此拿不到任何 key。isolated=True 的子作业不注册,杜绝 job 套 job。
if request.workflow_mode and not request.isolated and _sbx_sess:
+ from core.llm.ports import snapshot_parent_runtime
from core.llm.tools.job_tool import register_run_job
- from orchestration.jobs.agent import snapshot_parent_runtime
register_run_job(
toolkit,
@@ -266,31 +238,11 @@ async def native_tools(
),
)
- # ── Phase 3.9: get_data_context (the "data dictionary" tool for direct-DB data retrieval) ──
- # Three gates combined: (1) a direct DB server is enabled this run
- # (db_query / es_query); (2) the external NL2SQL black box is excluded
- # (query_database isn't in the set, so it naturally doesn't trigger);
- # (3) the corresponding data source has annotation content. If any is
- # unmet, don't attach — avoid adding a useless tool that misleads the
- # model. The metadata only ever appears as a tool return value, never in
- # the system prompt. See db_metadata_service.
- _db_servers_on = {"db_query", "es_query"} & set(enabled_mcp_keys)
- if _db_servers_on:
- try:
- from core.services import db_metadata_service as _dbmeta
-
- _eligible_ds = await asyncio.to_thread(
- _dbmeta.eligible_datasource_ids, _db_servers_on
- )
- except Exception as _e: # noqa: BLE001
- _eligible_ds = []
- _log.warning("[factory] eligible_datasource_ids failed: %s", _e)
- if _eligible_ds:
- register_get_data_context(toolkit, _eligible_ds)
+ await install_tools("after_workflow", toolkit, contribution_context)
if not request.disable_tools and _eval_scope is not None:
- from core.llm.evaluation_tools import register_evaluation_tools
- from core.vision import resolve_vision_mode
+ from core.harness.registry.evaluation import register_evaluation_tools
+ from core.models.vision import resolve_vision_mode
register_evaluation_tools(
toolkit,
@@ -301,7 +253,7 @@ async def native_tools(
)
if not request.disable_tools and _api_scope is not None:
- from core.llm.agent_api_tools import register_agent_api_tools
+ from core.llm.api_runtime.agent_tools import register_agent_api_tools
register_sandboxed_view_text_file(
toolkit,
diff --git a/src/backend/core/llm/factory/tools/skill_registration.py b/src/backend/core/llm/factory/tools/skill_registration.py
index bc0f9ef1..eaf9cffe 100644
--- a/src/backend/core/llm/factory/tools/skill_registration.py
+++ b/src/backend/core/llm/factory/tools/skill_registration.py
@@ -5,8 +5,7 @@
import asyncio
import os
-from core.capabilities import runtime as capability_runtime
-from core.capabilities.paths import capabilities_enabled
+from core.harness.registry.run_capabilities import has_run_capabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.selection import capabilities as factory_capabilities
@@ -16,12 +15,12 @@ async def skill_registration(
):
skill_ids_to_register = request.enabled_skill_ids
if skill_ids_to_register is None:
- skill_ids_to_register = factory_capabilities._effective_main_available_skills()
+ skill_ids_to_register = factory_capabilities.effective_main_available_skills()
# The fallback resolves from the static catalog, which today carries no
# evolution-authored ids — but the exposure gate is applied here anyway
# rather than relying on that. A gate that only covers the paths we
# happened to think of is not a gate.
- skill_ids_to_register = factory_capabilities._filter_skill_ids_for_user(
+ skill_ids_to_register = factory_capabilities.filter_skill_ids_for_user(
skill_ids_to_register, request.current_user_id
)
# Note: a subagent's (user_agent) enabled_skill_ids is always a list ([]
@@ -30,15 +29,8 @@ async def skill_registration(
# config, no inheriting the full catalog set.
if _prepared_capabilities is not None:
- missing = set(skill_ids_to_register or []) - set(_prepared_capabilities.bindings)
- if missing:
- from core.capabilities.errors import PackageMissing
-
- raise PackageMissing(
- "selected skills are absent from this prepared run",
- details={"skills": sorted(missing)},
- )
- loader = await asyncio.to_thread(capability_runtime.frozen_loader, _prepared_capabilities)
+ _prepared_capabilities.validate_selected_skills(skill_ids_to_register)
+ loader = await asyncio.to_thread(_prepared_capabilities.skill_loader)
allowed_skill_dirs: list[str] = []
if not request.disable_tools and skill_ids_to_register:
@@ -52,16 +44,12 @@ async def skill_registration(
if (
not request.disable_tools
- and not capabilities_enabled()
+ and not has_run_capabilities()
and _api_scope is None
and _eval_scope is None
and request.project_environment is None
):
- from core.agent_skills.config import (
- get_enabled_skill_sources,
- get_sandbox_skills_dir,
- get_user_skills_dir,
- )
+ from core.capabilities.skills.config import get_enabled_skill_sources, get_sandbox_skills_dir, get_user_skills_dir
for src in get_enabled_skill_sources():
root = str(src.root_dir)
diff --git a/src/backend/core/llm/factory/tools/tool_surface.py b/src/backend/core/llm/factory/tools/tool_surface.py
index fdbb4249..18a7b697 100644
--- a/src/backend/core/llm/factory/tools/tool_surface.py
+++ b/src/backend/core/llm/factory/tools/tool_surface.py
@@ -6,11 +6,11 @@
from typing import Any, Dict, Optional
from agentscope.tool import Toolkit
-from core.capabilities import runtime as capability_runtime
+from core.capabilities.skills.runtime_loader import ParallelRuntimeSkillLoader
from core.llm.factory.models import RequiredCapabilities
from core.llm.factory.request import AgentRequest
-from core.ontology.toolkit import OntologyFilteredToolkit
-from core.ontology.validator import register_runtime_asset_tags
+from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit
+from core.ontology.runtime_pack import register_runtime_asset_tags
async def tool_surface(
@@ -24,9 +24,8 @@ async def tool_surface(
required: RequiredCapabilities,
_required_connector_tool_names,
_required_plugin_mcp_tool_names,
- http_clients,
+ mcp_resources,
loader,
- mcp_clients,
skill_ids_to_register,
toolkit,
):
@@ -72,7 +71,7 @@ async def tool_surface(
}
def _build_toolkit(*, include_skills: bool = True) -> Toolkit:
- # ``toolkit`` here is still the ToolCollector; construct the real Toolkit from the current collection state.
+ # Construct the native Toolkit from the assembled native tool values.
#
# "Skill" (AgentScope's builtin SkillViewer) is hidden unconditionally —
# distinct from the ontology forbidden_tools above. The Toolkit injects
@@ -82,20 +81,10 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit:
# observability). A model that called Skill instead would bypass all of
# that and receive backend-path content unusable in the sandbox — so the
# schema is pure per-round prefill waste plus a wrong door.
- _visible_mcps = [*mcp_clients, *http_clients]
- if _prepared_capabilities is not None:
- from core.llm.capability_tools import AvailableMCPClient
-
- _live_run = (
- capability_runtime.get(
- _prepared_capabilities.run_id, scope_id=request.capability_scope
- )
- or _prepared_capabilities
- )
- _visible_mcps = [AvailableMCPClient(client, _live_run) for client in _visible_mcps]
- selected_tools = toolkit.function_tools
+ mcp_tools = [tool for resource in mcp_resources for tool in resource.tools]
+ selected_tools = list(toolkit)
if request.profile:
- from core.extensions import agent_profile
+ from core.harness.registry import agent_profile
_profile = agent_profile(request.profile)
if _profile.tool_filter is not None:
@@ -103,16 +92,21 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit:
# The existing MCP resolver already intersects the explicit per-agent list.
# A profile that does not inherit defaults gets none when the list is empty.
if not _profile.inherit_default_mcps and not request.enabled_mcp_ids:
- _visible_mcps = []
+ mcp_tools = []
if not _profile.native_skill_prompt:
include_skills = False
- return OntologyFilteredToolkit(
- tools=selected_tools,
- mcps=_visible_mcps,
- skills_or_loaders=(toolkit.skill_loaders or None) if include_skills else None,
+ names = [tool.name for tool in [*selected_tools, *mcp_tools]]
+ if len(names) != len(set(names)):
+ raise ValueError("Duplicate model-facing tool names")
+ built = OntologyFilteredToolkit(
+ tools=[*selected_tools, *mcp_tools],
+ skills_or_loaders=[ParallelRuntimeSkillLoader(toolkit.skill_loaders)] if include_skills and toolkit.skill_loaders else None,
skill_instruction_template=_SKILL_INSTRUCTION_TEMPLATE,
hidden_tools={*_ontology_hidden_tools, "Skill"},
)
+ if _prepared_capabilities is not None:
+ built.capability_guard = _prepared_capabilities.tool_guard(scope_id=request.capability_scope)
+ return built
# Compute schemas first in the "subagent tools not yet registered" state
# (consistent with 1.x: subagent tools are registered after
@@ -162,7 +156,7 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit:
# Progressive loading normally persists this during its resolution.
# Persist here as well so explicit activation is sticky in restricted
# modes, dedicated-agent chats and when progressive loading is disabled.
- from core.plugins import runtime as _plugin_activation
+ from core.capabilities.plugins import runtime as _plugin_activation
await asyncio.to_thread(
_plugin_activation.record_plugin_activation,
@@ -171,7 +165,7 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit:
user_id=str(request.current_user_id or ""),
)
if request.chat_id and (_required_skill_registered or _required_connector_tool_names):
- from core.llm.session_capabilities import record_session_capability_activation
+ from core.llm.runtime.session_capabilities import record_session_capability_activation
await asyncio.to_thread(
record_session_capability_activation,
diff --git a/src/backend/core/llm/finish_guard.py b/src/backend/core/llm/finish_guard.py
deleted file mode 100644
index 46a1feb3..00000000
--- a/src/backend/core/llm/finish_guard.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""finish_pin_guard helpers: auto-pin the files a model forgot when it "dumbly exits".
-
-Models often finish generating files and then exit directly, forgetting the
-``pin_to_workspace`` delivery. ``FinishPinGuardMiddleware`` (core.llm.middlewares),
-when a reasoning turn makes no tool calls at all (i.e. the model is about to
-finish), calls this module's ``_collect_unpinned`` + ``_direct_pin`` and
-**writes to the ContextVar by calling ``workspace.pin()`` directly**, bypassing
-the ReAct main loop's _acting mechanism. ``chat_run_executor`` calls
-``workspace.get_pinned()`` when wrapping up the stream to pick up this line and
-deliver it as an attachment over SSE.
-
-Design trade-offs:
-- **Do not inject a reminder** to the model — it has already decided to exit,
- and an injection into context would not survive this turn; the next turn's new
- agent would not see it. The auto-pin side effect itself is the only remedy the
- user perceives.
-- **At most one auto-pin per reply turn** (the middleware uses a ``_fired`` flag
- to prevent an infinite loop when all file_ids are garbage); when every pin
- fails, ``_fired`` is not set, leaving it for retry.
-"""
-from __future__ import annotations
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def _collect_unpinned() -> list[str]:
- """From the file_id set already collected by pin_hint, pick those not yet pinned and return a sorted list."""
- try:
- from core.llm import workspace as _ws
- from core.llm.hooks import _get_pin_hint_state
- state = _get_pin_hint_state()
- seen = state.get("seen") or set()
- if not seen:
- return []
- pinned = set(_ws.get_pinned_file_ids())
- return sorted(seen - pinned)
- except Exception as exc: # noqa: BLE001
- logger.warning("[finish_guard] _collect_unpinned failed: %s", exc)
- return []
-
-
-def _direct_pin(file_ids: list[str]) -> int:
- """Write file_ids directly to the workspace ContextVar, bypassing the
- pin_to_workspace tool + _acting path. Returns the number of pins that actually succeeded.
-
- Metadata comes from ``core.artifacts.store.get_artifact`` — on failure that id is skipped.
- """
- try:
- from core.artifacts.store import get_artifact
- from core.llm import workspace as _ws
- except Exception as exc: # noqa: BLE001
- logger.warning("[finish_guard] _direct_pin import failed: %s", exc)
- return 0
-
- count = 0
- for fid in file_ids:
- try:
- art = get_artifact(fid)
- if not art:
- logger.warning("[finish_guard] artifact not found: %s", fid)
- continue
- ok = _ws.pin(
- fid,
- name=art.get("name"),
- mime_type=art.get("mime_type"),
- size=art.get("size") or art.get("size_bytes"),
- url=f"/files/{fid}",
- )
- if ok:
- count += 1
- except Exception as exc: # noqa: BLE001
- logger.warning("[finish_guard] _direct_pin failed for %s: %s", fid, exc)
- # Only mark_active when at least one pin succeeded — avoid an empty set still
- # calling mark_active, which would leave the workspace in an "active but pinned=[]"
- # state (although downstream currently does not rely on the active flag for
- # gating, this is a more robust invariant).
- if count > 0:
- try:
- from core.llm import workspace as _ws
- _ws.mark_active()
- except Exception:
- pass
- return count
diff --git a/src/backend/core/llm/hooks.py b/src/backend/core/llm/hooks.py
deleted file mode 100644
index 684ce668..00000000
--- a/src/backend/core/llm/hooks.py
+++ /dev/null
@@ -1,588 +0,0 @@
-"""Pure-logic helpers: file-context building, per-turn state, model resolution.
-
-AgentScope 2.0 replaced the old hook factories with the Middleware classes in
-``core.llm.middlewares``; this module has been reduced to the pure-function
-helpers those middlewares reuse, independent of the agentscope version
-(``_build_file_context`` / ``_build_historical_files_context`` /
-``_get_main_model`` / ``_resolve_chat_mode`` / pin-hint & read-budget state, etc.).
-The runtime context moved from 1.x's ``agent._jx_context`` to ``agent.state``
-(AgentRuntimeState).
-"""
-
-from __future__ import annotations
-
-import base64
-import logging
-import re
-import threading
-from functools import wraps
-from contextvars import ContextVar
-from typing import Any, Dict, List, Optional
-
-logger = logging.getLogger(__name__)
-
-MAX_FILE_CONTENT_CHARS = 50_000 # per-turn read_artifact budget per file
-
-# Per-turn cumulative read budget for the read_artifact tool.
-# Mapping ``file_id → chars already returned this turn``. Reset by the
-# file_context pre_reply hook on every turn boundary so a single user turn
-# can read up to ``MAX_FILE_CONTENT_CHARS`` per file via paginated calls.
-#
-# The contextvar pattern works because AgentScope dispatches tool calls in
-# the same async task as the agent's reply(), so the value set in the hook
-# propagates to read_artifact.
-_artifact_read_chars_per_turn: ContextVar[Optional[Dict[str, int]]] = ContextVar(
- "_artifact_read_chars_per_turn", default=None
-)
-
-# Per-turn cache for the workspace-pin reminder hook:
-# seen: file_ids the agent has observed in tool results this turn
-# last_reminded_sig: signature of the unpinned set the last reminder covered
-# (used for in-hook dedup to avoid spamming when the same unpinned set
-# persists across multiple tool calls).
-_pin_hint_state: ContextVar[Optional[Dict[str, Any]]] = ContextVar("_pin_hint_state", default=None)
-
-
-def get_artifact_read_state() -> Dict[str, int]:
- """Return the current per-turn read tracker, lazily initializing it.
-
- Used by ``read_artifact`` to enforce the cumulative cap.
- """
- state = _artifact_read_chars_per_turn.get()
- if state is None:
- state = {}
- _artifact_read_chars_per_turn.set(state)
- return state
-
-
-def reset_artifact_read_state() -> None:
- """Clear the per-turn read tracker. Called from file_context_pre_reply."""
- _artifact_read_chars_per_turn.set({})
-
-
-def reset_pin_hint_state() -> None:
- """Clear the per-turn pin-reminder cache. Called at turn boundary."""
- _pin_hint_state.set({"seen": set(), "last_reminded_sig": ""})
-
-
-def _get_pin_hint_state() -> Dict[str, Any]:
- state = _pin_hint_state.get()
- if state is None:
- state = {"seen": set(), "last_reminded_sig": ""}
- _pin_hint_state.set(state)
- return state
-
-
-# ── DynamicModel helper (used by DynamicModelMiddleware) ─────────────────
-
-# Desktop models also capture a cloud session in their request hook. Cached
-# instances must never cross that identity boundary, even when DB config is unchanged.
-_model_cache: dict[tuple[Any, ...], Any] = {}
-_cached_version: int = -1
-
-
-_model_cache_lock = threading.RLock()
-
-
-def _serialized_model_cache(function):
- @wraps(function)
- def locked(*args, **kwargs):
- with _model_cache_lock:
- return function(*args, **kwargs)
-
- return locked
-
-
-def _check_version():
- """Invalidate cached model instances when ModelConfigService version changes."""
- global _model_cache, _cached_version
- from core.services.model_config import ModelConfigService
-
- current = ModelConfigService.get_instance().version
- if current != _cached_version:
- _model_cache = {}
- _cached_version = current
-
-
-def _model_cache_key(name: str) -> tuple[Any, ...]:
- from core.auth.desktop_bridge import bridge_enabled
-
- import asyncio
-
- try:
- loop = asyncio.get_running_loop()
- except RuntimeError:
- loop = None
- if not bridge_enabled():
- return (name, loop, None)
- from core.services.desktop_cloud_bridge import _state_fingerprint, get_state
-
- identity = _state_fingerprint(get_state())
- # Eviction only releases this cache's reference. In-flight runs keep their
- # captured model and its existing request hook, which rejects identity changes.
- for key in list(_model_cache):
- if isinstance(key, tuple) and key[0] == name and key[2] != identity:
- _model_cache.pop(key, None)
- return name, loop, identity
-
-
-@_serialized_model_cache
-def release_loop_models() -> None:
- import asyncio
-
- loop = asyncio.get_running_loop()
- for key in list(_model_cache):
- if key[1] is loop:
- _model_cache.pop(key, None)
-
-
-@_serialized_model_cache
-def _get_main_model(mode: str = "medium"):
- """Get the main agent model for the given chat mode (fast/medium/high/max)."""
- from core.llm.chat_models import get_default_model
- from core.llm.failover import with_failover
- from core.services.model_config import ModelConfigService
-
- _check_version()
- key = _model_cache_key(f"main:{mode}")
- cached = _model_cache.get(key)
- if cached is not None:
- return cached
- if mode in ("fast", "turbo"):
- instance = get_default_model(disable_thinking=True, stream=True)
- elif mode in ("low", "high", "xhigh", "max"):
- instance = get_default_model(reasoning_effort=mode, stream=True)
- else: # medium or unknown
- # medium uses the "thinking model", but picks chat_template_kwargs based on
- # supports_reasoning_effort (chat_models internally distinguishes
- # effort=medium vs None)
- instance = get_default_model(reasoning_effort="medium", stream=True)
- instance = with_failover(
- instance, ModelConfigService.get_instance().resolve("main_agent"), mode=mode
- )
- _model_cache[key] = instance
- return instance
-
-
-@_serialized_model_cache
-def _get_provider_model(provider_id: str, mode: str = "medium"):
- """Get a user-selected active chat provider model for the given chat mode."""
- from core.llm.chat_models import build_model_for_mode
- from core.llm.failover import with_failover
- from core.services.model_config import ModelConfigService
-
- pid = (provider_id or "").strip()
- if not pid:
- return _get_main_model(mode)
- _check_version()
- key = _model_cache_key(f"provider:{pid}:{mode}")
- cached = _model_cache.get(key)
- if cached is not None:
- return cached
- resolved = ModelConfigService.get_instance().resolve_provider(pid)
- if resolved is None:
- return _get_main_model(mode)
-
- instance = with_failover(
- build_model_for_mode(resolved, mode=mode, stream=True), resolved, mode=mode
- )
- _model_cache[key] = instance
- return instance
-
-
-def _resolve_chat_mode(ctx) -> str:
- """Resolve the final chat_mode (turbo/fast/medium/high/max) from agent.state (AgentRuntimeState)."""
- raw = getattr(ctx, "chat_mode", None)
- if raw in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"):
- return raw
- # Fallback for legacy clients
- return "medium" if getattr(ctx, "enable_thinking", True) else "fast"
-
-
-# ── FileContext hook (replaces FileContextMiddleware) ─────────────────────
-
-
-def _is_image(f: Dict[str, Any]) -> bool:
- mime = (f.get("mime_type") or "").lower()
- return mime.startswith("image/")
-
-
-def _download_artifact_bytes(
- file_id: str,
- fallback_name: str,
- log_prefix: str,
- user_id: Optional[str] = None,
-) -> Optional[bytes]:
- """Resolve an artifact's storage key and download its raw bytes.
-
- Returns None on any failure (missing artifact, storage error, etc.) and
- logs at WARNING. Used by the image and xlsx-preview hook branches that
- both need raw bytes (not parsed text).
-
- When ``user_id`` is supplied, the artifact's owner is verified via
- :func:`load_artifact_meta` before any storage round-trip. This is the
- defense-in-depth gate for hook-level reads: the agent context is built
- from ``request.attachments[].file_id`` (client-supplied), and the
- backend boundary doesn't re-verify ownership before reaching the
- hook. Without this check, a forged ``file_id`` belonging to another
- user would have its raw bytes downloaded and silently materialized
- into the agent's memory (image base64 or xlsx preview block). The
- matching ``fetch_parsed_text`` already does the same check for parsed
- text — this brings the binary paths to parity.
-
- ``user_id=None`` preserves the legacy "trust the caller" behavior
- used by ``core.llm.tools.read_artifact_tool._probe_xlsx_sheet_names`` /
- ``_probe_pptx_slide_count`` where ``fetch_parsed_text(user_id=...)``
- is the authoritative gate downstream.
- """
- from core.content.artifact_reader import (
- load_artifact_meta,
- resolve_artifact_storage,
- )
- from core.infra.exceptions import StorageError
- from core.storage import get_storage
-
- if user_id:
- meta = load_artifact_meta(file_id, user_id=user_id)
- if meta is None:
- logger.warning(
- "%s: artifact %s denied for user %s (not owner / missing / deleted)",
- log_prefix,
- file_id,
- user_id,
- )
- return None
-
- from core.artifacts.local_reference import authorized_reference, is_local_file_ref
- if is_local_file_ref(file_id):
- item = authorized_reference(file_id, user_id)
- if not item:
- return None
- try:
- from pathlib import Path
- return Path(item["path"]).read_bytes()
- except OSError:
- return None
-
- storage_key, _ = resolve_artifact_storage(file_id, fallback_name)
- if not storage_key:
- return None
-
- try:
- return get_storage().download_bytes(storage_key)
- except StorageError as e:
- logger.warning(f"{log_prefix}: storage download failed for {file_id}: {e}")
- return None
- except Exception as e:
- logger.warning(f"{log_prefix}: unexpected download error for {file_id}: {e}")
- return None
-
-
-def _fetch_image_base64(
- f: Dict[str, Any],
- user_id: Optional[str] = None,
-) -> Optional[tuple[str, str]]:
- """Fetch image data for a file attachment and return (base64_data, mime_type)."""
- file_id = f.get("file_id") or ""
- if not file_id:
- return None
- mime_type = (f.get("mime_type") or "image/png").lower()
- raw = _download_artifact_bytes(
- file_id,
- f.get("name") or "image",
- "Image hook",
- user_id=user_id,
- )
- if raw is None:
- return None
- return base64.b64encode(raw).decode("utf-8"), mime_type
-
-
-def _is_xlsx(f: Dict[str, Any]) -> bool:
- """Detect xlsx by mime_type or filename extension.
-
- We only special-case xlsx (not xls/csv) because:
- - xlsx is the dominant batch-execution upload format
- - parse_xlsx_preview uses openpyxl, same engine as the batch path
- - xls would need a separate xlrd-based preview helper (out of scope)
- """
- mime = (f.get("mime_type") or "").lower()
- if "spreadsheetml.sheet" in mime:
- return True
- name = (f.get("name") or f.get("filename") or "").lower()
- return name.endswith(".xlsx")
-
-
-def _build_xlsx_preview_block(
- f: Dict[str, Any],
- user_id: Optional[str] = None,
-) -> Optional[str]:
- """Build the bounded preview + metadata + directive block for a single xlsx.
-
- Returns None if the file can't be read (caller falls back to the regular
- parsed_text path, so the agent at least sees *something*).
- """
- from core.content.file_parser import parse_xlsx_preview
-
- file_id = f.get("file_id") or ""
- name = f.get("name") or f.get("filename") or "未命名表格"
- if not file_id:
- return None
-
- file_bytes = _download_artifact_bytes(
- file_id,
- name,
- "xlsx preview",
- user_id=user_id,
- )
- if file_bytes is None:
- return None
-
- try:
- from core.content.artifact_reader import (
- ATTACHMENT_PREVIEW_MAX_CHARS,
- fetch_parsed_text,
- )
-
- # Reuse the bytes already downloaded for the structural preview to
- # populate parsed_text + summary without a second storage round-trip.
- fetch_parsed_text(file_id, user_id=user_id, prefetched_bytes=file_bytes)
- info = parse_xlsx_preview(file_bytes, char_budget=ATTACHMENT_PREVIEW_MAX_CHARS)
- except RuntimeError as e:
- logger.warning(f"xlsx preview: parse failed for {file_id}: {e}")
- return None
- except Exception as e:
- logger.warning(f"xlsx preview: unexpected parse error for {file_id}: {e}")
- return None
-
- total_rows = info["total_rows"]
- total_cols = info["total_columns"]
- preview_rows = info["preview_rows"]
- omitted = max(0, total_rows - preview_rows)
- cols_str = ", ".join(info["header"]) if info["header"] else "(无列名)"
-
- lines: List[str] = [
- f"[文件: {name}]",
- f" file_id: {file_id}",
- f" 总规模: {total_rows} 行 × {total_cols} 列",
- f" 列名: [{cols_str}]",
- f" 已展示: 前 {preview_rows} 行(约 {len(info['preview_md'])} 字符,达到预览预算)",
- ]
- if info.get("truncated_columns"):
- lines.append(f" 注意: 共 {total_cols} 列,预览仅含前 {len(info['header'])} 列")
- if info.get("truncated_cells"):
- lines.append(f" 注意: {info['truncated_cells']} 个超长单元格已截断到 500 字以内")
- lines.append("")
- lines.append(info["preview_md"])
- if omitted:
- lines.append(f"... (剩余 {omitted} 行未展示) ...")
- lines.append("")
- lines.append("[操作指引]")
- lines.append(
- f"- 需要预览之外的正文时,调用 read_artifact(file_id='{file_id}', "
- "offset=N, limit=4000) 按需读取"
- )
- lines.append("- 以上只是有界预览,不要把未展示的行当作不存在,也不要推断未读取内容")
- return "\n".join(lines)
-
-
-def _build_file_context(
- uploaded_files: List[Dict[str, Any]],
- user_id: Optional[str] = None,
-) -> str:
- """Assemble the text-attachment list into the context text injected into the model (non-image files only).
-
- Files are resolved exclusively by ``file_id``. The backend lazily parses and
- caches the artifact, then injects only metadata plus a bounded preview. The
- request body never supplies parsed content.
-
- ``user_id`` is used as the ownership check when fetching ``fetch_parsed_text`` /
- the xlsx preview, preventing a forged request-body ``attachments[].file_id``
- from reading across users — see the :func:`_download_artifact_bytes` docstring
- for the core defense point.
- """
- text_files = [f for f in uploaded_files if not _is_image(f)]
- if not text_files:
- return ""
-
- file_descriptions = []
- for f in text_files:
- name = f.get("name", "未知文件")
- fid = f.get("file_id", "")
- desc = f"- {name}"
- if fid:
- desc += f" (file_id: {fid})"
- file_descriptions.append(desc)
- from core.content.artifact_reader import (
- ATTACHMENT_PREVIEW_MAX_CHARS,
- fetch_parsed_text,
- )
-
- file_content_parts: List[str] = []
- for f in text_files:
- # ── xlsx: bounded preview + metadata + directive (no full content) ──
- if _is_xlsx(f):
- block = _build_xlsx_preview_block(f, user_id=user_id)
- if block is not None:
- file_content_parts.append(block)
- continue
- # Fall through to regular path on preview failure so the agent
- # at least sees the parsed markdown (degraded but non-fatal).
-
- fid = (f.get("file_id") or "").strip()
- mime_type = f.get("mime_type") or ""
- if not fid:
- file_content_parts.append(
- f"[文件: {f.get('name', '未知文件')}]\n"
- " 状态: 缺少 file_id,无法读取正文"
- )
- continue
-
- logger.info("Document hook: loading preview for %s (%s)", f.get("name"), fid)
- content = fetch_parsed_text(fid, user_id=user_id)
- if not content:
- file_content_parts.append(
- f"[文件: {f.get('name', '未知文件')}]\n"
- f" file_id: {fid}\n"
- f" MIME: {mime_type or '(未知)'}\n"
- " 状态: 文件不可访问或解析失败"
- )
- continue
-
- total_chars = len(content)
- preview = content[:ATTACHMENT_PREVIEW_MAX_CHARS]
- truncated = total_chars > len(preview)
- block = [
- f"[文件: {f.get('name', '未知文件')}]",
- f" file_id: {fid}",
- f" MIME: {mime_type or '(未知)'}",
- f" 解析文本: 共 {total_chars} 字符,本轮自动展示 {len(preview)} 字符",
- "",
- preview,
- ]
- if truncated:
- block.append(f"... (其余 {total_chars - len(preview)} 字符未展示) ...")
- block.extend(
- [
- "",
- "[读取指引]",
- f"- 需要未展示的正文时,调用 read_artifact(file_id='{fid}', offset=N, limit=4000)",
- "- 不要推断尚未读取的内容",
- ]
- )
- file_content_parts.append("\n".join(block))
- file_content = "\n\n---\n\n".join(file_content_parts)
-
- return (
- f"[file name]: {chr(10).join(file_descriptions)}\n"
- f"[file content begin]\n"
- f"{file_content}\n"
- f"[file content end]\n"
- "以上是用户附件的元数据和有界预览;需要预览之外的正文时必须按需调用 read_artifact。"
- )
-
-
-def _source_label(source: str) -> str:
- from core.content.artifact_reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD
-
- return {SOURCE_USER_UPLOAD: "用户上传", SOURCE_AI_GENERATED: "AI 生成"}.get(source, "")
-
-
-def _build_historical_files_context(historical_files: List[Dict[str, Any]]) -> str:
- """Build a compact summary block for files from previous turns.
-
- Includes both user-uploaded files and AI-generated files (reports,
- charts, code output, etc.) produced by tools in earlier turns.
- Each entry is labeled by provenance so the agent knows what kind of
- file it's referencing.
-
- Agent gets only `{filename, file_id, source, summary}`; full content
- is fetched on-demand via the `read_artifact` tool. This keeps prompt
- size bounded as the conversation grows.
- """
- if not historical_files:
- return ""
-
- lines: List[str] = []
- for f in historical_files:
- file_id = f.get("file_id") or ""
- name = f.get("name") or f.get("filename") or "未命名文件"
- mime = f.get("mime_type") or ""
- summary = (f.get("summary") or "").strip()
- source = f.get("source") or ""
- deleted = bool(f.get("deleted"))
-
- source_label = _source_label(source)
- header_parts = [f"file_id: {file_id}"]
- if source_label:
- header_parts.append(source_label)
- if mime:
- header_parts.append(mime)
- header = f"- {name} ({', '.join(header_parts)})"
-
- if deleted:
- lines.append(header + " [文件已删除,无法读取]")
- continue
-
- lines.append(header)
- # Key point: tell the model the stable /myspace/ path so it can operate
- # directly with Read/Edit/Write — better suited than read_artifact for
- # "modify HTML/CSS/MD/source code" text-editing scenarios, and changes sync
- # back to "My Space" immediately.
- if name and source == _ai_source():
- lines.append(f" 沙盒路径: /myspace/{name} ← 用 Read/Edit/Write 直接读改写")
- if summary:
- indented = "\n".join(f" {ln}" for ln in summary.splitlines())
- lines.append(f" 摘要:\n{indented}")
- else:
- lines.append(" 摘要:(尚未生成)")
-
- header = (
- "[历史文件清单](本会话中之前轮次涉及的文件,包含用户上传和 AI 生成的两类,以下仅为摘要)\n"
- "访问方式:\n"
- '- **要修改文本/HTML/MD/代码** → 用 `Read("/myspace/")` 读完整内容,'
- "再用 `Edit` / `Write` 改;改完会自动同步回「我的空间」(同一 file_id)。\n"
- "- **要看 docx/pdf/xlsx 等需要解析的二进制文件** → 用 "
- "`read_artifact(file_id, offset, limit)` 拿解析后的文本。"
- )
- return header + "\n\n" + "\n".join(lines)
-
-
-def _ai_source() -> str:
- """Lazy resolve to avoid circular import at module load."""
- from core.content.artifact_reader import SOURCE_AI_GENERATED
-
- return SOURCE_AI_GENERATED
-
-
-# ── WorkspacePinHint hook ─────────────────────────────────────────────────
-# Strict-mode `pin_to_workspace` gating means generated files are hidden
-# from users until the agent explicitly pins them. Multiple input paths
-# (MCP tools that return file_id, `sandbox_get_artifact` for bash-produced
-# files, skill scripts) all share this responsibility, and the model
-# (especially smaller / flash variants under long contexts) often forgets
-# the pin step — see trace aeb643c6-5a8f-4874-a4b4-3d472c581fa6 where 6
-# excel_create_workbook + pdf_create + excel_export_pdf
-# produced 8 file_ids, all delivered to the user as exactly zero files.
-#
-# This `post_acting` hook scans every tool result for `"file_id"` mentions,
-# diffs them against the workspace state's pinned set, and appends a
-# concise system reminder to memory listing the unpinned ids. The reminder
-# is internal (added directly via `memory.add`, not via `print`) so it
-# doesn't surface to the SSE stream.
-
-# Pattern picks up top-level `"file_id": ""` keys in tool result JSON.
-# We deliberately match the JSON key form so we ignore `artifact_id`-style
-# echoes from `Bash` / `sandbox_put_artifact` etc., which
-# operate on already-existing artifacts and don't need pinning.
-_FILE_ID_RE = re.compile(r'"file_id"\s*:\s*"([A-Za-z0-9_\-]+)"')
-
-# Tools whose results legitimately mention `file_id` but never produce a
-# new deliverable artifact — skipping them avoids spurious reminders.
-_PIN_HINT_SKIP_TOOLS: frozenset[str] = frozenset(
- {
- "pin_to_workspace", # the pin tool itself
- "read_artifact", # echoes the input file_id; reading, not producing
- "generate_response", # the ReActAgent finish function
- }
-)
diff --git a/src/backend/core/llm/host_services.py b/src/backend/core/llm/host_services.py
new file mode 100644
index 00000000..95796e18
--- /dev/null
+++ b/src/backend/core/llm/host_services.py
@@ -0,0 +1,31 @@
+"""Compose model-owned services at the application host boundary."""
+from core.harness.registry.model_services import register_model_services
+from core.llm import prompt_kinds, prompt_versions, skill_selection
+
+
+def active_prompt_versions():
+ versions = []
+ for kind in list(prompt_kinds.KIND_SPECS):
+ try:
+ active = prompt_versions.get_active_version(kind)
+ except Exception:
+ continue
+ if active:
+ versions.append((kind, active))
+ return versions
+
+
+def register():
+ from core.harness.registry.mcp_clients import register_mcp_client_factory
+ from core.llm.mcp.lifetime import discover
+ register_mcp_client_factory(discover)
+ from core.models.config import ModelConfigService
+ from core.llm.prompt_rendering import render_active_prompt_part
+ from core.llm.runtime.review_text_agent import run_text_agent
+ register_model_services(
+ ranker=lambda **kwargs: skill_selection.select_skills_for_query(**kwargs),
+ prompt_reader=active_prompt_versions,
+ prompt_renderer=render_active_prompt_part,
+ text_agent=run_text_agent,
+ environment=lambda: ModelConfigService.get_instance().get_mcp_env_overlay(),
+ )
diff --git a/src/backend/core/llm/management_tool.py b/src/backend/core/llm/management_tool.py
deleted file mode 100644
index 617bbc70..00000000
--- a/src/backend/core/llm/management_tool.py
+++ /dev/null
@@ -1,84 +0,0 @@
-"""Local execution of an explicitly declared, installed manager plugin contract."""
-import asyncio
-import json
-from agentscope.tool import ToolBase, ToolChunk
-from agentscope.message import TextBlock, ToolResultState
-from agentscope.permission import PermissionBehavior, PermissionDecision
-from core.services.management_contract import declared, MANAGERS
-from core.services.management_errors import management_error
-
-
-class ManagementTool(ToolBase):
- is_mcp = True
- is_state_injected = False
- is_external_tool = False
- is_concurrency_safe = True
-
- def __init__(self, server_id, tool, manager, headers, *, cloud_backed):
- self.mcp_name, self.name = server_id, tool.name
- self.description, self.input_schema = tool.description, tool.inputSchema
- self.manager, self.headers = manager, dict(headers)
- self.cloud_backed = cloud_backed
- self.is_read_only = self.name.startswith(("get_", "list_"))
-
- async def check_permissions(self, *_args, **_kwargs):
- return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK,
- message="管理当前用户的本机技能或插件")
-
- def _execute(self, arguments):
- from core.services import desktop_cloud_bridge as bridge
- from core.capabilities import skills
- from core.auth.desktop_bridge import bridge_enabled
- state = None
- if self.cloud_backed and bridge_enabled():
- state = bridge.get_state()
- bridge.require_current_account(state)
- user_id = skills.current_local_user_id()
- context = bridge._bridge_context() or {}
- server = next((s for s in context.get("servers", []) if s["server_id"] == self.mcp_name), None)
- if not server or server.get("source_plugin") != self.manager:
- raise PermissionError("management plugin is no longer installed or authorized")
- # upload is a local contribution authorized by the same declared install capability.
- op = "install_skill" if self.name == "upload_skill_to_cloud" else self.name
- contract = next((t for t in server.get("tools", []) if t["name"] == op), None)
- if contract is None or declared(contract, self.manager) != self.manager:
- raise PermissionError("management plugin contract changed; synchronize capabilities")
- else:
- user_id = next((v for k, v in self.headers.items() if k.lower() == "x-current-user-id"), None)
- from core.capabilities.local_plugin_runtime import authorizes_manager
- if not authorizes_manager(user_id, self.mcp_name, self.manager):
- raise PermissionError("local management plugin is not installed")
- if not user_id:
- raise PermissionError("authentication required")
- from jsonschema import validate
- validate(arguments, self.input_schema)
- if state:
- bridge.require_current_account(state)
- if self.name == "upload_skill_to_cloud":
- from core.services.local_skill_upload import upload
- return upload(user_id, **arguments)
- kind = MANAGERS[self.manager]
- from core.services import local_skill_service
- from core.plugins.local import service as local_plugin_service
- service = local_skill_service if kind == "skill" else local_plugin_service
- action = self.name.removesuffix("_" + kind)
- if self.name == "list_" + kind + "s":
- return {"ok": True, "items": getattr(service, "list_" + kind + "s")(user_id)}
- args = dict(arguments)
- if "source" in args:
- source = args.pop("source")
- if source.get("kind") != "local_path":
- raise ValueError("local installation requires source.kind=local_path")
- args["source_path"] = source["path"]
- return getattr(service, action)(user_id, **args)
-
- async def __call__(self, **kwargs):
- try:
- result = await asyncio.to_thread(self._execute, kwargs)
- return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))],
- state=ToolResultState.SUCCESS,
- metadata={"origin": "local", "capabilities_changed": not self.is_read_only})
- except Exception as exc:
- message = management_error(exc)
- return ToolChunk(content=[TextBlock(text=json.dumps({"ok": False, "error": message}, ensure_ascii=False))],
- state=ToolResultState.ERROR, metadata={"origin": "local"})
diff --git a/src/backend/core/llm/mcp/__init__.py b/src/backend/core/llm/mcp/__init__.py
new file mode 100644
index 00000000..62e5be1f
--- /dev/null
+++ b/src/backend/core/llm/mcp/__init__.py
@@ -0,0 +1 @@
+"""MCP client pool, tool invocation and the gateway tool for agents."""
diff --git a/src/backend/core/llm/mcp/callbacks.py b/src/backend/core/llm/mcp/callbacks.py
new file mode 100644
index 00000000..2499287a
--- /dev/null
+++ b/src/backend/core/llm/mcp/callbacks.py
@@ -0,0 +1,168 @@
+"""Run-bound MCP interactions. Remote services never receive the main conversation.
+
+A server request (sampling, elicitation) is answered only on behalf of the outer
+tool call that caused it. On 2026-07-28 connections the SDK runs those callbacks
+inside the calling task, so `CURRENT_CALL` names the call exactly; handshake-era
+servers deliver them from the session task, where only a sole in-flight call can
+be credited.
+"""
+import asyncio
+import json
+from contextlib import aclosing, nullcontext
+from contextvars import ContextVar
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+import jsonschema
+from agentscope.message import Msg, TextBlock
+from mcp import types
+from core.llm.tools import user_questions
+
+CURRENT_CALL: ContextVar["CallScope | None"] = ContextVar("mcp_current_call", default=None)
+
+
+def route(session):
+ call = CURRENT_CALL.get()
+ if call is not None and call.session is session:
+ return call
+ return next(iter(session.in_flight)) if len(session.in_flight) == 1 else None
+
+
+@dataclass
+class Interactions:
+ """What one run allows a server to ask of it; shared by that run's calls."""
+ request: object
+ server: str
+ interactive: bool
+ agent: object = None
+ samples: int = 0
+
+ def open_call(self, session=None):
+ return CallScope(self, session)
+
+
+class CallScope:
+ """One outer tool call; server requests made on its behalf end with it."""
+
+ def __init__(self, run, session=None):
+ self.run, self.session = run, session
+ self.active = True
+ self.pending = set()
+ self.budget = None
+
+ async def dispatch(self, method, context, params):
+ task = asyncio.create_task(getattr(self, method)(context, params))
+ self.pending.add(task)
+ try:
+ return await task
+ finally:
+ self.pending.discard(task)
+
+ async def finish(self):
+ self.active = False
+ tasks = list(self.pending)
+ for task in tasks:
+ task.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def validate(self):
+ if not self.active or self.run.agent is None:
+ raise RuntimeError("MCP interaction is outside an active run")
+ prepared = getattr(self.run.agent, "_jx_prepared_capabilities", None)
+ if prepared is not None:
+ await asyncio.to_thread(prepared.validate, user_id=str(self.run.request.current_user_id or ""))
+ if not await asyncio.to_thread(prepared.validate_tool_call, self.run.server):
+ raise PermissionError("MCP capability was revoked")
+ guard = getattr(getattr(self.run.agent, "toolkit", None), "capability_guard", None)
+ if guard is not None and not await asyncio.to_thread(guard, self.run.server):
+ raise PermissionError("MCP capability was revoked")
+
+ async def sampling(self, _context, params):
+ if not self.active or self.run.agent is None or self.run.samples >= 4:
+ return types.ErrorData(code=-32600, message="Sampling unavailable or budget exhausted")
+ if params.tools or params.include_context not in (None, "none"):
+ return types.ErrorData(code=-32602, message="Sampling accepts isolated text without tools or shared context")
+ messages = []
+ for message in params.messages:
+ content = message.content if isinstance(message.content, list) else [message.content]
+ if any(not isinstance(item, types.TextContent) for item in content):
+ return types.ErrorData(code=-32602, message="Only text sampling is enabled")
+ messages.append(Msg(name=self.run.server, role=message.role,
+ content=[TextBlock(text=item.text) for item in content]))
+ if params.system_prompt:
+ messages.insert(0, Msg(name=self.run.server, role="system", content=params.system_prompt))
+ if len(params.model_dump_json()) > 32000 or not 0 < params.max_tokens <= 4096:
+ return types.ErrorData(code=-32602, message="Sampling request exceeds the run budget")
+ self.run.samples += 1
+ await self.validate()
+ from core.models.usage import model_usage_scope
+ from core.harness.ledger import HarnessUsageLedger
+ with model_usage_scope(self.run.request.run_id or "", HarnessUsageLedger()):
+ async with asyncio.timeout(60):
+ response = await self.run.agent.model(messages, tools=None, max_tokens=params.max_tokens)
+ response_value = None
+ if hasattr(response, "__aiter__"):
+ async with aclosing(response):
+ async for chunk in response:
+ response_value = chunk
+ else:
+ response_value = response
+ if response_value is None:
+ return types.ErrorData(code=-32603, message="Sampling model returned no response")
+ if not self.active:
+ return types.ErrorData(code=-32600, message="Outer MCP invocation ended")
+ await self.validate()
+ text = "".join(block.text for block in response_value.content if isinstance(block, TextBlock))
+ return types.CreateMessageResult(role="assistant", content=types.TextContent(text=text),
+ model=str(self.run.agent.model.model), stop_reason="endTurn")
+
+ async def elicitation(self, _context, params):
+ if not self.active or not self.run.interactive:
+ return types.ElicitResult(action="decline")
+ await self.validate()
+ if params.mode == "url":
+ parsed = urlsplit(params.url)
+ if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
+ return types.ElicitResult(action="decline")
+ questions = [{"id": "external", "question": f"{self.run.server}: {params.message}\n{params.url}",
+ "options": [{"label": "已在外部页面完成"}, {"label": "拒绝"}]}]
+ else:
+ schema = params.requested_schema
+ props = schema.get("properties", {})
+ if not props or len(props) > user_questions.MAX_QUESTIONS:
+ return types.ElicitResult(action="decline")
+ questions = []
+ for key, spec in props.items():
+ if spec.get("format") in {"password", "secret"} or spec.get("writeOnly"):
+ return types.ElicitResult(action="decline")
+ values = spec.get("enum", [True, False] if spec.get("type") == "boolean" else [])
+ questions.append({"id": key, "question": f"{self.run.server}: {params.message} — {spec.get('title', key)}",
+ "options": [{"label": json.dumps(v, ensure_ascii=False)} for v in values]})
+ # Only the human wait is excluded: validation, sampling and server
+ # work before/after this block still consume the original budget.
+ with self.budget.waiting_for_user() if self.budget is not None else nullcontext():
+ outcome = await user_questions.ask(chat_id=self.run.request.chat_id, questions=questions, interactive=True)
+ if not self.active or outcome["status"] != "answered":
+ return types.ElicitResult(action="cancel")
+ await self.validate()
+ answers = outcome["answers"]
+ if params.mode == "url":
+ accepted = answers[0].get("selected_labels") == ["已在外部页面完成"]
+ return types.ElicitResult(action="accept" if accepted else "decline")
+ content = {}
+ try:
+ for answer in answers:
+ if answer.get("skipped"):
+ continue
+ key = answer["id"]
+ selected = answer.get("selected_labels") or []
+ value = answer.get("custom")
+ if value is None and selected:
+ value = json.loads(selected[0])
+ elif props[key].get("type") != "string":
+ value = json.loads(value)
+ content[key] = value
+ jsonschema.validate(content, schema)
+ except (ValueError, TypeError, jsonschema.ValidationError, jsonschema.SchemaError):
+ return types.ElicitResult(action="decline")
+ return types.ElicitResult(action="accept", content=content)
diff --git a/src/backend/core/llm/mcp/chat_sessions.py b/src/backend/core/llm/mcp/chat_sessions.py
new file mode 100644
index 00000000..69527178
--- /dev/null
+++ b/src/backend/core/llm/mcp/chat_sessions.py
@@ -0,0 +1,181 @@
+"""Bounded pool of chat-scoped STDIO sessions shared by that chat's concurrent runs.
+
+A session is keyed by the trusted executing user, chat and capability scope plus the
+server configuration. Runs hold counted leases; the call gate inside the session
+orders writes, so a run never waits for another run to finish. Idle sessions
+expire, and are reclaimed oldest-first when capacity is needed.
+"""
+import asyncio
+import hashlib
+import json
+import time
+from dataclasses import dataclass
+
+IDLE_SECONDS = 60.0
+MAX_SESSIONS = 32
+MAX_USER_SESSIONS = 4
+_entries = {}
+
+
+@dataclass
+class Entry:
+ key: tuple
+ loop: object
+ session: object = None
+ opening: asyncio.Task | None = None
+ expiry: asyncio.Task | None = None
+ holders: int = 0
+ idle_at: float = 0.0
+ retired: bool = False
+
+ @property
+ def idle(self):
+ return self.holders == 0 and self.opening is None
+
+ def detach(self):
+ """Stop handing this session out; its holders keep it until they release."""
+ if _entries.get(self.key) is self:
+ del _entries[self.key]
+ self.retired = True
+ if self.expiry is not None:
+ self.expiry.cancel()
+ self.expiry = None
+
+
+class Lease:
+ def __init__(self, entry):
+ self.entry, self.session = entry, entry.session
+ self._released = False
+
+ async def release(self, *, abort=False):
+ if self._released:
+ return
+ self._released = True
+ entry = self.entry
+ entry.holders -= 1
+ if abort or self.session.stopped:
+ entry.detach()
+ if entry.holders:
+ return
+ if entry.retired:
+ await self.session.close()
+ else:
+ _park(entry)
+
+
+async def _expire(entry):
+ await asyncio.sleep(IDLE_SECONDS)
+ if entry.idle and _entries.get(entry.key) is entry:
+ entry.detach()
+ await entry.session.close()
+
+
+async def _reclaim(user):
+ """Make room for one more session, closing idle sessions oldest-first."""
+ while True:
+ user_entries = [e for k, e in _entries.items() if k[0] == user]
+ if len(user_entries) >= MAX_USER_SESSIONS:
+ candidates = user_entries
+ elif len(_entries) >= MAX_SESSIONS:
+ candidates = list(_entries.values())
+ else:
+ return
+ idle = [e for e in candidates if e.idle]
+ if not idle:
+ raise RuntimeError("MCP STDIO session capacity reached")
+ victim = min(idle, key=lambda e: e.idle_at)
+ victim.detach()
+ await victim.session.close()
+
+
+def _key(scope, name, cfg):
+ from mcp.client.stdio import get_default_environment
+ cfg = {**cfg, "env": get_default_environment() | (cfg.get("env") or {})}
+ cfg.pop("oauth_provider", None)
+ fingerprint = hashlib.sha256(json.dumps(cfg, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest()
+ return (*scope, name, fingerprint)
+
+
+def _park(entry):
+ """An unheld session waits out its idle period, or closes now if it was retired."""
+ if entry.retired:
+ return asyncio.create_task(entry.session.close())
+ entry.idle_at = time.monotonic()
+ entry.expiry = asyncio.create_task(_expire(entry))
+
+
+async def _open(entry, name, cfg):
+ from core.llm.mcp.session import Session
+ try:
+ session = await Session.open(name, cfg)
+ except BaseException:
+ if entry.opening is asyncio.current_task():
+ entry.opening = None
+ entry.detach()
+ raise
+ entry.opening = None
+ entry.session = session
+ if entry.retired:
+ await session.close()
+ raise RuntimeError("MCP chat session was revoked")
+ if entry.holders == 0:
+ _park(entry)
+ return session
+
+
+async def acquire(scope, name, cfg):
+ # Scope contains trusted executing user, chat and capability scope, not remote metadata.
+ key = _key(scope, name, cfg)
+ for old in [e for k, e in _entries.items() if k[:-1] == key[:-1] and k != key]:
+ old.detach()
+ if old.idle and old.session is not None:
+ await old.session.close()
+ entry = _entries.get(key)
+ if entry is not None and (entry.retired or entry.session is not None and entry.session.stopped):
+ entry.detach()
+ entry = None
+ if entry is None:
+ await _reclaim(scope[0])
+ entry = _entries.setdefault(key, Entry(key, asyncio.get_running_loop()))
+ if entry.expiry is not None:
+ entry.expiry.cancel()
+ entry.expiry = None
+ entry.holders += 1
+ try:
+ if entry.session is None:
+ if entry.opening is None:
+ entry.opening = asyncio.create_task(_open(entry, name, cfg), name=f"mcp-open:{name}")
+ await asyncio.shield(entry.opening)
+ if entry.retired or entry.session.stopped:
+ raise RuntimeError("MCP chat session was revoked")
+ return Lease(entry)
+ except BaseException:
+ entry.holders -= 1
+ if entry.holders == 0 and entry.session is not None and entry.expiry is None:
+ _park(entry)
+ raise
+
+
+async def close_chat(chat_id=None):
+ """Revoke every session of a chat (or all); holders' next calls fail as closed."""
+ for entry in [e for k, e in list(_entries.items()) if chat_id is None or k[1] == chat_id]:
+ entry.detach()
+ if entry.opening is not None:
+ opening, entry.opening = entry.opening, None
+ opening.cancel()
+ await asyncio.gather(opening, return_exceptions=True)
+ if entry.session is not None:
+ entry.session.failed = True
+ await entry.session.close()
+
+
+def invalidate_chat(chat_id):
+ """Deletion can run on a sync API worker; dispatch teardown to the owning loop."""
+ entries = [entry for key, entry in list(_entries.items()) if key[1] == chat_id]
+ for entry in entries:
+ # Plain flag writes take effect at once; dict and task work stays on the owning loop.
+ entry.retired = True
+ if entry.session is not None:
+ entry.session.failed = True
+ for loop in {entry.loop for entry in entries}:
+ loop.call_soon_threadsafe(lambda: asyncio.create_task(close_chat(chat_id)))
diff --git a/src/backend/core/llm/mcp/client.py b/src/backend/core/llm/mcp/client.py
new file mode 100644
index 00000000..9106c556
--- /dev/null
+++ b/src/backend/core/llm/mcp/client.py
@@ -0,0 +1,149 @@
+"""SDK-native connections; project authorization is attached at the transport boundary."""
+import asyncio
+import hashlib
+import json
+import math
+import os
+import weakref
+from contextlib import asynccontextmanager
+from typing import Any
+import httpx2
+from mcp import Client, StdioServerParameters
+from mcp.client.caching import CacheConfig, InMemoryResponseCacheStore
+from mcp.client.sse import sse_client
+from mcp.client.streamable_http import streamable_http_client
+from mcp.shared._httpx_utils import create_mcp_http_client
+
+
+HTTP_TRANSPORTS = frozenset({"streamable_http", "sse"})
+_KB_SERVER_ID = "retrieve_dataset_content"
+
+
+def _positive_timeout(value: Any, default: float) -> float:
+ try:
+ parsed = float(value)
+ return parsed if parsed > 0 else default
+ except (TypeError, ValueError):
+ return default
+
+
+def _execution_timeout(name: str, cfg: dict) -> float:
+ """Resolve a hard tool-call deadline, with a tighter KB-specific default."""
+ if cfg.get("execution_timeout") is not None:
+ return _positive_timeout(cfg["execution_timeout"], 120.0)
+ if name == _KB_SERVER_ID:
+ return _positive_timeout(os.getenv("KB_MCP_EXECUTION_TIMEOUT_SECONDS"), 75.0)
+ return _positive_timeout(os.getenv("MCP_TOOL_EXECUTION_TIMEOUT_SECONDS"), 120.0)
+
+
+def _http_transport_timeout(name: str, cfg: dict, execution_timeout: float) -> float:
+ if cfg.get("transport_timeout") is not None:
+ return _positive_timeout(cfg["transport_timeout"], 30.0)
+ if name == _KB_SERVER_ID:
+ # Leave a small margin beyond the MCP server's own structured timeout
+ # so callers receive its tool_result instead of a transport exception.
+ return execution_timeout + 5.0
+ if name == "internet_search":
+ # Search returns per-query timeouts after its 45-second batch deadline.
+ return 50.0
+ return 30.0
+
+
+def is_http_cfg(cfg: dict) -> bool:
+ return cfg.get("transport") in HTTP_TRANSPORTS
+
+
+@asynccontextmanager
+async def authorized_transport(name, cfg):
+ from core.llm.mcp.invocation import HEADER, PLUGIN_HEADER, for_url
+ provider = cfg.get("oauth_provider")
+ headers = cfg.get("headers") or {}
+ signed = any(key.lower() == HEADER.lower() for key in headers)
+
+ async def authorize(request):
+ from core.llm.mcp.callbacks import CURRENT_CALL
+ call = CURRENT_CALL.get()
+ if call is not None and call.budget is not None:
+ # This request is bounded by the invocation execution budget. HTTP
+ # read time includes human waits; retain connect/write/pool limits.
+ request.extensions["timeout"] = {**request.extensions.get("timeout", {}), "read": None}
+ if provider is not None:
+ await provider.mcp_request_hook(request)
+ if signed:
+ request.headers.update(for_url(
+ cfg["url"], request.headers.get("x-current-user-id"),
+ request.headers.get("x-chat-id"), request.headers.get(PLUGIN_HEADER, ""),
+ ))
+
+ timeout = _http_transport_timeout(name, cfg, _execution_timeout(name, cfg))
+ if cfg["transport"] == "sse":
+ def factory(headers=None, timeout=None, auth=None):
+ return httpx2.AsyncClient(headers=headers, timeout=timeout, auth=auth,
+ event_hooks={"request": [authorize]})
+ # One SSE stream serves all invocations; a shared idle deadline cannot
+ # distinguish human waits. Discovery and invocation budgets bound work.
+ async with sse_client(cfg["url"], headers=headers, timeout=timeout,
+ sse_read_timeout=math.inf, auth=provider, httpx_client_factory=factory) as streams:
+ yield streams
+ else:
+ async with create_mcp_http_client(headers=headers, timeout=httpx2.Timeout(timeout),
+ auth=provider) as http:
+ http.event_hooks["request"].append(authorize)
+ async with streamable_http_client(cfg["url"], http_client=http) as streams:
+ yield streams
+
+
+# Listings outlive a run: each run opens its own HTTP client, so discovery results
+# are shared through one store per event loop, partitioned by authorization context.
+FIRST_PARTY_LISTING_TTL_MS = 60_000
+_VOLATILE_HEADERS = frozenset({"x-chat-id", "x-conversation-id", "x-hugagent-invocation"})
+_listing_stores = weakref.WeakKeyDictionary()
+
+
+def listing_cache(cfg: dict):
+ """Cache config for an HTTP server, or None to keep the SDK's per-client default."""
+ if not is_http_cfg(cfg) or cfg.get("oauth_provider") is not None:
+ return None
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ return None
+ from core.llm.mcp.invocation import audience_for
+ headers = {str(k).lower(): str(v) for k, v in (cfg.get("headers") or {}).items()
+ if str(k).lower() not in _VOLATILE_HEADERS}
+ partition = hashlib.sha256(json.dumps(headers, sort_keys=True).encode()).hexdigest()
+ store = _listing_stores.setdefault(loop, InMemoryResponseCacheStore())
+ # Built-in servers send no freshness hints; their listings change only on deploy.
+ ttl = FIRST_PARTY_LISTING_TTL_MS if audience_for(cfg["url"]) else 0
+ return CacheConfig(store=store, partition=partition, target_id=cfg["url"], default_ttl_ms=ttl)
+
+
+class AuthorizedClient(Client):
+ """Reject schema changes within a frozen run instead of replaying an old permission."""
+ def freeze(self, descriptors):
+ self.frozen_tools = {d.name: d.model_dump(mode="json") for d in descriptors}
+
+ async def list_tools(self, **kwargs):
+ page = await super().list_tools(**kwargs)
+ frozen = getattr(self, "frozen_tools", None)
+ if frozen is not None:
+ for tool in page.tools:
+ if frozen.get(tool.name) != tool.model_dump(mode="json"):
+ raise ValueError("MCP tool schema changed; a new authorized run is required")
+ return page
+
+
+def make_client(name: str, cfg: dict, *, sampling_callback=None, elicitation_callback=None) -> Client:
+ """One native client per authorized lifetime; negotiation and caches belong to the SDK."""
+ transport = (authorized_transport(name, cfg) if is_http_cfg(cfg) else
+ StdioServerParameters(command=cfg.get("command", "python"),
+ args=cfg.get("args", []), env=cfg.get("env") or None,
+ cwd=cfg.get("cwd")))
+ cache = listing_cache(cfg)
+ # Interactive sessions also serve resources/prompts, whose SDK methods expose
+ # no per-request timeout. Session.open bounds discovery, and SessionTool
+ # bounds each invocation; never mutate a shared SDK timer during a callback.
+ sdk_timeout = math.inf if elicitation_callback is not None else _execution_timeout(name, cfg)
+ return AuthorizedClient(transport, mode="auto", read_timeout_seconds=sdk_timeout,
+ sampling_callback=sampling_callback, elicitation_callback=elicitation_callback,
+ **({"cache": cache} if cache is not None else {}))
diff --git a/src/backend/core/llm/mcp/context_tools.py b/src/backend/core/llm/mcp/context_tools.py
new file mode 100644
index 00000000..75776f26
--- /dev/null
+++ b/src/backend/core/llm/mcp/context_tools.py
@@ -0,0 +1,69 @@
+"""Expose advertised MCP resources and prompts as permission-governed tools."""
+import hashlib
+import json
+import jsonschema
+from mcp.shared.uri_template import UriTemplate
+from agentscope.message import TextBlock
+from agentscope.tool import ToolChunk
+from agentscope.permission import PermissionBehavior, PermissionDecision
+from core.llm.mcp.results import content_blocks
+from core.llm.mcp.tools import SessionTool
+
+
+class ContextTool(SessionTool):
+ is_read_only = True
+ is_concurrency_safe = True
+
+ def __init__(self, resource, kind, descriptor, timeout):
+ super().__init__(resource, timeout)
+ server = resource.name
+ self.kind, self.descriptor = kind, descriptor
+ identity = (str(descriptor.uri) if kind == "resource" else descriptor.uri_template
+ if kind == "resource_template" else descriptor.name)
+ self.name = "mcp_" + kind + "_" + hashlib.sha256(f"{server}:{identity}".encode()).hexdigest()[:16]
+ self.description = f"{kind} from {server}: {descriptor.description or identity}"
+ self.input_schema = {"type": "object", "properties": {}, "additionalProperties": False}
+ if kind == "resource_template":
+ self.template = UriTemplate.parse(descriptor.uri_template)
+ self.input_schema["properties"] = {v.name: ({"type": "array", "items": {"type": "string"}}
+ if v.explode else {"type": "string"}) for v in self.template.variables}
+ self.input_schema["required"] = [v for v in self.template.variable_names
+ if v not in self.template.query_variable_names]
+ if kind == "prompt":
+ self.input_schema["properties"] = {a.name: {"type": "string", "description": a.description or a.name}
+ for a in descriptor.arguments or []}
+ self.input_schema["required"] = [a.name for a in descriptor.arguments or [] if a.required]
+
+ async def check_permissions(self, *_args, **_kwargs):
+ return PermissionDecision(behavior=PermissionBehavior.ASK,
+ message="Read remote MCP context from the selected server")
+
+ async def call(self, **kwargs):
+ return await self._invoke(lambda: self._read(**kwargs), exclusive=False)
+
+ async def _read(self, **kwargs):
+ jsonschema.validate(kwargs, self.input_schema)
+ client = self.resource.session.client
+ if self.kind in {"resource", "resource_template"}:
+ uri = str(self.descriptor.uri) if self.kind == "resource" else self.template.expand(kwargs)
+ # An explicit read wants current content; only listings are cached.
+ result = await client.read_resource(uri, cache_mode="bypass")
+ blocks = [TextBlock(text=item.model_dump_json(by_alias=True, exclude_none=True)) for item in result.contents]
+ else:
+ result = await client.get_prompt(self.descriptor.name, arguments=kwargs)
+ blocks = []
+ for message in result.messages:
+ blocks.append(TextBlock(text=json.dumps({"role": message.role}, ensure_ascii=False)))
+ blocks.extend(content_blocks([message.content]))
+ return ToolChunk(content=blocks, metadata={"mcp": result.model_dump(mode="json", by_alias=True, exclude_none=True)})
+
+
+def context_tools(resource, timeout):
+ tools, names = [], set()
+ for kind, descriptor in resource.session.context:
+ tool = ContextTool(resource, kind, descriptor, timeout)
+ if tool.name in names:
+ raise ValueError("Duplicate MCP context identity")
+ names.add(tool.name)
+ tools.append(tool)
+ return tools
diff --git a/src/backend/core/llm/mcp/deadline.py b/src/backend/core/llm/mcp/deadline.py
new file mode 100644
index 00000000..95d119d7
--- /dev/null
+++ b/src/backend/core/llm/mcp/deadline.py
@@ -0,0 +1,40 @@
+"""Per-invocation execution time, excluding time spent waiting for a person."""
+import asyncio
+from contextlib import contextmanager
+
+
+class InvocationBudget:
+ """One task deadline; overlapping human waits pause the same clock once."""
+
+ def __init__(self, seconds: float):
+ self._timeout = asyncio.timeout(seconds)
+ self._remaining = seconds
+ self._waiters = 0
+ self._active = False
+
+ async def __aenter__(self):
+ await self._timeout.__aenter__()
+ self._active = True
+ return self
+
+ async def __aexit__(self, *exc):
+ self._active = False
+ return await self._timeout.__aexit__(*exc)
+
+ @contextmanager
+ def waiting_for_user(self):
+ # A late callback must not resurrect a completed or already expired call.
+ if not self._active or self._timeout.expired():
+ raise TimeoutError("MCP invocation is no longer active")
+ if self._waiters == 0:
+ self._remaining = self._timeout.when() - asyncio.get_running_loop().time()
+ if self._remaining <= 0:
+ raise TimeoutError("MCP execution budget exhausted")
+ self._timeout.reschedule(None)
+ self._waiters += 1
+ try:
+ yield
+ finally:
+ self._waiters -= 1
+ if self._waiters == 0 and self._active and not self._timeout.expired():
+ self._timeout.reschedule(asyncio.get_running_loop().time() + self._remaining)
diff --git a/src/backend/core/llm/mcp_invocation.py b/src/backend/core/llm/mcp/invocation.py
similarity index 87%
rename from src/backend/core/llm/mcp_invocation.py
rename to src/backend/core/llm/mcp/invocation.py
index faa66c06..85778848 100644
--- a/src/backend/core/llm/mcp_invocation.py
+++ b/src/backend/core/llm/mcp/invocation.py
@@ -26,10 +26,14 @@ def issue(audience, user_id, chat_id, plugin_id=""):
signature = hmac.new(key(), body.encode(), hashlib.sha256).hexdigest()
return {HEADER: body + "." + signature}
-def for_url(url, user_id, chat_id, plugin_id=""):
- from core.config.mcp_config import _mcp_http_url
+def audience_for(url):
+ """The built-in server a URL addresses, or None for any other server."""
+ from core.config.mcp_config import mcp_http_url as _mcp_http_url
from extensions.mcp_servers._ports import PORTS
- audience = next((name for name in PORTS if _mcp_http_url(name).rstrip("/") == str(url).rstrip("/")), None)
+ return next((name for name in PORTS if _mcp_http_url(name).rstrip("/") == str(url).rstrip("/")), None)
+
+def for_url(url, user_id, chat_id, plugin_id=""):
+ audience = audience_for(url)
if not audience:
return {}
try:
diff --git a/src/backend/core/llm/mcp/lifetime.py b/src/backend/core/llm/mcp/lifetime.py
new file mode 100644
index 00000000..c08240c0
--- /dev/null
+++ b/src/backend/core/llm/mcp/lifetime.py
@@ -0,0 +1,68 @@
+"""A run's handle on an MCP session: its tools, its interaction authority and its release."""
+import asyncio
+from dataclasses import dataclass, field
+from core.llm.mcp.client import _execution_timeout, is_http_cfg
+
+MANIFEST_SOURCES = frozenset({"cloud_manifest", "plugin_manifest"})
+
+
+@dataclass
+class MCPResource:
+ name: str
+ tools: list = field(default_factory=list)
+ session: object = None
+ interactions: object = None
+ # Write calls whose outcome is unknown in this run; exact replays are refused.
+ unknown_outcomes: set = field(default_factory=set)
+ closed: bool = False
+ _lease: object = None
+ _closing: asyncio.Task | None = None
+
+ @property
+ def descriptors(self):
+ return self.session.descriptors if self.session is not None else []
+
+ async def close(self, *, abort=False):
+ """Close a run-owned session, or return a pooled one; abort also retires the pooled session."""
+ # Tools reject calls at once; teardown finishes even if the caller is cancelled again.
+ self.closed = True
+ if self._closing is None:
+ self._closing = asyncio.create_task(self._close(abort))
+ await asyncio.shield(self._closing)
+
+ async def _close(self, abort):
+ if self._lease is not None:
+ await self._lease.release(abort=abort)
+ elif self.session is not None:
+ await self.session.close()
+
+
+def _bind_tools(resource, cfg):
+ from core.llm.mcp.context_tools import context_tools
+ from core.llm.mcp.tools import RemoteTool
+ timeout = _execution_timeout(resource.name, cfg)
+ enabled, disabled = cfg.get("enable_tools"), set(cfg.get("disable_tools") or [])
+ tools = [RemoteTool(resource, descriptor, timeout) for descriptor in resource.session.descriptors
+ if (enabled is None or descriptor.name in enabled) and descriptor.name not in disabled]
+ return tools + context_tools(resource, timeout)
+
+
+async def discover(name, cfg, *, interactions=None, scope=None):
+ """Bind a run to a session. Chat-scoped STDIO servers share a pooled process;
+ HTTP and unscoped servers get a run-owned session."""
+ if cfg.get("schema_source") in MANIFEST_SOURCES:
+ from core.llm.tools.cloud_manifest import manifest_tools
+ return MCPResource(name, tools=manifest_tools(name, cfg))
+ if scope and not is_http_cfg(cfg):
+ from core.llm.mcp.chat_sessions import acquire
+ lease = await acquire(scope, name, cfg)
+ resource = MCPResource(name, session=lease.session, interactions=interactions, _lease=lease)
+ else:
+ from core.llm.mcp.session import Session
+ resource = MCPResource(name, session=await Session.open(name, cfg), interactions=interactions)
+ try:
+ resource.tools = _bind_tools(resource, cfg)
+ except BaseException:
+ await resource.close(abort=True)
+ raise
+ return resource
diff --git a/src/backend/core/llm/mcp/manager.py b/src/backend/core/llm/mcp/manager.py
new file mode 100644
index 00000000..f7492d00
--- /dev/null
+++ b/src/backend/core/llm/mcp/manager.py
@@ -0,0 +1,62 @@
+"""Factory failure cleanup and run resource teardown."""
+from __future__ import annotations
+import asyncio
+import logging
+from contextvars import ContextVar
+from functools import wraps
+logger = logging.getLogger(__name__)
+
+# The factory owns transient clients until it returns them to its caller.
+# ContextVar values propagate into parallel HTTP probes, so partial construction
+# has one cleanup owner even when cancellation interrupts asyncio.gather.
+_factory_clients: ContextVar[list | None] = ContextVar("factory_mcp_clients", default=None)
+
+
+def own_factory_client(client):
+ clients = _factory_clients.get()
+ if clients is not None:
+ clients.append(client)
+ return client
+
+
+def close_factory_clients_on_error(function):
+ @wraps(function)
+ async def construct(*args, **kwargs):
+ clients = []
+ token = _factory_clients.set(clients)
+ try:
+ return await function(*args, **kwargs)
+ except BaseException:
+ await close_clients(list(reversed(clients)), abort=True)
+ raise
+ finally:
+ _factory_clients.reset(token)
+
+ return construct
+
+
+async def close_clients(clients: list, *, abort=False) -> None:
+ """Close every client in its owning task, deferring repeated cancellation."""
+ cancelled = None
+ task = asyncio.current_task()
+ for client in clients:
+ while True:
+ before = task.cancelling() if task else 0
+ try:
+ await client.close(abort=True) if abort else await client.close()
+ break
+ except asyncio.CancelledError as exc:
+ cancelled = exc
+ if task is not None and task.cancelling() > before:
+ # An external repeat-cancel interrupted this close. Finish
+ # this same client's cleanup on the same task, then the rest.
+ continue
+ # A client scope itself signalled cancellation. It must not
+ # prevent independent clients from releasing their resources.
+ logger.debug("MCP client close signalled cancellation")
+ break
+ except Exception as exc:
+ logger.debug("Error closing MCP client: %s", exc)
+ break
+ if cancelled is not None:
+ raise cancelled
diff --git a/src/backend/core/llm/mcp/results.py b/src/backend/core/llm/mcp/results.py
new file mode 100644
index 00000000..5f5f39f2
--- /dev/null
+++ b/src/backend/core/llm/mcp/results.py
@@ -0,0 +1,67 @@
+"""Lossless MCP result metadata with explicit model-facing content conversion."""
+import json
+import jsonschema
+from agentscope.message import TextBlock, DataBlock, Base64Source
+from agentscope.tool import ToolChunk
+from agentscope.tool._response import ToolResultState
+from mcp import MCPError, types
+from mcp.types import CONNECTION_CLOSED, REQUEST_TIMEOUT
+
+
+def content_blocks(contents):
+ blocks = []
+ for item in contents:
+ if isinstance(item, types.TextContent):
+ blocks.append(TextBlock(text=item.text))
+ elif isinstance(item, (types.ImageContent, types.AudioContent)):
+ blocks.append(DataBlock(source=Base64Source(media_type=item.mime_type, data=item.data)))
+ else:
+ # Resource URIs are references, never automatically fetched as network URLs.
+ # Preserve links, embedded resources and their provenance for the model.
+ blocks.append(TextBlock(text=item.model_dump_json(by_alias=True, exclude_none=True)))
+ return blocks
+
+
+def _carried_by_text(structured, blocks):
+ """Whether a text block already holds the structured value (SDKs wrap non-object returns as {"result": ...})."""
+ for block in blocks:
+ if not isinstance(block, TextBlock):
+ continue
+ if structured == {"result": block.text}:
+ return True
+ try:
+ value = json.loads(block.text)
+ except ValueError:
+ continue
+ if value == structured or structured == {"result": value}:
+ return True
+ return False
+
+
+def tool_result(result, descriptor):
+ content = content_blocks(result.content)
+ metadata = {"mcp": result.model_dump(mode="json", by_alias=True, exclude_none=True),
+ "output_schema": descriptor.output_schema}
+ # The model needs the structured value once; skip it only when text already carries it.
+ if result.structured_content is not None and not _carried_by_text(result.structured_content, content):
+ content.append(TextBlock(text=json.dumps(result.structured_content, ensure_ascii=False,
+ separators=(",", ":"))))
+ return ToolChunk(content=content, metadata=metadata,
+ state=ToolResultState.ERROR if result.is_error else ToolResultState.RUNNING)
+
+
+def error_result(text, **metadata):
+ return ToolChunk(content=[TextBlock(text=text)], state=ToolResultState.ERROR,
+ metadata={"origin": "mcp", **metadata})
+
+
+def is_timeout(exc):
+ return isinstance(exc, TimeoutError) or (isinstance(exc, MCPError) and exc.code == REQUEST_TIMEOUT)
+
+
+def poisons_session(exc):
+ """A request-level failure leaves the session usable; a lost peer, a changed
+ schema or a cancellation does not."""
+ if isinstance(exc, MCPError):
+ return exc.code == CONNECTION_CLOSED
+ return not isinstance(exc, (TimeoutError, jsonschema.ValidationError))
diff --git a/src/backend/core/llm/mcp/session.py b/src/backend/core/llm/mcp/session.py
new file mode 100644
index 00000000..a2309797
--- /dev/null
+++ b/src/backend/core/llm/mcp/session.py
@@ -0,0 +1,147 @@
+"""One live SDK session: its owner task, discovered surface and call admission."""
+import asyncio
+import contextvars
+from contextlib import AsyncExitStack, asynccontextmanager
+from mcp import types
+from core.llm.mcp.callbacks import route
+from core.llm.mcp.client import make_client
+
+MAX_PAGES = 100
+
+
+class CallGate:
+ """Read-only calls share the session; writes run alone."""
+
+ def __init__(self):
+ self._readers = 0
+ self._writer = False
+ self._changed = asyncio.Condition()
+
+ @asynccontextmanager
+ async def _hold(self, exclusive):
+ async with self._changed:
+ if exclusive:
+ await self._changed.wait_for(lambda: not self._writer and not self._readers)
+ self._writer = True
+ else:
+ await self._changed.wait_for(lambda: not self._writer)
+ self._readers += 1
+ try:
+ yield
+ finally:
+ async with self._changed:
+ if exclusive:
+ self._writer = False
+ else:
+ self._readers -= 1
+ self._changed.notify_all()
+
+ def shared(self):
+ return self._hold(False)
+
+ def exclusive(self):
+ return self._hold(True)
+
+
+async def _pages(method, field):
+ items, cursor, seen = [], None, set()
+ for _ in range(MAX_PAGES):
+ page = await method(cursor=cursor)
+ items.extend(getattr(page, field))
+ cursor = page.next_cursor
+ if cursor is None:
+ return items
+ if cursor in seen:
+ raise ValueError("MCP pagination cursor repeated")
+ seen.add(cursor)
+ raise ValueError(f"MCP {field} listing exceeded {MAX_PAGES} pages")
+
+
+class Session:
+ """Owns one SDK client from a dedicated task, so any run's task may call through it."""
+
+ def __init__(self, name, cfg):
+ self.name, self.cfg = name, cfg
+ self.client = None
+ self.descriptors = []
+ self.context = [] # (kind, descriptor) for advertised resources, templates and prompts
+ self.calls = CallGate()
+ self.in_flight = set()
+ self.failed = False
+ self._stop = asyncio.Event()
+ self._owner = None
+
+ @property
+ def stopped(self):
+ return self._stop.is_set() or self.failed or (self._owner is not None and self._owner.done())
+
+ async def _sampling(self, context, params):
+ call = route(self)
+ if call is None:
+ return types.ErrorData(code=-32600, message="No active sampling scope")
+ return await call.dispatch("sampling", context, params)
+
+ async def _elicitation(self, context, params):
+ call = route(self)
+ if call is None:
+ return types.ElicitResult(action="decline")
+ return await call.dispatch("elicitation", context, params)
+
+ async def _discover(self, client):
+ capabilities = client.server_capabilities
+ if capabilities.tools is not None:
+ descriptors = await _pages(client.list_tools, "tools")
+ names = [d.name for d in descriptors]
+ if len(names) != len(set(names)):
+ raise ValueError("Duplicate MCP tool name")
+ self.descriptors = descriptors
+ client.freeze(descriptors)
+ listings = []
+ if capabilities.resources is not None:
+ listings += [("resource", client.list_resources, "resources"),
+ ("resource_template", client.list_resource_templates, "resource_templates")]
+ if capabilities.prompts is not None:
+ listings.append(("prompt", client.list_prompts, "prompts"))
+ found = await asyncio.gather(*(_pages(method, field) for _, method, field in listings))
+ self.context = [(kind, item) for (kind, _, _), items in zip(listings, found) for item in items]
+
+ async def _own(self, client, ready):
+ try:
+ async with AsyncExitStack() as stack:
+ await stack.enter_async_context(client)
+ self.client = client
+ await self._discover(client)
+ ready.set_result(None)
+ await self._stop.wait()
+ except BaseException as exc:
+ if not ready.done():
+ ready.set_exception(exc)
+ else:
+ raise
+
+ @classmethod
+ async def open(cls, name, cfg):
+ session = cls(name, cfg)
+ client = make_client(name, cfg, sampling_callback=session._sampling,
+ elicitation_callback=session._elicitation)
+ ready = asyncio.get_running_loop().create_future()
+ # A fresh context keeps run-scoped ContextVars out of the long-lived session task.
+ session._owner = asyncio.create_task(session._own(client, ready), name=f"mcp-session:{name}",
+ context=contextvars.Context())
+ try:
+ await asyncio.wait_for(asyncio.shield(ready), timeout=float(cfg.get("discovery_timeout") or 30))
+ except BaseException:
+ session._stop.set()
+ from core.infra.task_lifecycle import cancel_and_join
+ try:
+ await cancel_and_join(session._owner)
+ finally:
+ if ready.done() and not ready.cancelled():
+ ready.exception()
+ raise
+ return session
+
+ async def close(self):
+ self._stop.set()
+ if self._owner is not None:
+ await asyncio.shield(self._owner)
diff --git a/src/backend/core/llm/mcp/tools.py b/src/backend/core/llm/mcp/tools.py
new file mode 100644
index 00000000..1f42a7e6
--- /dev/null
+++ b/src/backend/core/llm/mcp/tools.py
@@ -0,0 +1,113 @@
+"""AgentScope tool boundary over a live native MCP session."""
+import hashlib
+import json
+import math
+from agentscope.tool import ToolBase
+from agentscope.permission import PermissionBehavior, PermissionDecision
+from core.llm.mcp.deadline import InvocationBudget
+from core.llm.mcp.callbacks import CURRENT_CALL
+from core.llm.mcp.results import error_result, is_timeout, poisons_session, tool_result
+
+MAX_QUEUED_PROGRESS = 32
+
+
+def _call_key(name, arguments):
+ body = json.dumps(arguments, sort_keys=True, ensure_ascii=False, default=str)
+ return hashlib.sha256(f"{name}\0{body}".encode()).hexdigest()
+
+
+def queue_progress(queue, event):
+ """Keep only the newest event per call; the stream loop drains the queue between events."""
+ for index, queued in enumerate(queue):
+ if queued["tool_id"] == event["tool_id"]:
+ queue[index] = event
+ return
+ if len(queue) < MAX_QUEUED_PROGRESS:
+ queue.append(event)
+
+
+class SessionTool(ToolBase):
+ """A run's tool over a session; the run handle decides whether it may still call."""
+ is_mcp = True
+
+ def __init__(self, resource, timeout):
+ super().__init__()
+ self.resource, self.mcp_name, self.timeout = resource, resource.name, timeout
+
+ def _require_open(self):
+ if self.resource.closed or self.resource.session.stopped:
+ raise RuntimeError("MCP run lease is closed")
+
+ async def _invoke(self, operation, *, exclusive, write_key=None):
+ """Run one request; only a lost session is fatal, other failures become results."""
+ self._require_open()
+ resource, session = self.resource, self.resource.session
+ if write_key is not None and write_key in resource.unknown_outcomes:
+ return error_result("该写操作上次以相同参数调用时结果未知,已拒绝重复执行。请先核对服务端状态。",
+ server=self.mcp_name, outcome_unknown=True)
+ async with session.calls.exclusive() if exclusive else session.calls.shared():
+ self._require_open()
+ call = resource.interactions.open_call(session) if resource.interactions else None
+ token = CURRENT_CALL.set(call)
+ if call is not None:
+ session.in_flight.add(call)
+ try:
+ async with InvocationBudget(self.timeout) as budget:
+ if call is not None:
+ call.budget = budget
+ return await operation()
+ except BaseException as exc:
+ if poisons_session(exc):
+ session.failed = True
+ raise
+ if write_key is not None and is_timeout(exc):
+ resource.unknown_outcomes.add(write_key)
+ return error_result(
+ f"MCP 写操作未在预期内完成({type(exc).__name__}),结果未知。请核对服务端状态,不要直接重试。",
+ server=self.mcp_name, outcome_unknown=True)
+ return error_result(f"MCP 调用失败:{type(exc).__name__}: {exc}"[:2000], server=self.mcp_name)
+ finally:
+ CURRENT_CALL.reset(token)
+ if call is not None:
+ session.in_flight.discard(call)
+ await call.finish()
+
+
+class RemoteTool(SessionTool):
+ def __init__(self, resource, descriptor, timeout):
+ super().__init__(resource, timeout)
+ self.name = descriptor.name
+ self.description = descriptor.description or descriptor.name
+ self.input_schema = descriptor.input_schema
+ self.is_read_only = bool(descriptor.annotations and descriptor.annotations.read_only_hint)
+ # Read-only calls have no effects to order, so the agent may run them together.
+ self.is_concurrency_safe = self.is_read_only
+ self.descriptor = descriptor
+
+ async def check_permissions(self, *_args, **_kwargs):
+ return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK,
+ message="MCP operation is subject to the run execution policy")
+
+ async def call(self, **kwargs):
+ from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID
+ tool_call_id = CURRENT_TOOL_CALL_ID.get()
+ interactions = self.resource.interactions
+
+ async def progress(value, total, message):
+ if interactions is None or interactions.agent is None:
+ return
+ queue_progress(interactions.agent.state.pending_mcp_progress, {
+ "tool_id": tool_call_id, "name": self.name, "server_id": self.mcp_name,
+ "progress": value, "total": total, "message": str(message or "")[:500]})
+
+ async def operation():
+ result = await self.resource.session.client.call_tool(
+ self.descriptor.name, kwargs,
+ # The outer pausable budget owns execution time. A second SDK
+ # wall clock would expire during legacy server-to-client elicitation.
+ read_timeout_seconds=math.inf if interactions is not None else self.timeout,
+ progress_callback=progress)
+ return tool_result(result, self.descriptor)
+
+ return await self._invoke(operation, exclusive=not self.is_read_only,
+ write_key=None if self.is_read_only else _call_key(self.name, kwargs))
diff --git a/src/backend/core/llm/mcp_manager.py b/src/backend/core/llm/mcp_manager.py
deleted file mode 100644
index 879f0a35..00000000
--- a/src/backend/core/llm/mcp_manager.py
+++ /dev/null
@@ -1,336 +0,0 @@
-"""MCP client pool manager for AgentScope 2.0.
-
-Manages MCPClient instances with TTL caching.
-
-Migration notes (1.x → 2.0)
----------------------------
-- ``StdIOStatefulClient(name, command, args, env)`` →
- ``MCPClient(name=, is_stateful=True, mcp_config=StdioMCPConfig(command, args, env))``.
-- In 2.0 ``Toolkit`` is constructed **once** (``Toolkit(tools=, mcps=, ...)``); there are
- no incremental ``register_mcp_client`` / ``register_tool_function`` methods and no
- ``namesake_strategy``. Stateful clients must ``connect()`` **before** being passed
- into ``Toolkit``.
-- Therefore ``connect_mcp_clients`` only connects and returns the client list; the
- Toolkit is built once in agent_factory via
- ``Toolkit(tools=[...FunctionTool...], mcps=clients)``.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import time
-from contextvars import ContextVar
-from functools import wraps
-from typing import Any, Dict, List
-
-import httpx
-import mcp.types
-from mcp.client.sse import sse_client
-from mcp.client.streamable_http import streamable_http_client
-from pydantic import ConfigDict, Field
-
-from agentscope.mcp import MCPClient, StdioMCPConfig
-from agentscope.permission import PermissionBehavior, PermissionDecision
-from agentscope.tool import MCPTool, ToolBase, ToolChunk
-
-logger = logging.getLogger(__name__)
-
-
-def has_usable_schema(tool: Any) -> bool:
- """Whether a manifest tool entry carries the server's real ``inputSchema``.
-
- A tool the platform only knows by name and description (a plugin manifest
- lists its tools for display, without schemas) must not be handed to a model:
- the parameter object would be empty, so the model has nowhere to put its
- arguments and every call arrives blank.
-
- The marker is whether a schema was captured at all, not whether it declares
- parameters: a server is free to describe an argument-less tool as
- ``{"type": "object"}``, while an entry that never reached the server carries
- no ``inputSchema`` key, which normalizes to an empty dict.
- """
- if not isinstance(tool, dict):
- return False
- schema = tool.get("inputSchema") or tool.get("input_schema")
- return isinstance(schema, dict) and bool(schema)
-
-
-def _cyfunc_probe() -> None: # after Cython compilation its type is cython_function_or_method
- pass
-
-
-# Cython compiles methods into cython_function_or_method; pydantic v2 does not recognize
-# it as a method and treats it as an "un-annotated field", raising PydanticUserError,
-# which makes the whole module fail to import after hardened compilation and fall back to
-# plaintext. Registering that type in ignored_types makes pydantic ignore compiled
-# methods; under pure Python it is just FunctionType, so there is no side effect.
-_CYFUNCTION_TYPE = type(_cyfunc_probe)
-
-# TTL for the instance-level memoization of ``list_tools``. Within a single turn,
-# agent_factory's HTTP MCP liveness probe (``_connect_http``, parallel list_tools) and
-# the subsequent ``Toolkit.get_tool_schemas()`` (AgentScope ``_get_available_tools``
-# does a **serial** ``await list_tools()`` per client) each enumerate tools once per
-# server — stateless HTTP creates a new connection every time, and the serial
-# re-enumeration is the main cause of the ~2s agent build time (11 servers, 5 slow
-# search MCPs at ~450ms each, chained serially). Memoization cuts the second
-# enumeration to ~0ms (the probe pass stays parallel at ~465ms). Per-request HTTP
-# clients get a fresh instance every turn → cache lifetime = one turn, no header
-# leakage across turns/users; pooled stdio clients survive across turns, so the TTL
-# is the safety net (tool definitions are static anyway; config changes rebuild the
-# pool → new instances).
-_LIST_TOOLS_TTL_S = 300.0
-
-
-class BareNameMCPClient(MCPClient):
- """Restore the server-side bare name of MCP tools (``internet_search`` rather than ``mcp__internet_search__internet_search``).
-
- AgentScope 2.0's ``MCPTool`` adapter rewrites the outward-facing name to
- ``mcp____``, but this project's display-name mapping
- (core/config/display_names), citation extraction (orchestration/citation_anchor
- dispatches on bare names like ``internet_search``), catalog gating, tool
- references in system prompts and SKILL.md, and frontend icons/panels/renderers
- are all built on the 1.x bare names. ``MCPTool.__call__`` actually calls the
- server via ``self._tool.name`` (the bare name), so rewriting the adapter's
- ``.name`` only affects the LLM-visible name and the SSE event stream; the call
- path is unaffected.
- """
-
- model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,), arbitrary_types_allowed=True)
- oauth_provider: Any = Field(default=None, exclude=True)
-
- def _create_http_client(self):
- """Attach the SDK OAuth provider without forking AgentScope's client."""
- from core.llm.mcp_invocation import HEADER, PLUGIN_HEADER, for_url
- config = self.mcp_config
- signed = any(key.lower() == HEADER.lower() for key in (config.headers or {}))
- if self.oauth_provider is None and not signed:
- return super()._create_http_client()
- oauth_hook = getattr(self.oauth_provider, "mcp_request_hook", None)
- async def request_hook(request):
- if oauth_hook:
- await oauth_hook(request)
- if signed:
- request.headers.update(for_url(config.url, request.headers.get("x-current-user-id"), request.headers.get("x-chat-id"), request.headers.get(PLUGIN_HEADER, "")))
- event_hooks = {"request": [request_hook]}
- if config.url.endswith("/sse") or config.url.endswith("/messages/"):
- def _http_client_factory(headers=None, timeout=None, auth=None):
- return httpx.AsyncClient(
- headers=headers,
- timeout=timeout,
- auth=auth,
- event_hooks=event_hooks,
- )
-
- return sse_client(
- url=config.url,
- headers=config.headers,
- timeout=config.timeout,
- auth=self.oauth_provider,
- httpx_client_factory=_http_client_factory,
- )
- http_client = httpx.AsyncClient(
- headers=config.headers,
- timeout=config.timeout,
- auth=self.oauth_provider,
- event_hooks=event_hooks,
- )
- return streamable_http_client(url=config.url, http_client=http_client)
-
- async def get_tool(self, name: str) -> MCPTool:
- tool = await super().get_tool(name)
- tool.name = tool._tool.name
- # Use AgentScope's concurrent batch executor for every MCP call.
- # Permission checks remain independent of the execution policy.
- tool.is_concurrency_safe = True
- return tool
-
- async def list_tools(self) -> List[MCPTool]:
- """Instance-level TTL memoization to eliminate duplicate list_tools for the same server within a turn.
-
- See the ``_LIST_TOOLS_TTL_S`` comment at the top of the module: the liveness
- probe and ``get_tool_schemas`` each call once, and the second serial
- re-enumeration is the main cause of the ~2s agent build time. The cache hangs
- off the instance (``__slots__`` includes ``__dict__``, so arbitrary attributes
- can be set); a new client per turn → naturally scoped to a single turn.
- """
- cached = getattr(self, "_lt_cache", None)
- if cached is not None:
- expires_at, tools = cached
- if time.monotonic() < expires_at:
- return tools
- tools = await super().list_tools()
- self._lt_cache = (time.monotonic() + _LIST_TOOLS_TTL_S, tools)
- return tools
-
-
-from core.llm.gateway_mcp_tool import GatewayMCPTool
-
-
-class ManifestMCPClient(BareNameMCPClient):
- """Network-free MCP discovery backed only by the current cloud manifest."""
-
- manifest_tools: List[Dict[str, Any]] = Field(default_factory=list, exclude=True)
- gateway_invoke_url: str = Field(exclude=True)
- schema_hash: str = Field(exclude=True)
- gateway_transport: Any = Field(default=None, exclude=True)
- gateway_plugin: str = Field(default="", exclude=True)
- manager_cloud_available: bool = Field(default=False, exclude=True)
- read_only: bool = Field(default=False, exclude=True)
-
- def _raw_manifest_tools(self) -> List[mcp.types.Tool]:
- tools: List[mcp.types.Tool] = []
- from core.capabilities.paths import capabilities_enabled
- from core.services.management_contract import localize, declared, tools as manager_tools
- items = list(self.manifest_tools)
- from extensions.mcp_servers.registry import gateway_server
- if self.read_only and gateway_server(self.name) is not None:
- items = [item for item in items if item.get("annotations", {}).get("readOnlyHint") is True]
- if capabilities_enabled() and self.gateway_plugin == "skill-manager":
- if any(t.get("name") == "install_skill" and declared(t, self.gateway_plugin) for t in items):
- items += [t for t in manager_tools("skill-manager", "local") if t["name"] == "upload_skill_to_cloud" and not any(x.get("name") == t["name"] for x in items)]
- for item in items:
- if capabilities_enabled():
- item = localize(item, self.gateway_plugin, cloud_available=self.manager_cloud_available or bool(self.gateway_invoke_url))
- if item is None:
- continue
- try:
- tool = mcp.types.Tool.model_validate(item)
- except Exception as exc: # noqa: BLE001 - isolate one malformed tool
- logger.warning("Invalid manifest tool in MCP '%s': %s", self.name, exc)
- continue
- if self.enable_tools is not None and tool.name not in self.enable_tools:
- continue
- if self.disable_tools is not None and tool.name in self.disable_tools:
- continue
- if not has_usable_schema(item):
- # A manifest entry that never captured the server's inputSchema
- # would become a zero-parameter tool: the model then has nowhere
- # to put its arguments and retries the same empty call forever.
- # Withhold it instead of offering a tool that can never succeed.
- logger.warning(
- "Manifest tool '%s' in MCP '%s' has no captured inputSchema; withheld",
- tool.name,
- self.name,
- )
- continue
- tools.append(tool)
- return tools
-
- async def list_raw_tools(self) -> List[mcp.types.Tool]:
- return self._raw_manifest_tools()
-
- async def get_tool(self, name: str) -> GatewayMCPTool:
- for tool in self._raw_manifest_tools():
- if tool.name == name:
- from core.capabilities.paths import capabilities_enabled
- from core.services.management_contract import declared
- if capabilities_enabled() and declared(tool.model_dump(by_alias=True), self.gateway_plugin):
- from core.llm.management_tool import ManagementTool
- return ManagementTool(self.name, tool, self.gateway_plugin, self.mcp_config.headers or {}, cloud_backed=bool(self.gateway_invoke_url))
- return GatewayMCPTool(
- mcp_name=self.name,
- tool=tool,
- invoke_url=self.gateway_invoke_url,
- schema_hash=self.schema_hash,
- headers=dict(self.mcp_config.headers or {}),
- timeout=float(self.execution_timeout or 120.0),
- transport=self.gateway_transport,
- source_plugin=self.gateway_plugin,
- )
- raise ValueError(f"Tool '{name}' not found in cloud manifest MCP '{self.name}'")
-
- async def list_tools(self) -> List[GatewayMCPTool]:
- return [await self.get_tool(tool.name) for tool in self._raw_manifest_tools()]
-
-
-def make_stdio_client(server_name: str, server_cfg: dict) -> MCPClient:
- """Build a stdio MCPClient (not yet connected) from a server config in configs/mcp_config.py."""
- return BareNameMCPClient(
- name=server_name,
- is_stateful=True,
- mcp_config=StdioMCPConfig(
- command=server_cfg.get("command", "python"),
- args=server_cfg.get("args", []),
- env=server_cfg.get("env") or None,
- ),
- )
-
-
-# The factory owns transient clients until it returns them to its caller.
-# ContextVar values propagate into parallel HTTP probes, so partial construction
-# has one cleanup owner even when cancellation interrupts asyncio.gather.
-_factory_clients: ContextVar[list | None] = ContextVar("factory_mcp_clients", default=None)
-
-
-def own_factory_client(client):
- clients = _factory_clients.get()
- if clients is not None:
- clients.append(client)
- return client
-
-
-def close_factory_clients_on_error(function):
- @wraps(function)
- async def construct(*args, **kwargs):
- clients = []
- token = _factory_clients.set(clients)
- try:
- return await function(*args, **kwargs)
- except BaseException:
- await close_clients(list(reversed(clients)))
- raise
- finally:
- _factory_clients.reset(token)
-
- return construct
-
-
-async def connect_mcp_clients(
- mcp_servers: Dict[str, dict],
-) -> List[MCPClient]:
- """Connect to MCP servers and return the connected MCPClient list."""
- clients: List[MCPClient] = []
- for server_name, server_cfg in mcp_servers.items():
- client = make_stdio_client(server_name, server_cfg)
- try:
- await client.connect()
- clients.append(own_factory_client(client))
- logger.debug("MCP client '%s' connected", server_name)
- except asyncio.CancelledError:
- await client.close()
- raise
- except Exception as exc:
- logger.warning("Failed to connect MCP server '%s': %s", server_name, exc)
- try:
- await client.close()
- except Exception:
- pass
- return clients
-
-
-async def close_clients(clients: List[MCPClient]) -> None:
- """Close every client in its owning task, deferring repeated cancellation."""
- cancelled = None
- task = asyncio.current_task()
- for client in clients:
- while True:
- before = task.cancelling() if task else 0
- try:
- await client.close()
- break
- except asyncio.CancelledError as exc:
- cancelled = exc
- if task is not None and task.cancelling() > before:
- # An external repeat-cancel interrupted this close. Finish
- # this same client's cleanup on the same task, then the rest.
- continue
- # A client scope itself signalled cancellation. It must not
- # prevent independent clients from releasing their resources.
- logger.debug("MCP client close signalled cancellation")
- break
- except Exception as exc:
- logger.debug("Error closing MCP client: %s", exc)
- break
- if cancelled is not None:
- raise cancelled
diff --git a/src/backend/core/llm/mcp_pool.py b/src/backend/core/llm/mcp_pool.py
deleted file mode 100644
index dc98f03e..00000000
--- a/src/backend/core/llm/mcp_pool.py
+++ /dev/null
@@ -1,387 +0,0 @@
-"""Global MCP connection pool (AgentScope 2.0).
-
-Keeps stable MCP servers connected across requests to eliminate the 1-7s
-subprocess spawn overhead. Per-request servers (e.g. retrieve_dataset_content
-with runtime KB env vars) are spawned fresh each time.
-
-Migration notes (1.x → 2.0)
----------------------------
-- ``StdIOStatefulClient`` / ``HttpStatefulClient`` → ``MCPClient(is_stateful=,
- mcp_config=StdioMCPConfig|HttpMCPConfig)``.
-- The 2.0 ``Toolkit`` is constructed in one shot (``Toolkit(tools=, mcps=, ...)``);
- there is **no** ``register_mcp_client`` / ``register_tool_function`` and **no**
- ``MCPClient.get_callable_function`` — so the 1.x "cache MCPToolFunction,
- register on demand" optimization cannot be ported directly. This pool now only
- handles **connection reuse** (stable clients keep their connection across
- requests, saving the subprocess spawn), returning a list of connected
- ``MCPClient`` per request, and ``agent_factory`` builds
- ``Toolkit(tools=[...FunctionTool...], mcps=clients)`` in one go.
-- HTTP per-request (KB switches headers per user): use an MCPClient with
- ``is_stateful=False``; each call opens a fresh underlying connection, no
- pooling needed (see agent_factory).
-
-Usage:
- pool = MCPConnectionPool.get_instance()
- await pool.initialize(server_configs) # once at startup
-
- # Per request (stable reuses connections, per-request spawns fresh):
- clients, transient = await pool.get_request_clients(enabled_keys, per_request_cfg)
- toolkit = Toolkit(tools=fn_tools, mcps=clients) # in agent_factory
- ...
- await pool.close_transient(transient)
-
- await pool.shutdown() # on shutdown
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import os
-from threading import Lock
-from typing import Any, Dict, List, Optional, Set, Tuple
-
-from agentscope.mcp import HttpMCPConfig, MCPClient, StdioMCPConfig
-from core.llm.mcp_manager import BareNameMCPClient, ManifestMCPClient, has_usable_schema
-
-logger = logging.getLogger(__name__)
-
-HTTP_TRANSPORTS = frozenset({"streamable_http", "sse"})
-_KB_SERVER_ID = "retrieve_dataset_content"
-
-
-def _positive_timeout(value: Any, default: float) -> float:
- try:
- parsed = float(value)
- return parsed if parsed > 0 else default
- except (TypeError, ValueError):
- return default
-
-
-def _execution_timeout(name: str, cfg: dict) -> float:
- """Resolve a hard tool-call deadline, with a tighter KB-specific default."""
- if cfg.get("execution_timeout") is not None:
- return _positive_timeout(cfg["execution_timeout"], 120.0)
- if name == _KB_SERVER_ID:
- return _positive_timeout(os.getenv("KB_MCP_EXECUTION_TIMEOUT_SECONDS"), 75.0)
- return _positive_timeout(os.getenv("MCP_TOOL_EXECUTION_TIMEOUT_SECONDS"), 120.0)
-
-
-def _http_transport_timeout(name: str, cfg: dict, execution_timeout: float) -> float:
- if cfg.get("transport_timeout") is not None:
- return _positive_timeout(cfg["transport_timeout"], 30.0)
- if name == _KB_SERVER_ID:
- # Leave a small margin beyond the MCP server's own structured timeout
- # so callers receive its tool_result instead of a transport exception.
- return execution_timeout + 5.0
- if name == "internet_search":
- # Search returns per-query timeouts after its 45-second batch deadline.
- return 50.0
- return 30.0
-
-
-def is_http_cfg(cfg: dict) -> bool:
- return cfg.get("transport") in HTTP_TRANSPORTS
-
-
-def uses_manifest_schema(cfg: dict) -> bool:
- """Return whether a config contains a usable dynamic cloud manifest.
-
- At least one tool must carry the server's real ``inputSchema``; entries known
- only by name and description are withheld when the client enumerates tools
- (see ``has_usable_schema``), so a manifest holding nothing else describes no
- callable tool at all.
- """
- tools = cfg.get("manifest_tools")
- if cfg.get("schema_source") == "plugin_manifest":
- from core.services.management_contract import declared
- return bool(tools and cfg.get("origin") == "local_plugin" and all(
- declared(tool, cfg.get("gateway_plugin")) for tool in tools))
- return bool(
- cfg.get("schema_source") == "cloud_manifest"
- and str(cfg.get("gateway_invoke_url") or "").strip()
- and str(cfg.get("schema_hash") or "").strip()
- and isinstance(tools, list)
- and any(
- isinstance(tool, dict)
- and bool(str(tool.get("name") or "").strip())
- and has_usable_schema(tool)
- for tool in tools
- )
- )
-
-
-def make_client(
- name: str,
- cfg: dict,
- *,
- is_stateful: bool = True,
- client_cls: type[BareNameMCPClient] = BareNameMCPClient,
-) -> MCPClient:
- """Build an MCPClient (HTTP or stdio) from a config dict.
-
- HTTP servers may set ``is_stateful=False`` for a fresh per-request connection
- (multi-user KB). stdio must be stateful (enforced by 2.0). ``client_cls`` lets
- callers substitute a ``BareNameMCPClient`` subclass (e.g. the confirmation-gated
- client) while reusing the same URL/config construction.
- """
- execution_timeout = _execution_timeout(name, cfg)
- if is_http_cfg(cfg):
- # ⚠️ The 2.0 streamable_http MCP client does not follow 307 redirects. Many MCP
- # servers 307-redirect ``/mcp/`` (trailing slash) to ``/mcp`` → 2.0 throws
- # HTTPStatusError outright and triggers an anyio cancel-scope crash. Strip the
- # trailing slash to match the server's canonical form and avoid the redirect.
- _url = (cfg.get("url") or "").rstrip("/")
- client_kwargs = {
- "name": name,
- "is_stateful": is_stateful,
- "oauth_provider": cfg.get("oauth_provider"),
- "mcp_config": HttpMCPConfig(
- url=_url,
- headers=cfg.get("headers") or None,
- timeout=_http_transport_timeout(name, cfg, execution_timeout),
- ),
- "execution_timeout": execution_timeout,
- "enable_tools": cfg.get("enable_tools"),
- }
- if cfg.get("schema_source") in ("cloud_manifest", "plugin_manifest"):
- if not uses_manifest_schema(cfg):
- raise ValueError(f"MCP '{name}' has an invalid cloud capability manifest")
- return ManifestMCPClient(
- **client_kwargs,
- manifest_tools=list(cfg["manifest_tools"]),
- gateway_invoke_url=str(cfg["gateway_invoke_url"]),
- schema_hash=str(cfg["schema_hash"]),
- gateway_transport=cfg.get("gateway_transport"),
- gateway_plugin=str(cfg.get("gateway_plugin") or ""),
- manager_cloud_available=bool(cfg.get("manager_cloud_available")),
- read_only=bool(cfg.get("read_only")),
- )
- return client_cls(
- **client_kwargs,
- )
- return client_cls(
- name=name,
- is_stateful=True,
- mcp_config=StdioMCPConfig(
- command=cfg.get("command", "python"),
- args=cfg.get("args", []),
- env=cfg.get("env") or None,
- ),
- execution_timeout=execution_timeout,
- )
-
-
-class MCPConnectionPool:
- """Singleton MCP connection pool (connection reuse only, 2.0)."""
-
- _instance: Optional[MCPConnectionPool] = None
- _instance_lock = Lock()
-
- def __init__(self) -> None:
- self._stable_clients: Dict[str, MCPClient] = {}
- self._stable_configs: Dict[str, dict] = {}
- self._stable_server_ids: Set[str] = set()
- self._initialized = False
- self._config_version: int = 0
- self._lock = asyncio.Lock()
-
- @classmethod
- def get_instance(cls) -> MCPConnectionPool:
- if cls._instance is None:
- with cls._instance_lock:
- if cls._instance is None:
- cls._instance = cls()
- return cls._instance
-
- async def initialize(self, server_configs: Optional[Dict[str, dict]] = None) -> None:
- """Connect to all stable MCP servers."""
- async with self._lock:
- if server_configs is None:
- from core.services.mcp_service import McpServerConfigService
-
- server_configs = McpServerConfigService.get_instance().get_all_servers()
-
- # ⚠️ 2.0: only stdio stable servers are pooled. HTTP MCP is **not pooled** even
- # when marked is_stable — the 2.0 stateful HTTP client is task-bound, and reusing
- # it across request tasks crashes the cancel scope; HTTP always goes per-request
- # with is_stateful=False (see agent_factory._connect_http).
- self._stable_server_ids = {
- name
- for name, cfg in server_configs.items()
- if cfg.get("is_stable", False) and not is_http_cfg(cfg)
- }
- await self._close_all_stable()
-
- stable_items = [
- (name, cfg)
- for name, cfg in server_configs.items()
- if name in self._stable_server_ids
- ]
-
- async def _connect_one(name: str, cfg: dict) -> bool:
- try:
- client = make_client(name, cfg)
- await client.connect()
- self._stable_clients[name] = client
- self._stable_configs[name] = cfg
- logger.info(
- "[mcp_pool] Connected stable server: %s (%s)",
- name,
- cfg.get("transport", "stdio"),
- )
- return True
- except Exception as exc:
- logger.warning("[mcp_pool] Failed to connect stable server '%s': %s", name, exc)
- return False
-
- results = await asyncio.gather(*(_connect_one(n, c) for n, c in stable_items))
- connected = sum(1 for ok in results if ok)
- self._initialized = True
- self._config_version += 1
- logger.info(
- "[mcp_pool] Initialized with %d/%d stable servers",
- connected,
- len(self._stable_server_ids),
- )
-
- async def reinitialize_if_config_changed(self, new_server_configs: Dict[str, dict]) -> None:
- new_stable_ids = {
- name
- for name, cfg in new_server_configs.items()
- if cfg.get("is_stable", False) and not is_http_cfg(cfg)
- }
- changed = new_stable_ids != self._stable_server_ids
- if not changed:
- for name in new_stable_ids:
- if self._stable_configs.get(name) != new_server_configs.get(name):
- changed = True
- break
- if changed:
- logger.info("[mcp_pool] Config change detected, reinitializing stable connections")
- await self.initialize(new_server_configs)
-
- @property
- def has_cached_tools(self) -> bool:
- # No tool-func cache in 2.0; property kept for caller compatibility (means stable servers are connected).
- return bool(self._stable_clients)
-
- async def get_request_clients(
- self,
- enabled_keys: List[str],
- per_request_servers_cfg: Optional[Dict[str, dict]] = None,
- ) -> Tuple[List[MCPClient], List[MCPClient]]:
- """Return the connected MCPClient list needed for this request + the transient list (for closing).
-
- - stable servers: reuse the already-connected client (no re-spawn).
- - per-request stdio servers: spawn fresh (concurrently).
- The caller (agent_factory) uses the returned clients in a single
- ``Toolkit(tools=, mcps=clients)``.
- """
- request_clients: List[MCPClient] = []
- transient_clients: List[MCPClient] = []
-
- # Phase 1: spawn per-request servers concurrently
- spawn_tasks: Dict[str, asyncio.Task] = {}
- for key in enabled_keys:
- if key not in self._stable_server_ids:
- cfg = (per_request_servers_cfg or {}).get(key)
- if cfg is not None:
- spawn_tasks[key] = asyncio.create_task(self._spawn_transient(key, cfg))
-
- # Phase 2: collect stable clients (already connected, reconnect if needed)
- for key in enabled_keys:
- if key not in self._stable_server_ids:
- continue
- client = self._stable_clients.get(key)
- if client is None:
- client = await self._reconnect_stable(key)
- if client is not None:
- request_clients.append(client)
-
- # Phase 3: await per-request spawns
- for key, task in spawn_tasks.items():
- try:
- client = await task
- if client is not None:
- request_clients.append(client)
- transient_clients.append(client)
- except Exception as exc:
- logger.warning("[mcp_pool] Failed per-request server '%s': %s", key, exc)
-
- return request_clients, transient_clients
-
- # Backward-compat alias: old callers took (toolkit, transient); now returns (clients, transient).
- async def build_toolkit_from_cache(
- self,
- enabled_keys: List[str],
- per_request_servers_cfg: Optional[Dict[str, dict]] = None,
- ) -> Tuple[List[MCPClient], List[MCPClient]]:
- return await self.get_request_clients(enabled_keys, per_request_servers_cfg)
-
- async def close_transient(self, transient_clients: List[MCPClient]) -> None:
- for client in transient_clients:
- try:
- await client.close()
- except Exception as exc:
- logger.debug("[mcp_pool] close transient failed: %s", exc)
-
- async def _spawn_transient(self, key: str, cfg: dict) -> Optional[MCPClient]:
- client = make_client(key, cfg)
- try:
- await client.connect()
- return client
- except Exception as exc:
- logger.warning("[mcp_pool] spawn_transient '%s' connect failed: %s", key, exc)
- try:
- await client.close()
- except Exception:
- pass
- raise
-
- async def _reconnect_stable(self, name: str) -> Optional[MCPClient]:
- cfg = self._stable_configs.get(name)
- if cfg is None:
- return None
- old = self._stable_clients.pop(name, None)
- if old is not None:
- try:
- await old.close()
- except Exception:
- pass
- try:
- client = make_client(name, cfg)
- await client.connect()
- self._stable_clients[name] = client
- logger.info("[mcp_pool] Reconnected stable server: %s", name)
- return client
- except Exception as exc:
- logger.warning("[mcp_pool] Reconnection failed for '%s': %s", name, exc)
- return None
-
- async def _close_all_stable(self) -> None:
- for name, client in list(self._stable_clients.items()):
- try:
- await client.close()
- except Exception as exc:
- logger.warning("[mcp_pool] close failed for '%s': %s", name, exc)
- self._stable_clients.clear()
- self._stable_configs.clear()
-
- async def shutdown(self) -> None:
- async with self._lock:
- await self._close_all_stable()
- self._initialized = False
- logger.info("[mcp_pool] Shut down")
-
- @property
- def is_initialized(self) -> bool:
- return self._initialized
-
- @property
- def stable_client_count(self) -> int:
- return len(self._stable_clients)
-
- @property
- def stable_server_ids(self) -> frozenset[str]:
- return frozenset(self._stable_server_ids)
diff --git a/src/backend/core/llm/middleware/__init__.py b/src/backend/core/llm/middleware/__init__.py
new file mode 100644
index 00000000..a22094fc
--- /dev/null
+++ b/src/backend/core/llm/middleware/__init__.py
@@ -0,0 +1 @@
+"""Record of the AgentScope middleware chain replaced by HookBus hooks."""
diff --git a/src/backend/core/llm/middleware_migration.py b/src/backend/core/llm/middleware/migration_map.py
similarity index 51%
rename from src/backend/core/llm/middleware_migration.py
rename to src/backend/core/llm/middleware/migration_map.py
index 2c1c228d..929b5011 100644
--- a/src/backend/core/llm/middleware_migration.py
+++ b/src/backend/core/llm/middleware/migration_map.py
@@ -18,82 +18,91 @@ class MiddlewareMigration:
MIDDLEWARE_MIGRATION: tuple[MiddlewareMigration, ...] = (
MiddlewareMigration(
"DynamicModelMiddleware",
- (HookStage.BEFORE_RUN, HookStage.BEFORE_MODEL),
- "adapter-covered",
- "neutral model-selection policy reaches parity",
+ (HookStage.BEFORE_RUN,),
+ "neutral-hook",
+ "removed; model selection runs through HookSpec",
),
MiddlewareMigration(
"FileContextMiddleware",
(HookStage.TRANSFORM_CONTEXT,),
- "adapter-covered",
- "neutral attachment transformer reaches parity",
+ "neutral-hook",
+ "removed; attachments emit neutral context items",
),
MiddlewareMigration(
"SteerMiddleware",
- (HookStage.BEFORE_MODEL, HookStage.BEFORE_TOOL),
- "adapter-covered",
- "durable steer policy consumes Invocation only",
+ (HookStage.TRANSFORM_CONTEXT, HookStage.AFTER_CONTEXT, HookStage.BEFORE_TOOL),
+ "neutral-hook",
+ "removed; acknowledge only after context application",
),
MiddlewareMigration(
"WorkspacePinHintMiddleware",
- (HookStage.AFTER_TOOL, HookStage.BEFORE_MODEL),
- "adapter-covered",
- "pin reminder becomes a neutral context patch",
+ (HookStage.BEFORE_RUN, HookStage.TRANSFORM_CONTEXT),
+ "neutral-hook",
+ "removed; pin reminder emits neutral context items",
),
MiddlewareMigration(
"IterBudgetReminderMiddleware",
- (HookStage.BEFORE_MODEL,),
- "adapter-covered",
- "turn-budget reminder becomes a neutral context patch",
+ (HookStage.TRANSFORM_CONTEXT,),
+ "neutral-hook",
+ "removed; budget context and tool choice are patched before binding",
),
MiddlewareMigration(
"StallInterventionMiddleware",
- (HookStage.BEFORE_MODEL,),
- "adapter-covered",
- "stall policy consumes neutral attempt history",
+ (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL),
+ "neutral-hook",
+ "removed; evolution policy consumes neutral attempt history",
),
MiddlewareMigration(
"OntologyGateMiddleware",
(HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL),
- "adapter-covered",
- "ontology decision returns reject/modify Decision",
+ "neutral-hook",
+ "removed; ontology admission returns reject/modify Decision",
),
MiddlewareMigration(
"ToolPermissionMiddleware",
- (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL),
- "adapter-covered",
- "neutral tool runner owns permission tickets and approval routing",
+ (HookStage.BEFORE_TOOL,),
+ "neutral-hook",
+ "removed; neutral admission policy owns tickets and execution scope",
),
MiddlewareMigration(
"CitationAnchorMiddleware",
(HookStage.AFTER_TOOL,),
- "adapter-covered",
+ "neutral-hook",
"citation annotation returns tool_result patch",
),
MiddlewareMigration(
"ActingToolCallIdMiddleware",
(HookStage.BEFORE_TOOL,),
- "adapter-covered",
+ "neutral-hook",
"tool Invocation carries call id",
),
MiddlewareMigration(
"ToolEffectMiddleware",
(HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL),
- "adapter-covered",
+ "neutral-hook",
"effect gateway is invoked by neutral tool runner",
),
MiddlewareMigration(
"JobLedgerReminderMiddleware",
- (HookStage.BEFORE_MODEL,),
- "adapter-covered",
- "job projection becomes a neutral context patch",
+ (HookStage.TRANSFORM_CONTEXT,),
+ "neutral-hook",
+ "removed; execution host contributes the ledger hook",
),
MiddlewareMigration(
"FinishPinGuardMiddleware",
(HookStage.BEFORE_FINISH,),
- "adapter-covered",
+ "neutral-hook",
"finish guard returns an explicit Decision",
),
+ MiddlewareMigration("CommandCompletionMiddleware",
+ (HookStage.TRANSFORM_CONTEXT, HookStage.BEFORE_FINISH),
+ "neutral-hook", "removed; completion decisions continue the active run"),
+ MiddlewareMigration("ProjectEnvironmentMiddleware", (HookStage.BEFORE_TOOL,),
+ "neutral-hook", "removed; admission returns an explicit denial"),
+ MiddlewareMigration("ScreenshotPruningMiddleware", (HookStage.TRANSFORM_CONTEXT,),
+ "neutral-hook", "removed; profile contributes a plain context transform"),
+ MiddlewareMigration("PlanStaleReminderMiddleware", (HookStage.TRANSFORM_CONTEXT,),
+ "neutral-hook", "removed; plan reminder consumes a context snapshot"),
)
diff --git a/src/backend/core/llm/middlewares.py b/src/backend/core/llm/middlewares.py
deleted file mode 100644
index fbdd63c5..00000000
--- a/src/backend/core/llm/middlewares.py
+++ /dev/null
@@ -1,1450 +0,0 @@
-"""AgentScope 2.0 middlewares — replace the 1.x hooks (pre_reply / post_acting / post_reasoning).
-
-1.x → 2.0 migration mapping:
- - dynamic_model (pre_reply) → DynamicModelMiddleware.on_reply
- - file_context (pre_reply) → FileContextMiddleware.on_reply
- - workspace_pin_hint (post_acting) → WorkspacePinHintMiddleware.on_reasoning
- (⚠️ on_acting cannot write to context; must use on_reasoning to inject the reminder before the next reasoning round)
- - finish_pin_guard (post_reasoning)→ FinishPinGuardMiddleware.on_reasoning
- - iter_budget (new in 2.0, no 1.x predecessor) → IterBudgetReminderMiddleware.on_reasoning
-
-The runtime context moved from ``agent._jx_context`` to ``agent.state`` (fields on an AgentRuntimeState subclass).
-Pure-logic helpers still reuse hooks.py / finish_guard.py (they are agentscope-version independent).
-"""
-
-from __future__ import annotations
-
-import asyncio
-import base64
-import json
-import logging
-import time
-from contextlib import suppress
-from contextvars import ContextVar
-from pathlib import Path
-from typing import Any, List, Optional
-
-from agentscope.agent import Agent
-from agentscope.event import ToolCallEndEvent
-from agentscope.message import ToolResultState
-from agentscope.middleware import MiddlewareBase
-from agentscope.state import AgentState
-from agentscope.tool._response import ToolChunk, ToolResponse
-from core.llm.context_adapter import (
- append_context_text,
- append_state_context_item,
- next_context_sequence,
- render_text_block,
-)
-from core.llm.context_ir import (
- KIND_ATTACHMENT,
- KIND_PROJECT,
- KIND_REMINDER,
- POLICY_HEAD_TAIL,
- POLICY_NEVER,
- ContextItem,
-)
-from core.llm.hooks import (
- _FILE_ID_RE,
- _PIN_HINT_SKIP_TOOLS,
- _build_file_context,
- _build_historical_files_context,
- _fetch_image_base64,
- _get_main_model,
- _get_pin_hint_state,
- _get_provider_model,
- _is_image,
- _resolve_chat_mode,
- reset_artifact_read_state,
- reset_pin_hint_state,
-)
-from core.llm.plan_update_tool import parse_plan_update_args
-from pydantic import ConfigDict, Field
-
-logger = logging.getLogger(__name__)
-
-
-def _append_reminder(agent: Agent, text: str, *, origin: str) -> None:
- append_context_text(
- agent,
- f"\n{text}\n ",
- kind=KIND_REMINDER,
- origin=origin,
- trust="system",
- priority=850,
- token_budget=4_000,
- truncation_policy=POLICY_HEAD_TAIL,
- )
-
-
-def _append_state_payload(
- state: AgentState,
- content: Any,
- *,
- kind: str,
- origin: str,
- trust: str,
- priority: int = 650,
- token_budget: int = 8_000,
-) -> None:
- created_seq = next_context_sequence(state.context)
- append_state_context_item(
- state,
- ContextItem.create(
- item_id=f"{origin}:{created_seq}",
- kind=kind,
- origin=origin,
- trust=trust,
- visibility="model",
- priority=priority,
- token_budget=token_budget,
- truncation_policy=POLICY_HEAD_TAIL,
- content=content,
- cache_class="dynamic",
- created_seq=created_seq,
- render_role="user",
- message_group=f"{origin}:{created_seq}",
- ),
- )
-
-
-def _cyfunc_probe() -> None: # once compiled by Cython, its type is cython_function_or_method
- pass
-
-
-# Cython compiles methods into cython_function_or_method; pydantic v2 doesn't recognize it as a
-# method and treats it as an "unannotated field", raising PydanticUserError — so after hardened
-# compilation the whole module fails to import and falls back to plaintext. Registering that type
-# in ignored_types makes pydantic ignore the compiled methods. Under pure Python it is just a
-# regular FunctionType (pydantic already ignores methods), so there is no side effect.
-_CYFUNCTION_TYPE = type(_cyfunc_probe)
-
-
-# ── Current tool-call id seam ───────────────────────────────────────────────
-# When a tool function needs to know its own tool_call_id (e.g. call_subagent attaches the
-# subagent's streaming events to its own card via parent_tool_id), AgentScope does not pass it
-# down (toolkit.call_tool does not inject the id into tool kwargs). Use an on_acting middleware
-# to write the current tool_call.id into a ContextVar before each tool executes — the tool runs
-# in the **same task call chain**, so it can read it.
-# Concurrent tool calls each run in separate tasks spawned by asyncio.gather (each with its own
-# copy of the context), so ContextVars don't cross-contaminate — naturally aligned with parallel subagents.
-CURRENT_TOOL_CALL_ID: ContextVar[str] = ContextVar("jx_current_tool_call_id", default="")
-CURRENT_RUN_BINDING: ContextVar[tuple[str, str] | None] = ContextVar(
- "jx_current_run_binding", default=None
-)
-
-
-class ActingToolCallIdMiddleware(MiddlewareBase):
- """on_acting: expose the current tool_call.id to tool functions (see above). Pure pass-through, does not change tool behavior."""
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- prepared = getattr(agent, "_jx_prepared_capabilities", None)
- if prepared is not None:
- from core.capabilities.runtime import validate
-
- await asyncio.to_thread(
- validate, prepared, user_id=str(getattr(agent.state, "user_id", "") or "")
- )
- tc = input_kwargs.get("tool_call")
- tcid = getattr(tc, "id", "") or "" if tc is not None else ""
- if prepared is not None:
- from core.capabilities.runtime import record_tool_scope
-
- await asyncio.to_thread(
- record_tool_scope, prepared, tcid, str(getattr(tc, "name", "") or "")
- )
- token = CURRENT_TOOL_CALL_ID.set(tcid) if tcid else None
- try:
- async for item in next_handler(**input_kwargs):
- yield item
- finally:
- if token is not None:
- try:
- CURRENT_TOOL_CALL_ID.reset(token)
- except Exception: # noqa: BLE001
- pass
-
-
-class ToolEffectMiddleware(MiddlewareBase):
- """Commit durable Intent before every harness-owned tool invocation.
-
- The final ``ToolResponse`` is committed before it leaves this middleware,
- so the existing AgentScope event/SSE mapping remains unchanged while the
- database becomes authoritative for replay and reconciliation.
- """
-
- def __init__(self, session_factory=None, registry=None) -> None: # noqa: ANN001
- from core.db.engine import SessionLocal
- from core.services.tool_effect_ledger import (
- DEFAULT_TOOL_RECOVERY_REGISTRY,
- ToolEffectGateway,
- ToolEffectJournal,
- )
-
- self._ledger = ToolEffectJournal(session_factory or SessionLocal)
- self._registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY
- self._gateway = ToolEffectGateway(self._ledger, self._registry)
- from core.services.harness_ledger import HarnessUsageLedger
-
- self._usage_ledger = HarnessUsageLedger(session_factory or SessionLocal)
-
- @staticmethod
- def _args(tool_call) -> dict: # noqa: ANN001
- raw = getattr(tool_call, "input", "") or "{}"
- try:
- parsed = json.loads(raw) if isinstance(raw, str) else raw
- except (TypeError, json.JSONDecodeError):
- parsed = {"_raw": str(raw)}
- return parsed if isinstance(parsed, dict) else {"_value": parsed}
-
- @staticmethod
- def _response_payload(response: ToolResponse) -> dict:
- return {"tool_response": response.model_dump(mode="json")}
-
- @staticmethod
- def _response_from_payload(payload: Any) -> ToolResponse:
- from agentscope.message import DataBlock, TextBlock
-
- raw = payload.get("tool_response") if isinstance(payload, dict) else None
- if not isinstance(raw, dict):
- return ToolResponse(
- content=[render_text_block(json.dumps(payload, ensure_ascii=False, default=str))],
- state=ToolResultState.SUCCESS,
- )
- blocks = []
- for item in raw.get("content") or []:
- if not isinstance(item, dict):
- blocks.append(render_text_block(str(item)))
- elif item.get("type") == "data":
- blocks.append(DataBlock.model_validate(item))
- else:
- blocks.append(TextBlock.model_validate(item))
- state = ToolResultState(str(raw.get("state") or "success"))
- return ToolResponse(
- id=str(raw.get("id") or ""),
- content=blocks,
- state=state,
- metadata=dict(raw.get("metadata") or {}),
- )
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- from core.services.tool_effect_ledger import ToolIntentCommitError
-
- run_id = str(getattr(agent.state, "run_id", "") or "")
- run_owner = str(getattr(agent.state, "journal_owner", "") or "")
- tool_call = input_kwargs.get("tool_call")
- tool_name = str(getattr(tool_call, "name", "") or "")
- tool_call_id = str(getattr(tool_call, "id", "") or "")
- if not run_id or not run_owner or not tool_name:
- raise ToolIntentCommitError(
- "tool execution requires a durable run_id and journal_owner binding"
- )
-
- args = self._args(tool_call)
- if tool_name == "create_scheduled_task":
- from core.services.automation_tool_routing import creation_arguments
- args = creation_arguments(args, str(getattr(agent.state, "user_id", "") or ""),
- str(getattr(agent.state, "chat_id", "") or ""))
- live_chunks: asyncio.Queue = asyncio.Queue()
-
- async def _invoke():
- from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT
- from core.harness.usage import (
- UsageAttempt,
- attempt_status_for_exception,
- record_usage_safely,
- )
-
- final = None
- started = time.monotonic()
- usage_status = "failed"
- binding_token = CURRENT_RUN_BINDING.set((run_id, run_owner))
- invoke_kwargs = input_kwargs
- effect = CURRENT_TOOL_EFFECT.get()
- if effect is not None and tool_name in {
- "create_scheduled_task",
- "update_scheduled_task",
- "delete_scheduled_task",
- }:
- adapter_args = dict(args)
- adapter_args["tool_effect_id"] = effect.effect_id
- adapter_call = tool_call.model_copy(
- update={"input": json.dumps(adapter_args, ensure_ascii=False)}
- )
- invoke_kwargs = {**input_kwargs, "tool_call": adapter_call}
- try:
- outcome_unknown = False
- async for item in next_handler(**invoke_kwargs):
- # Toolkit normally converts adapter exceptions into ordinary
- # error responses. An ambiguous remote write must retain its
- # ledger meaning across that conversion and stop this run.
- ambiguous = bool(
- (getattr(item, "metadata", None) or {}).get("gateway_outcome_unknown")
- )
- outcome_unknown = outcome_unknown or ambiguous
- if isinstance(item, ToolResponse):
- final = item
- elif not ambiguous:
- await live_chunks.put(item)
- if outcome_unknown:
- raise RuntimeError("cloud write outcome unknown; reconciliation required")
- usage_status = (
- "success"
- if final is not None and final.state == ToolResultState.SUCCESS
- else "failed"
- )
- except asyncio.CancelledError:
- usage_status = "cancelled"
- raise
- except Exception as exc:
- usage_status = attempt_status_for_exception(exc)
- raise
- finally:
- CURRENT_RUN_BINDING.reset(binding_token)
- if effect is not None:
- try:
- await record_usage_safely(
- self._usage_ledger,
- UsageAttempt(
- run_id=run_id,
- kind="tool",
- operation_name=tool_name,
- effect_id=effect.effect_id,
- status=usage_status,
- latency_ms=int((time.monotonic() - started) * 1_000),
- metadata={"tool_call_id": tool_call_id},
- ),
- )
- except Exception: # usage cannot change tool execution
- logger.debug("tool usage attempt persistence failed", exc_info=True)
- if final is None:
- from core.services.tool_effect_ledger import ToolEffectError
-
- raise ToolEffectError(f"tool {tool_name} ended without ToolResponse")
- return self._response_payload(final)
-
- def _response_failed(payload: Any) -> bool:
- raw = payload.get("tool_response") if isinstance(payload, dict) else None
- return bool(isinstance(raw, dict) and str(raw.get("state") or "success") != "success")
-
- task = asyncio.create_task(
- self._gateway.execute_outcome(
- run_id=run_id,
- owner=run_owner,
- tool_call_id=tool_call_id,
- tool_name=tool_name,
- args=args,
- invoke=_invoke,
- classify_failure=_response_failed,
- )
- )
- try:
- while not task.done() or not live_chunks.empty():
- try:
- item = await asyncio.wait_for(live_chunks.get(), timeout=0.02)
- except asyncio.TimeoutError:
- continue
- yield item
- outcome = await task
- finally:
- if not task.done():
- task.cancel()
- with suppress(BaseException):
- await task
- elif not task.cancelled():
- task.exception()
- response = self._response_from_payload(outcome.result)
- if outcome.result_state == "error" and response.state == ToolResultState.SUCCESS:
- response.state = ToolResultState.ERROR
- linkage = {
- "effect_id": outcome.effect_id,
- "result_id": outcome.result_id,
- }
- response.metadata = {**dict(response.metadata or {}), **linkage}
- links = getattr(agent.state, "tool_effect_links", None)
- if isinstance(links, dict):
- links[tool_call_id] = linkage
- if not outcome.invoked:
- yield ToolChunk(
- content=list(response.content),
- state=response.state,
- metadata=dict(response.metadata),
- )
- yield response
-
-
-class CitationAnchorMiddleware(MiddlewareBase):
- """on_acting: 统一证据锚点——工具结果回给模型前完成 提取 → 发号 → cite_id 回注。
-
- 协议依据(agentscope 2.0 toolkit.call_tool):中间 ToolChunk 是增量、最后一个
- yield 是**累积完整**的 ToolResponse;SSE 侧(orchestration/streaming.py)只在
- ToolResultEndEvent 时把 delta 累积成一条 tool_result,不实时消费中间增量。
- 因此这里缓冲中间块,在最终 ToolResponse 上注号后:
- 1) 合成一个携带完整注号文本的 ToolChunk(SSE 累积到的就是注号后文本);
- 2) 用注号文本替换 ToolResponse.content(模型上下文 / 落库拿到同一份)。
- 两侧看到的 cite_id 因此严格一致。跳过名单、非纯文本内容、非 SUCCESS 状态、
- 任何异常 → 原样放行(引用功能降级,绝不影响工具本身)。
- """
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- from orchestration.citation_anchor import (
- SKIP_TOOLS,
- annotate_tool_result,
- resolve_allocator,
- )
-
- tool_call = input_kwargs.get("tool_call")
- tool_name = str(getattr(tool_call, "name", "") or "")
- tool_id = str(getattr(tool_call, "id", "") or "")
- if not tool_name or tool_name in SKIP_TOOLS:
- async for item in next_handler(**input_kwargs):
- yield item
- return
-
- buffered: list = []
- final: ToolResponse | None = None
- async for item in next_handler(**input_kwargs):
- if isinstance(item, ToolResponse):
- final = item
- break # per call_tool protocol the ToolResponse is the last yield
- buffered.append(item)
-
- if final is None:
- for item in buffered:
- yield item
- return
-
- annotated = False
- if final.state == ToolResultState.SUCCESS:
- try:
- blocks = list(final.content or [])
- text_blocks = [b for b in blocks if getattr(b, "type", "") == "text"]
- if text_blocks and len(text_blocks) == len(blocks):
- # 发号器绑在 agent 上(run 入口注入);缺失时就地建并绑定,
- # 保证编号在该 agent 的整条流里唯一
- allocator = resolve_allocator(agent)
- full_text = "".join((b.text or "") for b in text_blocks)
- new_text, items = annotate_tool_result(tool_name, tool_id, full_text, allocator)
- if items:
- allocator.register(tool_id, items)
- final.content = [render_text_block(new_text)]
- yield ToolChunk(
- content=[render_text_block(new_text)],
- state=final.state,
- metadata=dict(final.metadata or {}),
- )
- annotated = True
- except Exception: # noqa: BLE001
- logger.warning(
- "[citation-anchor] middleware annotate failed tool=%s",
- tool_name,
- exc_info=True,
- )
-
- if not annotated:
- for item in buffered:
- yield item
- yield final
-
-
-class AgentRuntimeState(AgentState):
- """Extends AgentState to carry the runtime fields of the former ModelContext (replaces agent._jx_context)."""
-
- model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,))
-
- model_name: str = ""
- model_provider_id: str = ""
- # When a subagent explicitly configures a model (model_provider_id) → the factory sets this
- # True, and DynamicModel must not override it with the main model based on chat_mode
- # (otherwise the subagent's own model is effectively useless — neither channels nor web get it).
- model_pinned: bool = False
- user_id: str | None = None
- chat_id: str | None = None
- run_id: str | None = None
- capability_scope: str = ""
- journal_owner: str | None = None
- tool_effect_links: dict = Field(default_factory=dict)
- enable_thinking: bool = True
- chat_mode: str | None = None
- uploaded_files: List[dict] = Field(default_factory=list)
- historical_files: List[dict] = Field(default_factory=list)
- user_message_text: str = ""
- # Vision evidence transcribed ahead of the model call by the streaming entry
- # point; FileContextMiddleware injects it verbatim instead of redoing the
- # network round-trip. Empty = transcribe inline (non-streaming callers).
- vision_evidence_text: str = ""
- ontology_enabled: bool = False
- ontology_runtime: dict = Field(default_factory=dict)
- # Set by SteerMiddleware immediately before the next reasoning round. The
- # streaming adapter turns it into one ``steer_applied`` event, then clears
- # it. Keeping this on the typed runtime state avoids process-global queues.
- steer_delivery: dict | None = None
- # Model responses recorded by the hook adapter the moment they complete,
- # drained by the streaming adapter into ``model_step`` events. Each entry
- # is one ``core.llm.model_steps`` assistant record.
- pending_model_steps: List[dict] = Field(default_factory=list)
-
- def apply_request_context(self, context: dict, user_message_text: str) -> None:
- """Populate per-request runtime fields from the request ``context`` dict (replaces the 1.x agent._jx_context).
-
- Shared by both entry points — streaming and workflow (non-streaming) — to keep two
- hand-copied field mappings from silently drifting (streaming once didn't lowercase
- chat_mode while workflow did).
- """
- # A child may use the shared role or its own provider. Request context
- # still describes the parent selection; it must not relabel a pinned
- # child (vision routing and audit read these fields).
- if not self.model_pinned:
- self.model_name = str(context.get("model_name", "") or "")
- self.model_provider_id = str(context.get("model_provider_id", "") or "")
- self.user_id = str(context.get("user_id", "") or "") or None
- self.chat_id = str(context.get("chat_id", "") or "") or None
- self.run_id = str(context.get("run_id", "") or "") or None
- self.journal_owner = str(context.get("journal_owner", "") or "") or None
- self.enable_thinking = bool(context.get("enable_thinking", True))
- cm = str(context.get("chat_mode") or "").lower() or None
- if cm:
- self.chat_mode = cm
- self.uploaded_files = list(context.get("uploaded_files", []) or [])
- self.historical_files = list(context.get("historical_files", []) or [])
- self.user_message_text = user_message_text or ""
- self.ontology_enabled = bool(context.get("ontology_enabled", False))
- runtime = context.get("ontology_runtime")
- self.ontology_runtime = runtime if isinstance(runtime, dict) else {}
-
-
-class OntologyGateMiddleware(MiddlewareBase):
- """L-a deterministic gate: validate every visible tool call without an LLM."""
-
- def __init__(self, runtime: dict | None = None) -> None:
- self.runtime = runtime if isinstance(runtime, dict) else {}
- self.completed_tools: set[str] = set()
- self.denial_counts: dict[str, int] = {}
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- state_runtime = getattr(agent.state, "ontology_runtime", None)
- runtime = state_runtime if isinstance(state_runtime, dict) else self.runtime
- if not runtime.get("enabled"):
- async for item in next_handler(**input_kwargs):
- yield item
- return
-
- from core.ontology.validator import (
- activate_runtime_for_asset,
- evaluate_tool_call,
- render_runtime_prompt,
- )
- from core.services.ontology_service import resolve_runtime_asset_tags
-
- started = time.monotonic()
- tool_call = input_kwargs.get("tool_call")
- tool_name = str(getattr(tool_call, "name", "") or "")
- raw_input = getattr(tool_call, "input", "{}")
- try:
- tool_input = raw_input if isinstance(raw_input, dict) else json.loads(raw_input or "{}")
- except (TypeError, json.JSONDecodeError):
- tool_input = {}
- asset_kind, asset_id = self._resolve_invoked_asset(tool_name, tool_input)
- await asyncio.to_thread(
- resolve_runtime_asset_tags,
- runtime=runtime,
- kind=asset_kind,
- asset_id=asset_id,
- user_id=str(getattr(agent.state, "user_id", "") or ""),
- )
- activations = activate_runtime_for_asset(
- runtime,
- kind=asset_kind,
- asset_id=asset_id,
- )
- if activations:
- agent.state.ontology_enabled = True
- agent.state.ontology_runtime = runtime
- contract = render_runtime_prompt(runtime)
- if contract:
- _append_reminder(
- agent,
- "检测到受领域本体治理的运行时资产,策略已升级且本轮不可降级。\n" f"{contract}",
- origin="harness:ontology_gate",
- )
- await self._audit_activations(agent, runtime, activations)
- decision = evaluate_tool_call(
- runtime,
- tool_name=tool_name,
- tool_input=tool_input,
- completed_tools=self.completed_tools,
- )
- if not decision.allowed:
- key = ",".join(decision.matched_rule_ids) or tool_name
- self.denial_counts[key] = self.denial_counts.get(key, 0) + 1
- denial_count = self.denial_counts[key]
- thresholds = [pack.get("config", {}) for pack in runtime.get("packs", [])]
- strategy_threshold = min(
- (int(item.get("repeated_denial_threshold", 2)) for item in thresholds),
- default=2,
- )
- breaker_threshold = min(
- (int(item.get("circuit_breaker_threshold", 5)) for item in thresholds),
- default=5,
- )
- guidance = list(decision.suggestions)
- if denial_count >= strategy_threshold:
- guidance.append("同一规则已重复拦截,请改变执行策略,不要原样重试。")
- if denial_count >= breaker_threshold:
- guidance.append("本体门禁已触发熔断;停止调用该工具,并向用户说明缺失条件。")
- await self._audit(
- agent,
- runtime,
- tool_name,
- decision,
- started,
- denial_count=denial_count,
- circuit_breaker=denial_count >= breaker_threshold,
- )
- if denial_count == strategy_threshold and getattr(agent.state, "chat_id", None):
- try:
- from core.services.ontology_evolution_service import (
- schedule_ontology_evolution,
- )
-
- schedule_ontology_evolution(
- user_id=str(getattr(agent.state, "user_id", "") or "system")
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("[ontology] evolution scheduling failed: %s", exc)
- payload = {
- "code": "ONTOLOGY_GATE_DENIED",
- "tool": tool_name,
- "violations": decision.violations,
- "suggestions": guidance,
- "denial_count": denial_count,
- "circuit_breaker": denial_count >= breaker_threshold,
- }
- yield ToolChunk(
- content=[render_text_block(json.dumps(payload, ensure_ascii=False))],
- state=ToolResultState.DENIED,
- metadata={"ontology_gate": payload},
- )
- return
-
- await self._audit(agent, runtime, tool_name, decision, started)
-
- last_state = None
- async for item in next_handler(**input_kwargs):
- last_state = getattr(item, "state", last_state)
- yield item
- if last_state not in {
- ToolResultState.ERROR,
- ToolResultState.DENIED,
- ToolResultState.INTERRUPTED,
- }:
- self.completed_tools.add(tool_name)
-
- async def _audit(
- self,
- agent: Agent,
- runtime: dict[str, Any],
- tool_name: str,
- decision,
- started: float,
- *,
- denial_count: int = 0,
- circuit_breaker: bool = False,
- ) -> None: # noqa: ANN001
- if not decision.violations and not decision.matched_rule_ids:
- return
- runtime.setdefault("runtime_events", []).append(
- {
- "type": "ontology_gate",
- "status": "completed",
- "governance_run_id": runtime.get("governance_run_id"),
- "stage": "tool",
- "decision": decision.decision,
- "tool_name": tool_name,
- "matched_rule_ids": list(decision.matched_rule_ids),
- "violations": list(decision.violations),
- "suggestions": list(decision.suggestions),
- "denial_count": denial_count,
- "circuit_breaker": circuit_breaker,
- }
- )
- try:
- from core.services.ontology_service import record_enforcement_event
-
- first = decision.violations[0] if decision.violations else {}
- pack_id = first.get("pack_id")
- version_id = None
- for pack in runtime.get("packs", []):
- if pack.get("pack_id") == pack_id:
- version_id = pack.get("version_id")
- break
- payload = {
- "user_id": getattr(agent.state, "user_id", None),
- "chat_id": getattr(agent.state, "chat_id", None),
- "pack_id": pack_id,
- "version_id": version_id,
- "rule_id": first.get("rule_id")
- or (decision.matched_rule_ids[0] if decision.matched_rule_ids else None),
- "stage": "tool",
- "event_type": "tool_call_gate",
- "decision": decision.decision,
- "mode": first.get("mode", "enforce"),
- "target": tool_name,
- "latency_ms": round((time.monotonic() - started) * 1000),
- "details": {
- "governance_run_id": runtime.get("governance_run_id"),
- "violations": decision.violations,
- "matched_rule_ids": decision.matched_rule_ids,
- "suggestions": decision.suggestions,
- "denial_count": denial_count,
- "circuit_breaker": circuit_breaker,
- },
- }
- await asyncio.to_thread(record_enforcement_event, payload)
- except Exception as exc: # noqa: BLE001
- logger.warning("[ontology] audit event persistence failed: %s", exc)
-
- @staticmethod
- def _resolve_invoked_asset(tool_name: str, tool_input: dict[str, Any]) -> tuple[str, str]:
- if tool_name == "call_subagent" and tool_input.get("agent_id"):
- return "subagent", str(tool_input["agent_id"])
- if tool_name == "view_text_file":
- file_path = str(tool_input.get("file_path") or "")
- path = Path(file_path)
- if path.name == "SKILL.md" and path.parent.name:
- return "skill", path.parent.name
- return "tool", tool_name
-
- @staticmethod
- async def _audit_activations(
- agent: Agent,
- runtime: dict[str, Any],
- activations: list[dict[str, Any]],
- ) -> None:
- try:
- from core.services.ontology_service import record_runtime_activation
-
- for event in activations:
- await asyncio.to_thread(
- record_runtime_activation,
- event,
- runtime,
- user_id=getattr(agent.state, "user_id", None),
- chat_id=getattr(agent.state, "chat_id", None),
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("[ontology] activation audit persistence failed: %s", exc)
-
-
-# ── DynamicModel ──────────────────────────────────────────────────────────
-class DynamicModelMiddleware(MiddlewareBase):
- async def on_reply(self, agent: Agent, input_kwargs: dict, next_handler):
- try:
- # The subagent pinned its own model → skip the chat_mode-based main-model override
- # and respect its configuration. Otherwise the subagent model built by the factory
- # from model_provider_id would be unconditionally replaced with the main model here —
- # symptom: "subagent channel binding doesn't take effect / web and channel behave differently".
- if not getattr(agent.state, "model_pinned", False):
- mode = _resolve_chat_mode(agent.state)
- provider_id = getattr(agent.state, "model_provider_id", "") or ""
- if provider_id:
- agent.model = _get_provider_model(provider_id, mode=mode)
- else:
- agent.model = _get_main_model(mode=mode)
- run_id = str(getattr(agent.state, "run_id", "") or "")
- if run_id:
- from core.llm.model_usage import instrument_model_usage
-
- instrument_model_usage(agent.model)
- except Exception as exc: # noqa: BLE001
- logger.warning("[dynamic_model] failed: %s", exc)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
-
-# ── FileContext ───────────────────────────────────────────────────────────
-class FileContextMiddleware(MiddlewareBase):
- def __init__(self) -> None:
- self._injected = False
-
- async def on_reply(self, agent: Agent, input_kwargs: dict, next_handler):
- try:
- await self._inject(agent)
- except Exception as exc: # noqa: BLE001
- logger.warning("[file_context] failed: %s", exc, exc_info=True)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
- async def _inject(self, agent: Agent) -> None:
- # Reset per-turn state at each reply-turn boundary (read_artifact budget / pin-reminder bookkeeping)
- reset_artifact_read_state()
- reset_pin_hint_state()
-
- st = agent.state
- uploaded_files = list(getattr(st, "uploaded_files", None) or [])
- historical_files = list(getattr(st, "historical_files", None) or [])
- if not uploaded_files and not historical_files:
- return
- # user_id serves as the ownership-verification gate for attachment reads (prevents forged
- # file_id cross-user reads; see the _download_artifact_bytes / _build_file_context docstrings in hooks).
- user_id = getattr(st, "user_id", None) or None
- if self._injected:
- return
- self._injected = True
-
- # 0. Historical files digest
- if historical_files:
- hist_context = _build_historical_files_context(historical_files)
- if hist_context:
- _append_state_payload(
- st,
- hist_context,
- kind=KIND_PROJECT,
- origin="workspace:historical_files",
- trust="workspace",
- )
-
- # 1. Current-turn text files.
- # Off the event loop: on a parse-cache miss _build_file_context downloads the
- # artifact and POSTs it to the external file-parser service with blocking
- # ``requests`` — measured at 79s for an 8MB PDF, and the service timeout allows
- # 120s. Called inline it pinned the whole loop for that long: every SSE stream
- # (all chats, all users) emitted nothing, queued requests all landed in the same
- # millisecond once it returned, and even /health stalled. The visible symptom is
- # "streaming stopped working" on the very turn a document is attached.
- text_context = (
- await asyncio.to_thread(_build_file_context, uploaded_files, user_id=user_id)
- if uploaded_files
- else ""
- )
- if text_context:
- _append_state_payload(
- st,
- text_context,
- kind=KIND_ATTACHMENT,
- origin="user:uploaded_files",
- trust="user",
- )
-
- # 2. Images. Two paths, decided by whether the *effective* model can see:
- # - natively multimodal → pass the bytes through as DataBlocks (best fidelity)
- # - text-only → vision bridge transcribes them into text evidence
- # (core/vision). This is the only place the choice is made: downstream
- # never degrades media, so a text-only model that received raw bytes
- # would fail the request outright rather than silently go blind.
- image_files = [f for f in uploaded_files if _is_image(f)]
- if not image_files:
- return
- if _effective_model_supports_vision(st):
- # Same reason: each image is downloaded from storage (S3/OSS in deployments
- # that use them) and base64-encoded — blocking I/O plus CPU on the loop.
- await asyncio.to_thread(self._inject_native_images, st, image_files, user_id)
- else:
- await self._inject_vision_evidence(st, image_files, user_id)
-
- # ── image injection paths ──────────────────────────────────────────────
-
- @staticmethod
- def _inject_native_images(st: AgentState, image_files: list, user_id) -> None:
- """Natively multimodal model: append framework-neutral image blocks."""
- image_blocks: list = []
- image_names: list[str] = []
- for f in image_files:
- result = _fetch_image_base64(f, user_id=user_id)
- if result:
- b64_data, mime_type = result
- image_blocks.append(
- {
- "type": "data",
- "source": {
- "type": "base64",
- "media_type": mime_type,
- "data": b64_data,
- },
- }
- )
- image_names.append(f.get("name", "图片"))
- if not image_blocks:
- return
- names_str = "、".join(image_names)
- prefix_block = {
- "type": "text",
- "text": f"[用户上传了 {len(image_blocks)} 张图片:{names_str}]",
- }
- _append_state_payload(
- st,
- [prefix_block, *image_blocks],
- kind=KIND_ATTACHMENT,
- origin="user:uploaded_images",
- trust="user",
- token_budget=16_000,
- )
-
- @staticmethod
- async def _inject_vision_evidence(st: AgentState, image_files: list, user_id) -> None:
- """Text-only model: inject the images as text evidence from the vision bridge.
-
- The streaming entry point usually transcribes ahead of time (so it can report
- "reading image" progress while the network call is in flight) and leaves the
- result on ``st.vision_evidence_text``. When that is absent — the non-streaming
- workflow, channel bots — this transcribes inline as before.
- """
- precomputed = (getattr(st, "vision_evidence_text", "") or "").strip()
- if precomputed:
- _append_state_payload(
- st,
- precomputed,
- kind=KIND_ATTACHMENT,
- origin="vision:transcription",
- trust="tool",
- )
- return
-
- from core.vision.attachments import transcribe_attachments
-
- result = await transcribe_attachments(image_files, user_id=user_id)
- if result is None or not result.text:
- return
- _append_state_payload(
- st,
- result.text,
- kind=KIND_ATTACHMENT,
- origin="vision:transcription",
- trust="tool",
- )
-
-
-def _effective_model_supports_vision(st: AgentState) -> bool:
- """Whether the model this turn will actually run on can read images natively."""
- from core.vision import resolve_vision_mode
-
- return resolve_vision_mode(getattr(st, "model_provider_id", "") or "") == "native"
-
-
-# ── WorkspacePinHint ───────────────────────────────────────────────────────
-def _recent_tool_results(context: list, scan: int = 12) -> list:
- """Return the most recent tool_result blocks at the tail of context (2.0: pydantic blocks on assistant messages)."""
- blocks = []
- for msg in context[-scan:]:
- try:
- if msg.has_content_blocks("tool_result"):
- blocks.extend(msg.get_content_blocks("tool_result"))
- except Exception: # noqa: BLE001
- continue
- return blocks
-
-
-def _tool_result_text(block: Any) -> str:
- from core.llm.message_compat import flatten_tool_output
-
- return flatten_tool_output(getattr(block, "output", None))
-
-
-class WorkspacePinHintMiddleware(MiddlewareBase):
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler):
- try:
- self._scan_and_remind(agent)
- except Exception as exc: # noqa: BLE001
- logger.warning("[pin_hint] failed: %s", exc, exc_info=True)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
- def _scan_and_remind(self, agent: Agent) -> None:
- from core.llm import workspace as _ws
-
- results = _recent_tool_results(agent.state.context)
- if not results:
- return
- state = _get_pin_hint_state()
- seen: set = state["seen"]
- for block in results:
- name = getattr(block, "name", "") or ""
- if name in _PIN_HINT_SKIP_TOOLS:
- continue
- text_blob = _tool_result_text(block)
- if '"file_id"' in text_blob:
- seen.update(_FILE_ID_RE.findall(text_blob))
- if not seen:
- return
- pinned = set(_ws.get_pinned_file_ids())
- unpinned = seen - pinned
- if not unpinned:
- return
- sig = ",".join(sorted(unpinned))
- if sig == state.get("last_reminded_sig"):
- return
- preview = sorted(unpinned)[:6]
- preview_str = ", ".join(preview)
- if len(unpinned) > len(preview):
- preview_str += f", …(+{len(unpinned) - len(preview)} 个)"
- reminder = (
- f"沙盒里有 {len(unpinned)} 个 file_id 还没 pin:[{preview_str}]。"
- f"若是给用户的最终产物,必须调 `pin_to_workspace(file_ids=[...])` 才能交付。"
- )
- _append_reminder(agent, reminder, origin="harness:workspace_pin")
- state["last_reminded_sig"] = sig
-
-
-# ── PlanStaleReminder ──────────────────────────────────────────────────────
-# 停滞满 N 轮催一次,催完重新计时——等价于 Claude Code 那两个都取 10 的阈值。
-_PLAN_REMINDER_INTERVAL = 10
-
-_PLAN_STALE_REMINDER_TEMPLATE = """`update_plan` 已经 {rounds} 轮没有用过了。当前计划栏停在 {done}/{total}:
-{checklist}
-
-如果其中有步骤其实已经做完,可以调用一次 `update_plan` 把状态补上(传全量列表);如果这份
-清单已经不符合你正在做的事,重写它比让它停在原地更好。仅在确实相关时才使用——这只是一条
-温和的提醒,不适用就忽略,不要为了回应它而打断手上的工作。不要向用户提起这条提醒。"""
-
-
-def _block_attr(block: Any, name: str, default: Any = None) -> Any:
- """内容块在 AS2 里是 pydantic 对象,但历史路径上也出现过 dict —— 两种都读。"""
- if isinstance(block, dict):
- return block.get(name, default)
- return getattr(block, name, default)
-
-
-def _latest_plan_call(context: list) -> tuple[Optional[dict], str]:
- """倒着找最近一次 ``update_plan`` 调用,返回(解析后的计划, 这次调用的指纹)。
-
- 指纹用 tool_call 的 id:id 变了就说明模型刚更新过清单,计时要清零。拿不到 id 时
- 退化成清单内容本身——同样能区分"又调了一次且内容有变",只在"重复提交完全相同的
- 清单"时无法分辨,而那种情况本来也不该重置计时。
- """
- for msg in reversed(list(context or [])):
- try:
- if hasattr(msg, "has_content_blocks") and not msg.has_content_blocks("tool_call"):
- continue
- blocks = list(msg.get_content_blocks("tool_call") or [])
- except Exception: # noqa: BLE001
- continue
- for b in reversed(blocks):
- if _block_attr(b, "name", "") != "update_plan":
- continue
- raw = _block_attr(b, "input", None)
- if isinstance(raw, str):
- try:
- raw = json.loads(raw)
- except (json.JSONDecodeError, ValueError):
- raw = None
- plan = parse_plan_update_args(raw)
- fingerprint = str(_block_attr(b, "id", "") or "") or json.dumps(
- plan or {}, ensure_ascii=False, sort_keys=True
- )
- return plan, fingerprint
- return None, ""
-
-
-class PlanStaleReminderMiddleware(MiddlewareBase):
- """计划栏停滞满 ``_PLAN_REMINDER_INTERVAL`` 轮时,把当前清单软提醒给模型一次。
-
- 模型没建过清单、或清单已全部结算,都不打扰;模型自己更新了清单则重新计时。
- """
-
- def __init__(self, *, interval: int = _PLAN_REMINDER_INTERVAL) -> None:
- self._interval = max(1, int(interval))
- self._since_last = 0
- self._last_fingerprint: Optional[str] = None
-
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler):
- try:
- self._maybe_remind(agent)
- except Exception as exc: # noqa: BLE001
- logger.warning("[plan_stale] failed: %s", exc)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
- def _maybe_remind(self, agent: Agent) -> None:
- plan, fingerprint = _latest_plan_call(agent.state.context)
- if not plan:
- return
- if fingerprint != self._last_fingerprint:
- self._last_fingerprint = fingerprint
- self._since_last = 0
- return
- steps = plan.get("steps") or []
- if not any(s.get("status") in ("pending", "in_progress") for s in steps):
- return
- self._since_last += 1
- if self._since_last < self._interval:
- return
- done = sum(1 for s in steps if s.get("status") == "completed")
- checklist = "\n".join(
- f" {i}. [{s.get('status', 'pending')}] {s.get('title', '')}"
- for i, s in enumerate(steps, start=1)
- )
- _append_reminder(
- agent,
- _PLAN_STALE_REMINDER_TEMPLATE.format(
- done=done,
- total=len(steps),
- rounds=self._since_last,
- checklist=checklist,
- ),
- origin="harness:plan_stale",
- )
- logger.info(
- "[plan_stale] reminded at %d/%d after %d idle rounds",
- done,
- len(steps),
- self._since_last,
- )
- self._since_last = 0
-
-
-# ── IterBudgetReminder ─────────────────────────────────────────────────────
-class IterBudgetReminderMiddleware(MiddlewareBase):
- """Inject a wrap-up reminder as ReAct approaches max_iters, so the model soft-lands instead of erroring on the hard limit.
-
- When AgentScope 2.0 runs max_iters dry it only yields ExceedMaxItersEvent + one fixed
- English error string, and everything produced in the turn is discarded (in the subagent
- scenario the error string is returned to the main agent as the tool result). This
- middleware appends a system-reminder to context when remaining rounds <= threshold,
- with two levels of wording:
- - N (>1) rounds left: stop exploring/retrying, consolidate the information gathered so far and wrap up;
- - final round: no further tool calls allowed — the framework loop gives no more reasoning
- chances before exiting, so a tool call in the final round is guaranteed to trip the hard
- limit (_agent.py main-loop semantics).
-
- Dedup key is (reply_id, cur_iter): remind only once per round; across rounds the wording
- escalates in urgency as remaining rounds decrease. When max_iters is very small
- (<= threshold+1, e.g. plan wrap-up style micro budgets), don't remind — avoid nagging to
- wrap up right after starting.
- """
-
- def __init__(self, *, threshold: int = 2) -> None:
- self._threshold = threshold
- self._last_key: tuple | None = None
- # Env is immutable for the process lifetime — parse the kill-switch once
- # per middleware (one per agent build), not on every reasoning round.
- from core.config.settings import _bool, _env
-
- self._force_text = _bool(_env("CHAT_FINAL_ITER_FORCE_TEXT", "true"))
-
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler):
- max_iters, cur_iter, remaining = self._budget(agent)
- try:
- self._maybe_remind(agent, max_iters, cur_iter, remaining)
- except Exception as exc: # noqa: BLE001
- logger.warning("[iter_budget] failed: %s", exc)
- try:
- input_kwargs = self._maybe_force_text(input_kwargs, max_iters, remaining)
- except Exception as exc: # noqa: BLE001
- logger.warning("[iter_budget] force-text failed: %s", exc)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
- @staticmethod
- def _budget(agent: Agent) -> tuple[int, int, int]:
- """Return (max_iters, cur_iter, remaining-including-current-round)."""
- max_iters = int(getattr(agent.react_config, "max_iters", 0) or 0)
- cur_iter = int(getattr(agent.state, "cur_iter", 0) or 0)
- return max_iters, cur_iter, max_iters - cur_iter
-
- def _maybe_force_text(self, input_kwargs: dict, max_iters: int, remaining: int) -> dict:
- """Final round: force tool_choice="none" so the model can only produce text.
-
- The reminder alone is advisory — a model that still emits a tool call in
- the final round trips AgentScope's hard limit and the whole turn's output
- is discarded. Forcing text turns the hard error into a guaranteed clean
- exit (the loop sees no tool calls and yields the reply Msg). Skipped for
- micro budgets (same guard as the reminder) and when a caller already
- pinned an explicit tool_choice. Kill-switch: CHAT_FINAL_ITER_FORCE_TEXT=false.
- """
- if remaining > 1 or max_iters <= self._threshold + 1 or not self._force_text:
- return input_kwargs
- if input_kwargs.get("tool_choice") is not None:
- return input_kwargs
- from agentscope.tool import ToolChoice
-
- logger.info(
- "[iter_budget] final round (max_iters=%d): forcing tool_choice=none for clean delivery",
- max_iters,
- )
- return {**input_kwargs, "tool_choice": ToolChoice(mode="none")}
-
- def _maybe_remind(self, agent: Agent, max_iters: int, cur_iter: int, remaining: int) -> None:
- if max_iters <= self._threshold + 1:
- return
- if remaining > self._threshold:
- return
- key = (getattr(agent.state, "reply_id", "") or "", cur_iter)
- if key == self._last_key:
- return
- self._last_key = key
- if remaining <= 1:
- reminder = (
- "这是本次回复的最后一轮推理:不要再调用任何工具(再调用会被系统强制"
- "中断,已完成的工作将无法交付)。请直接基于已获得的信息输出最终回复;"
- "若任务未全部完成,如实汇报已产出的成果、当前进展和未完成的部分。"
- )
- else:
- reminder = (
- f"注意:推理-工具调用轮次即将用尽,包含本轮在内最多还剩 {remaining} 轮。"
- "请停止新的探索,不要再重试已反复失败的操作,立即整合已获得的信息进行"
- "收尾;若任务无法在剩余轮次内全部完成,优先输出已有成果与进展说明。"
- )
- _append_reminder(agent, reminder, origin="harness:iteration_budget")
-
-
-# ── JobLedgerReminder ──────────────────────────────────────────────────────
-class JobLedgerReminderMiddleware(MiddlewareBase):
- """本会话存在未收敛的批量作业时,每轮把台账状态回灌进上下文。
-
- 为什么要 harness 主动回灌,而不是指望模型记着:进度是**外部事实**,写在
- ``job_items`` 表里,不在模型的记忆里。模型隔了十几轮之后对"还剩多少没做"的印象
- 只会越来越糊,最后就演变成"边际收益递减,先交付吧"——上一轮 568 行只补 66 行
- 就是这么停的。
-
- 每轮注入一条 ````,内容是可查证的数字(总计/已完成/待办/失败)
- 和明确的下一步。同一轮只注入一次;台账已收敛(待办与失败都为 0)就不再打扰。
- """
-
- def __init__(self, *, chat_id: Optional[str], user_id: Optional[str]) -> None:
- self._chat_id = chat_id or ""
- self._user_id = user_id or ""
- self._last_key: tuple | None = None
-
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler):
- try:
- self._maybe_remind(agent)
- except Exception as exc: # noqa: BLE001 —— 提醒失败绝不该拖垮主链路
- logger.warning("[job_ledger] reminder failed: %s", exc)
- async for evt in next_handler(**input_kwargs):
- yield evt
-
- def _maybe_remind(self, agent: Agent) -> None:
- if not self._chat_id:
- return
- key = (
- getattr(agent.state, "reply_id", "") or "",
- int(getattr(agent.state, "cur_iter", 0) or 0),
- )
- if key == self._last_key:
- return
-
- from core.db.engine import SessionLocal
- from core.db.models import Job
- from core.services.job_service import JobService
-
- with SessionLocal() as db:
- jobs = (
- db.query(Job)
- .filter(Job.chat_id == self._chat_id, Job.user_id == self._user_id)
- .order_by(Job.created_at.desc())
- .limit(3)
- .all()
- )
- if not jobs:
- return
- svc = JobService(db)
- lines = []
- for job in jobs:
- stats = svc.stats(job.job_id)
- unsettled = int(stats.get("pending", 0)) + int(stats.get("failed", 0))
- if unsettled <= 0 and job.status in ("completed", "cancelled"):
- continue
- lines.append(
- f"- 作业「{job.name or job.job_id}」({job.job_id},状态 {job.status}):"
- f"总计 {stats.get('total', 0)},已完成 {stats.get('done', 0)},"
- f"查无 {stats.get('not_found', 0)},待办 {stats.get('pending', 0)},"
- f"失败 {stats.get('failed', 0)}"
- )
- if not lines:
- return
-
- self._last_key = key
- reminder = (
- "本会话有尚未结算完的批量作业(数字取自台账,是可查证的事实,不是估计):\n"
- + "\n".join(lines)
- + "\n\n还有待办或失败项时:用 run_job(action='resume', job_id=...) 续跑"
- "(已完成的项不会重做),或先查 run_job(action='status', job_id=...) 看明细。"
- "**不得**把未完成当作完成来交付;确实要收尾,也必须如实报出分母、完成数与未覆盖清单。"
- )
- _append_reminder(agent, reminder, origin="harness:job_ledger")
-
-
-# ── StallIntervention ──────────────────────────────────────────────────────
-class StallInterventionMiddleware(MiddlewareBase):
- """Apply the active profile's intervention rules to the **ReAct loop**.
-
- The rules existed before this and only ever reached ``autonomous_loop`` — the
- long-running self-driving product — through
- :mod:`core.evolution.policies`. That left the orchestration profile with one
- field that governed nothing on the axis nearly all traffic takes, and it is a
- fair reading of "you are still tuning a workflow": every other part of the
- profile shapes the ReAct assembly, this one did not.
-
- What a ReAct turn can actually observe is narrower than what a multi-run loop
- can, and pretending otherwise is how a rule ends up never firing:
-
- * ``repeated_actions`` — the same tool called with the same arguments again;
- * ``tool_error_streak`` — consecutive failing tool calls;
- * ``no_progress`` — consecutive calls that returned nothing usable.
-
- ``no_diff`` and ``reviewer_score_flat`` need a file tree and a reviewer
- verdict, which a single answer has neither of; those stay loop-only and are
- declared as such in :mod:`core.evolution.agent_profile`.
-
- The intervention itself is a ```` appended to the context —
- the same mechanism the iteration-budget reminder uses. A middleware cannot
- reach into the model's plan, and it should not: the model is told what has
- been observed and what to do about it, and stays responsible for the how.
- """
-
- def __init__(self, rules: List[Any] | None = None) -> None:
- self._rules = list(rules or [])
- self._signals: dict[str, int] = {}
- self._last_call: tuple | None = None
- self._fired: set[str] = set()
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- if not self._rules:
- async for item in next_handler(**input_kwargs):
- yield item
- return
-
- tool_call = input_kwargs.get("tool_call")
- tool_name = str(getattr(tool_call, "name", "") or "")
- try:
- signature = (
- tool_name,
- json.dumps(getattr(tool_call, "input", None), sort_keys=True, default=str),
- )
- except Exception: # noqa: BLE001
- signature = (tool_name, "")
-
- if signature == self._last_call:
- self._signals["repeated_actions"] = self._signals.get("repeated_actions", 0) + 1
- else:
- # A streak is consecutive by definition; a different call breaks it.
- self._signals["repeated_actions"] = 0
- self._last_call = signature
-
- last_state = None
- async for item in next_handler(**input_kwargs):
- last_state = getattr(item, "state", last_state)
- yield item
-
- failed = last_state in {
- ToolResultState.ERROR,
- ToolResultState.DENIED,
- ToolResultState.INTERRUPTED,
- }
- if failed:
- self._signals["tool_error_streak"] = self._signals.get("tool_error_streak", 0) + 1
- self._signals["no_progress"] = self._signals.get("no_progress", 0) + 1
- else:
- self._signals["tool_error_streak"] = 0
- self._signals["no_progress"] = 0
-
- try:
- self._maybe_intervene(agent)
- except Exception as exc: # noqa: BLE001
- logger.warning("[stall-intervention] failed: %s", exc)
-
- def _maybe_intervene(self, agent: Agent) -> None:
- from core.evolution.agent_profile import (
- ACTION_CHANGE_STRATEGY,
- ACTION_DELEGATE,
- ACTION_ESCALATE,
- ACTION_NARROW_SCOPE,
- ACTION_RETRY,
- ACTION_ROLLBACK_AND_FORK,
- ACTION_STOP,
- )
-
- applicable = [
- rule
- for rule in self._rules
- if self._signals.get(getattr(rule, "signal", ""), 0)
- >= int(getattr(rule, "threshold", 0) or 0)
- and int(getattr(rule, "threshold", 0) or 0) > 0
- ]
- if not applicable:
- return
- # Most-specific first: a long stall escalates rather than repeating the
- # mild response that already failed to help.
- rule = max(applicable, key=lambda r: int(getattr(r, "threshold", 0) or 0))
- action = str(getattr(rule, "action", ""))
- signal = str(getattr(rule, "signal", ""))
-
- # Fire once per (signal, action). Repeating the same reminder every
- # iteration turns it into noise the model learns to skip.
- key = f"{signal}:{action}"
- if key in self._fired:
- return
- self._fired.add(key)
-
- observed = {
- "repeated_actions": "你已经用完全相同的参数重复调用同一个工具",
- "tool_error_streak": "连续多次工具调用失败",
- "no_progress": "连续多轮没有取得任何可用结果",
- }.get(signal, "当前执行已停滞")
-
- guidance = {
- ACTION_CHANGE_STRATEGY: "换一条思路:不要重试同样的调用,改用别的工具或别的切入角度。",
- ACTION_NARROW_SCOPE: "缩小范围:先只解决其中最小的一个子问题,拿到确定结果再往下走。",
- ACTION_RETRY: "可以再试一次,但必须改变参数或前置条件,原样重试不会有不同结果。",
- ACTION_DELEGATE: "把这段工作交给合适的子智能体处理,只取回结论。",
- ACTION_ESCALATE: "停止尝试,向用户说明卡在哪里、还缺什么信息。",
- ACTION_STOP: "停止继续尝试,如实汇报已完成的部分和未完成的原因。",
- # Loop-only: a single answer has nothing to roll back to.
- ACTION_ROLLBACK_AND_FORK: "放弃当前路径,回到上一个可用状态重新规划。",
- }.get(action, "")
- if not guidance:
- return
-
- _append_reminder(
- agent,
- f"{observed}。{guidance}\n"
- f"(该干预来自当前生效的编排配置:{signal} ≥ "
- f"{getattr(rule, 'threshold', 0)} → {action})",
- origin="harness:stall_intervention",
- )
- logger.info(
- "[stall-intervention] %s >= %s -> %s",
- signal,
- getattr(rule, "threshold", 0),
- action,
- )
-
-
-# ── FinishPinGuard ─────────────────────────────────────────────────────────
-class FinishPinGuardMiddleware(MiddlewareBase):
- def __init__(self) -> None:
- self._fired = False
-
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler):
- had_tool_call = False
- async for evt in next_handler(**input_kwargs):
- if isinstance(evt, ToolCallEndEvent):
- had_tool_call = True
- yield evt
- if had_tool_call or self._fired:
- return
- try:
- from core.llm.finish_guard import _collect_unpinned, _direct_pin
-
- unpinned = _collect_unpinned()
- if unpinned:
- pinned_now = _direct_pin(unpinned)
- if pinned_now > 0:
- self._fired = True
- logger.info(
- "[finish_guard] auto-pinned %d/%d file_id(s)",
- pinned_now,
- len(unpinned),
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("[finish_guard] failed: %s", exc)
-
-
-__all__ = [
- "AgentRuntimeState",
- "DynamicModelMiddleware",
- "FileContextMiddleware",
- "WorkspacePinHintMiddleware",
- "IterBudgetReminderMiddleware",
- "FinishPinGuardMiddleware",
-]
diff --git a/src/backend/core/llm/persona_distiller.py b/src/backend/core/llm/persona_distiller.py
deleted file mode 100644
index ff85c3ea..00000000
--- a/src/backend/core/llm/persona_distiller.py
+++ /dev/null
@@ -1,226 +0,0 @@
-"""Persona distiller — map/reduce LLM caller for colleague / personal skill jobs.
-
-Two stages (driven by core/services/persona_distillation_service.py):
-- map: summarize_session(trajectory) → per-session digest JSON (lightweight call)
-- reduce: distill_persona(digests, ...) → final SKILL.md (single synthesis call)
-
-Prompts live as independent parts of the 'distillation' kind:
- session_digest / colleague_distiller / personal_distiller
-DB active version (per part) → filesystem fallback.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from openai import AsyncOpenAI
-
-from core.config.distillation import DistillationConfig, get_config
-from core.llm._distill_shared import (
- estimate_cost_usd,
- parse_strict_json,
- resolve_distiller_model,
-)
-from core.services.distillation_service import Trajectory, trajectory_to_dict
-
-logger = logging.getLogger(__name__)
-
-_PROMPT_DIR = Path(__file__).resolve().parents[2] / "prompts" / "prompt_text" / "distillation"
-
-_DIGEST_KEYS = ("topics", "demonstrated_skills", "decision_points", "style_samples", "tool_sequences")
-
-
-def _load_prompt(part_id: str) -> str:
- try:
- from core.services import prompt_version_service as pvs
- rendered = pvs.render_active_prompt_part("distillation", part_id)
- if rendered:
- return rendered
- except Exception as exc:
- logger.debug("persona_distiller: prompt_version_service miss for %s (%s)", part_id, exc)
-
- fp = _PROMPT_DIR / f"{part_id}.system.md"
- try:
- return fp.read_text(encoding="utf-8")
- except Exception as exc:
- logger.error("persona_distiller: failed to load prompt %s (%s)", part_id, exc)
- raise RuntimeError(f"persona_distiller: prompt '{part_id}' unavailable") from exc
-
-
-# client is cached by (api_key, base_url, timeout) to reuse the connection pool; auto-swapped on hot model config change
-_CLIENT_CACHE: dict[tuple, AsyncOpenAI] = {}
-
-
-def _get_client(api_key: str, base_url: str, timeout: float) -> AsyncOpenAI:
- key = (api_key, base_url, timeout)
- client = _CLIENT_CACHE.get(key)
- if client is None:
- client = AsyncOpenAI(api_key=api_key, base_url=base_url, timeout=timeout)
- _CLIENT_CACHE.clear() # after a config change the old client is no longer needed, avoid accumulation
- _CLIENT_CACHE[key] = client
- return client
-
-
-async def _call_llm(system_prompt: str, max_tokens: int, cfg: DistillationConfig) -> tuple[str, int, int, str]:
- """One chat call. Returns (content, prompt_tokens, completion_tokens, model_name)."""
- model_cfg = resolve_distiller_model()
- client = _get_client(
- model_cfg.api_key or "DUMMY",
- model_cfg.base_url,
- float(cfg.llm_timeout_s),
- )
- resp = await client.chat.completions.create(
- model=model_cfg.model_name,
- messages=[
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": "请按规则输出严格 JSON。"},
- ],
- temperature=cfg.llm_temperature,
- max_tokens=max_tokens,
- )
- content = resp.choices[0].message.content or ""
- usage = resp.usage
- ptok = getattr(usage, "prompt_tokens", 0) if usage else 0
- ctok = getattr(usage, "completion_tokens", 0) if usage else 0
- return content, ptok, ctok, model_cfg.model_name
-
-
-# ─────────────────────── map: per-session digest ────────────────────────
-
-
-async def summarize_session(
- trajectory: Trajectory,
- cfg: Optional[DistillationConfig] = None,
- hint: str = "",
-) -> tuple[Optional[Dict[str, Any]], float]:
- """Summarize one session. Returns (digest_dict | None, cost_usd).
-
- None means the LLM output couldn't be parsed — caller skips the session
- but still accounts the cost. Retries once on parse failure (reasoning models
- occasionally truncate / drift in format, and a single failure would
- silently drop the entire session out of the distillation scope).
- """
- cfg = cfg or get_config()
- system_prompt = (
- _load_prompt("session_digest")
- .replace(
- "{trajectory_json}",
- json.dumps(trajectory_to_dict(trajectory), ensure_ascii=False, indent=2),
- )
- .replace("{hint}", hint or "(无)")
- )
- cost = 0.0
- content = ""
- parsed: Optional[Dict[str, Any]] = None
- for attempt in range(2):
- content, ptok, ctok, model_name = await _call_llm(
- system_prompt, cfg.persona_digest_max_tokens, cfg
- )
- cost += estimate_cost_usd(model_name, ptok, ctok)
- parsed = parse_strict_json(content)
- if parsed is not None:
- break
- logger.warning(
- "persona_distiller: digest parse failed (attempt %d) chat=%s head=%r",
- attempt + 1, trajectory.chat_id, (content or "")[:120],
- )
- if parsed is None:
- return None, cost
-
- digest: Dict[str, Any] = {"chat_id": trajectory.chat_id}
- for key in _DIGEST_KEYS:
- val = parsed.get(key)
- digest[key] = [str(x)[:300] for x in val][:5] if isinstance(val, list) else []
- digest["low_value"] = bool(parsed.get("low_value", False))
- return digest, cost
-
-
-# ─────────────────────── reduce: persona synthesis ────────────────────────
-
-
-@dataclass
-class PersonaDistillOutput:
- skill: Optional[Dict[str, Any]] # {id, frontmatter, instructions_md}
- digest_text: str # persona_digest (colleague) / notes (personal)
- confidence: float
- raw_response: str
- cost_usd: float
- error: Optional[str] = None # reason for parse failure
-
-
-async def distill_persona(
- kind: str,
- digests: List[Dict[str, Any]],
- memories_grouped: Optional[Dict[str, Any]],
- assigned_identity: str,
- hint: str,
- sampled_ratio: float,
- cfg: Optional[DistillationConfig] = None,
-) -> PersonaDistillOutput:
- """Reduce stage: digests (+ grouped memories) → SKILL.md.
-
- kind: 'colleague' | 'personal' — picks the prompt template.
- Retries once on JSON-parse failure (costs accumulate).
- """
- cfg = cfg or get_config()
- part_id = "colleague_distiller" if kind == "colleague" else "personal_distiller"
- useful = [d for d in digests if not d.get("low_value")]
-
- system_prompt = (
- _load_prompt(part_id)
- .replace("{assigned_identity}", assigned_identity or "(由你决定)")
- .replace("{digest_count}", str(len(useful)))
- .replace("{sampled_ratio}", f"{sampled_ratio:.2f}")
- .replace("{digests_json}", json.dumps(useful, ensure_ascii=False, indent=1))
- .replace(
- "{memories_json}",
- json.dumps(memories_grouped or {}, ensure_ascii=False, indent=1)
- if memories_grouped else "(无)",
- )
- .replace("{hint}", hint or "(无)")
- )
-
- total_cost = 0.0
- content = ""
- parsed: Optional[Dict[str, Any]] = None
- for attempt in range(2):
- content, ptok, ctok, model_name = await _call_llm(
- system_prompt, cfg.persona_reduce_max_tokens, cfg
- )
- total_cost += estimate_cost_usd(model_name, ptok, ctok)
- parsed = parse_strict_json(content)
- if parsed is not None:
- break
- logger.warning(
- "persona_distiller: reduce parse failed (attempt %d) head=%r",
- attempt + 1, (content or "")[:200],
- )
-
- if parsed is None:
- return PersonaDistillOutput(
- skill=None,
- digest_text="",
- confidence=0.0,
- raw_response=content,
- cost_usd=total_cost,
- error="failed_parse",
- )
-
- skill = parsed.get("skill") if isinstance(parsed.get("skill"), dict) else None
- try:
- confidence = max(0.0, min(1.0, float(parsed.get("confidence") or 0.0)))
- except (TypeError, ValueError):
- confidence = 0.0
- digest_text = str(parsed.get("persona_digest") or parsed.get("notes") or "")[:1000]
- return PersonaDistillOutput(
- skill=skill,
- digest_text=digest_text,
- confidence=confidence,
- raw_response=content,
- cost_usd=total_cost,
- error=None if skill else "missing_skill",
- )
diff --git a/src/backend/core/llm/ports.py b/src/backend/core/llm/ports.py
new file mode 100644
index 00000000..644f6364
--- /dev/null
+++ b/src/backend/core/llm/ports.py
@@ -0,0 +1,79 @@
+"""Execution-layer services injected into the model runtime by its host."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Callable, Protocol
+
+
+class ToolEffectOutcome(Protocol):
+ result: Any
+ effect_id: str
+ result_id: str
+ result_state: str
+ invoked: bool
+
+
+class ToolEffectRuntime(Protocol):
+ async def execute_outcome(self, **kwargs: Any) -> ToolEffectOutcome: ...
+ def binding_error(self, message: str) -> Exception: ...
+ def response_error(self, message: str) -> Exception: ...
+
+
+def register_tool_effect_runtime(factory: Callable[..., ToolEffectRuntime]) -> None:
+ process_state().tool_effect_factory = factory
+
+
+def tool_effect_runtime(session_factory=None, registry=None) -> ToolEffectRuntime:
+ if process_state().tool_effect_factory is None:
+ raise RuntimeError("Tool effect execution service is not registered")
+ return process_state().tool_effect_factory(session_factory, registry)
+
+
+class CommandExecution(Protocol):
+ async def start(self, provider: Any, request: Any, **kwargs: Any) -> dict: ...
+ async def cancel(self, task_id: str, user_id: str, session_id: str) -> Any: ...
+ async def wait(self, task_id: str, user_id: str, session_id: str, timeout_ms: int) -> Any: ...
+ async def adopt(self, provider: Any, reference: str, **kwargs: Any) -> dict: ...
+
+
+class CommandRecords(Protocol):
+ def resolve_id(self, reference: str, user_id: str, session_id: str) -> str: ...
+ def read(self, task_id: str, user_id: str, session_id: str) -> dict: ...
+ def list_owned(self, user_id: str, chat_id: str) -> list[dict]: ...
+
+
+def register_execution_services(*, commands: CommandExecution, records: CommandRecords,
+ parent_snapshot: Callable[..., dict]) -> None:
+ process_state().commands, process_state().command_records, process_state().parent_snapshot = commands, records, parent_snapshot
+
+
+def command_execution() -> CommandExecution:
+ if process_state().commands is None:
+ raise RuntimeError("Command execution service is not registered")
+ return process_state().commands
+
+
+def command_records() -> CommandRecords:
+ if process_state().command_records is None:
+ raise RuntimeError("Command record service is not registered")
+ return process_state().command_records
+
+
+def snapshot_parent_runtime(request: Any, **kwargs: Any) -> dict:
+ if process_state().parent_snapshot is None:
+ raise RuntimeError("Parent runtime snapshot service is not registered")
+ return process_state().parent_snapshot(request, **kwargs)
+
+
+@dataclass
+class PortsState:
+ tool_effect_factory: Callable[..., ToolEffectRuntime] | None = None
+ commands: CommandExecution | None = None
+ command_records: CommandRecords | None = None
+ parent_snapshot: Callable[..., dict] | None = None
+
+
+_STATE = PortsState()
+
+
+def process_state() -> PortsState:
+ return _STATE
diff --git a/src/backend/core/llm/project_environment_middleware.py b/src/backend/core/llm/project_environment_middleware.py
deleted file mode 100644
index 5aa2f685..00000000
--- a/src/backend/core/llm/project_environment_middleware.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""Recheck project authorization immediately before tool I/O."""
-
-import asyncio
-
-from agentscope.message import TextBlock, ToolResultState
-from agentscope.middleware import MiddlewareBase
-from agentscope.tool._response import ToolChunk
-from core.llm.project_environment import validate_project_environment
-from fastapi import HTTPException
-
-
-class ProjectEnvironmentMiddleware(MiddlewareBase):
- def __init__(self, request):
- self.request = request
-
- async def on_acting(self, agent, input_kwargs, next_handler):
- try:
- await asyncio.to_thread(validate_project_environment, self.request)
- except HTTPException as error:
- yield ToolChunk(
- content=[TextBlock(type="text", text=str(error.detail))],
- state=ToolResultState.DENIED,
- metadata={"project_environment_denied": True},
- )
- return
- async for item in next_handler(**input_kwargs):
- yield item
diff --git a/src/backend/core/llm/prompt_files.py b/src/backend/core/llm/prompt_files.py
new file mode 100644
index 00000000..98a79ab5
--- /dev/null
+++ b/src/backend/core/llm/prompt_files.py
@@ -0,0 +1,138 @@
+"""Read prompt parts from markdown files on disk."""
+from __future__ import annotations
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from core.llm import prompt_kinds
+from core.llm import prompt_storage
+from core.llm import prompt_versions_state
+
+def backend_root() -> Path:
+ # src/backend
+ return Path(__file__).resolve().parents[2]
+
+
+def fs_dir(kind: str) -> Path:
+ spec = prompt_kinds.KIND_SPECS.get(kind)
+ if spec is None:
+ raise ValueError(f"unknown kind: {kind}")
+ if spec.root is not None:
+ return spec.root
+ return backend_root().joinpath("prompts", "prompt_text", *spec.subdir)
+
+
+def fs_signature(dirp: Path) -> tuple:
+ """目录里每个 md 的 (文件名, mtime, 大小)。用它判断要不要重读磁盘。"""
+ try:
+ names = sorted(
+ [p.name for p in dirp.glob("*.system.md")]
+ + [p.relative_to(dirp).as_posix() for p in dirp.glob("*/system.md")]
+ )
+ except OSError:
+ return ()
+ sig: List[tuple] = []
+ for name in names:
+ try:
+ st = os.stat(dirp / name)
+ except OSError:
+ continue
+ sig.append((name, st.st_mtime_ns, st.st_size))
+ return tuple(sig)
+
+
+def read_fs_parts(kind: str) -> List[Dict[str, Any]]:
+ """Read on-disk markdown into a parts[] list.
+
+ 布局由 :data:`KIND_SPECS` 的 ``layout`` 决定(concat / independent / single)。
+ concat 版的 sort_order 取文件名的数字前缀(``05_x`` → 5),新增文件落到它本该
+ 在的位置,不挤动相邻段。
+
+ 结果按目录内容签名缓存:这些 md 是只读的部署产物,而本函数落在每轮对话的
+ 提示词兜底路径和管理台的每次 GET 上,不做缓存就是反复读同一批文件。
+ """
+ dirp = fs_dir(kind)
+ sig = fs_signature(dirp)
+ with prompt_versions_state.process_state().fs_parts_cache_lock:
+ cached = prompt_versions_state.process_state().fs_parts_cache.get(kind)
+ if cached is not None and cached[0] == sig:
+ return prompt_storage.clone(cached[1])
+
+ parts = build_fs_parts(kind, dirp, [name for name, _, _ in sig])
+ with prompt_versions_state.process_state().fs_parts_cache_lock:
+ prompt_versions_state.process_state().fs_parts_cache[kind] = (sig, parts)
+ return prompt_storage.clone(parts)
+
+
+def prompt_part_id(filename: str) -> str:
+ """Map a flat prompt file or an agent package prompt to its stable part ID."""
+ if filename.endswith("/system.md"):
+ return Path(filename).parent.name
+ return filename.removesuffix(".system.md")
+
+
+def build_fs_parts(kind: str, dirp: Path, files: List[str]) -> List[Dict[str, Any]]:
+ layout = prompt_kinds.KIND_SPECS[kind].layout
+
+ if layout == "single":
+ fname = f"{kind}.system.md"
+ if fname not in files:
+ return []
+ return [
+ {
+ "part_id": kind,
+ "display_name": kind,
+ "content": (dirp / fname).read_text(encoding="utf-8"),
+ "sort_order": 0,
+ "is_enabled": True,
+ }
+ ]
+
+ if layout == "independent":
+ # 每个文件是一份独立提示词(按 part_id 单取,见 render_active_prompt_part),
+ # 不像 concat 那样拼成一份。skill_distiller 固定排头,兼容旧数据。
+ if kind == "distillation":
+ files = sorted(files, key=lambda f: (f != "skill_distiller.system.md", f))
+ return [
+ {
+ "part_id": prompt_part_id(fname),
+ "display_name": prompt_part_id(fname),
+ "content": (dirp / fname).read_text(encoding="utf-8"),
+ "sort_order": idx * 10,
+ "is_enabled": True,
+ }
+ for idx, fname in enumerate(files)
+ ]
+
+ parts: List[Dict[str, Any]] = []
+ for idx, fname in enumerate(files):
+ name = prompt_part_id(fname)
+ prefix = name.split("_", 1)[0]
+ parts.append(
+ {
+ "part_id": f"system/{name}",
+ "display_name": name,
+ "content": (dirp / fname).read_text(encoding="utf-8"),
+ "sort_order": int(prefix) if prefix.isdigit() else idx * 10,
+ "is_enabled": True,
+ }
+ )
+ return parts
+
+
+def default_version_for_kind(kind: str, version_id: Optional[str] = None) -> Dict[str, Any]:
+ """Build a new version dict from filesystem for a given kind."""
+ spec = prompt_kinds.KIND_SPECS.get(kind)
+ if spec is None:
+ raise ValueError(f"unknown kind: {kind}")
+
+ now = datetime.now(timezone.utc).isoformat()
+ return {
+ "id": version_id or prompt_versions_state.DEFAULT_VERSION_ID,
+ "kind": kind,
+ "name": spec.name,
+ "description": spec.desc,
+ "parts": read_fs_parts(kind),
+ "created_at": now,
+ "updated_at": now,
+ }
diff --git a/src/backend/core/llm/prompt_kinds.py b/src/backend/core/llm/prompt_kinds.py
new file mode 100644
index 00000000..96782bc6
--- /dev/null
+++ b/src/backend/core/llm/prompt_kinds.py
@@ -0,0 +1,193 @@
+"""Prompt kinds: built-in specs plus admin-defined kinds."""
+from __future__ import annotations
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+from extensions.paths import DEFAULT_AGENTS_DIR
+from sqlalchemy.orm import Session
+from core.llm import prompt_storage
+
+@dataclass(frozen=True)
+class KindSpec:
+ """一个内置 kind 的全部差异都集中在这里,别处不再按 kind 分支。
+
+ ``layout`` 决定目录里的 md 文件怎么变成 parts:
+
+ - ``concat``:目录下每个 ``*.system.md`` 是一段,拼起来构成一份提示词
+ - ``independent``:每个文件是一份互不相干的独立提示词,按 part_id 单取
+ - ``single``:整个 kind 只有 ``.system.md`` 一个文件
+ """
+
+ subdir: tuple[str, ...]
+ layout: str
+ label: str
+ name: str
+ desc: str
+ visible: bool = True
+ preview: str | None = None
+ #: 兜底 md 的绝对目录。可选模块的提示词随模块代码放(不进 CE),不在 prompt_text/ 下。
+ root: Path | None = None
+
+
+def register_kind(key: str, spec: KindSpec) -> None:
+ """可选模块注册自己的内置 kind(模块关闭时不注册,Config 台就不出现这个 tab)。
+
+ 须在启动播种(``seed_from_filesystem``)之前调用,即放在模块的 ``setup()`` 里。
+ """
+ KIND_SPECS[key] = spec
+
+
+def list_custom_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]:
+ """管理员自建的 kind(``[{"key","label"}]``)。
+
+ 存在版本池同一个 ContentBlock 的 ``custom_kinds`` 下——它们和内置 kind 共用
+ versions/active 两张表,差别只在于没有文件系统兜底、可以删。
+ 「对话模式」就是靠这个扩展的:管理员在提示词管理新开一个 tab,模式那边就能绑它。
+ """
+ payload = prompt_storage.load_payload(db)
+ out: List[Dict[str, str]] = []
+ for item in payload.get("custom_kinds") or []:
+ if not isinstance(item, dict):
+ continue
+ key = str(item.get("key") or "").strip()
+ if key and key not in KIND_SPECS:
+ out.append({"key": key, "label": str(item.get("label") or key)})
+ return out
+
+
+def all_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]:
+ """内置 + 自定义的完整 kind 清单,供管理端渲染 tab 与模式绑定下拉。"""
+ items = [{"key": k, "label": s.label, "builtin": True, "visible": s.visible, "preview": s.preview, "description": s.desc} for k, s in KIND_SPECS.items()]
+ items += [{**c, "builtin": False} for c in list_custom_kinds(db)]
+ return items
+
+
+def is_valid_kind(kind: str, db: Optional[Session] = None) -> bool:
+ if kind in KIND_SPECS:
+ return True
+ return any(c["key"] == kind for c in list_custom_kinds(db))
+
+
+def create_custom_kind(
+ key: str, label: str, db: Optional[Session] = None, updated_by: str = "admin"
+) -> Dict[str, str]:
+ """新开一个 kind(tab),并给它建一个空的 ``default`` 版本并激活。
+
+ 没有初始版本的 kind 在管理端会是个点进去什么都没有的空 tab,所以这里顺手建上,
+ 管理员进去直接就能写正文。
+ """
+ cleaned = "".join(
+ ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "_" for ch in (key or "").strip().lower()
+ ).strip("-_")
+ if not cleaned:
+ raise ValueError("kind key required")
+ if cleaned in KIND_SPECS:
+ raise ValueError(f"「{cleaned}」是内置 kind,不能重复创建")
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ customs = payload.setdefault("custom_kinds", [])
+ if any(str((c or {}).get("key")) == cleaned for c in customs):
+ raise ValueError(f"kind「{cleaned}」已存在")
+ customs.append({"key": cleaned, "label": (label or cleaned).strip()[:60]})
+
+ # 空的初始版本:一个可编辑的 part,内容留空由管理员填。
+ versions = payload.setdefault("versions", [])
+ now = datetime.now(timezone.utc).isoformat()
+ versions.append(
+ {
+ "kind": cleaned,
+ "id": "default",
+ "name": (label or cleaned).strip()[:60],
+ "description": "",
+ "parts": [{"part_id": cleaned, "display_name": label or cleaned, "content": ""}],
+ "created_at": now,
+ "updated_at": now,
+ "updated_by": updated_by,
+ }
+ )
+ payload.setdefault("active", {})[cleaned] = "default"
+ prompt_storage.save_payload(payload, db=db, updated_by=updated_by)
+ return {"key": cleaned, "label": (label or cleaned).strip()[:60]}
+
+
+def delete_custom_kind(key: str, db: Optional[Session] = None) -> bool:
+ """删掉一个自定义 kind 及其全部版本。内置 kind 拒绝删除。
+
+ 注意:绑了这个 kind 的对话模式会退回默认提示词装配(``render_kind_segment(fs_fallback=False)``
+ 取不到就返回空串,装配侧按"没配提示词"处理),不会让对话起不来。
+ """
+ if key in KIND_SPECS:
+ raise ValueError("内置 kind 不可删除")
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ customs = payload.get("custom_kinds") or []
+ remaining = [c for c in customs if str((c or {}).get("key")) != key]
+ if len(remaining) == len(customs):
+ return False
+ payload["custom_kinds"] = remaining
+ payload["versions"] = [v for v in (payload.get("versions") or []) if v.get("kind") != key]
+ (payload.get("active") or {}).pop(key, None)
+ prompt_storage.save_payload(payload, db=db)
+ return True
+
+
+KIND_SPECS: Dict[str, KindSpec] = {
+ "desktop": KindSpec(("desktop",), "independent", "桌面端", "default - 桌面端提示词", "本机环境、文件、项目与工具规则;仅桌面端装配", preview="desktop-preview"),
+ "system": KindSpec(
+ ("default", "system"),
+ "concat",
+ "系统提示词",
+ "default - 标准系统提示词",
+ "当前默认系统提示词,包含 role / constraints / tools / workflow / format",
+ ),
+ "turbo": KindSpec(
+ ("turbo",),
+ "single",
+ "极速模式",
+ "default - 极速模式",
+ "极速模式(快速查询)系统提示词:仅联网搜索/网页抓取/知识库检索三类工具,"
+ "1-2 轮并行调用后直接作答",
+ ),
+ "plan_mode": KindSpec(
+ ("plan_mode",),
+ "single",
+ "计划模式",
+ "default - 计划模式",
+ "Plan 模式下用于拆解用户任务为可执行步骤的 sub-agent 系统提示词",
+ ),
+ "plan_tool": KindSpec(
+ ("plan_tool",),
+ "single",
+ "任务计划清单",
+ "default - 任务计划清单",
+ "顶层对话注册 update_plan 工具时追加的计划清单说明",
+ ),
+ "code_exec": KindSpec(
+ ("code_exec", "system"),
+ "concat",
+ "代码执行",
+ "default - 代码执行 (沙盒)",
+ "Lab 代码执行模式的系统提示词(沙盒环境、工具能力、执行规范等)",
+ ),
+ "distillation": KindSpec(
+ ("distillation",),
+ "independent",
+ "蒸馏",
+ "default - 技能蒸馏",
+ "从对话轨迹蒸馏出可复用技能的系统提示词",
+ ),
+ "subagents": KindSpec(
+ ("subagents",),
+ "independent",
+ "平台默认子智能体",
+ "default - 平台默认子智能体",
+ "探索员、执行员和审查员三个平台内置角色的独立系统提示词",
+ visible=False,
+ root=DEFAULT_AGENTS_DIR,
+ ),
+}
+
+
+BUILTIN_KINDS = tuple(KIND_SPECS)
+
+
+VALID_KINDS = BUILTIN_KINDS
diff --git a/src/backend/core/llm/prompt_rendering.py b/src/backend/core/llm/prompt_rendering.py
new file mode 100644
index 00000000..61c00d0d
--- /dev/null
+++ b/src/backend/core/llm/prompt_rendering.py
@@ -0,0 +1,137 @@
+"""Render the active prompt version, or the file defaults, into text."""
+from __future__ import annotations
+from core.llm import prompt_versions
+from typing import Any, Dict, List, Optional
+from sqlalchemy.orm import Session
+import logging
+from core.llm import prompt_files
+from core.llm import prompt_kinds
+from core.llm import prompt_versions_state
+
+logger = logging.getLogger(__name__)
+
+def render_active_prompt(kind: str, db: Optional[Session] = None) -> Optional[str]:
+ """Concatenate enabled parts of the active version into a single string.
+
+ Returns None if no active version exists in DB (caller should fall back).
+ """
+ v = prompt_versions.get_active_version(kind, db=db)
+ if not v:
+ return None
+ parts = v.get("parts") or []
+ chunks: List[str] = []
+ for p in parts:
+ if not p.get("is_enabled", True):
+ continue
+ content = (p.get("content") or "").strip()
+ if content:
+ chunks.append(content)
+ return "\n\n".join(chunks) if chunks else None
+
+
+def render_active_prompt_part(
+ kind: str, part_id: str, db: Optional[Session] = None
+) -> Optional[str]:
+ """Return a single named part of the active version (independent prompts).
+
+ Used by distillation and platform-subagent prompts where each part is a standalone
+ system prompt (skill_distiller / session_digest / colleague_distiller /
+ personal_distiller) rather than a concatenation segment.
+
+ Back-compat: an active version that predates the multi-part layout holds
+ exactly one part (the legacy skill_distiller prompt) — only a request for
+ ``skill_distiller`` may fall back to it; other part_ids must not silently
+ receive the wrong prompt.
+ Returns None if no active version / no match (caller falls back to fs).
+ """
+ v = prompt_versions.get_active_version(kind, db=db)
+ if not v:
+ return None
+ parts = v.get("parts") or []
+ for p in parts:
+ if p.get("part_id") == part_id:
+ content = (p.get("content") or "").strip()
+ return content or None
+ if len(parts) == 1 and part_id == "skill_distiller":
+ content = (parts[0].get("content") or "").strip()
+ return content or None
+ return None
+
+
+def skipped_parts(kind: str) -> set:
+ try:
+ from core.config.local_mode import local_mode_enabled
+
+ if local_mode_enabled():
+ from prompts.desktop_workspace import SKIP_PARTS
+ return SKIP_PARTS.get(kind) or set()
+ except Exception:
+ logger.debug("local-mode check failed while assembling %s", kind, exc_info=True)
+ return set()
+
+
+def render_kind_segment(
+ kind: str, db: Optional[Session] = None, *, fs_fallback: bool = True
+) -> str:
+ """某个 kind 装配好的正文:DB 激活版本 → 文件系统默认,都没有就返回空串。
+
+ 这是「一个 kind 的正文怎么取」的唯一实现——代码执行段、极速模式、任务计划清单
+ 以及 Config 管理台 ``/v1/admin/prompts/preview`` 的预览都走它,预览与智能体
+ 实际看到的因此不会漂移。
+
+ 文件系统这一步读的就是播种用的那批 md,所以尚未播种的部署渲染出来的,与它之后
+ 存进库里的是同一份。``fs_fallback=False`` 用于「对话模式」绑定的 kind:绑了个
+ 空 kind 就该退回默认装配,而不是套用别的 kind 的话术。
+ """
+ skip = skipped_parts(kind)
+ try:
+ rendered = render_active_parts(kind, db=db, skip=skip)
+ if rendered:
+ return rendered
+ except Exception:
+ logger.debug("render %s active prompt failed", kind, exc_info=True)
+ if not fs_fallback or kind not in prompt_kinds.KIND_SPECS:
+ return ""
+ chunks = [
+ (p.get("content") or "").strip()
+ for p in prompt_files.read_fs_parts(kind)
+ if p.get("is_enabled", True)
+ and (p.get("content") or "").strip()
+ and (p.get("part_id") or "") not in skip
+ ]
+ return "\n\n".join(chunks)
+
+
+def render_active_parts(kind: str, *, db: Optional[Session], skip: set) -> Optional[str]:
+ """激活版本的正文,按 ``skip`` 剔除若干段。``skip`` 为空时等同 render_active_prompt。"""
+ if not skip:
+ return render_active_prompt(kind, db=db)
+ version = prompt_versions.get_active_version(kind, db=db)
+ if not version:
+ return None
+ chunks = [
+ (p.get("content") or "").strip()
+ for p in (version.get("parts") or [])
+ if p.get("is_enabled", True)
+ and (p.get("content") or "").strip()
+ and (p.get("part_id") or "").strip() not in skip
+ ]
+ return "\n\n".join(chunks) or None
+
+
+def render_turbo_system_prompt(db: Optional[Session] = None) -> str:
+ """极速模式(快速查询)的整份系统提示词。
+
+ 极速模式不做常规装配:智能体只带检索类工具,默认提示词的工具/流程段都不适用。
+ """
+ return render_kind_segment("turbo", db=db) or prompt_versions_state.TURBO_LAST_RESORT
+
+
+def effective_parts(kind: str, db: Optional[Session] = None) -> List[Dict[str, Any]]:
+ """Read the active version, with whole-version filesystem fallback only."""
+ try:
+ version = prompt_versions.get_active_version(kind, db=db)
+ except Exception:
+ logger.warning("Prompt version unavailable for %s; using filesystem", kind, exc_info=True)
+ version = None
+ return version.get("parts", []) if version is not None else prompt_files.read_fs_parts(kind)
diff --git a/src/backend/core/llm/prompt_storage.py b/src/backend/core/llm/prompt_storage.py
new file mode 100644
index 00000000..c646de50
--- /dev/null
+++ b/src/backend/core/llm/prompt_storage.py
@@ -0,0 +1,99 @@
+"""Load and cache the stored prompt pool for this worker."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+from core.files.content.content_blocks_constants import DEFAULT_PROMPT_VERSIONS, PROMPT_VERSIONS_BLOCK_ID
+from core.db.engine import SessionLocal
+from core.db.models import ContentBlock
+from sqlalchemy.orm import Session
+from core.llm import prompt_versions_state
+
+def load_payload(db: Optional[Session] = None) -> Dict[str, Any]:
+ """Read the committed pool revision before reusing a worker-local payload.
+
+ Workers do not share Python memory. Local invalidation alone leaves other
+ workers serving old versions indefinitely, including on read-modify-write.
+ Query columns directly so an existing ORM identity-map entry cannot hide a
+ newer payload. The payload and its revision are loaded in the same query.
+ """
+ pass # State is owned by its responsibility module.
+ own_session = db is None
+ if own_session:
+ db = SessionLocal()
+ try:
+ revision = (
+ db.query(ContentBlock.updated_at)
+ .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID)
+ .first()
+ )
+ cache_key = (db.get_bind(), tuple(revision) if revision is not None else None)
+ with prompt_versions_state.process_state().payload_cache_lock:
+ if prompt_versions_state.process_state().payload_cache is not None and prompt_versions_state.process_state().payload_cache_key == cache_key:
+ return clone(prompt_versions_state.process_state().payload_cache)
+
+ row = (
+ db.query(ContentBlock.updated_at, ContentBlock.payload)
+ .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID)
+ .first()
+ )
+ payload = (
+ row.payload
+ if row is not None and isinstance(row.payload, dict)
+ else clone(DEFAULT_PROMPT_VERSIONS)
+ )
+ # A write may have committed between the two SELECTs. Cache the revision
+ # returned with the payload, never the earlier revision probe.
+ cache_key = (db.get_bind(), (row.updated_at,) if row is not None else None)
+ with prompt_versions_state.process_state().payload_cache_lock:
+ prompt_versions_state.process_state().payload_cache = payload
+ prompt_versions_state.process_state().payload_cache_key = cache_key
+ return clone(payload)
+ finally:
+ if own_session:
+ db.close()
+
+
+def save_payload(
+ payload: Dict[str, Any],
+ *,
+ db: Optional[Session] = None,
+ updated_by: str = "system",
+) -> None:
+ own_session = db is None
+ if own_session:
+ db = SessionLocal()
+ try:
+ row = db.query(ContentBlock).filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID).first()
+ now = datetime.now(timezone.utc)
+ if row:
+ row.payload = payload
+ row.updated_at = now
+ row.updated_by = updated_by
+ else:
+ row = ContentBlock(
+ id=PROMPT_VERSIONS_BLOCK_ID,
+ payload=payload,
+ updated_at=now,
+ updated_by=updated_by,
+ )
+ db.add(row)
+ db.commit()
+ finally:
+ if own_session:
+ db.close()
+
+ invalidate_cache()
+
+
+def invalidate_cache() -> None:
+ """Drop this worker's cache; other workers detect the DB revision on read."""
+ pass # State is owned by its responsibility module.
+ with prompt_versions_state.process_state().payload_cache_lock:
+ prompt_versions_state.process_state().payload_cache = None
+ prompt_versions_state.process_state().payload_cache_key = None
+
+
+def clone(obj: Any) -> Any:
+ import copy
+
+ return copy.deepcopy(obj)
diff --git a/src/backend/core/llm/prompt_versions.py b/src/backend/core/llm/prompt_versions.py
new file mode 100644
index 00000000..86242df3
--- /dev/null
+++ b/src/backend/core/llm/prompt_versions.py
@@ -0,0 +1,296 @@
+"""Create, activate and seed prompt versions."""
+from __future__ import annotations
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+from sqlalchemy.orm import Session
+import logging
+from core.llm import prompt_files
+from core.llm import prompt_kinds
+from core.llm import prompt_storage
+from core.llm import prompt_versions_state
+
+logger = logging.getLogger(__name__)
+
+def list_versions(kind: Optional[str] = None, db: Optional[Session] = None) -> List[Dict[str, Any]]:
+ payload = prompt_storage.load_payload(db)
+ versions = payload.get("versions") or []
+ active = payload.get("active") or {}
+ items = []
+ for v in versions:
+ if kind and v.get("kind") != kind:
+ continue
+ items.append(
+ {
+ "id": v.get("id"),
+ "kind": v.get("kind"),
+ "name": v.get("name") or v.get("id"),
+ "description": v.get("description") or "",
+ "parts_count": len(v.get("parts") or []),
+ "is_active": active.get(v.get("kind")) == v.get("id"),
+ "created_at": v.get("created_at"),
+ "updated_at": v.get("updated_at"),
+ }
+ )
+ return items
+
+
+def get_version(
+ kind: str, version_id: str, db: Optional[Session] = None
+) -> Optional[Dict[str, Any]]:
+ if not prompt_kinds.is_valid_kind(kind, db):
+ return None
+ payload = prompt_storage.load_payload(db)
+ active = payload.get("active") or {}
+ for v in payload.get("versions") or []:
+ if v.get("kind") == kind and v.get("id") == version_id:
+ data = prompt_storage.clone(v)
+ data["is_active"] = active.get(kind) == version_id
+ return data
+ return None
+
+
+def get_active_version(kind: str, db: Optional[Session] = None) -> Optional[Dict[str, Any]]:
+ """Return the active version for a kind, or None if not found / DB empty."""
+ if not prompt_kinds.is_valid_kind(kind, db):
+ return None
+ payload = prompt_storage.load_payload(db)
+ active_id = (payload.get("active") or {}).get(kind)
+ if not active_id:
+ return None
+ for v in payload.get("versions") or []:
+ if v.get("kind") == kind and v.get("id") == active_id:
+ return prompt_storage.clone(v)
+ return None
+
+
+def upsert_version(
+ kind: str,
+ version_id: str,
+ *,
+ name: Optional[str] = None,
+ description: Optional[str] = None,
+ parts: Optional[List[Dict[str, Any]]] = None,
+ from_id: Optional[str] = None,
+ db: Optional[Session] = None,
+ updated_by: str = "admin",
+) -> Dict[str, Any]:
+ if not prompt_kinds.is_valid_kind(kind, db):
+ raise ValueError(f"invalid kind: {kind}")
+ if not version_id:
+ raise ValueError("version_id required")
+
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ versions: List[Dict[str, Any]] = payload.setdefault("versions", [])
+ now = datetime.now(timezone.utc).isoformat()
+
+ existing = next(
+ (v for v in versions if v.get("kind") == kind and v.get("id") == version_id), None
+ )
+
+ # If cloning, look up source (optional)
+ source_parts: List[Dict[str, Any]] = []
+ if from_id:
+ src = next((v for v in versions if v.get("kind") == kind and v.get("id") == from_id), None)
+ if src:
+ source_parts = prompt_storage.clone(src.get("parts") or [])
+ if name is None:
+ name = f"{src.get('name') or from_id} 副本"
+ if description is None:
+ description = src.get("description") or ""
+
+ if existing:
+ if name is not None:
+ existing["name"] = name
+ if description is not None:
+ existing["description"] = description
+ if parts is not None:
+ existing["parts"] = normalize_parts(parts)
+ existing["updated_at"] = now
+ saved = existing
+ else:
+ # On create: if from_id is passed and parts is empty (empty list / not passed),
+ # always treat it as a clone.
+ # Only explicitly passing non-empty parts overrides the clone's source content.
+ if parts and len(parts) > 0:
+ final_parts = normalize_parts(parts)
+ elif from_id and source_parts:
+ final_parts = source_parts
+ else:
+ final_parts = normalize_parts(parts) if parts is not None else []
+ saved = {
+ "id": version_id,
+ "kind": kind,
+ "name": name or version_id,
+ "description": description or "",
+ "parts": final_parts,
+ "created_at": now,
+ "updated_at": now,
+ }
+ versions.append(saved)
+
+ prompt_storage.save_payload(payload, db=db, updated_by=updated_by)
+ return saved
+
+
+def delete_version(kind: str, version_id: str, db: Optional[Session] = None) -> None:
+ if not prompt_kinds.is_valid_kind(kind, db):
+ raise ValueError(f"invalid kind: {kind}")
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ active = payload.get("active") or {}
+ if active.get(kind) == version_id:
+ raise ValueError("cannot delete the currently active version; activate another first")
+
+ versions: List[Dict[str, Any]] = payload.get("versions") or []
+ new_list = [v for v in versions if not (v.get("kind") == kind and v.get("id") == version_id)]
+ if len(new_list) == len(versions):
+ raise KeyError(f"version not found: {kind}/{version_id}")
+ payload["versions"] = new_list
+ prompt_storage.save_payload(payload, db=db)
+
+
+def activate_version(kind: str, version_id: str, db: Optional[Session] = None) -> None:
+ if not prompt_kinds.is_valid_kind(kind, db):
+ raise ValueError(f"invalid kind: {kind}")
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ versions: List[Dict[str, Any]] = payload.get("versions") or []
+ if not any(v.get("kind") == kind and v.get("id") == version_id for v in versions):
+ raise KeyError(f"version not found: {kind}/{version_id}")
+ payload.setdefault("active", {})[kind] = version_id
+ prompt_storage.save_payload(payload, db=db)
+ # Downstream prompt builders cache by (active_id, updated_at); poke their cache too.
+ try:
+ from prompts.prompt_runtime import invalidate_prompt_cache
+
+ invalidate_prompt_cache()
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.prompt_versions:activate_version', fallback_error)
+
+
+def normalize_parts(parts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ result: List[Dict[str, Any]] = []
+ for idx, p in enumerate(parts or []):
+ pid = (p.get("part_id") or "").strip()
+ if not pid:
+ continue
+ result.append(
+ {
+ "part_id": pid,
+ "display_name": p.get("display_name") or pid.split("/")[-1],
+ "content": p.get("content") or "",
+ "sort_order": int(
+ p.get("sort_order") if p.get("sort_order") is not None else idx * 10
+ ),
+ "is_enabled": bool(p.get("is_enabled", True)),
+ }
+ )
+ # Stable sort by sort_order
+ result.sort(key=lambda x: x["sort_order"])
+ return result
+
+
+def seed_from_filesystem(
+ *,
+ db: Optional[Session] = None,
+) -> Dict[str, Any]:
+ """Create default versions from filesystem if not already present."""
+ payload = prompt_storage.clone(prompt_storage.load_payload(db))
+ versions: List[Dict[str, Any]] = payload.setdefault("versions", [])
+ active: Dict[str, str] = payload.setdefault("active", {})
+ changed = False
+
+ def exists(kind: str, vid: str) -> bool:
+ return any(v.get("kind") == kind and v.get("id") == vid for v in versions)
+
+ added: List[str] = []
+
+ # ── One-time migration: rename legacy system/v4 → system/default ──
+ # Keeps the user's existing edits; the name change aligns the in-code
+ # default version id with other kinds (code_exec/default, distillation/default, …).
+ v4_row = next((v for v in versions if v.get("kind") == "system" and v.get("id") == "v4"), None)
+ default_row = next(
+ (v for v in versions if v.get("kind") == "system" and v.get("id") == "default"), None
+ )
+ if v4_row and not default_row:
+ v4_row["id"] = "default"
+ # Refresh display name only if it still looks like the factory label
+ if v4_row.get("name", "").startswith("v4 - 标准版"):
+ v4_row["name"] = "default - 标准系统提示词"
+ if active.get("system") == "v4":
+ active["system"] = "default"
+ added.append("system/v4 → system/default (renamed)")
+ changed = True
+
+ # ── One-time migration: extract system/90_plan_mode from every system
+ # version into a new plan_mode/default version (if not already seeded).
+ # After extraction, the system/90_plan_mode part is REMOVED from system
+ # versions so the main agent prompt no longer duplicates it.
+ if not exists("plan_mode", "default"):
+ extracted_content: Optional[str] = None
+ for v in versions:
+ if v.get("kind") != "system":
+ continue
+ new_parts: List[Dict[str, Any]] = []
+ for p in v.get("parts") or []:
+ if (p.get("part_id") or "").strip() == "system/90_plan_mode":
+ if extracted_content is None and (p.get("content") or "").strip():
+ extracted_content = p["content"]
+ changed = True
+ continue
+ new_parts.append(p)
+ v["parts"] = new_parts
+ if extracted_content:
+ now = datetime.now(timezone.utc).isoformat()
+ versions.append(
+ {
+ "id": "default",
+ "kind": "plan_mode",
+ "name": "default - 计划模式(从 v4 迁移)",
+ "description": "由历史 v4 system/90_plan_mode 片段迁移而来的 plan_mode 默认版本",
+ "parts": [
+ {
+ "part_id": "plan_mode",
+ "display_name": "plan_mode",
+ "content": extracted_content,
+ "sort_order": 0,
+ "is_enabled": True,
+ }
+ ],
+ "created_at": now,
+ "updated_at": now,
+ }
+ )
+ active["plan_mode"] = "default"
+ added.append("plan_mode/default (migrated)")
+ changed = True
+
+ # 每个内置 kind 的默认版本:没有就按文件系统建;已有就把**新增**的 md 文件补进去。
+ # 补这一步是因为默认版本只在首次冷启动时从文件系统建一次,之后新加的 *.system.md
+ # 再也进不了已建库的部署。已有 part 的正文原样保留,管理员改过的不会被覆盖。
+ for kind in list(prompt_kinds.KIND_SPECS):
+ if not exists(kind, prompt_versions_state.DEFAULT_VERSION_ID):
+ default_row = prompt_files.default_version_for_kind(kind, prompt_versions_state.DEFAULT_VERSION_ID)
+ versions.append(default_row)
+ added.append(f"{kind}/{prompt_versions_state.DEFAULT_VERSION_ID}")
+ changed = True
+ else:
+ default_row = next(
+ v for v in versions if v.get("kind") == kind and v.get("id") == prompt_versions_state.DEFAULT_VERSION_ID
+ )
+ existing_pids = {
+ (p.get("part_id") or "").strip() for p in default_row.get("parts") or []
+ }
+ for fs_part in prompt_files.read_fs_parts(kind):
+ if fs_part["part_id"] not in existing_pids:
+ default_row.setdefault("parts", []).append(fs_part)
+ added.append(f"{kind}/{prompt_versions_state.DEFAULT_VERSION_ID}:{fs_part['part_id']}")
+ changed = True
+ if kind not in active:
+ active[kind] = prompt_versions_state.DEFAULT_VERSION_ID
+ changed = True
+
+ if added or changed:
+ prompt_storage.save_payload(payload, db=db, updated_by="system_seed")
+ if added:
+ logger.info("[prompt_version_service] seeded: %s", ", ".join(added))
+ return {"added": added, "active": active}
diff --git a/src/backend/core/llm/prompt_versions_state.py b/src/backend/core/llm/prompt_versions_state.py
new file mode 100644
index 00000000..3a529ffa
--- /dev/null
+++ b/src/backend/core/llm/prompt_versions_state.py
@@ -0,0 +1,30 @@
+"""Default version id and caches for prompt versions."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from threading import Lock
+from typing import Any, Dict, Optional
+
+DEFAULT_VERSION_ID = "default"
+
+
+TURBO_LAST_RESORT = (
+ "你是极速查询助手。收到问题后,最多发起 1-2 轮工具调用(同一轮内可并行"
+ "调用多个检索工具),拿到结果立即用中文简洁作答,并标注信息来源。"
+ "不要使用超出检索范围的工具,不要反复迭代。"
+)
+
+
+@dataclass
+class PromptVersionsState:
+ payload_cache: Optional[Dict[str, Any]] = None
+ payload_cache_key: Optional[tuple] = None
+ payload_cache_lock: object = field(default_factory=lambda: Lock())
+ fs_parts_cache: Dict[str, tuple] = field(default_factory=lambda: {})
+ fs_parts_cache_lock: object = field(default_factory=lambda: Lock())
+
+
+_STATE = PromptVersionsState()
+
+
+def process_state() -> PromptVersionsState:
+ return _STATE
diff --git a/src/backend/core/llm/providers/__init__.py b/src/backend/core/llm/providers/__init__.py
deleted file mode 100644
index be52242c..00000000
--- a/src/backend/core/llm/providers/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-"""Multi-vendor model provider support: registry + per-engine model implementations."""
-
-from .registry import ( # noqa: F401
- PROVIDER_SPECS,
- ProviderField,
- ProviderSpec,
- get_spec,
- is_known,
- list_specs,
- split_provider_extra,
- to_frontend_schema,
- validate_payload,
-)
diff --git a/src/backend/core/llm/runtime/__init__.py b/src/backend/core/llm/runtime/__init__.py
new file mode 100644
index 00000000..f2f8736a
--- /dev/null
+++ b/src/backend/core/llm/runtime/__init__.py
@@ -0,0 +1 @@
+"""The AgentScope execution boundary: streaming, hooks, state and tool adapters."""
diff --git a/src/backend/core/llm/agentscope_hook_adapter.py b/src/backend/core/llm/runtime/agentscope_hook_adapter.py
similarity index 63%
rename from src/backend/core/llm/agentscope_hook_adapter.py
rename to src/backend/core/llm/runtime/agentscope_hook_adapter.py
index 014413cf..25576bac 100644
--- a/src/backend/core/llm/agentscope_hook_adapter.py
+++ b/src/backend/core/llm/runtime/agentscope_hook_adapter.py
@@ -2,21 +2,18 @@
from __future__ import annotations
-import asyncio
import inspect
-import json
-import time
from typing import Any
from agentscope.message import Msg
from agentscope.middleware import MiddlewareBase
-from agentscope.tool._response import ToolResponse
from core.harness.events import Event, EventSink, thaw_value
from core.harness.hooks import HookBus, HookStage, Invocation
-from core.harness.usage import UsageAttempt, attempt_status_for_exception, record_usage_safely
-from core.llm.model_steps import record_assistant_step
-from core.llm.reasoning_replay import stamp_reasoning_origin
-from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger
+from core.llm.runtime.model_steps import record_assistant_step
+from core.models.reasoning_replay import stamp_reasoning_origin
+from core.llm.runtime.context_hook_adapter import attachment_input, begin_context_turn, append_hook_context, apply_reasoning_hooks
+from core.llm.runtime.model_selection_adapter import apply_model_selection, model_selection_input
+from core.harness.ledger import DurableEventStore, HarnessUsageLedger
def _plain(value: Any) -> Any:
@@ -159,123 +156,13 @@ def __init__(
self,
bus: HookBus | None = None,
*,
- legacy_middlewares: tuple[MiddlewareBase, ...] = (),
+ permission_policy=None,
+ durable_tools=False,
) -> None:
self.bus = bus or build_hook_bus()
- # Transitional policies execute only through this compatibility
- # boundary. They are no longer mounted beside the adapter in Agent.
- self.legacy_middlewares = tuple(legacy_middlewares)
-
- def _legacy_for(self, hook_name: str) -> tuple[MiddlewareBase, ...]:
- return tuple(
- middleware
- for middleware in self.legacy_middlewares
- if middleware.is_implemented(hook_name)
- )
-
- async def _record_legacy_hook(
- self,
- *,
- run_id: str,
- middleware: MiddlewareBase,
- hook_name: str,
- status: str,
- started: float,
- ) -> None:
- if not run_id or self.bus.usage_recorder is None:
- return
- await record_usage_safely(
- self.bus.usage_recorder,
- UsageAttempt(
- run_id=run_id,
- kind="hook",
- operation_name=f"{type(middleware).__name__}.{hook_name}",
- status=status,
- latency_ms=int((time.monotonic() - started) * 1_000),
- metadata={"adapter": "agentscope", "legacy": True},
- ),
- )
-
- async def _legacy_stream(
- self,
- hook_name: str,
- agent: Any,
- input_kwargs: dict,
- next_handler,
- ):
- middlewares = self._legacy_for(hook_name)
-
- async def execute(index: int, **kwargs):
- if index >= len(middlewares):
- async for item in next_handler(**kwargs):
- yield item
- return
- middleware = middlewares[index]
-
- async def following(**next_kwargs):
- async for item in execute(index + 1, **next_kwargs):
- yield item
-
- callback = getattr(middleware, hook_name)
- started = time.monotonic()
- status = "success"
- try:
- async for item in callback(agent, kwargs, following):
- yield item
- except asyncio.CancelledError:
- status = "cancelled"
- raise
- except Exception as exc:
- status = attempt_status_for_exception(exc)
- raise
- finally:
- await self._record_legacy_hook(
- run_id=self._run_id(agent),
- middleware=middleware,
- hook_name=hook_name,
- status=status,
- started=started,
- )
-
- async for item in execute(0, **input_kwargs):
- yield item
-
- async def _legacy_model_call(
- self,
- agent: Any,
- input_kwargs: dict,
- next_handler,
- ):
- middlewares = self._legacy_for("on_model_call")
-
- async def execute(index: int, **kwargs):
- if index >= len(middlewares):
- return await next_handler(**kwargs)
- middleware = middlewares[index]
-
- async def following(**next_kwargs):
- return await execute(index + 1, **next_kwargs)
-
- started = time.monotonic()
- status = "success"
- try:
- return await middleware.on_model_call(agent, kwargs, following)
- except asyncio.CancelledError:
- status = "cancelled"
- raise
- except Exception as exc:
- status = attempt_status_for_exception(exc)
- raise
- finally:
- await self._record_legacy_hook(
- run_id=self._run_id(agent),
- middleware=middleware,
- hook_name="on_model_call",
- status=status,
- started=started,
- )
-
- return await execute(0, **input_kwargs)
+ self.permission_policy = permission_policy
+ from core.llm.runtime.tool_execution_adapter import DurableToolDispatch
+ self.tool_dispatch = DurableToolDispatch() if durable_tools else None
@staticmethod
def _run_id(agent: Any) -> str:
@@ -302,7 +189,7 @@ async def _observe_event(self, event: Any, run_id: str) -> None:
invocation = adapt_agentscope_event(event, run_id=run_id)
if invocation is None:
return
- await self.bus.event_sink.append(
+ await self.bus.event_sink.publish(
Event.create(
run_id=run_id,
event_type="framework_invocation",
@@ -316,21 +203,25 @@ async def _observe_event(self, event: Any, run_id: str) -> None:
)
async def on_reply(self, agent: Any, input_kwargs: dict, next_handler):
- from core.llm.model_usage import model_usage_scope
+ from core.models.usage import model_usage_scope
run_id = self._run_id(agent)
with model_usage_scope(run_id, self.bus.usage_recorder):
- await self._stage(
+ selected = await self._stage(
HookStage.BEFORE_RUN,
agent,
operation_name="agent_reply",
- data={"metadata": {"framework": "agentscope"}},
+ data={"metadata": {"framework": "agentscope"}, "runtime": model_selection_input(agent)},
)
+ apply_model_selection(agent, selected)
+ begin_context_turn()
transformed = await self._stage(
HookStage.TRANSFORM_CONTEXT,
agent,
operation_name="agent_reply",
data={
+ "runtime": _plain(attachment_input(agent)),
+ "context_items": [],
"messages": _plain(input_kwargs.get("inputs")),
"context": _plain(
getattr(getattr(agent, "state", None), "context", None)
@@ -350,33 +241,31 @@ async def on_reply(self, agent: Any, input_kwargs: dict, next_handler):
state.context = _restore_context(
getattr(state, "context", None), transformed.data["context"]
)
- final_output = None
- async for event in self._legacy_stream(
- "on_reply", agent, invoke_kwargs, next_handler
- ):
- await self._observe_event(event, run_id)
- if isinstance(event, Msg):
- if final_output is not None:
- yield final_output
- final_output = event
- else:
- yield event
- await self._stage(
- HookStage.BEFORE_FINISH,
- agent,
- operation_name="agent_reply",
- data={"output": _plain(final_output), "metadata": {}},
- )
- if final_output is not None:
- yield final_output
+ append_hook_context(agent, transformed)
+ from core.llm.runtime.context_hook_adapter import finish_context_hooks
+ while True:
+ final_output = None
+ async for event in next_handler(**invoke_kwargs):
+ await self._observe_event(event, run_id)
+ if isinstance(event, Msg):
+ if final_output is not None:
+ yield final_output
+ final_output = event
+ else:
+ yield event
+ continue_run = await finish_context_hooks(self, agent, final_output, _plain)
+ if final_output is not None:
+ yield final_output
+ if not continue_run:
+ break
+ invoke_kwargs = {**invoke_kwargs, "inputs": None}
async def on_reasoning(self, agent: Any, input_kwargs: dict, next_handler):
- from core.llm.model_usage import model_usage_scope
+ from core.models.usage import model_usage_scope
with model_usage_scope(self._run_id(agent), self.bus.usage_recorder):
- async for event in self._legacy_stream(
- "on_reasoning", agent, input_kwargs, next_handler
- ):
+ input_kwargs = await apply_reasoning_hooks(self, agent, input_kwargs, _plain, _restore_context)
+ async for event in next_handler(**input_kwargs):
yield event
@staticmethod
@@ -423,7 +312,7 @@ async def on_model_call(self, agent: Any, input_kwargs: dict, next_handler):
"parameters": parameters,
},
)
- response = await self._legacy_model_call(agent, input_kwargs, next_handler)
+ response = await next_handler(**input_kwargs)
if not inspect.isasyncgen(response):
await self._stage(
HookStage.AFTER_MODEL,
@@ -453,32 +342,8 @@ async def patched_stream():
return patched_stream()
async def on_acting(self, agent: Any, input_kwargs: dict, next_handler):
- tool_call = input_kwargs.get("tool_call")
- raw_args = getattr(tool_call, "input", "") or "{}"
- try:
- args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
- except (TypeError, json.JSONDecodeError):
- args = {"_raw": str(raw_args)}
- name = str(getattr(tool_call, "name", "") or "tool")
- await self._stage(
- HookStage.BEFORE_TOOL,
- agent,
- operation_name=name,
- data={"tool_args": args, "metadata": {}},
- )
- final = None
- async for event in self._legacy_stream(
- "on_acting", agent, input_kwargs, next_handler
- ):
- if isinstance(event, ToolResponse):
- final = event
- else:
+ from contextlib import aclosing
+ from core.llm.runtime.tool_result_adapter import run_tool
+ async with aclosing(run_tool(self, agent, input_kwargs, next_handler, _plain)) as stream:
+ async for event in stream:
yield event
- await self._stage(
- HookStage.AFTER_TOOL,
- agent,
- operation_name=name,
- data={"tool_result": _plain(final), "metadata": {}},
- )
- if final is not None:
- yield final
diff --git a/src/backend/core/llm/runtime/context_hook_adapter.py b/src/backend/core/llm/runtime/context_hook_adapter.py
new file mode 100644
index 00000000..9989a268
--- /dev/null
+++ b/src/backend/core/llm/runtime/context_hook_adapter.py
@@ -0,0 +1,80 @@
+"""Translate neutral context additions at the framework boundary."""
+from core.harness.events import thaw_value
+from core.llm.context.items import ContextItem
+from core.llm.context.ir_constants import POLICY_HEAD_TAIL
+from core.llm.context.rendering import append_state_context_item, next_context_sequence
+from core.llm.runtime.turn_state import reset_artifact_read_state, reset_pin_hint_state
+
+
+def attachment_input(agent):
+ state = getattr(agent, "state", None)
+ return {name: getattr(state, name, default) for name, default in (
+ ("uploaded_files", []), ("historical_files", []), ("user_id", ""),
+ ("model_provider_id", ""), ("vision_evidence_text", ""),
+ )}
+
+
+def begin_context_turn():
+ reset_artifact_read_state()
+ reset_pin_hint_state()
+
+
+def append_hook_context(agent, invocation):
+ for item in invocation.data.get("context_items", ()):
+ item = thaw_value(item)
+ sequence = next_context_sequence(agent.state.context)
+ origin = item["origin"]
+ append_state_context_item(agent.state, ContextItem.create(
+ item_id=f"{origin}:{sequence}", kind=item["kind"], origin=origin,
+ trust=item["trust"], visibility="model", priority=item["priority"],
+ token_budget=item["token_budget"], truncation_policy=item.get("truncation_policy", POLICY_HEAD_TAIL),
+ content=item["content"], cache_class="dynamic", created_seq=sequence,
+ render_role="user", message_group=f"{origin}:{sequence}",
+ ))
+
+
+async def apply_reasoning_hooks(adapter, agent, input_kwargs, plain, restore_context):
+ from agentscope.tool import ToolChoice
+ from core.harness.hooks import HookStage
+ from core.llm.runtime.workspace import get_pinned_file_ids
+ state = agent.state
+ transformed = await adapter._stage(
+ HookStage.TRANSFORM_CONTEXT, agent, operation_name="agent_reasoning",
+ data={"identity": {"chat_id": getattr(state, "chat_id", None), "user_id": getattr(state, "user_id", None)},
+ "context": plain(state.context), "context_items": [],
+ "tool_choice": plain(input_kwargs.get("tool_choice")),
+ "runtime": {"pinned_file_ids": list(get_pinned_file_ids()), "cur_iter": getattr(state, "cur_iter", 0),
+ "reply_id": getattr(state, "reply_id", ""),
+ "max_iters": getattr(getattr(agent, "react_config", None), "max_iters", 0)}},
+ )
+ if transformed.data.get("context") is not None:
+ state.context = restore_context(state.context, transformed.data["context"])
+ append_hook_context(agent, transformed)
+ await adapter._stage(HookStage.AFTER_CONTEXT, agent, operation_name="agent_reasoning",
+ data={"steer_delivery": transformed.data.get("steer_delivery")})
+ if transformed.data.get("steer_delivery") is not None:
+ from core.harness.events import thaw_value
+ state.steer_delivery = thaw_value(transformed.data["steer_delivery"])
+ choice = transformed.data.get("tool_choice")
+ if choice is not None:
+ from core.harness.events import thaw_value
+ return {**input_kwargs, "tool_choice": ToolChoice.model_validate(thaw_value(choice))}
+ return input_kwargs
+
+
+async def finish_context_hooks(adapter, agent, output, plain):
+ from core.harness.hooks import HookStage
+ from core.llm.runtime.workspace import get_pinned_file_ids
+ result = await adapter._stage(HookStage.BEFORE_FINISH, agent, operation_name="agent_reply",
+ data={"output": plain(output), "metadata": {},
+ "identity": {"chat_id": getattr(agent.state, "chat_id", None), "user_id": getattr(agent.state, "user_id", None)},
+ "runtime": {"pinned_file_ids": list(get_pinned_file_ids()),
+ "finish_pins_completed": getattr(adapter, "finish_pins_completed", False)}})
+ if result.data.get("continue_run"):
+ return True
+ pins = result.data.get("pin_file_ids") or ()
+ if pins:
+ from core.llm.runtime.finish_guard import pin_files
+ if pin_files(list(pins)) > 0:
+ adapter.finish_pins_completed = True
+ return False
diff --git a/src/backend/core/llm/runtime/delta_coalescer.py b/src/backend/core/llm/runtime/delta_coalescer.py
new file mode 100644
index 00000000..ed6365b7
--- /dev/null
+++ b/src/backend/core/llm/runtime/delta_coalescer.py
@@ -0,0 +1,106 @@
+"""Merge consecutive streaming deltas so every event downstream is worth a write.
+
+A model streams answer text, thinking and tool arguments a few characters at a
+time, and each event the stream yields becomes one durable run-event append
+(several Redis round trips). :class:`DeltaCoalescer` sits between the event
+mapper and the consumer and merges a run of same-stream deltas into one event:
+
+* a delta goes out at once when nothing was flushed for
+ ``DELTA_FLUSH_INTERVAL_S`` (so the first token is never delayed), otherwise
+ it waits in the buffer until that interval has passed or
+ ``DELTA_FLUSH_CHARS`` have accumulated (see :mod:`streaming_constants`);
+* only one stream is buffered at a time — a delta for a different stream, or
+ any other event (tool start/end, results, errors, the end of the stream),
+ flushes the buffer first, so the merged stream is the original stream in the
+ original order with adjacent same-stream deltas concatenated;
+* :meth:`deadline` tells a consumer that waits on its source with a timeout
+ when the buffered tail is due, so text never sits behind a silent model.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Hashable, List, Optional, Tuple
+
+from core.llm.runtime import streaming_constants
+
+Event = Tuple[str, Any]
+
+
+def _stream_key(kind: str, payload: Any) -> Optional[Hashable]:
+ """Which delta stream an event belongs to, or None when it is not a delta."""
+ if kind in ("text_delta", "thinking_delta"):
+ return kind
+ if kind == "tool_call_delta":
+ return (kind, payload.get("id"))
+ return None
+
+
+def _merge(kind: str, first: Any, text: str) -> Any:
+ if kind == "tool_call_delta":
+ return {**first, "delta": text}
+ return text
+
+
+def _text(kind: str, payload: Any) -> str:
+ return (payload.get("delta") if kind == "tool_call_delta" else payload) or ""
+
+
+class DeltaCoalescer:
+ def __init__(self) -> None:
+ self._key: Optional[Hashable] = None
+ self._kind = ""
+ self._first: Any = None
+ self._parts: List[str] = []
+ self._size = 0
+ self._last_flush = float("-inf")
+
+ def feed(self, event: Event, now: Optional[float] = None) -> List[Event]:
+ """Take one mapped event; return the events to deliver now, in order."""
+ kind, payload = event
+ key = _stream_key(kind, payload)
+ if key is None:
+ return [*self.drain(now), event]
+ text = _text(kind, payload)
+ if not text:
+ return []
+ now = time.monotonic() if now is None else now
+ out = self.drain(now) if key != self._key else []
+ if self._key is None:
+ self._key, self._kind, self._first = key, kind, payload
+ self._parts.append(text)
+ self._size += len(text)
+ if (
+ self._size >= streaming_constants.DELTA_FLUSH_CHARS
+ or now - self._last_flush >= streaming_constants.DELTA_FLUSH_INTERVAL_S
+ ):
+ out.extend(self.drain(now))
+ return out
+
+ def drain(self, now: Optional[float] = None) -> List[Event]:
+ """Flush whatever is buffered."""
+ if self._key is None:
+ return []
+ event = (self._kind, _merge(self._kind, self._first, "".join(self._parts)))
+ self._key, self._kind, self._first = None, "", None
+ self._parts, self._size = [], 0
+ self._last_flush = time.monotonic() if now is None else now
+ return [event]
+
+ def deadline(self) -> Optional[float]:
+ """Monotonic time the buffered delta is due, or None when nothing waits."""
+ if self._key is None:
+ return None
+ return self._last_flush + streaming_constants.DELTA_FLUSH_INTERVAL_S
+
+ def flush_due(self, now: Optional[float] = None) -> List[Event]:
+ """Flush the buffer if its deadline has passed."""
+ now = time.monotonic() if now is None else now
+ due = self.deadline()
+ return self.drain(now) if due is not None and now >= due else []
+
+ def reset(self) -> None:
+ """Discard the buffer (the stream was abandoned) and forget the clock."""
+ self._key, self._kind, self._first = None, "", None
+ self._parts, self._size = [], 0
+ self._last_flush = float("-inf")
diff --git a/src/backend/core/llm/runtime/file_context.py b/src/backend/core/llm/runtime/file_context.py
new file mode 100644
index 00000000..8f0c26fe
--- /dev/null
+++ b/src/backend/core/llm/runtime/file_context.py
@@ -0,0 +1,277 @@
+"""Turn file attachments into context: text, images and spreadsheet previews."""
+from __future__ import annotations
+from core.files import artifact_bytes
+import base64
+from typing import Any, Dict, List, Optional
+import logging
+logger = logging.getLogger(__name__)
+
+def fetch_image_base64(
+ f: Dict[str, Any],
+ user_id: Optional[str] = None,
+) -> Optional[tuple[str, str]]:
+ """Fetch image data for a file attachment and return (base64_data, mime_type)."""
+ file_id = f.get("file_id") or ""
+ if not file_id:
+ return None
+ mime_type = (f.get("mime_type") or "image/png").lower()
+ raw = artifact_bytes.download_artifact_bytes(
+ file_id,
+ f.get("name") or "image",
+ "Image hook",
+ user_id=user_id,
+ )
+ if raw is None:
+ return None
+ return base64.b64encode(raw).decode("utf-8"), mime_type
+
+
+def is_xlsx(f: Dict[str, Any]) -> bool:
+ """Detect xlsx by mime_type or filename extension.
+
+ We only special-case xlsx (not xls/csv) because:
+ - xlsx is the dominant batch-execution upload format
+ - parse_xlsx_preview uses openpyxl, same engine as the batch path
+ - xls would need a separate xlrd-based preview helper (out of scope)
+ """
+ mime = (f.get("mime_type") or "").lower()
+ if "spreadsheetml.sheet" in mime:
+ return True
+ name = (f.get("name") or f.get("filename") or "").lower()
+ return name.endswith(".xlsx")
+
+
+def build_xlsx_preview_block(
+ f: Dict[str, Any],
+ user_id: Optional[str] = None,
+) -> Optional[str]:
+ """Build the bounded preview + metadata + directive block for a single xlsx.
+
+ Returns None if the file can't be read (caller falls back to the regular
+ parsed_text path, so the agent at least sees *something*).
+ """
+ from core.files.content.spreadsheet_parser import parse_xlsx_preview
+
+ file_id = f.get("file_id") or ""
+ name = f.get("name") or f.get("filename") or "未命名表格"
+ if not file_id:
+ return None
+
+ file_bytes = artifact_bytes.download_artifact_bytes(
+ file_id,
+ name,
+ "xlsx preview",
+ user_id=user_id,
+ )
+ if file_bytes is None:
+ return None
+
+ try:
+ from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS, fetch_parsed_text
+
+ # Reuse the bytes already downloaded for the structural preview to
+ # populate parsed_text + summary without a second storage round-trip.
+ fetch_parsed_text(file_id, user_id=user_id, prefetched_bytes=file_bytes)
+ info = parse_xlsx_preview(file_bytes, char_budget=ATTACHMENT_PREVIEW_MAX_CHARS)
+ except RuntimeError as e:
+ logger.warning(f"xlsx preview: parse failed for {file_id}: {e}")
+ return None
+ except Exception as e:
+ logger.warning(f"xlsx preview: unexpected parse error for {file_id}: {e}")
+ return None
+
+ total_rows = info["total_rows"]
+ total_cols = info["total_columns"]
+ preview_rows = info["preview_rows"]
+ omitted = max(0, total_rows - preview_rows)
+ cols_str = ", ".join(info["header"]) if info["header"] else "(无列名)"
+
+ lines: List[str] = [
+ f"[文件: {name}]",
+ f" file_id: {file_id}",
+ f" 总规模: {total_rows} 行 × {total_cols} 列",
+ f" 列名: [{cols_str}]",
+ f" 已展示: 前 {preview_rows} 行(约 {len(info['preview_md'])} 字符,达到预览预算)",
+ ]
+ if info.get("truncated_columns"):
+ lines.append(f" 注意: 共 {total_cols} 列,预览仅含前 {len(info['header'])} 列")
+ if info.get("truncated_cells"):
+ lines.append(f" 注意: {info['truncated_cells']} 个超长单元格已截断到 500 字以内")
+ lines.append("")
+ lines.append(info["preview_md"])
+ if omitted:
+ lines.append(f"... (剩余 {omitted} 行未展示) ...")
+ lines.append("")
+ lines.append("[操作指引]")
+ lines.append(
+ f"- 需要预览之外的正文时,调用 read_artifact(file_id='{file_id}', "
+ "offset=N, limit=4000) 按需读取"
+ )
+ lines.append("- 以上只是有界预览,不要把未展示的行当作不存在,也不要推断未读取内容")
+ return "\n".join(lines)
+
+
+def build_file_context(
+ uploaded_files: List[Dict[str, Any]],
+ user_id: Optional[str] = None,
+) -> str:
+ """Assemble the text-attachment list into the context text injected into the model (non-image files only).
+
+ Files are resolved exclusively by ``file_id``. The backend lazily parses and
+ caches the artifact, then injects only metadata plus a bounded preview. The
+ request body never supplies parsed content.
+
+ ``user_id`` is used as the ownership check when fetching ``fetch_parsed_text`` /
+ the xlsx preview, preventing a forged request-body ``attachments[].file_id``
+ from reading across users — see the :func:`artifact_bytes.download_artifact_bytes` docstring
+ for the core defense point.
+ """
+ text_files = [f for f in uploaded_files if not artifact_bytes.is_image(f)]
+ if not text_files:
+ return ""
+
+ file_descriptions = []
+ for f in text_files:
+ name = f.get("name", "未知文件")
+ fid = f.get("file_id", "")
+ desc = f"- {name}"
+ if fid:
+ desc += f" (file_id: {fid})"
+ file_descriptions.append(desc)
+ from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS, fetch_parsed_text
+
+ file_content_parts: List[str] = []
+ for f in text_files:
+ # ── xlsx: bounded preview + metadata + directive (no full content) ──
+ if is_xlsx(f):
+ block = build_xlsx_preview_block(f, user_id=user_id)
+ if block is not None:
+ file_content_parts.append(block)
+ continue
+ # Fall through to regular path on preview failure so the agent
+ # at least sees the parsed markdown (degraded but non-fatal).
+
+ fid = (f.get("file_id") or "").strip()
+ mime_type = f.get("mime_type") or ""
+ if not fid:
+ file_content_parts.append(
+ f"[文件: {f.get('name', '未知文件')}]\n"
+ " 状态: 缺少 file_id,无法读取正文"
+ )
+ continue
+
+ logger.info("Document hook: loading preview for %s (%s)", f.get("name"), fid)
+ content = fetch_parsed_text(fid, user_id=user_id)
+ if not content:
+ file_content_parts.append(
+ f"[文件: {f.get('name', '未知文件')}]\n"
+ f" file_id: {fid}\n"
+ f" MIME: {mime_type or '(未知)'}\n"
+ " 状态: 文件不可访问或解析失败"
+ )
+ continue
+
+ total_chars = len(content)
+ preview = content[:ATTACHMENT_PREVIEW_MAX_CHARS]
+ truncated = total_chars > len(preview)
+ block = [
+ f"[文件: {f.get('name', '未知文件')}]",
+ f" file_id: {fid}",
+ f" MIME: {mime_type or '(未知)'}",
+ f" 解析文本: 共 {total_chars} 字符,本轮自动展示 {len(preview)} 字符",
+ "",
+ preview,
+ ]
+ if truncated:
+ block.append(f"... (其余 {total_chars - len(preview)} 字符未展示) ...")
+ block.extend(
+ [
+ "",
+ "[读取指引]",
+ f"- 需要未展示的正文时,调用 read_artifact(file_id='{fid}', offset=N, limit=4000)",
+ "- 不要推断尚未读取的内容",
+ ]
+ )
+ file_content_parts.append("\n".join(block))
+ file_content = "\n\n---\n\n".join(file_content_parts)
+
+ return (
+ f"[file name]: {chr(10).join(file_descriptions)}\n"
+ f"[file content begin]\n"
+ f"{file_content}\n"
+ f"[file content end]\n"
+ "以上是用户附件的元数据和有界预览;需要预览之外的正文时必须按需调用 read_artifact。"
+ )
+
+
+def describe_source(source: str) -> str:
+ from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD
+
+ return {SOURCE_USER_UPLOAD: "用户上传", SOURCE_AI_GENERATED: "AI 生成"}.get(source, "")
+
+
+def build_historical_files_context(historical_files: List[Dict[str, Any]]) -> str:
+ """Build a compact summary block for files from previous turns.
+
+ Includes both user-uploaded files and AI-generated files (reports,
+ charts, code output, etc.) produced by tools in earlier turns.
+ Each entry is labeled by provenance so the agent knows what kind of
+ file it's referencing.
+
+ Agent gets only `{filename, file_id, source, summary}`; full content
+ is fetched on-demand via the `read_artifact` tool. This keeps prompt
+ size bounded as the conversation grows.
+ """
+ if not historical_files:
+ return ""
+
+ lines: List[str] = []
+ for f in historical_files:
+ file_id = f.get("file_id") or ""
+ name = f.get("name") or f.get("filename") or "未命名文件"
+ mime = f.get("mime_type") or ""
+ summary = (f.get("summary") or "").strip()
+ source = f.get("source") or ""
+ deleted = bool(f.get("deleted"))
+
+ source_label = describe_source(source)
+ header_parts = [f"file_id: {file_id}"]
+ if source_label:
+ header_parts.append(source_label)
+ if mime:
+ header_parts.append(mime)
+ header = f"- {name} ({', '.join(header_parts)})"
+
+ if deleted:
+ lines.append(header + " [文件已删除,无法读取]")
+ continue
+
+ lines.append(header)
+ # Key point: tell the model the stable /myspace/ path so it can operate
+ # directly with Read/Edit/Write — better suited than read_artifact for
+ # "modify HTML/CSS/MD/source code" text-editing scenarios, and changes sync
+ # back to "My Space" immediately.
+ if name and source == ai_source():
+ lines.append(f" 沙盒路径: /myspace/{name} ← 用 Read/Edit/Write 直接读改写")
+ if summary:
+ indented = "\n".join(f" {ln}" for ln in summary.splitlines())
+ lines.append(f" 摘要:\n{indented}")
+ else:
+ lines.append(" 摘要:(尚未生成)")
+
+ header = (
+ "[历史文件清单](本会话中之前轮次涉及的文件,包含用户上传和 AI 生成的两类,以下仅为摘要)\n"
+ "访问方式:\n"
+ '- **要修改文本/HTML/MD/代码** → 用 `Read("/myspace/")` 读完整内容,'
+ "再用 `Edit` / `Write` 改;改完会自动同步回「我的空间」(同一 file_id)。\n"
+ "- **要看 docx/pdf/xlsx 等需要解析的二进制文件** → 用 "
+ "`read_artifact(file_id, offset, limit)` 拿解析后的文本。"
+ )
+ return header + "\n\n" + "\n".join(lines)
+
+
+def ai_source() -> str:
+ """Lazy resolve to avoid circular import at module load."""
+ from core.files.artifacts.reader import SOURCE_AI_GENERATED
+
+ return SOURCE_AI_GENERATED
diff --git a/src/backend/core/llm/runtime/finish_guard.py b/src/backend/core/llm/runtime/finish_guard.py
new file mode 100644
index 00000000..9c56f9f6
--- /dev/null
+++ b/src/backend/core/llm/runtime/finish_guard.py
@@ -0,0 +1,54 @@
+"""Apply neutral finish-pin decisions in the parent execution context.
+
+Workspace pins use ContextVars and must be written in the framework task, after
+all finish hooks return. A failed pin leaves the guard eligible for retry.
+"""
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def pin_files(file_ids: list[str]) -> int:
+ """Write file_ids directly to the workspace ContextVar, bypassing the
+ pin_to_workspace tool + _acting path. Returns the number of pins that actually succeeded.
+
+ Metadata comes from ``core.files.artifacts.store.get_artifact`` — on failure that id is skipped.
+ """
+ try:
+ from core.files.artifacts.store import get_artifact
+ from core.llm.runtime import workspace as _ws
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[finish_guard] _direct_pin import failed: %s", exc)
+ return 0
+
+ count = 0
+ for fid in file_ids:
+ try:
+ art = get_artifact(fid)
+ if not art:
+ logger.warning("[finish_guard] artifact not found: %s", fid)
+ continue
+ ok = _ws.pin(
+ fid,
+ name=art.get("name"),
+ mime_type=art.get("mime_type"),
+ size=art.get("size") or art.get("size_bytes"),
+ url=f"/files/{fid}",
+ )
+ if ok:
+ count += 1
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[finish_guard] _direct_pin failed for %s: %s", fid, exc)
+ # Only mark_active when at least one pin succeeded — avoid an empty set still
+ # calling mark_active, which would leave the workspace in an "active but pinned=[]"
+ # state (although downstream currently does not rely on the active flag for
+ # gating, this is a more robust invariant).
+ if count > 0:
+ try:
+ from core.llm.runtime import workspace as _ws
+ _ws.mark_active()
+ except Exception:
+ logger.warning("[finish_guard] failed to mark workspace active", exc_info=True)
+ return count
diff --git a/src/backend/core/llm/runtime/hooks_constants.py b/src/backend/core/llm/runtime/hooks_constants.py
new file mode 100644
index 00000000..3b78f011
--- /dev/null
+++ b/src/backend/core/llm/runtime/hooks_constants.py
@@ -0,0 +1,17 @@
+"""Limits for file content injected by context hooks."""
+from __future__ import annotations
+import re
+
+MAX_FILE_CONTENT_CHARS = 50_000 # per-turn read_artifact budget per file
+
+
+FILE_ID_RE = re.compile(r'"file_id"\s*:\s*"([A-Za-z0-9_\-]+)"')
+
+
+PIN_HINT_SKIP_TOOLS: frozenset[str] = frozenset(
+ {
+ "pin_to_workspace", # the pin tool itself
+ "read_artifact", # echoes the input file_id; reading, not producing
+ "generate_response", # the ReActAgent finish function
+ }
+)
diff --git a/src/backend/core/llm/human_interaction.py b/src/backend/core/llm/runtime/human_interaction.py
similarity index 86%
rename from src/backend/core/llm/human_interaction.py
rename to src/backend/core/llm/runtime/human_interaction.py
index fbb81bd5..6a8d5b03 100644
--- a/src/backend/core/llm/human_interaction.py
+++ b/src/backend/core/llm/runtime/human_interaction.py
@@ -28,9 +28,9 @@ def has_pending(chat_id: Optional[str]) -> bool:
if not chat_id:
return False
- from core.llm.tools import _tool_confirmation, user_questions
+ from core.llm.tools import confirmation, user_questions
- return _tool_confirmation.has_pending(chat_id) or user_questions.has_pending(chat_id)
+ return confirmation.has_pending(chat_id) or user_questions.has_pending(chat_id)
__all__ = [
diff --git a/src/backend/core/llm/interaction_store.py b/src/backend/core/llm/runtime/interaction_store.py
similarity index 100%
rename from src/backend/core/llm/interaction_store.py
rename to src/backend/core/llm/runtime/interaction_store.py
diff --git a/src/backend/core/llm/manifest_agent.py b/src/backend/core/llm/runtime/manifest_agent.py
similarity index 93%
rename from src/backend/core/llm/manifest_agent.py
rename to src/backend/core/llm/runtime/manifest_agent.py
index 140d3af9..aea1c7bb 100644
--- a/src/backend/core/llm/manifest_agent.py
+++ b/src/backend/core/llm/runtime/manifest_agent.py
@@ -6,11 +6,13 @@
import logging
from typing import Any, Callable, Optional
-from agentscope.agent import Agent
-from core.llm.context_adapter import AgentScopeContextAdapter
-from core.llm.context_ir import VISIBILITY_MANIFEST_ONLY, ContextAssembler, estimate_context_tokens
-from core.llm.context_manager import FALLBACK_CONTEXT_WINDOW, usable_context_window
-from core.llm.context_safety import enforce_context_budget
+from agentscope.agent import Agent, InjectionConfig
+from core.llm.context.adapter import AgentScopeContextAdapter
+from core.llm.context.ir_constants import VISIBILITY_MANIFEST_ONLY
+from core.llm.context.ir import ContextAssembler
+from core.llm.context.content import estimate_context_tokens
+from core.models.context_window import FALLBACK_CONTEXT_WINDOW, usable_context_window
+from core.llm.context.safety import enforce_context_budget
logger = logging.getLogger(__name__)
@@ -21,6 +23,7 @@ class ManifestBoundAgent(Agent):
def __init__(
self, *args, context_adapter_factory=None, **kwargs
) -> None: # noqa: ANN002,ANN003
+ kwargs.setdefault("injection_config", InjectionConfig(inject_runtime_state=False))
super().__init__(*args, **kwargs)
self._context_adapter_factory = context_adapter_factory or AgentScopeContextAdapter
self._tool_reserve_memo: Optional[tuple[int, int]] = None
@@ -29,7 +32,7 @@ def __init__(
def context_adapter_factory(self): # noqa: ANN201
return self._context_adapter_factory
- async def compress_context(self, context_config=None) -> None: # noqa: ANN001
+ async def compress_context(self, context_config=None, instructions=None) -> None: # noqa: ANN001
"""Disable AgentScope's second, opaque context selector.
Durable pre-turn compaction may replace history with an explicit
@@ -38,7 +41,7 @@ async def compress_context(self, context_config=None) -> None: # noqa: ANN001
candidate decision. Running AgentScope compression here would delete
candidates before the final manifest and make exclusions unauditable.
"""
- del context_config
+ del context_config, instructions
@property
def request_context_manifest(self) -> dict[str, Any]:
diff --git a/src/backend/core/llm/message_compat.py b/src/backend/core/llm/runtime/message_compat.py
similarity index 93%
rename from src/backend/core/llm/message_compat.py
rename to src/backend/core/llm/runtime/message_compat.py
index 7fe33ecd..f3bcfa4a 100644
--- a/src/backend/core/llm/message_compat.py
+++ b/src/backend/core/llm/runtime/message_compat.py
@@ -3,9 +3,11 @@
from __future__ import annotations
import json
+from collections.abc import Mapping
from typing import Any, Dict, List
+from agentscope.message import Msg
-from core.llm.context_adapter import AgentScopeContextAdapter
+from core.llm.context.adapter import AgentScopeContextAdapter
@@ -28,7 +30,7 @@ def dict_to_msg(d: Dict[str, Any], *, created_seq: int = 0) -> Any:
# 1.x practice of putting tool_result on "tool"→"user" is rejected by Msg
# validation in 2.0, so "tool" now maps to "assistant".
# (The dict layer still keeps the "tool" marker so compaction can tell a
- # result carrier from a step boundary — see core/llm/model_steps.py.)
+ # result carrier from a step boundary — see core/llm/runtime/model_steps.py.)
role_map = {"human": "user", "ai": "assistant", "tool": "assistant"}
role = role_map.get(role, role)
@@ -56,12 +58,12 @@ def session_to_msgs(session_messages: List[Dict[str, Any]]) -> List[Any]:
``await load_session_into_memory(history, agent.memory)``.
This is also where tool-returned media stops being a reference and becomes
- bytes again (:func:`core.llm.tool_media_store.hydrate_rows`). Everything
+ bytes again (:func:`core.llm.runtime.tool_media_store.hydrate_rows`). Everything
upstream — replay, the compaction checkpoint, the run's recovery snapshot —
carries the sha256 only, because none of them needs the pixels and all of
them get JSON-serialized; this is the first point where something does.
"""
- from core.llm.tool_media_store import hydrate_rows
+ from core.llm.runtime.tool_media_store import hydrate_rows
return [
dict_to_msg(message, created_seq=index)
@@ -80,10 +82,10 @@ def flatten_tool_output(output: Any) -> str:
"""
if isinstance(output, str):
return output
- if isinstance(output, list):
+ if isinstance(output, (list, tuple)):
parts: list[str] = []
for item in output:
- if isinstance(item, dict):
+ if isinstance(item, Mapping):
text_val = item.get("text")
parts.append(str(text_val) if text_val is not None else str(item))
elif getattr(item, "type", None) == "text":
@@ -101,7 +103,7 @@ def _format_tool_output(output: Any) -> str:
# In 2.0, output is often list[TextBlock|dict] (pydantic blocks); json.dumps
# would fail and degrade to repr. Extract text via flatten_tool_output
# first, then truncate.
- if isinstance(output, list):
+ if isinstance(output, (list, tuple)):
text = flatten_tool_output(output)
return text[:2000] if len(text) > 2000 else text
try:
diff --git a/src/backend/core/llm/runtime/model_cache.py b/src/backend/core/llm/runtime/model_cache.py
new file mode 100644
index 00000000..aa66cbd3
--- /dev/null
+++ b/src/backend/core/llm/runtime/model_cache.py
@@ -0,0 +1,133 @@
+"""Cache model instances per chat mode and invalidate them on config changes."""
+from __future__ import annotations
+import threading
+from dataclasses import dataclass, field
+from functools import wraps
+from typing import Any
+
+def serialized_model_cache(function):
+ @wraps(function)
+ def locked(*args, **kwargs):
+ with process_state().model_cache_lock:
+ return function(*args, **kwargs)
+
+ return locked
+
+
+def check_version():
+ """Invalidate cached model instances when ModelConfigService version changes."""
+ from core.models.config import ModelConfigService
+
+ current = ModelConfigService.get_instance().version
+ if current != process_state().cached_version:
+ process_state().model_cache = {}
+ process_state().cached_version = current
+
+
+def model_cache_key(name: str) -> tuple[Any, ...]:
+ from core.harness.registry.host_runtime import requires_remote_execution
+
+ import asyncio
+
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
+ if not requires_remote_execution():
+ return (name, loop, None)
+ from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint, get_state
+
+ identity = _state_fingerprint(get_state())
+ # Eviction only releases this cache's reference. In-flight runs keep their
+ # captured model and its existing request hook, which rejects identity changes.
+ for key in list(process_state().model_cache):
+ if isinstance(key, tuple) and key[0] == name and key[2] != identity:
+ process_state().model_cache.pop(key, None)
+ return name, loop, identity
+
+
+@serialized_model_cache
+def release_loop_models() -> None:
+ import asyncio
+
+ loop = asyncio.get_running_loop()
+ for key in list(process_state().model_cache):
+ if key[1] is loop:
+ process_state().model_cache.pop(key, None)
+
+
+@serialized_model_cache
+def get_main_model(mode: str = "medium"):
+ """Get the main agent model for the given chat mode (fast/medium/high/max)."""
+ from core.models.chat_models import get_default_model
+ from core.models.failover import with_failover
+ from core.models.config import ModelConfigService
+
+ check_version()
+ key = model_cache_key(f"main:{mode}")
+ cached = process_state().model_cache.get(key)
+ if cached is not None:
+ return cached
+ if mode in ("fast", "turbo"):
+ instance = get_default_model(disable_thinking=True, stream=True)
+ elif mode in ("low", "high", "xhigh", "max"):
+ instance = get_default_model(reasoning_effort=mode, stream=True)
+ else: # medium or unknown
+ # medium uses the "thinking model", but picks chat_template_kwargs based on
+ # supports_reasoning_effort (chat_models internally distinguishes
+ # effort=medium vs None)
+ instance = get_default_model(reasoning_effort="medium", stream=True)
+ instance = with_failover(
+ instance, ModelConfigService.get_instance().resolve("main_agent"), mode=mode
+ )
+ process_state().model_cache[key] = instance
+ return instance
+
+
+@serialized_model_cache
+def get_provider_model(provider_id: str, mode: str = "medium"):
+ """Get a user-selected active chat provider model for the given chat mode."""
+ from core.models.chat_models import build_model_for_mode
+ from core.models.failover import with_failover
+ from core.models.config import ModelConfigService
+
+ pid = (provider_id or "").strip()
+ if not pid:
+ return get_main_model(mode)
+ check_version()
+ key = model_cache_key(f"provider:{pid}:{mode}")
+ cached = process_state().model_cache.get(key)
+ if cached is not None:
+ return cached
+ resolved = ModelConfigService.get_instance().resolve_provider(pid)
+ if resolved is None:
+ return get_main_model(mode)
+
+ instance = with_failover(
+ build_model_for_mode(resolved, mode=mode, stream=True), resolved, mode=mode
+ )
+ process_state().model_cache[key] = instance
+ return instance
+
+
+def resolve_chat_mode(ctx) -> str:
+ """Resolve the final chat_mode (turbo/fast/medium/high/max) from agent.state (AgentRuntimeState)."""
+ raw = getattr(ctx, "chat_mode", None)
+ if raw in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"):
+ return raw
+ # Fallback for legacy clients
+ return "medium" if getattr(ctx, "enable_thinking", True) else "fast"
+
+
+@dataclass
+class ModelCacheState:
+ model_cache: dict[tuple[Any, ...], Any] = field(default_factory=lambda: {})
+ cached_version: int = field(default_factory=lambda: -1)
+ model_cache_lock: object = field(default_factory=lambda: threading.RLock())
+
+
+_STATE = ModelCacheState()
+
+
+def process_state() -> ModelCacheState:
+ return _STATE
diff --git a/src/backend/core/llm/runtime/model_selection_adapter.py b/src/backend/core/llm/runtime/model_selection_adapter.py
new file mode 100644
index 00000000..214ad220
--- /dev/null
+++ b/src/backend/core/llm/runtime/model_selection_adapter.py
@@ -0,0 +1,28 @@
+"""Translate model-selection decisions into the framework's model reference."""
+import logging
+from core.llm.runtime import model_cache
+from core.models.usage import instrument_model_usage
+
+logger = logging.getLogger(__name__)
+
+
+def model_selection_input(agent):
+ state = getattr(agent, "state", None)
+ return {name: getattr(state, name, default) for name, default in (
+ ("model_pinned", False), ("model_provider_id", ""), ("chat_mode", None),
+ )}
+
+
+def apply_model_selection(agent, invocation):
+ selection = invocation.data.get("model_selection")
+ if selection is None:
+ return
+ try:
+ provider = selection["provider_id"]
+ agent.model = (model_cache.get_provider_model(provider, mode=selection["mode"])
+ if provider else model_cache.get_main_model(mode=selection["mode"]))
+ if invocation.run_id:
+ instrument_model_usage(agent.model)
+ except Exception as exc:
+ # Model refresh is optional; retain the factory's existing model on error.
+ logger.warning("[dynamic_model] failed: %s", exc)
diff --git a/src/backend/core/llm/model_steps.py b/src/backend/core/llm/runtime/model_steps.py
similarity index 98%
rename from src/backend/core/llm/model_steps.py
rename to src/backend/core/llm/runtime/model_steps.py
index ef9ac27b..3d0d8a9b 100644
--- a/src/backend/core/llm/model_steps.py
+++ b/src/backend/core/llm/runtime/model_steps.py
@@ -67,7 +67,7 @@ def _persistable_output(output: Any) -> Any:
"""Tool output as the model saw it, media included.
Media is the one part too big to inline: the bytes go to the blob table
- keyed by their own sha256 (:mod:`core.llm.tool_media_store`) and the record
+ keyed by their own sha256 (:mod:`core.llm.runtime.tool_media_store`) and the record
keeps a reference, so a later turn replays the same pixels the model was
shown rather than a note saying a picture used to be here. The reference
travels all the way to context assembly, which resolves it; replay itself
@@ -92,7 +92,7 @@ def _persistable_output(output: Any) -> Any:
if block_type == "text":
blocks.append({"type": "text", "text": str(data.get("text") or "")})
elif block_type == "data":
- from core.llm import tool_media_store
+ from core.llm.runtime import tool_media_store
stored = tool_media_store.store(data)
if stored is not None:
@@ -142,6 +142,7 @@ def record_tool_result_step(block: Any) -> Dict[str, Any]:
"id": str(data.get("id") or ""),
"name": str(data.get("name") or ""),
"output": _persistable_output(data.get("output")),
+ **({"metadata": _plain(data["metadata"])} if "mcp" in (data.get("metadata") or {}) else {}),
"state": str(data.get("state") or "success"),
}
],
diff --git a/src/backend/core/llm/project_environment.py b/src/backend/core/llm/runtime/project_environment.py
similarity index 88%
rename from src/backend/core/llm/project_environment.py
rename to src/backend/core/llm/runtime/project_environment.py
index e81b7de2..30c9d24a 100644
--- a/src/backend/core/llm/project_environment.py
+++ b/src/backend/core/llm/runtime/project_environment.py
@@ -3,19 +3,18 @@
from __future__ import annotations
import asyncio
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from threading import Lock
-from core.project_environment.config import FIELDS
-from core.services.project_environment import ProjectEnvironmentService
+from core.projects.environment.config import FIELDS
+from core.projects.environment.service import ProjectEnvironmentService
from fastapi import HTTPException
# Compare-and-store only; resource lookups never run under this lock.
-_snapshot_lock = Lock()
def _snapshot(context, key, value):
- with _snapshot_lock:
+ with process_state().snapshot_lock:
if context.get(key) is None:
context[key] = value
return context[key]
@@ -41,8 +40,8 @@ def check_invocation(self, request):
raise HTTPException(403, "所选能力未绑定到此项目,或当前用户没有使用权限")
def restrict(self, request, *, resolve_defaults=True):
- from core.config.catalog import get_enabled_ids
- from core.llm.factory.selection.capabilities import _effective_main_available_skills
+ from core.capabilities.catalog.api import get_enabled_ids
+ from core.llm.factory.selection.capabilities import effective_main_available_skills as _effective_main_available_skills
if resolve_defaults and request.enabled_skill_ids is None:
request.enabled_skill_ids = _effective_main_available_skills()
@@ -81,7 +80,7 @@ def restrict_plugins(self, resolution, request):
plugin.mcp_ids = [m for m in plugin.mcp_ids if m in self.mcp_ids]
plugin.bound_mcp_ids = [m for m in plugin.bound_mcp_ids if m in self.mcp_ids]
setattr(resolution, field, entries)
- from core.plugins.runtime.resolution import _finalize_resolution
+ from core.capabilities.plugins.runtime.resolution import finalize_resolution as _finalize_resolution
_finalize_resolution(resolution, resolution.directory)
# A disallowed plugin sharing an allowed loose skill must not consume
@@ -101,11 +100,11 @@ def restrict_plugins(self, resolution, request):
def _resolve(request):
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
from core.db.engine import SessionLocal
from core.db.models import ChatSession, Project
- from core.project_environment.dependencies import expand_environment
- from core.services.project_environment_resources import environment_resources
+ from core.projects.environment.dependencies import expand_environment
+ from core.projects.environment_resources import environment_resources
context = request.project_ctx if request.project_ctx is not None else {}
project_id = context.get("project_id")
@@ -169,3 +168,15 @@ def validate_project_environment(request):
"""Current authorization may narrow a run; project setting edits cannot widen it."""
if request.project_environment is not None and _resolve(request) != request.project_environment:
raise HTTPException(403, "项目资源权限已变更,本轮已暂停,请重新发起对话")
+
+
+@dataclass
+class ProjectEnvironmentState:
+ snapshot_lock: object = field(default_factory=lambda: Lock())
+
+
+_STATE = ProjectEnvironmentState()
+
+
+def process_state() -> ProjectEnvironmentState:
+ return _STATE
diff --git a/src/backend/core/llm/runtime/project_environment_hook.py b/src/backend/core/llm/runtime/project_environment_hook.py
new file mode 100644
index 00000000..9b422762
--- /dev/null
+++ b/src/backend/core/llm/runtime/project_environment_hook.py
@@ -0,0 +1,19 @@
+"""Recheck project authorization before the framework starts tool I/O."""
+import asyncio
+from fastapi import HTTPException
+from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage
+from core.llm.runtime.project_environment import validate_project_environment
+
+
+def project_environment_hook(request):
+ async def authorize(invocation):
+ try:
+ await asyncio.to_thread(validate_project_environment, request)
+ except HTTPException as error:
+ return Decision(DecisionKind.REJECT, reason=str(error.detail), patch={
+ "tool_response": {"content": [{"type": "text", "text": str(error.detail)}],
+ "state": "denied", "metadata": {"project_environment_denied": True}}})
+ return Decision()
+ return HookSpec("project_environment_authorization", HookStage.BEFORE_TOOL,
+ authorize, priority=50, timeout_ms=30000,
+ mutable_fields=frozenset({"tool_response"}))
diff --git a/src/backend/core/llm/runtime/review_text_agent.py b/src/backend/core/llm/runtime/review_text_agent.py
new file mode 100644
index 00000000..d8044b2e
--- /dev/null
+++ b/src/backend/core/llm/runtime/review_text_agent.py
@@ -0,0 +1,51 @@
+"""SDK execution adapter for a tool-free independent ontology review."""
+from typing import Any
+from core.harness.registry.capability_runtime import child_scope
+
+async def run_text_agent(
+ prompt: str,
+ *,
+ user_id: str,
+ model_name: str | None,
+ model_provider_id: str | None,
+ runtime: dict[str, Any],
+ capability_scope: str = "",
+) -> str:
+ from core.llm.factory import create_agent_executor
+ from core.llm.mcp.manager import close_clients
+ from core.llm.runtime.streaming import StreamingAgent
+
+ agent, clients = await create_agent_executor(
+ disable_tools=True,
+ enabled_skill_ids=[],
+ current_user_id=user_id,
+ model_name=model_name,
+ model_provider_id=model_provider_id,
+ chat_mode="fast",
+ max_iters=2,
+ isolated=True,
+ ontology_runtime=runtime,
+ capability_scope=child_scope(capability_scope, "ontology-model", "text"),
+ )
+ stream = StreamingAgent(agent, clients)
+ text = ""
+ try:
+ async for event_type, payload in stream.stream(
+ [{"role": "user", "content": prompt}],
+ {
+ "user_id": user_id,
+ "model_name": model_name or "",
+ "model_provider_id": model_provider_id or "",
+ "enable_thinking": False,
+ "chat_mode": "fast",
+ "ontology_enabled": True,
+ "ontology_runtime": runtime,
+ },
+ ):
+ if event_type == "text_delta":
+ text += payload
+ elif event_type == "error":
+ raise RuntimeError(str(payload))
+ finally:
+ await close_clients(clients)
+ return text
diff --git a/src/backend/core/llm/session_capabilities.py b/src/backend/core/llm/runtime/session_capabilities.py
similarity index 97%
rename from src/backend/core/llm/session_capabilities.py
rename to src/backend/core/llm/runtime/session_capabilities.py
index 1d19007c..f3422643 100644
--- a/src/backend/core/llm/session_capabilities.py
+++ b/src/backend/core/llm/runtime/session_capabilities.py
@@ -81,7 +81,7 @@ def resolve_session_activated_capabilities(
if not user_id or not chat_id:
return result
try:
- from core.config.catalog_resolver import resolve_explicit_runtime_capabilities
+ from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities
from core.db.engine import SessionLocal
from core.db.models import ChatSession
diff --git a/src/backend/core/llm/runtime/state.py b/src/backend/core/llm/runtime/state.py
new file mode 100644
index 00000000..bb26cf26
--- /dev/null
+++ b/src/backend/core/llm/runtime/state.py
@@ -0,0 +1,75 @@
+"""Typed AgentScope state at the framework boundary."""
+from typing import List
+from agentscope.state import AgentState
+from pydantic import ConfigDict, Field
+
+def _cyfunc_probe():
+ return None
+
+_CYFUNCTION_TYPE = type(_cyfunc_probe)
+
+class AgentRuntimeState(AgentState):
+ """Extends AgentState to carry the runtime fields of the former ModelContext (replaces agent._jx_context)."""
+
+ model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,))
+
+ model_name: str = ""
+ model_provider_id: str = ""
+ # When a subagent explicitly configures a model (model_provider_id) → the factory sets this
+ # True, and The model selection hook must not override it with the main model based on chat_mode
+ # (otherwise the subagent's own model is effectively useless — neither channels nor web get it).
+ model_pinned: bool = False
+ user_id: str | None = None
+ chat_id: str | None = None
+ run_id: str | None = None
+ capability_scope: str = ""
+ journal_owner: str | None = None
+ tool_effect_links: dict = Field(default_factory=dict)
+ enable_thinking: bool = True
+ chat_mode: str | None = None
+ uploaded_files: List[dict] = Field(default_factory=list)
+ historical_files: List[dict] = Field(default_factory=list)
+ user_message_text: str = ""
+ # Vision evidence transcribed ahead of the model call by the streaming entry
+ # point; The attachment hook injects it verbatim instead of redoing the
+ # network round-trip. Empty = transcribe inline (non-streaming callers).
+ vision_evidence_text: str = ""
+ ontology_enabled: bool = False
+ ontology_runtime: dict = Field(default_factory=dict)
+ # Set by The steering hook immediately before the next reasoning round. The
+ # streaming adapter turns it into one ``steer_applied`` event, then clears
+ # it. Keeping this on the typed runtime state avoids process-global queues.
+ steer_delivery: dict | None = None
+ # Model responses recorded by the hook adapter the moment they complete,
+ # drained by the streaming adapter into ``model_step`` events. Each entry
+ # is one ``core.llm.runtime.model_steps`` assistant record.
+ pending_model_steps: List[dict] = Field(default_factory=list)
+ pending_mcp_progress: List[dict] = Field(default_factory=list)
+
+ def apply_request_context(self, context: dict, user_message_text: str) -> None:
+ """Populate per-request runtime fields from the request ``context`` dict (replaces the 1.x agent._jx_context).
+
+ Shared by both entry points — streaming and workflow (non-streaming) — to keep two
+ hand-copied field mappings from silently drifting (streaming once didn't lowercase
+ chat_mode while workflow did).
+ """
+ # A child may use the shared role or its own provider. Request context
+ # still describes the parent selection; it must not relabel a pinned
+ # child (vision routing and audit read these fields).
+ if not self.model_pinned:
+ self.model_name = str(context.get("model_name", "") or "")
+ self.model_provider_id = str(context.get("model_provider_id", "") or "")
+ self.user_id = str(context.get("user_id", "") or "") or None
+ self.chat_id = str(context.get("chat_id", "") or "") or None
+ self.run_id = str(context.get("run_id", "") or "") or None
+ self.journal_owner = str(context.get("journal_owner", "") or "") or None
+ self.enable_thinking = bool(context.get("enable_thinking", True))
+ cm = str(context.get("chat_mode") or "").lower() or None
+ if cm:
+ self.chat_mode = cm
+ self.uploaded_files = list(context.get("uploaded_files", []) or [])
+ self.historical_files = list(context.get("historical_files", []) or [])
+ self.user_message_text = user_message_text or ""
+ self.ontology_enabled = bool(context.get("ontology_enabled", False))
+ runtime = context.get("ontology_runtime")
+ self.ontology_runtime = runtime if isinstance(runtime, dict) else {}
diff --git a/src/backend/core/llm/runtime/stream_events.py b/src/backend/core/llm/runtime/stream_events.py
new file mode 100644
index 00000000..123e0865
--- /dev/null
+++ b/src/backend/core/llm/runtime/stream_events.py
@@ -0,0 +1,261 @@
+"""Map AgentScope reply events to SSE events."""
+from __future__ import annotations
+from core.llm.runtime.stream_text import looks_like_tool_error as _looks_like_tool_error, strip_thinking_answer as _strip_thinking_answer
+from core.harness import log_writer
+import time
+from datetime import datetime, timezone
+from typing import Any, AsyncIterator, Dict, Optional, Tuple
+import logging
+logger = logging.getLogger(__name__)
+
+async def map_event(
+ self, ev: Any, *, result_step: Optional[Dict[str, Any]] = None
+) -> AsyncIterator[Tuple[str, Any]]:
+ """Map a single reply_stream event into 0..N SSE events."""
+ nm = type(ev).__name__
+
+ if nm == "TextBlockDeltaEvent":
+ delta = getattr(ev, "delta", "") or ""
+ if not delta:
+ return
+ # Normal state (enable_thinking): 2.0 events are already incremental — forward
+ # directly, no accumulate+recompute needed (the frontend parses itself),
+ # avoiding an O(n) scan of the full answer on every delta.
+ if self._enable_thinking:
+ if self._round_index == 0:
+ self._first_round_text += delta
+ yield ("text_delta", delta)
+ return
+ # Suppression state: may span multiple deltas — accumulate, then strip out the answer after the closing tag.
+ self._raw_text += delta
+ answer, self._in_thinking = _strip_thinking_answer(
+ self._raw_text, self._enable_thinking, self._in_thinking
+ )
+ if answer and answer != self._emitted_answer:
+ out = (
+ answer[len(self._emitted_answer) :]
+ if answer.startswith(self._emitted_answer)
+ else answer
+ )
+ if out:
+ yield ("text_delta", out)
+ self._emitted_answer = answer
+ return
+
+ if nm == "ThinkingBlockDeltaEvent":
+ if self._enable_thinking:
+ d = getattr(ev, "delta", "") or ""
+ if d:
+ yield ("thinking_delta", d)
+ return
+
+ if nm == "ToolCallStartEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ name = getattr(ev, "tool_call_name", "") or "unknown"
+ self._tool_name_buf[tid] = name
+ self._tool_args_buf[tid] = ""
+ self._pending_tool_calls[tid] = {
+ "tool_name": name,
+ "tool_args": None,
+ "started_monotonic": time.monotonic(),
+ "started_at": datetime.now(timezone.utc),
+ }
+ yield ("tool_call_start", {"name": name, "id": tid})
+ return
+
+ if nm == "ToolCallDeltaEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ delta = getattr(ev, "delta", "") or ""
+ if not delta:
+ return
+ self._tool_args_buf[tid] = self._tool_args_buf.get(tid, "") + delta
+ yield (
+ "tool_call_delta",
+ {"name": self._tool_name_buf.get(tid, "unknown"), "id": tid, "delta": delta},
+ )
+ return
+
+ if nm == "ToolCallEndEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ name = self._tool_name_buf.pop(tid, "unknown")
+ args_str = self._tool_args_buf.pop(tid, "")
+ import json
+
+ try:
+ args = json.loads(args_str) if args_str else {}
+ except json.JSONDecodeError:
+ args = {"_raw": args_str}
+ rec = self._pending_tool_calls.get(tid)
+ if rec is not None:
+ rec["tool_args"] = args
+ try:
+ from core.llm.runtime.tool_callbacks import note_tool_call
+
+ note_tool_call(self.__dict__.setdefault("_tool_warn_state", {}), name, args)
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.runtime.stream_events:map_event', fallback_error)
+ yield ("tool_call", {"name": name, "args": args, "id": tid})
+ return
+
+ if nm == "ToolResultTextDeltaEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ self._tool_result_buf[tid] = self._tool_result_buf.get(tid, "") + (
+ getattr(ev, "delta", "") or ""
+ )
+ return
+
+ if nm == "ToolResultEndEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ content = self._tool_result_buf.pop(tid, "")
+ if result_step is None:
+ result_step = await self._tool_result_step(tid)
+ if result_step is None:
+ raise RuntimeError(f"Tool result {tid} is missing from model context")
+ raw_state = getattr(ev, "state", "") or ""
+ state = str(getattr(raw_state, "value", raw_state) or "")
+ pending = self._pending_tool_calls.pop(tid, None)
+ name = (
+ getattr(ev, "tool_call_name", "")
+ or (pending or {}).get("tool_name")
+ or self._tool_name_buf.get(tid)
+ or "unknown"
+ )
+ try:
+ is_error = state in {
+ "error",
+ "denied",
+ "interrupted",
+ } or _looks_like_tool_error(content)
+ effect_link = dict(
+ (getattr(self.agent.state, "tool_effect_links", None) or {}).pop(tid, {})
+ )
+ started_mono = pending.get("started_monotonic") if pending else None
+ duration_ms = (
+ int((time.monotonic() - started_mono) * 1000) if started_mono else None
+ )
+ log_writer.schedule_tool_call_write(
+ {
+ # See the matching comment in _produce: carry user_id/chat_id explicitly, never rely on contextvars.
+ "user_id": self.agent.state.user_id or None,
+ "chat_id": self.agent.state.chat_id or None,
+ "run_id": self.agent.state.run_id or None,
+ "effect_id": effect_link.get("effect_id"),
+ "result_id": effect_link.get("result_id"),
+ "tool_name": (pending or {}).get("tool_name") or name,
+ "tool_call_id": tid,
+ "tool_args": (pending or {}).get("tool_args"),
+ "tool_result": content,
+ "status": "failed" if is_error else "success",
+ "error_message": content if is_error else None,
+ "duration_ms": duration_ms,
+ "started_at": (pending or {}).get("started_at"),
+ }
+ )
+ except Exception: # noqa: BLE001
+ logger.debug("tool_call log persist failed", exc_info=True)
+ # Persist the canonical result before any UI filtering (notably
+ # update_plan). The consumer must finish this event before the
+ # display-only result can announce completion.
+ yield ("model_step", result_step)
+ yield (
+ "tool_result",
+ {
+ "name": name,
+ "id": tid,
+ "content": content,
+ "status": state,
+ "mcp": (result_step["blocks"][0].get("metadata") or {}).get("mcp"),
+ },
+ )
+ return
+
+ if nm == "ModelCallStartEvent":
+ # Marks the wire as "waiting on the model" — see _model_call_inflight.
+ self._model_call_inflight = True
+ # Agent construction and manifest-backed tool assembly happen before
+ # this event. Surface the provider-dispatch boundary accurately.
+ yield ("model_call_start", None)
+ return
+
+ if nm == "ModelCallEndEvent":
+ self._model_call_inflight = False
+ # The hook adapter recorded this response before the event was
+ # queued; hand the record downstream in the same order.
+ for step in self._drain_model_steps():
+ yield ("model_step", step)
+ _prompt_tokens = int(getattr(ev, "input_tokens", 0) or 0)
+ _completion_tokens = int(getattr(ev, "output_tokens", 0) or 0)
+ self._usage_records.append(
+ {
+ "prompt_tokens": _prompt_tokens,
+ "completion_tokens": _completion_tokens,
+ }
+ )
+ from core.llm.context.usage import build_context_usage_snapshot
+
+ self._context_usage = build_context_usage_snapshot(
+ self.agent,
+ prompt_tokens=_prompt_tokens,
+ completion_tokens=_completion_tokens,
+ model_call_index=len(self._usage_records),
+ )
+ # Compaction usage is anchored by CompactingAgent._save_to_context
+ # after the response is saved. AgentScope emits this event first;
+ # observing here would count that response again as new content.
+ # 首轮权威判定:思考模式下整轮正文没有出现 → 该模型不内联思考
+ # (结构化 reasoning 通道,或本轮确实没思考)。补发协议标记,前端据此
+ # 把误当思考缓冲/展示的正文重归正文区(bug:无思考时正文进思考块)。
+ # 只看首轮——混合形态模型工具后省略 属既有启发式管辖,不在此误判。
+ if (
+ self._enable_thinking
+ and self._round_index == 0
+ and not self._structured_marker_sent
+ and self._first_round_text
+ and " " not in self._first_round_text
+ ):
+ self._structured_marker_sent = True
+ yield ("reasoning_protocol", {"structured_reasoning": True})
+ yield ("context_usage", dict(self._context_usage))
+ self._round_index += 1
+ self._first_round_text = ""
+ # New model call round → reset answer accumulation (the next text segment computes deltas from scratch)
+ self._raw_text = ""
+ self._emitted_answer = ""
+ self._in_thinking = False
+ return
+
+ if nm == "RequireUserConfirmEvent":
+ # 2.0 native HITL (distinct from the myspace gate); our tools default to ALLOW, so this rarely triggers.
+ try:
+ yield (
+ "file_confirm",
+ {
+ "reply_id": getattr(ev, "reply_id", ""),
+ "tool_calls": [
+ {
+ "id": getattr(tc, "id", ""),
+ "name": getattr(tc, "name", ""),
+ "input": getattr(tc, "input", ""),
+ }
+ for tc in (getattr(ev, "tool_calls", []) or [])
+ ],
+ },
+ )
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.runtime.stream_events:map_event', fallback_error)
+ return
+
+ if nm == "ExceedMaxItersEvent":
+ yield (
+ "error",
+ {"kind": "exceed_max_iters", "name": getattr(ev, "name", "")},
+ )
+ return
+
+ # Everything else is never forwarded, internal only: lifecycle (ReplyStart/End,
+ # the various *Start/*End) + DataBlockStart/Delta (model-produced
+ # image_chunk — currently no downstream consumer and the configured models only take
+ # images as input, so intentionally dropped; add a branch when support is needed).
+ return
diff --git a/src/backend/core/llm/runtime/stream_loop.py b/src/backend/core/llm/runtime/stream_loop.py
new file mode 100644
index 00000000..528e943f
--- /dev/null
+++ b/src/backend/core/llm/runtime/stream_loop.py
@@ -0,0 +1,332 @@
+"""Drive one agent reply and yield its SSE events."""
+from __future__ import annotations
+from core.llm.runtime import streaming_constants
+from core.harness import log_writer
+import asyncio
+import time
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
+from agentscope.message import Msg
+from core.infra.logging import LogContext
+from core.llm.context.rendering import next_request_sequence, render_context_item, render_session_input
+from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER, SESSION_CONTEXT_META_KEY
+from core.llm.context.items import ContextItem
+from core.llm.runtime.message_compat import session_to_msgs
+import logging
+logger = logging.getLogger(__name__)
+
+async def stream(
+ self,
+ session_messages: List[Dict[str, Any]],
+ context: Dict[str, Any],
+ *,
+ effective_user_message: Optional[str] = None,
+) -> AsyncIterator[Tuple[str, Any]]:
+ agent = self.agent
+
+ # Commands keep their short spelling in persisted history, while this
+ # request must execute the expanded instructions. Copy the current row
+ # so neither stored history nor earlier user turns are rewritten.
+ if effective_user_message is not None:
+ session_messages = list(session_messages)
+ if session_messages and session_messages[-1].get("role") in ("user", "human"):
+ session_messages[-1] = {
+ **session_messages[-1],
+ "content": effective_user_message,
+ }
+ else:
+ session_messages.append({"role": "user", "content": effective_user_message})
+
+ _last_user_text = ""
+ if session_messages:
+ for _m in reversed(session_messages):
+ if _m.get("role") in ("user", "human"):
+ _last_user_text = str(_m.get("content") or "")
+ break
+
+ # ctx → agent.state (AgentRuntimeState, replaces the 1.x agent._jx_context = ModelContext)
+ st = agent.state
+ try:
+ st.apply_request_context(context, _last_user_text)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[stream] set agent.state failed: %s", exc)
+ self._enable_thinking = bool(context.get("enable_thinking", True))
+
+ # Vision bridge: transcribe uploaded images *before* handing off to the model,
+ # so the UI can say "读图中" while it happens. Reading an image is a plain
+ # network wait of several seconds; folded inside the agent turn it is
+ # indistinguishable from the model thinking, and the user just watches a
+ # generic spinner count up. FileContextMiddleware injects the result.
+ async for _vision_event in self._prepare_vision_evidence(st):
+ yield _vision_event
+
+ _log_ctx = LogContext(user_id=st.user_id or None, chat_id=st.chat_id or None)
+ _log_ctx.__enter__()
+
+ # Load history into context (excluding the last user message — reply_stream's inputs carries it, avoiding duplication)
+ history = list(session_messages)
+ last_user_content = ""
+ last_user_row = None
+ if history and history[-1].get("role") in ("user", "human"):
+ last_user_row = history.pop()
+ last_user_content = last_user_row.get("content", "")
+ if history:
+ try:
+ agent.state.context.extend(session_to_msgs(history))
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[stream] load history failed: %s", exc)
+
+ user_msg: Optional[Msg] = None
+ if last_user_content:
+ request_seq = next_request_sequence(agent.state.context)
+ if isinstance((last_user_row or {}).get(SESSION_CONTEXT_META_KEY), dict):
+ user_msg = render_session_input(last_user_row, created_seq=request_seq)
+ else:
+ user_msg = render_context_item(
+ ContextItem.create(
+ item_id=f"request:user_input:{request_seq}",
+ kind=KIND_USER_INPUT,
+ origin="user:chat",
+ trust="user",
+ visibility="model",
+ priority=1_000,
+ token_budget=100_000,
+ truncation_policy=POLICY_NEVER,
+ content=last_user_content,
+ cache_class="dynamic",
+ created_seq=request_seq,
+ render_role="user",
+ render_name="user",
+ message_group=f"request:user_input:{request_seq}",
+ )
+ )
+
+ # myspace write-confirmation gate (in-house ContextVar, distinct from 2.0 native HITL)
+ from core.llm.tools import confirmation as _mc
+
+ _confirm_chat_id = st.chat_id or None
+
+ def _drain_confirm_signals() -> list:
+ out: list = []
+ q = _mc.get_ui_queue(_confirm_chat_id)
+ if q is None:
+ return out
+ while True:
+ try:
+ out.append(q.get_nowait())
+ except asyncio.QueueEmpty:
+ break
+ return out
+
+ from core.llm.tools import user_questions as _user_questions
+
+ def _drain_user_question_signals() -> list:
+ out: list = []
+ q = _user_questions.get_ui_queue(_confirm_chat_id)
+ if q is None:
+ return out
+ while True:
+ try:
+ out.append(q.get_nowait())
+ except asyncio.QueueEmpty:
+ break
+ return out
+
+ def _drain_human_interaction_events() -> list:
+ """Map every queued human-interaction signal to a stream event."""
+
+ events: list = []
+ for signal in _drain_confirm_signals():
+ event_type = (
+ "design_pick"
+ if (signal or {}).get("kind") == _mc.KIND_DESIGN_PICK
+ else "file_confirm"
+ )
+ events.append((event_type, signal))
+ for signal in _drain_user_question_signals():
+ signal_event = signal.get("event")
+ if signal_event == "requested":
+ events.append(("user_question", signal))
+ elif signal_event == "resolved":
+ events.append(("user_question_resolved", signal))
+ return events
+
+ # reply_stream runs as a background task feeding the queue; the main loop drains confirm signals concurrently
+ event_q: asyncio.Queue = asyncio.Queue()
+ _DONE = object()
+
+ # Subagent streaming bypass: register this run's event_q so call_subagent (separate thread)
+ # can deliver the subagent's thinking/tool_call/... events back into this main queue in real time.
+ from core.llm.subagents import stream as _subagent_stream
+
+ _subagent_stream.attach(st.chat_id, asyncio.get_running_loop(), event_q)
+
+ async def _produce():
+ try:
+ async for ev in agent.reply_stream(inputs=user_msg):
+ result_step = None
+ if type(ev).__name__ == "ToolResultEndEvent":
+ tid = getattr(ev, "tool_call_id", "") or ""
+ # Snapshot while the producer is still at the tool
+ # boundary, before another model step/steer/compaction
+ # can replace the live context.
+ result_step = await self._tool_result_step(tid)
+ if result_step is None:
+ raise RuntimeError(f"Tool result {tid} is missing from model context")
+ await event_q.put(("ev", (ev, result_step)))
+ except asyncio.CancelledError:
+ raise
+ except BaseException as e: # noqa: BLE001
+ import traceback
+
+ logger.error("Agent reply_stream failed: %r\n%s", e, traceback.format_exc())
+ await event_q.put(("err", e))
+ finally:
+ await event_q.put(("done", _DONE))
+
+ prod_task = asyncio.create_task(_produce())
+ self._producer_task = prod_task
+
+ _stream_start = time.monotonic()
+ _first_event_logged = False
+ _poll_interval = streaming_constants.QUEUE_POLL_INTERVAL_S
+ # Last model_progress emission — throttles the liveness signal emitted
+ # when upstream events map to nothing. Only the swallowed-event branch
+ # reads/updates the clock, keeping the mapped hot path free of it.
+ _last_progress_ts = _stream_start
+
+ # Every event leaves through the coalescer, so a buffered delta is always
+ # flushed before whatever follows it and the stream keeps its order.
+ deltas = self._deltas
+ try:
+ while True:
+ for progress in st.pending_mcp_progress:
+ if progress.get("tool_id") in self._pending_tool_calls:
+ for out in deltas.feed(("tool_progress", progress)):
+ yield out
+ st.pending_mcp_progress = []
+ for _human_event in _drain_human_interaction_events():
+ for out in deltas.feed(_human_event):
+ yield out
+ _timeout = _poll_interval
+ _due = deltas.deadline()
+ if _due is not None:
+ _timeout = min(_timeout, max(0.0, _due - time.monotonic()))
+ try:
+ kind, payload = await asyncio.wait_for(event_q.get(), timeout=_timeout)
+ except asyncio.TimeoutError:
+ _flushed = deltas.flush_due()
+ if _flushed:
+ for out in _flushed:
+ yield out
+ continue
+ if _due is not None:
+ continue
+ if self._model_call_inflight:
+ # Model call in flight with a silent wire — count as
+ # liveness (throttled) so the inactivity watchdog waits
+ # for the model client's own timeout/retry outcome
+ # instead of misjudging the run as hung.
+ _now = time.monotonic()
+ if _now - _last_progress_ts >= streaming_constants.MODEL_PROGRESS_MIN_INTERVAL_S:
+ _last_progress_ts = _now
+ yield ("model_progress", None)
+ continue
+ yield ("heartbeat", None)
+ continue
+ if kind in {"done", "err"}:
+ # A human answer wakes the tool before AgentScope queues its
+ # terminal event. Drain once more here so an immediately
+ # finishing reply cannot strand the frontend in submitted
+ # state waiting for user_question_resolved.
+ for _human_event in _drain_human_interaction_events():
+ for out in deltas.feed(_human_event):
+ yield out
+ for out in deltas.drain():
+ yield out
+ if kind == "err":
+ yield ("error", payload)
+ break
+ if kind == _subagent_stream.QUEUE_KIND:
+ # Subagent bypass events — passed straight through, never into _map_event.
+ if not _first_event_logged:
+ _first_event_logged = True
+ for out in deltas.feed(("subagent_event", payload)):
+ yield out
+ continue
+ # kind == "ev"
+ payload, result_step = payload
+ steer_delivery = getattr(agent.state, "steer_delivery", None)
+ if isinstance(steer_delivery, dict):
+ # SteerMiddleware appends the user instruction immediately
+ # before this next reasoning round. Emit one acknowledgement
+ # before mapping ModelCallStart so the queued-card UI can
+ # switch from "waiting" to "applied" deterministically.
+ agent.state.steer_delivery = None
+ for out in deltas.feed(("steer_applied", dict(steer_delivery))):
+ yield out
+ reasoning_protocol = self._take_reasoning_protocol()
+ if reasoning_protocol is not None:
+ for out in deltas.feed(("reasoning_protocol", reasoning_protocol)):
+ yield out
+ _mapped_any = False
+ async for out in self._map_event(payload, result_step=result_step):
+ if not _first_event_logged:
+ _ttfe = (time.monotonic() - _stream_start) * 1000
+ logger.info("[stream] TTFE: %.0fms, type=%s", _ttfe, out[0])
+ _first_event_logged = True
+ _mapped_any = True
+ yield out
+ if not _mapped_any and deltas.deadline() is None:
+ # The model is alive (an upstream event just arrived) but
+ # nothing was forwarded — typical of long tool-call-arg /
+ # suppressed-thinking streaming. Emit a throttled liveness
+ # signal so the inactivity watchdog doesn't misjudge a
+ # healthy run as hung.
+ _now = time.monotonic()
+ if _now - _last_progress_ts >= streaming_constants.MODEL_PROGRESS_MIN_INTERVAL_S:
+ _last_progress_ts = _now
+ yield ("model_progress", None)
+ except Exception as e: # noqa: BLE001
+ for out in deltas.drain():
+ yield out
+ yield ("error", e)
+ finally:
+ await self.stop_producer()
+ # Deregister the subagent bypass — prevents late events from being written into a finished run's event_q.
+ try:
+ _subagent_stream.detach(st.chat_id)
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.runtime.stream_loop:stream', fallback_error)
+ for _tid, _rec in list(self._pending_tool_calls.items()):
+ try:
+ _started_mono = _rec.get("started_monotonic")
+ _dur = int((time.monotonic() - _started_mono) * 1000) if _started_mono else None
+ log_writer.schedule_tool_call_write(
+ {
+ # user_id/chat_id are taken explicitly from agent.state — contextvars are
+ # unreliable in the stream() generator frame (the agent runs in the context
+ # snapshot of _produce's create_task, while tool results are written in the
+ # generator's consumer frame; the two contexts don't sync), so _context_ids
+ # cannot be relied on.
+ "user_id": st.user_id or None,
+ "chat_id": st.chat_id or None,
+ "tool_name": _rec.get("tool_name", "unknown"),
+ "tool_call_id": _tid,
+ "tool_args": _rec.get("tool_args"),
+ "tool_result": None,
+ "status": "failed",
+ "error_message": "no tool_result received (stream ended)",
+ "duration_ms": _dur,
+ "started_at": _rec.get("started_at"),
+ }
+ )
+ except Exception:
+ logger.debug("pending tool_call flush failed", exc_info=True)
+ self._pending_tool_calls.clear()
+ deltas.reset()
+ try:
+ _log_ctx.__exit__(None, None, None)
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.runtime.stream_loop:stream', fallback_error)
diff --git a/src/backend/core/llm/runtime/stream_text.py b/src/backend/core/llm/runtime/stream_text.py
new file mode 100644
index 00000000..bebd8ece
--- /dev/null
+++ b/src/backend/core/llm/runtime/stream_text.py
@@ -0,0 +1,63 @@
+"""Split visible answer text from thinking and detect tool error text."""
+from __future__ import annotations
+from typing import Tuple
+from core.infra.model_output import THINK_BLOCK_RE
+from core.llm.runtime import streaming_constants
+
+def looks_like_tool_error(content: str) -> bool:
+ if not content:
+ return False
+ s = content.strip()
+ if not s:
+ return False
+ head = s[:64]
+ if s.startswith(('{"error"', "{'error'", '{"ok": false', '{"ok":false')):
+ return True
+ if '"ok": false' in head or '"ok":false' in head:
+ return True
+ if s.startswith(("Error executing tool", "Error: ", "Traceback (most recent call last)")):
+ return True
+ if "validation error for" in head:
+ return True
+ if "不存在或已删除" in s or "无权访问" in head:
+ return True
+ if streaming_constants.HTTP_ERR_RE.search(s):
+ return True
+ return False
+
+
+def strip_thinking_answer(raw: str, enable_thinking: bool, in_thinking: bool) -> Tuple[str, bool]:
+ """Extract the user-visible "answer" portion from the accumulated raw text.
+
+ Returns (answer, new_in_thinking). When enable_thinking=True, returns as-is (the frontend
+ parses itself); otherwise suppresses every … span and returns only
+ the visible text.
+
+ 一次模型调用里可以出现**多段**思考链(ReAct 模型常在工具调用前再想一轮)。
+ 早先的实现只取"最后一个 之后的内容":当第二段思考刚开了
+ 还没闭合时,rfind 命中的是第一段的闭标签,于是 ``vis1正在生成的思考…``
+ 被整段当成正文吐出去 —— 网页端 enable_thinking=False 只在快速模式出现(模型
+ 本来就不思考),而定时任务这条路径恰恰是"模型照常思考、正文里不该有思考",
+ 于是思考过程就跟着落进了自动化任务的输出正文。
+
+ 改成"先删掉所有闭合块,再处理孤儿标签",未闭合的那段思考不会再漏出来。
+ """
+ if enable_thinking:
+ return raw, False
+ # 1) 闭合的思考块整段丢掉(可能有多段)
+ text = THINK_BLOCK_RE.sub("", raw)
+ # 2) 孤儿闭标签:有些服务端把开标签吃进 chat template,正文以裸思考开头,
+ # 最后一个 之前的都是思考。
+ last_end = text.rfind(" ")
+ if last_end != -1:
+ text = text[last_end + len(" ") :]
+ return text, False
+ # 3) 还没闭合的思考块:它之后的内容都还在思考里,不出正文
+ open_idx = text.find("")
+ if open_idx != -1:
+ return text[:open_idx], True
+ # 4) 整段没有任何标签:沿用上一次的判定(服务端预填开标签的形态下,
+ # 在 到来之前只能靠这一位把裸思考压住)
+ if in_thinking:
+ return "", True
+ return text, False
diff --git a/src/backend/core/llm/runtime/streaming.py b/src/backend/core/llm/runtime/streaming.py
new file mode 100644
index 00000000..d6e0a124
--- /dev/null
+++ b/src/backend/core/llm/runtime/streaming.py
@@ -0,0 +1,287 @@
+"""Wrap an AgentScope agent so it streams SSE events."""
+from __future__ import annotations
+from contextlib import aclosing
+from core.llm.runtime import stream_loop
+from core.llm.runtime import stream_events
+from core.llm.runtime.delta_coalescer import DeltaCoalescer
+import asyncio
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
+from agentscope.agent import Agent
+from core.llm.mcp.lifetime import MCPResource
+import logging
+logger = logging.getLogger(__name__)
+
+class StreamingAgent:
+ """Wraps an AgentScope 2.0 ``Agent`` to produce streaming SSE events via reply_stream."""
+
+ def __init__(
+ self,
+ agent: Agent,
+ mcp_clients: List[MCPResource],
+ ):
+ self._producer_task = None
+ self.agent = agent
+ self.mcp_clients = mcp_clients
+ self._enable_thinking = False
+ # usage: accumulated from ModelCallEndEvent
+ self._usage_records: List[Dict[str, int]] = []
+ self._context_usage: Optional[Dict[str, Any]] = None
+ # tool_id → {tool_name, tool_args(str), started_monotonic, started_at}
+ self._pending_tool_calls: Dict[str, Dict[str, Any]] = {}
+ # tool_id → name (recorded at ToolCallStart), tool_id → accumulated args string (ToolCallDelta)
+ self._tool_name_buf: Dict[str, str] = {}
+ self._tool_args_buf: Dict[str, str] = {}
+ # Merges text / thinking / tool-argument deltas before they leave _map_event.
+ self._deltas = DeltaCoalescer()
+ # tool_id → accumulated result text (ToolResultTextDelta)
+ self._tool_result_buf: Dict[str, str] = {}
+ # Accumulated answer text (for dedup in the -suppression scenario)
+ self._raw_text = ""
+ self._emitted_answer = ""
+ # 首轮正文累计(enable_thinking 下):轮次结束时若整轮没有 ,
+ # 说明该模型不是"内联思考"形态(结构化 reasoning 通道、或本轮无思考)——
+ # 补发 structured_reasoning 标记,让前端把误缓冲成"思考"的正文流回正文区。
+ self._first_round_text = ""
+ self._round_index = 0
+ self._structured_marker_sent = False
+ self._in_thinking = False
+ self._reasoning_protocol_emitted = False
+ # True between ModelCallStartEvent and ModelCallEndEvent. While a call is
+ # in flight the LLM endpoint may legitimately produce nothing for minutes
+ # (long prefill on a huge prompt / hung gateway being retried); the queue
+ # poll then emits model_progress instead of heartbeat so the run
+ # inactivity watchdog doesn't kill a run that is merely waiting on the
+ # model — the model client's own httpx timeouts remain the backstop.
+ self._model_call_inflight = False
+
+ def _drain_model_steps(self) -> List[Dict[str, Any]]:
+ state = getattr(self.agent, "state", None)
+ pending = getattr(state, "pending_model_steps", None)
+ if not pending:
+ return []
+ drained = list(pending)
+ state.pending_model_steps = []
+ return drained
+
+ async def _tool_result_step(self, tool_call_id: str) -> Optional[Dict[str, Any]]:
+ """Record the tool result exactly as AgentScope saved it into context.
+
+ ``_execute_tool_call`` appends the (already bounded) ToolResultBlock to
+ the last assistant message before it yields ``ToolResultEndEvent``, so
+ by the time the event reaches this consumer the block is there.
+
+ Recording a result that carries media writes those bytes to the blob
+ table (see :mod:`core.llm.runtime.tool_media_store`), so the work goes to a
+ thread instead of blocking the event loop.
+ """
+ from core.llm.runtime.model_steps import record_tool_result_step
+
+ context = getattr(getattr(self.agent, "state", None), "context", None) or []
+ last = context[-1] if context else None
+ if last is None or getattr(last, "role", "") != "assistant":
+ logger.error("tool result %s ended with no assistant message in context", tool_call_id)
+ return None
+ match = None
+ for block in last.get_content_blocks("tool_result"):
+ if getattr(block, "id", "") == tool_call_id:
+ match = block
+ if match is None:
+ logger.error("tool result %s ended but its block is not in context", tool_call_id)
+ return None
+ return await asyncio.to_thread(record_tool_result_step, match)
+
+ def _take_reasoning_protocol(self) -> Optional[Dict[str, bool]]:
+ """Return the structured-reasoning marker once the active model is known.
+
+ DynamicModelMiddleware may replace ``agent.model`` at reply start, so this is
+ evaluated immediately before mapping the first AgentScope event rather than in
+ ``__init__``.
+ """
+ if self._reasoning_protocol_emitted:
+ return None
+ model = getattr(self.agent, "model", None)
+ if not bool(getattr(model, "structured_reasoning", False)):
+ return None
+ self._reasoning_protocol_emitted = True
+ return {"structured_reasoning": True}
+
+ def get_usage(self) -> Dict[str, int]:
+ """Return the in-memory fallback; async production paths use aget_usage."""
+ total_prompt = sum(r.get("prompt_tokens", 0) for r in self._usage_records)
+ total_completion = sum(r.get("completion_tokens", 0) for r in self._usage_records)
+ last = self._usage_records[-1] if self._usage_records else {}
+ return {
+ "prompt_tokens": total_prompt,
+ "completion_tokens": total_completion,
+ "total_tokens": total_prompt + total_completion,
+ "llm_call_count": len(self._usage_records),
+ "context_tokens": int(last.get("prompt_tokens", 0) or 0)
+ + int(last.get("completion_tokens", 0) or 0),
+ "last_prompt_tokens": int(last.get("prompt_tokens", 0) or 0),
+ "last_completion_tokens": int(last.get("completion_tokens", 0) or 0),
+ }
+
+ async def aget_usage(self) -> Dict[str, int]:
+ """Derive usage from append-only attempts without blocking the event loop."""
+ run_id = str(getattr(getattr(self.agent, "state", None), "run_id", "") or "")
+ if run_id:
+ try:
+ from core.harness.ledger import HarnessUsageLedger
+
+ attempts = await asyncio.wait_for(
+ asyncio.to_thread(HarnessUsageLedger().attempts, run_id),
+ timeout=1.0,
+ )
+ model_attempts = tuple(row for row in attempts if row.kind == "model")
+ if model_attempts:
+ primary_attempts = tuple(
+ row
+ for row in model_attempts
+ if row.metadata.get("source") not in {"vision", "compaction", "followup"}
+ )
+ last = (primary_attempts or model_attempts)[-1]
+ return {
+ "prompt_tokens": sum(row.usage.prompt_tokens for row in model_attempts),
+ "completion_tokens": sum(
+ row.usage.completion_tokens for row in model_attempts
+ ),
+ "cache_read_tokens": sum(
+ row.usage.cache_read_tokens for row in model_attempts
+ ),
+ "cache_write_tokens": sum(
+ row.usage.cache_write_tokens for row in model_attempts
+ ),
+ "total_tokens": sum(row.usage.total_tokens for row in model_attempts),
+ "llm_call_count": len(model_attempts),
+ "context_tokens": last.usage.prompt_tokens + last.usage.completion_tokens,
+ "last_prompt_tokens": last.usage.prompt_tokens,
+ "last_completion_tokens": last.usage.completion_tokens,
+ }
+ except Exception:
+ logger.debug("usage ledger aggregate unavailable", exc_info=True)
+ return self.get_usage()
+
+ def get_context_usage(self, usage: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+ """Return the latest provider measurement or canonical fallback."""
+ if self._context_usage and self._context_usage.get("exact"):
+ return dict(self._context_usage)
+ last = self._usage_records[-1] if self._usage_records else {}
+ if usage:
+ prompt = int(usage.get("last_prompt_tokens", 0) or 0)
+ completion = int(usage.get("last_completion_tokens", 0) or 0)
+ else:
+ prompt = int(last.get("prompt_tokens", 0) or 0)
+ completion = int(last.get("completion_tokens", 0) or 0)
+ from core.llm.context.usage import build_context_usage_snapshot
+
+ snapshot = build_context_usage_snapshot(
+ self.agent,
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ model_call_index=len(self._usage_records),
+ )
+ self._context_usage = snapshot
+ return dict(snapshot)
+
+ def restore_context_usage(self, snapshot: Optional[Dict[str, Any]]) -> None:
+ """Restore a snapshot after a transient call that is not replayed next turn."""
+ self._context_usage = dict(snapshot) if snapshot else None
+
+ async def _prepare_vision_evidence(self, st: Any) -> AsyncIterator[Tuple[str, Any]]:
+ """Transcribe this turn's image attachments, emitting progress around the wait.
+
+ Yields ``("vision_progress", {...})`` when there is actually something to
+ read, so the frontend can label the wait "图像理解中" instead of the generic
+ turn spinner. Silent (no events, no delay) when the turn has no images or the
+ running model reads images natively — in the native case the picture goes to
+ the model as-is and no transcription happens at all.
+
+ Never raises: a failure here degrades to the middleware's inline path.
+ """
+ try:
+ from core.llm.context.attachment_hook import supports_native_images
+ from core.models.vision.attachments import image_attachments, transcribe_attachments
+
+ from core.files import artifact_bytes
+ images = image_attachments(list(getattr(st, "uploaded_files", None) or []), is_image=artifact_bytes.is_image)
+ if not images or supports_native_images(getattr(st, "model_provider_id", "") or ""):
+ return
+
+ yield (
+ "vision_progress",
+ {
+ "status": "running",
+ "count": len(images),
+ "names": [f.get("name") or "图片" for f in images],
+ },
+ )
+ from core.models.usage import model_usage_scope
+ from core.harness.ledger import HarnessUsageLedger
+
+ run_id = str(getattr(st, "run_id", "") or "")
+ with model_usage_scope(run_id, HarnessUsageLedger()):
+ result = await transcribe_attachments(
+ images, download_bytes=artifact_bytes.download_artifact_bytes,
+ user_id=getattr(st, "user_id", None),
+ )
+ if result is not None and result.text:
+ st.vision_evidence_text = result.text
+ yield (
+ "vision_progress",
+ {
+ "status": "done",
+ "count": result.count if result else 0,
+ "ok": bool(result and result.ok),
+ "duration_seconds": round(result.duration_seconds, 2) if result else 0.0,
+ },
+ )
+ except Exception as exc: # noqa: BLE001 — fall back to inline transcription
+ logger.warning("[stream] vision pre-pass failed, deferring to middleware: %s", exc)
+ yield ("vision_progress", {"status": "done", "count": 0, "ok": False})
+
+ async def stream(
+ self,
+ session_messages: List[Dict[str, Any]],
+ context: Dict[str, Any],
+ *,
+ effective_user_message: Optional[str] = None,
+ ) -> AsyncIterator[Tuple[str, Any]]:
+ async with aclosing(stream_loop.stream(self, session_messages, context, effective_user_message=effective_user_message)) as events:
+ async for event in events:
+ yield event
+
+ async def _map_event(
+ self, ev: Any, *, result_step: Optional[Dict[str, Any]] = None
+ ) -> AsyncIterator[Tuple[str, Any]]:
+ """Map one AgentScope event, with deltas merged by ``self._deltas``.
+
+ A merged delta may still be buffered when this returns; whoever drives
+ the stream flushes it through ``self._deltas`` (on its deadline, before
+ its own events, and at the end).
+ """
+ async with aclosing(stream_events.map_event(self, ev, result_step=result_step)) as events:
+ async for event in events:
+ for out in self._deltas.feed(event):
+ yield out
+
+ async def stop_producer(self):
+ """Stop actual model/tool work before closing the stream or its clients."""
+ from core.infra.task_lifecycle import cancel_and_join
+
+ task = self._producer_task
+ if task is not None:
+ await cancel_and_join(task)
+ if self._producer_task is task:
+ self._producer_task = None
+
+ async def shutdown(self):
+ """Close transient (per-request) MCP clients."""
+ from core.llm.mcp.manager import close_clients
+
+ try:
+ await self.stop_producer()
+ finally:
+ try:
+ await close_clients(self.mcp_clients)
+ finally:
+ self.mcp_clients = []
diff --git a/src/backend/core/llm/runtime/streaming_constants.py b/src/backend/core/llm/runtime/streaming_constants.py
new file mode 100644
index 00000000..d60a3272
--- /dev/null
+++ b/src/backend/core/llm/runtime/streaming_constants.py
@@ -0,0 +1,20 @@
+"""Intervals and patterns used while streaming."""
+from __future__ import annotations
+import re
+
+HTTP_ERR_RE = re.compile(r"HTTP\s*[45]\d\d")
+
+
+MODEL_PROGRESS_MIN_INTERVAL_S = 5.0
+
+
+QUEUE_POLL_INTERVAL_S = 3.0
+
+
+# Streaming deltas (answer text, thinking, tool arguments) are merged by
+# ``delta_coalescer`` into at most one event per interval, or sooner once this
+# many characters wait.
+DELTA_FLUSH_INTERVAL_S = 0.05
+
+
+DELTA_FLUSH_CHARS = 256
diff --git a/src/backend/orchestration/tool_callbacks.py b/src/backend/core/llm/runtime/tool_callbacks.py
similarity index 70%
rename from src/backend/orchestration/tool_callbacks.py
rename to src/backend/core/llm/runtime/tool_callbacks.py
index 73c909e9..2f89c355 100644
--- a/src/backend/orchestration/tool_callbacks.py
+++ b/src/backend/core/llm/runtime/tool_callbacks.py
@@ -1,20 +1,4 @@
-"""Tool-call soft warnings (observation only — never blocks).
-
-Historical background: after the AgentScope migration, tool progress goes through
-routing/streaming.py's msg_queue polling, and the old callback mechanism was deprecated.
-This module is revived as **observation only**: once the code-execution capability is
-enabled in all modes (CODE_CAPABILITY_ENABLED), the default tool set grows larger, and the
-model may waver between semantically overlapping tools (docs/code-execution-merge-proposal
-§3.4 / §5.5). Here, during the gradual rollout, we record suspicious co-occurrences to help
-diagnose whether the model goes off track. **Does not block any call**, and never raises.
-
-Usage (called once per tool_call by the streaming layer):
-
- from orchestration.tool_callbacks import note_tool_call
- note_tool_call(state, tool_name, tool_args)
-
-``state`` is an arbitrary per-run mutable dict (held by StreamingAgent); this module manages its own keys.
-"""
+"Tool-call soft warnings (observation only — never blocks).\n\nHistorical background: after the AgentScope migration, tool progress goes through\nrouting/streaming.py's msg_queue polling, and the old callback mechanism was deprecated.\nThis module is revived as **observation only**: once the code-execution capability is\nenabled in all modes (CODE_CAPABILITY_ENABLED), the default tool set grows larger, and the\nmodel may waver between semantically overlapping tools (docs/code-execution-merge-proposal\n§3.4 / §5.5). Here, during the gradual rollout, we record suspicious co-occurrences to help\ndiagnose whether the model goes off track. **Does not block any call**, and never raises.\n\nUsage (called once per tool_call by the streaming layer):\n\n from core.llm.runtime.tool_callbacks import note_tool_call\n note_tool_call(state, tool_name, tool_args)\n\n``state`` is an arbitrary per-run mutable dict (held by StreamingAgent); this module manages its own keys.\n"
from __future__ import annotations
@@ -69,8 +53,7 @@ def note_tool_call(
sorted(seen),
)
- if "Read" in seen and "view_text_file" in seen \
- and "read_vs_view" not in warned:
+ if "Read" in seen and "view_text_file" in seen and "read_vs_view" not in warned:
warned.add("read_vs_view")
logger.warning(
"[tool-soft-warn] Read 与 view_text_file 同一 run 共现 "
@@ -83,7 +66,8 @@ def note_tool_call(
logger.warning(
"[tool-soft-warn] %s 写 /myspace 路径 %s "
"—— 合并后最高风险点(§13 硬保险负责拦截),灰度观测",
- tool_name, p,
+ tool_name,
+ p,
)
break
except Exception: # noqa: BLE001 — observation must never affect the main flow
diff --git a/src/backend/core/llm/runtime/tool_execution_adapter.py b/src/backend/core/llm/runtime/tool_execution_adapter.py
new file mode 100644
index 00000000..07de06c5
--- /dev/null
+++ b/src/backend/core/llm/runtime/tool_execution_adapter.py
@@ -0,0 +1,210 @@
+"""SDK stream and payload translation for the injected durable tool executor."""
+import asyncio
+import json
+import logging
+import time
+from contextlib import suppress
+from typing import Any
+from agentscope.message import ToolResultState
+from agentscope.tool._response import ToolChunk, ToolResponse
+from core.harness.tool_call_context import CURRENT_RUN_BINDING
+from core.llm.context.rendering import render_text_block
+
+logger = logging.getLogger(__name__)
+
+
+class DurableToolDispatch:
+ """Commit durable Intent before every harness-owned tool invocation.
+
+ The final ``ToolResponse`` is committed before it leaves this middleware,
+ so the existing AgentScope event/SSE mapping remains unchanged while the
+ database becomes authoritative for replay and reconciliation.
+ """
+
+ def __init__(self, session_factory=None, registry=None) -> None: # noqa: ANN001
+ from core.db.engine import SessionLocal
+ from core.llm.ports import tool_effect_runtime
+
+ self._gateway = tool_effect_runtime(session_factory, registry)
+ from core.harness.ledger import HarnessUsageLedger
+
+ self._usage_ledger = HarnessUsageLedger(session_factory or SessionLocal)
+
+ @staticmethod
+ def _args(tool_call) -> dict: # noqa: ANN001
+ raw = getattr(tool_call, "input", "") or "{}"
+ try:
+ parsed = json.loads(raw) if isinstance(raw, str) else raw
+ except (TypeError, json.JSONDecodeError):
+ parsed = {"_raw": str(raw)}
+ return parsed if isinstance(parsed, dict) else {"_value": parsed}
+
+ @staticmethod
+ def _response_payload(response: ToolResponse) -> dict:
+ return {"tool_response": response.model_dump(mode="json")}
+
+ @staticmethod
+ def _response_from_payload(payload: Any) -> ToolResponse:
+ from agentscope.message import DataBlock, TextBlock
+
+ raw = payload.get("tool_response") if isinstance(payload, dict) else None
+ if not isinstance(raw, dict):
+ return ToolResponse(
+ content=[render_text_block(json.dumps(payload, ensure_ascii=False, default=str))],
+ state=ToolResultState.SUCCESS,
+ )
+ blocks = []
+ for item in raw.get("content") or []:
+ if not isinstance(item, dict):
+ blocks.append(render_text_block(str(item)))
+ elif item.get("type") == "data":
+ blocks.append(DataBlock.model_validate(item))
+ else:
+ blocks.append(TextBlock.model_validate(item))
+ state = ToolResultState(str(raw.get("state") or "success"))
+ return ToolResponse(
+ id=str(raw.get("id") or ""),
+ content=blocks,
+ state=state,
+ metadata=dict(raw.get("metadata") or {}),
+ )
+
+ async def stream(self, agent: Any, input_kwargs: dict, next_handler): # noqa: ANN001
+ run_id = str(getattr(agent.state, "run_id", "") or "")
+ run_owner = str(getattr(agent.state, "journal_owner", "") or "")
+ tool_call = input_kwargs.get("tool_call")
+ tool_name = str(getattr(tool_call, "name", "") or "")
+ tool_call_id = str(getattr(tool_call, "id", "") or "")
+ if not run_id or not run_owner or not tool_name:
+ raise self._gateway.binding_error(
+ "tool execution requires a durable run_id and journal_owner binding"
+ )
+
+ args = self._args(tool_call)
+ from core.harness.registry.tool_operations import prepare_tool_arguments
+ args = prepare_tool_arguments(tool_name, args, str(getattr(agent.state, "user_id", "") or ""),
+ str(getattr(agent.state, "chat_id", "") or ""))
+ live_chunks: asyncio.Queue = asyncio.Queue()
+
+ async def _invoke():
+ from core.harness.usage import (
+ UsageAttempt,
+ attempt_status_for_exception,
+ record_usage_safely,
+ )
+ from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT
+
+ final = None
+ started = time.monotonic()
+ usage_status = "failed"
+ binding_token = CURRENT_RUN_BINDING.set((run_id, run_owner))
+ invoke_kwargs = input_kwargs
+ effect = CURRENT_TOOL_EFFECT.get()
+ if effect is not None and tool_name in {
+ "create_scheduled_task",
+ "update_scheduled_task",
+ "delete_scheduled_task",
+ }:
+ adapter_args = dict(args)
+ adapter_args["tool_effect_id"] = effect.effect_id
+ adapter_call = tool_call.model_copy(
+ update={"input": json.dumps(adapter_args, ensure_ascii=False)}
+ )
+ invoke_kwargs = {**input_kwargs, "tool_call": adapter_call}
+ try:
+ outcome_unknown = False
+ async for item in next_handler(**invoke_kwargs):
+ # Toolkit normally converts adapter exceptions into ordinary
+ # error responses. An ambiguous remote write must retain its
+ # ledger meaning across that conversion and stop this run.
+ ambiguous = bool(
+ (getattr(item, "metadata", None) or {}).get("gateway_outcome_unknown")
+ )
+ outcome_unknown = outcome_unknown or ambiguous
+ if isinstance(item, ToolResponse):
+ final = item
+ elif not ambiguous:
+ await live_chunks.put(item)
+ if outcome_unknown:
+ raise RuntimeError("cloud write outcome unknown; reconciliation required")
+ usage_status = (
+ "success"
+ if final is not None and final.state == ToolResultState.SUCCESS
+ else "failed"
+ )
+ except asyncio.CancelledError:
+ usage_status = "cancelled"
+ raise
+ except Exception as exc:
+ usage_status = attempt_status_for_exception(exc)
+ raise
+ finally:
+ CURRENT_RUN_BINDING.reset(binding_token)
+ if effect is not None:
+ try:
+ await record_usage_safely(
+ self._usage_ledger,
+ UsageAttempt(
+ run_id=run_id,
+ kind="tool",
+ operation_name=tool_name,
+ effect_id=effect.effect_id,
+ status=usage_status,
+ latency_ms=int((time.monotonic() - started) * 1_000),
+ metadata={"tool_call_id": tool_call_id},
+ ),
+ )
+ except Exception: # usage cannot change tool execution
+ logger.debug("tool usage attempt persistence failed", exc_info=True)
+ if final is None:
+ raise self._gateway.response_error(f"tool {tool_name} ended without ToolResponse")
+ return self._response_payload(final)
+
+ def _response_failed(payload: Any) -> bool:
+ raw = payload.get("tool_response") if isinstance(payload, dict) else None
+ return bool(isinstance(raw, dict) and str(raw.get("state") or "success") != "success")
+
+ task = asyncio.create_task(
+ self._gateway.execute_outcome(
+ run_id=run_id,
+ owner=run_owner,
+ tool_call_id=tool_call_id,
+ tool_name=tool_name,
+ args=args,
+ invoke=_invoke,
+ classify_failure=_response_failed,
+ )
+ )
+ try:
+ while not task.done() or not live_chunks.empty():
+ try:
+ item = await asyncio.wait_for(live_chunks.get(), timeout=0.02)
+ except asyncio.TimeoutError:
+ continue
+ yield item
+ outcome = await task
+ finally:
+ if not task.done():
+ task.cancel()
+ with suppress(BaseException):
+ await task
+ elif not task.cancelled():
+ task.exception()
+ response = self._response_from_payload(outcome.result)
+ if outcome.result_state == "error" and response.state == ToolResultState.SUCCESS:
+ response.state = ToolResultState.ERROR
+ linkage = {
+ "effect_id": outcome.effect_id,
+ "result_id": outcome.result_id,
+ }
+ response.metadata = {**dict(response.metadata or {}), **linkage}
+ links = getattr(agent.state, "tool_effect_links", None)
+ if isinstance(links, dict):
+ links[tool_call_id] = linkage
+ if not outcome.invoked:
+ yield ToolChunk(
+ content=list(response.content),
+ state=response.state,
+ metadata=dict(response.metadata),
+ )
+ yield response
diff --git a/src/backend/core/llm/runtime/tool_hook_adapter.py b/src/backend/core/llm/runtime/tool_hook_adapter.py
new file mode 100644
index 00000000..71973e92
--- /dev/null
+++ b/src/backend/core/llm/runtime/tool_hook_adapter.py
@@ -0,0 +1,81 @@
+"""Translate neutral tool decisions; arguments remain immutable after admission."""
+from agentscope.tool._response import ToolResponse
+from core.harness.events import thaw_value
+from core.harness.hooks import DecisionKind, HookRejected, HookPaused, HookStage, Invocation
+from core.llm.runtime.context_hook_adapter import append_hook_context
+
+
+async def _tool_admission(adapter, agent, tool_call, args, plain):
+ state = agent.state
+ outcome = await adapter.bus.run(Invocation.create(
+ run_id=adapter._run_id(agent), stage=HookStage.BEFORE_TOOL,
+ operation_name=str(getattr(tool_call, "name", "") or "tool"),
+ data={"tool_args": args, "raw_input": plain(getattr(tool_call, "input", None)),
+ "ontology_runtime": plain(getattr(state, "ontology_runtime", None)),
+ "identity": {"journal_owner": getattr(state, "journal_owner", ""), "user_id": getattr(state, "user_id", ""), "chat_id": getattr(state, "chat_id", "")},
+ "context_items": [], "metadata": {"tool_call_id": getattr(tool_call, "id", "")}},
+ ))
+ updated = outcome.invocation
+ if updated.data.get("ontology_runtime") is not None:
+ state.ontology_runtime = thaw_value(updated.data["ontology_runtime"])
+ state.ontology_enabled = bool(state.ontology_runtime.get("enabled"))
+ append_hook_context(agent, updated)
+ response = updated.data.get("tool_response")
+ if outcome.decision.kind == DecisionKind.REJECT:
+ if response is None:
+ raise HookRejected(outcome.decision.reason)
+ return ToolResponse.model_validate(thaw_value(response))
+ if outcome.decision.kind == DecisionKind.PAUSE:
+ raise HookPaused(outcome.decision.reason)
+ return None
+
+
+async def permission_dispatch(adapter, agent, input_kwargs, next_handler):
+ from contextlib import aclosing, nullcontext
+ call_id = str(getattr(input_kwargs.get("tool_call"), "id", "") or "")
+ policy = adapter.permission_policy
+ scope = policy.bind(call_id) if policy is not None else nullcontext(None)
+ async def dispatch(**kwargs):
+ async with aclosing(execute_tool(adapter, agent, kwargs, next_handler)) as executed:
+ async for event in executed:
+ yield event
+ with scope as audit:
+ async with aclosing(dispatch(**input_kwargs)) as stream:
+ async for item in stream:
+ if isinstance(item, ToolResponse) and audit is not None:
+ item.metadata = {**dict(item.metadata or {}), "permission": audit}
+ yield item
+
+
+async def tool_admission(adapter, agent, tool_call, args, plain):
+ call_id = str(getattr(tool_call, "id", "") or "")
+ try:
+ result = await _tool_admission(adapter, agent, tool_call, args, plain)
+ except BaseException:
+ if adapter.permission_policy is not None:
+ adapter.permission_policy.discard(call_id)
+ raise
+ if result is not None and adapter.permission_policy is not None:
+ adapter.permission_policy.discard(call_id)
+ return result
+
+
+async def execute_tool(adapter, agent, input_kwargs, next_handler):
+ import asyncio
+ from contextlib import aclosing
+ from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID
+ call = input_kwargs.get("tool_call")
+ call_id = str(getattr(call, "id", "") or "")
+ prepared = getattr(agent, "_jx_prepared_capabilities", None)
+ if prepared is not None:
+ await asyncio.to_thread(prepared.validate, user_id=str(getattr(agent.state, "user_id", "") or ""))
+ await asyncio.to_thread(prepared.record_tool_scope, call_id, str(getattr(call, "name", "") or ""))
+ token = CURRENT_TOOL_CALL_ID.set(call_id)
+ try:
+ stream = (adapter.tool_dispatch.stream(agent, input_kwargs, next_handler)
+ if adapter.tool_dispatch is not None else next_handler(**input_kwargs))
+ async with aclosing(stream):
+ async for item in stream:
+ yield item
+ finally:
+ CURRENT_TOOL_CALL_ID.reset(token)
diff --git a/src/backend/core/llm/tool_media_store.py b/src/backend/core/llm/runtime/tool_media_store.py
similarity index 95%
rename from src/backend/core/llm/tool_media_store.py
rename to src/backend/core/llm/runtime/tool_media_store.py
index 724efeda..a1f12fc3 100644
--- a/src/backend/core/llm/tool_media_store.py
+++ b/src/backend/core/llm/runtime/tool_media_store.py
@@ -36,6 +36,7 @@
import hashlib
import logging
from collections import OrderedDict
+from dataclasses import dataclass
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence
from core.db.engine import SessionLocal
@@ -54,7 +55,6 @@
# 命中率本就有限,而这份缓存是每进程一份,调大等于按 worker 数翻倍占用常驻内存。
_CACHE_MAX_BYTES = 32 * 1024 * 1024
_cache: "OrderedDict[str, str]" = OrderedDict()
-_cache_bytes = 0
def _cache_get(digest: str) -> Optional[str]:
@@ -65,14 +65,13 @@ def _cache_get(digest: str) -> Optional[str]:
def _cache_put(digest: str, data_b64: str) -> None:
- global _cache_bytes
if digest in _cache:
return
_cache[digest] = data_b64
- _cache_bytes += len(data_b64)
- while _cache_bytes > _CACHE_MAX_BYTES and len(_cache) > 1:
+ process_state().cache_bytes += len(data_b64)
+ while process_state().cache_bytes > _CACHE_MAX_BYTES and len(_cache) > 1:
_, evicted = _cache.popitem(last=False)
- _cache_bytes -= len(evicted)
+ process_state().cache_bytes -= len(evicted)
def _data_block(block: Mapping[str, Any], source: Dict[str, Any]) -> Dict[str, Any]:
@@ -236,6 +235,17 @@ def resolved(block: Mapping[str, Any]) -> Dict[str, Any]:
def reset_cache() -> None:
"""测试用:清空取回缓存。"""
- global _cache_bytes
_cache.clear()
- _cache_bytes = 0
+ process_state().cache_bytes = 0
+
+
+@dataclass
+class ToolMediaStoreState:
+ cache_bytes: object = 0
+
+
+_STATE = ToolMediaStoreState()
+
+
+def process_state() -> ToolMediaStoreState:
+ return _STATE
diff --git a/src/backend/core/llm/runtime/tool_result_adapter.py b/src/backend/core/llm/runtime/tool_result_adapter.py
new file mode 100644
index 00000000..bc993eec
--- /dev/null
+++ b/src/backend/core/llm/runtime/tool_result_adapter.py
@@ -0,0 +1,53 @@
+"""SDK result buffering and conversion around neutral tool hooks."""
+import json
+from contextlib import aclosing
+from agentscope.tool._response import ToolChunk, ToolResponse
+from core.harness.events import thaw_value
+from core.harness.hooks import HookStage
+from core.llm.citations.hook import citation_candidate
+from core.llm.citations.allocator import resolve_allocator
+from core.llm.runtime.context_hook_adapter import append_hook_context
+from core.llm.runtime.tool_hook_adapter import tool_admission, permission_dispatch
+
+
+async def run_tool(adapter, agent, input_kwargs, next_handler, plain):
+ call = input_kwargs.get("tool_call")
+ raw = getattr(call, "input", "") or "{}"
+ try:
+ args = json.loads(raw) if isinstance(raw, str) else raw
+ except (TypeError, json.JSONDecodeError):
+ args = {"_raw": str(raw)}
+ name = str(getattr(call, "name", "") or "tool")
+ final = await tool_admission(adapter, agent, call, args, plain)
+ buffered = []
+ buffer_result = citation_candidate(name)
+ exposed = False
+ if final is not None:
+ buffered.append(ToolChunk(content=list(final.content), state=final.state, metadata=dict(final.metadata or {})))
+ else:
+ async with aclosing(permission_dispatch(adapter, agent, input_kwargs, next_handler)) as stream:
+ async for event in stream:
+ if isinstance(event, ToolResponse):
+ final = event
+ elif buffer_result:
+ buffered.append(event)
+ else:
+ exposed = True
+ yield event
+ if buffer_result:
+ resolve_allocator(agent)
+ observed = await adapter._stage(
+ HookStage.AFTER_TOOL, agent, operation_name=name,
+ data={"tool_result": plain(final), "context_items": [], "result_exposed": exposed,
+ "metadata": {"tool_call_id": str(getattr(call, "id", "") or "")}},
+ )
+ append_hook_context(agent, observed)
+ patched = thaw_value(observed.data.get("tool_result"))
+ if patched is not None and patched != plain(final):
+ final = ToolResponse.model_validate(patched)
+ yield ToolChunk(content=list(final.content), state=final.state, metadata=dict(final.metadata or {}))
+ else:
+ for event in buffered:
+ yield event
+ if final is not None:
+ yield final
diff --git a/src/backend/core/llm/runtime/turn_state.py b/src/backend/core/llm/runtime/turn_state.py
new file mode 100644
index 00000000..f3b69d78
--- /dev/null
+++ b/src/backend/core/llm/runtime/turn_state.py
@@ -0,0 +1,41 @@
+"""Per-turn trackers for artifact reads and pin hints."""
+from __future__ import annotations
+from contextvars import ContextVar
+from typing import Any, Dict, Optional
+
+def get_artifact_read_state() -> Dict[str, int]:
+ """Return the current per-turn read tracker, lazily initializing it.
+
+ Used by ``read_artifact`` to enforce the cumulative cap.
+ """
+ state = artifact_read_chars_per_turn.get()
+ if state is None:
+ state = {}
+ artifact_read_chars_per_turn.set(state)
+ return state
+
+
+def reset_artifact_read_state() -> None:
+ """Clear the per-turn read tracker. Called from file_context_pre_reply."""
+ artifact_read_chars_per_turn.set({})
+
+
+def reset_pin_hint_state() -> None:
+ """Clear the per-turn pin-reminder cache. Called at turn boundary."""
+ pin_hint_state.set({"seen": set(), "last_reminded_sig": ""})
+
+
+def get_pin_hint_state() -> Dict[str, Any]:
+ state = pin_hint_state.get()
+ if state is None:
+ state = {"seen": set(), "last_reminded_sig": ""}
+ pin_hint_state.set(state)
+ return state
+
+
+artifact_read_chars_per_turn: ContextVar[Optional[Dict[str, int]]] = ContextVar(
+ "_artifact_read_chars_per_turn", default=None
+)
+
+
+pin_hint_state: ContextVar[Optional[Dict[str, Any]]] = ContextVar("_pin_hint_state", default=None)
diff --git a/src/backend/core/llm/workspace.py b/src/backend/core/llm/runtime/workspace.py
similarity index 100%
rename from src/backend/core/llm/workspace.py
rename to src/backend/core/llm/runtime/workspace.py
diff --git a/src/backend/core/agent_skills/selector.py b/src/backend/core/llm/skill_selection.py
similarity index 91%
rename from src/backend/core/agent_skills/selector.py
rename to src/backend/core/llm/skill_selection.py
index f7bbf3a6..1a75dba0 100644
--- a/src/backend/core/agent_skills/selector.py
+++ b/src/backend/core/llm/skill_selection.py
@@ -10,18 +10,13 @@
from agentscope.model import OpenAIChatModel
-from core.agent_skills.loader import get_skill_loader
-from core.agent_skills.registry import AgentSkillMetadata
-from core.llm.context_adapter import AgentScopeContextAdapter
-from core.llm.context_ir import (
- KIND_SYSTEM_RULE,
- KIND_USER_INPUT,
- POLICY_HEAD_TAIL,
- POLICY_NEVER,
- ContextAssembler,
- make_text_context_item,
-)
-from core.llm.context_manager import FALLBACK_CONTEXT_WINDOW, usable_context_window
+from core.capabilities.skills.loader_factory import get_skill_loader
+from core.capabilities.skills.registry import AgentSkillMetadata
+from core.llm.context.adapter import AgentScopeContextAdapter
+from core.llm.context.ir_constants import KIND_SYSTEM_RULE, KIND_USER_INPUT, POLICY_HEAD_TAIL, POLICY_NEVER
+from core.llm.context.ir import ContextAssembler
+from core.llm.context.items import make_text_context_item
+from core.models.context_window import FALLBACK_CONTEXT_WINDOW, usable_context_window
def select_skills_for_query(
@@ -62,7 +57,7 @@ def select_skills_for_query(
return sorted([s.id for s in candidates])[:max_skills]
# Use LLM to select relevant skills
- return _llm_select_skills(
+ return rank_skill_candidates(
user_query=user_query,
candidates=candidates,
model=model,
@@ -70,7 +65,7 @@ def select_skills_for_query(
)
-def _llm_select_skills(
+def rank_skill_candidates(
*,
user_query: str,
candidates: List[AgentSkillMetadata],
diff --git a/src/backend/core/llm/steer_middleware.py b/src/backend/core/llm/steer_middleware.py
deleted file mode 100644
index 2309277b..00000000
--- a/src/backend/core/llm/steer_middleware.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""Deliver user steering at safe model/tool boundaries."""
-import asyncio
-from agentscope.agent import Agent
-from agentscope.message import ToolResultState
-from agentscope.middleware import MiddlewareBase
-from agentscope.tool._response import ToolChunk, ToolResponse
-from core.llm.context_adapter import append_context_text, render_text_block
-from core.llm.context_ir import KIND_STEER, POLICY_NEVER
-
-
-class SteerMiddleware(MiddlewareBase):
- """Deliver a queued user instruction at the next safe ReAct boundary.
-
- ``on_reasoning`` is the primary insertion point: AgentScope calls it after
- the previous tool results have entered context and immediately before the
- next model call. ``on_acting`` is the earlier fallback for a steer that
- arrives while the model is constructing its next tool call; it marks that
- not-yet-started tool batch interrupted so the next reasoning round can
- replan. Both paths preserve a valid assistant-tool-result-user order.
- """
-
- def __init__(self, take_pending=None, acknowledge=None) -> None:
- self._take_pending = take_pending
- self._acknowledge = acknowledge
- self._lock = asyncio.Lock()
- self._delivery: dict | None = None
- self._interrupted_tools = False
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- run_id = str(getattr(agent.state, "run_id", "") or "")
- if not run_id:
- async for item in next_handler(**input_kwargs):
- yield item
- return
-
- async with self._lock:
- if self._delivery is None:
- from core.services.chat_steer_service import take_pending_steer
-
- self._delivery = await (self._take_pending or take_pending_steer)(run_id)
- self._interrupted_tools = self._delivery is not None
- delivery = self._delivery
-
- if delivery is None:
- async for item in next_handler(**input_kwargs):
- yield item
- return
-
- notice = "用户追加了新指令;本工具调用已在执行前中止,等待模型按新指令重新规划。"
- block = render_text_block(notice)
- yield ToolChunk(content=[block], state=ToolResultState.INTERRUPTED)
- yield ToolResponse(content=[block], state=ToolResultState.INTERRUPTED)
-
- async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- run_id = str(getattr(agent.state, "run_id", "") or "")
- async with self._lock:
- # A steer can arrive while a long-running tool is executing. There
- # is no later on_acting hook in that round, so poll again here,
- # after AgentScope saved the tool result and before it starts the
- # next model call. This is the normal "insert after this tool"
- # path; without it the steer waits until another tool call (or the
- # whole run) finishes.
- if self._delivery is None and run_id:
- from core.services.chat_steer_service import take_pending_steer
-
- self._delivery = await (self._take_pending or take_pending_steer)(run_id)
- delivery = self._delivery
- interrupted_tools = self._interrupted_tools
- self._delivery = None
- self._interrupted_tools = False
-
- if delivery is not None:
- message = str(delivery.get("message") or "").strip()
- if message:
- append_context_text(
- agent,
- "[用户在当前执行中追加的新指令]\n"
- f"{message}\n"
- + (
- "请立即按这条新指令调整后续计划;不要继续已经被中止的旧工具调用。"
- if interrupted_tools
- else "请立即按这条新指令调整后续计划;上一轮工具结果已经完成,可按需使用。"
- ),
- kind=KIND_STEER,
- origin="user:steer",
- trust="user",
- priority=950,
- token_budget=4_000,
- truncation_policy=POLICY_NEVER,
- )
- if self._acknowledge is not None:
- await self._acknowledge(delivery)
- agent.state.steer_delivery = dict(delivery)
-
- async for item in next_handler(**input_kwargs):
- yield item
diff --git a/src/backend/core/llm/subagents/__init__.py b/src/backend/core/llm/subagents/__init__.py
new file mode 100644
index 00000000..015345e9
--- /dev/null
+++ b/src/backend/core/llm/subagents/__init__.py
@@ -0,0 +1 @@
+"""Sub-agents: the call tool, their sessions, prompts and streamed execution."""
diff --git a/src/backend/core/llm/_subagent_execution.py b/src/backend/core/llm/subagents/execution.py
similarity index 90%
rename from src/backend/core/llm/_subagent_execution.py
rename to src/backend/core/llm/subagents/execution.py
index 6a995fcf..628270ca 100644
--- a/src/backend/core/llm/_subagent_execution.py
+++ b/src/backend/core/llm/subagents/execution.py
@@ -10,21 +10,15 @@
import logging
import threading
import time
-import uuid
-from concurrent.futures import ThreadPoolExecutor
from typing import Any, Callable, Dict, List, Optional, Tuple
-from agentscope.message import TextBlock
-from agentscope.tool import Toolkit
# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
-from core.infra.task_lifecycle import settle_task
-from core.llm import subagent_sessions
-from core.llm.context_adapter import next_request_sequence, render_context_item
-from core.llm.context_ir import KIND_USER_INPUT, POLICY_NEVER, make_text_context_item
+from core.llm.context.rendering import next_request_sequence, render_context_item
+from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER
+from core.llm.context.items import make_text_context_item
from core.llm.tools._common import resolve_sandbox_session
-from core.services import log_service as log_writer
+from core.db.threads import run_in_session
logger = logging.getLogger(__name__)
@@ -209,7 +203,7 @@ def _run_subagent_in_thread(
or they never reach the user-visible assistant message.
When ``emit`` is provided, the sub-agent is consumed **in streaming mode**
- (``agent._reply``, event by event) and every thinking/tool_call/tool_result/content
+ (``agent.reply_stream``, event by event) and every thinking/tool_call/tool_result/content
sub-step is bypass-forwarded back to the main SSE stream via ``emit``; with ``emit``
None the behavior matches the old implementation (final text only). The returned text
is always taken from the reply's **final Msg** — fully equivalent to the old
@@ -218,19 +212,18 @@ def _run_subagent_in_thread(
async def _inner() -> str:
from agentscope.message import Msg
- from core.db.engine import SessionLocal
- from core.llm.builtin_subagents import build_builtin_runtime_profile, get_builtin_subagent
+ from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent
from core.llm.factory import create_agent_executor
- from core.llm.mcp_manager import close_clients
- from core.llm.message_compat import extract_messages_from_context, session_to_msgs
- from core.services.user_agent_service import UserAgentService
- from orchestration.streaming import StreamingAgent
+ from core.llm.mcp.manager import close_clients
+ from core.llm.runtime.message_compat import extract_messages_from_context, session_to_msgs
+ from core.llm.runtime.streaming import StreamingAgent
+ from core.capabilities.agents.edition import UserAgentService
builtin_spec = get_builtin_subagent(agent_id)
if builtin_spec is not None:
user_agent = build_builtin_runtime_profile(builtin_spec, parent_runtime)
else:
- with SessionLocal() as db:
+ def _load_user_agent(db):
svc = UserAgentService(db)
user_agent = svc.get_raw_by_id(agent_id, user_id=current_user_id)
_ = user_agent.mcp_server_ids, user_agent.skill_ids, user_agent.kb_ids
@@ -241,6 +234,8 @@ async def _inner() -> str:
user_agent.max_tokens,
user_agent.timeout,
)
+ return user_agent
+ user_agent = await run_in_session(_load_user_agent)
# A conversation is the only sandbox boundary. Every built-in and user-defined
# child agent inherits the parent's sandbox session and therefore sees the same
@@ -251,7 +246,7 @@ async def _inner() -> str:
runtime.get("sandbox_session_id"),
runtime.get("chat_id"),
)
- from core.services.local_project_workspace import child_project_context
+ from core.projects.local_workspace import child_project_context
agent, mcp_clients = await create_agent_executor(
user_agent=user_agent,
@@ -278,7 +273,7 @@ async def _inner() -> str:
# Initialize one so the sub-agent's pin_to_workspace has somewhere to land; after the
# run, get_pinned() hands the files back to the main context for re-pinning —
# otherwise files pinned to the workspace are lost and never shown in the main conversation.
- from core.llm import workspace as _ws
+ from core.llm.runtime import workspace as _ws
_ws.init_state()
@@ -323,15 +318,13 @@ async def _inner() -> str:
extract_messages_from_context(agent.state.context),
)
- # Streaming path: consume _reply directly (reply_stream would discard the final
- # Msg, which we need as the authoritative return text). Non-Msg events are mapped
- # to sub-steps and bypass-forwarded via emit.
+ # Public stream includes the final message for child result extraction.
final_msg = None
mapper = _SubMapper()
stream_mapper = StreamingAgent(agent, mcp_clients)
stream_mapper._enable_thinking = True
_last_progress = time.monotonic()
- async for chunk in agent._reply(inputs=user_msg):
+ async for chunk in agent.reply_stream(inputs=user_msg, yield_final_msg=True):
if isinstance(chunk, Msg):
final_msg = chunk
continue
@@ -350,6 +343,9 @@ async def _inner() -> str:
Exception
): # noqa: BLE001 — bypass mapping must never take down the sub-agent
logger.debug("subagent event map failed (ignored)", exc_info=True)
+ for kind, payload in stream_mapper._deltas.drain():
+ for sub in mapper.feed(kind, payload):
+ emit(sub)
response_text = (final_msg.get_text_content() if final_msg else "") or ""
return (
True,
@@ -373,8 +369,8 @@ async def _owned_inner():
return await _inner()
finally:
from core.infra.redis import close_redis
- from core.llm.chat_models import close_loop_http_clients
- from core.llm.hooks import release_loop_models
+ from core.models.chat_models import close_loop_http_clients
+ from core.llm.runtime.model_cache import release_loop_models
release_loop_models()
try:
diff --git a/src/backend/core/llm/_subagent_prompts.py b/src/backend/core/llm/subagents/prompts.py
similarity index 100%
rename from src/backend/core/llm/_subagent_prompts.py
rename to src/backend/core/llm/subagents/prompts.py
diff --git a/src/backend/core/llm/subagent_sessions.py b/src/backend/core/llm/subagents/sessions.py
similarity index 98%
rename from src/backend/core/llm/subagent_sessions.py
rename to src/backend/core/llm/subagents/sessions.py
index 7cac4aa9..465546d3 100644
--- a/src/backend/core/llm/subagent_sessions.py
+++ b/src/backend/core/llm/subagents/sessions.py
@@ -18,7 +18,7 @@
from typing import Any, Dict, List, Optional
from core.infra.ephemeral import get_ephemeral_state
-from core.llm.human_interaction import STREAM_TTL_SECONDS
+from core.llm.runtime.human_interaction import STREAM_TTL_SECONDS
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/llm/_subagent_stream.py b/src/backend/core/llm/subagents/stream.py
similarity index 80%
rename from src/backend/core/llm/_subagent_stream.py
rename to src/backend/core/llm/subagents/stream.py
index d06d03c8..c21dd513 100644
--- a/src/backend/core/llm/_subagent_stream.py
+++ b/src/backend/core/llm/subagents/stream.py
@@ -1,14 +1,14 @@
"""Sub-agent streaming event side-channel — relays call_subagent's internal
thinking / tool_call / tool_result / content events into the main chat SSE stream in real time.
-Background: ``call_subagent`` (core/llm/subagent_tool.py) runs the sub-agent in a
+Background: ``call_subagent`` (core/llm/subagents/tool.py) runs the sub-agent in a
**separate thread + separate event loop** (to avoid anyio cancel-scope cross-task
errors). The sub-agent used to execute one-shot via ``agent.reply()``, swallowing all
intermediate events; the frontend only saw a single "call sub-agent" tool card and the
final summary text, with no visibility into its internal tool calls.
This module provides a per-chat side-channel shaped after the proven ``_tool_confirmation``:
-- ``stream()`` (orchestration/streaming.py) calls ``attach`` on startup with its own
+- ``stream()`` (core/llm/runtime/streaming.py) calls ``attach`` on startup with its own
``event_q`` + the loop it runs on, and ``detach`` on shutdown.
- ``call_subagent`` and its worker thread deliver sub-agent events back to the main
``event_q`` via ``push`` — using ``loop.call_soon_threadsafe`` for **cross-thread
@@ -29,7 +29,7 @@
import asyncio
import logging
import threading
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
@@ -46,24 +46,20 @@ class _Bus:
queue: asyncio.Queue
-_LOCK = threading.RLock()
-_BUSES: Dict[str, _Bus] = {}
-
-
def _key(chat_id: Optional[str]) -> str:
return str(chat_id) if chat_id else _NO_CHAT
def attach(chat_id: Optional[str], loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
"""Register this run's event outlet (event_q + its loop) when stream() starts."""
- with _LOCK:
- _BUSES[_key(chat_id)] = _Bus(loop=loop, queue=queue)
+ with process_state().LOCK:
+ process_state().BUSES[_key(chat_id)] = _Bus(loop=loop, queue=queue)
def detach(chat_id: Optional[str]) -> None:
"""Deregister when stream() winds down — prevents late events from writing to a finished run's event_q."""
- with _LOCK:
- _BUSES.pop(_key(chat_id), None)
+ with process_state().LOCK:
+ process_state().BUSES.pop(_key(chat_id), None)
def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None:
@@ -77,8 +73,8 @@ def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None:
never registered) or the loop is already closed; never raises and drags down
the sub-agent.
"""
- with _LOCK:
- bus = _BUSES.get(_key(chat_id))
+ with process_state().LOCK:
+ bus = process_state().BUSES.get(_key(chat_id))
if bus is None:
return
try:
@@ -92,5 +88,18 @@ def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None:
def is_active(chat_id: Optional[str]) -> bool:
"""Whether the chat currently has an active stream listening — lets call_subagent decide whether to emit streaming events."""
- with _LOCK:
- return _key(chat_id) in _BUSES
+ with process_state().LOCK:
+ return _key(chat_id) in process_state().BUSES
+
+
+@dataclass
+class StreamState:
+ LOCK: object = field(default_factory=lambda: threading.RLock())
+ BUSES: Dict[str, _Bus] = field(default_factory=lambda: {})
+
+
+_STATE = StreamState()
+
+
+def process_state() -> StreamState:
+ return _STATE
diff --git a/src/backend/core/llm/subagent_tool.py b/src/backend/core/llm/subagents/tool.py
similarity index 89%
rename from src/backend/core/llm/subagent_tool.py
rename to src/backend/core/llm/subagents/tool.py
index 16c9b4f8..2a14c825 100644
--- a/src/backend/core/llm/subagent_tool.py
+++ b/src/backend/core/llm/subagents/tool.py
@@ -5,54 +5,40 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
+from core.harness import log_writer
import asyncio
import logging
-import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
+# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk).
+from agentscope.tool._response import ToolChunk
from core.infra.task_lifecycle import settle_task
-from core.llm import subagent_sessions
-from core.llm.context_adapter import next_request_sequence, render_context_item
-from core.llm.context_ir import KIND_USER_INPUT, POLICY_NEVER, make_text_context_item
-from core.llm.tools._common import resolve_sandbox_session
-from core.services import log_service as log_writer
+from core.llm.subagents import sessions as subagent_sessions
-logger = logging.getLogger(__name__)
-from core.llm._subagent_execution import (
- _ChildExecution,
- _map_subagent_event,
- _run_subagent_in_thread,
- _shared_ontology_runtime,
- _SubMapper,
-)
-from core.llm._subagent_prompts import (
- _collapse_ids,
- _get_tools_desc,
- build_explicit_subagent_command_hint,
- build_subagent_mention_hint,
- build_subagent_prompt_section,
-)
+from core.llm.subagents.execution import _ChildExecution, _map_subagent_event, _run_subagent_in_thread, _shared_ontology_runtime, _SubMapper
+from core.llm.subagents.prompts import _get_tools_desc as get_tools_desc, build_explicit_subagent_command_hint, build_subagent_mention_hint, build_subagent_prompt_section
+
+logger = logging.getLogger(__name__)
_subagent_pool = ThreadPoolExecutor(max_workers=16, thread_name_prefix="subagent")
def _child_capability_runtime(parent_runtime, agent_id):
runtime = dict(parent_runtime or {})
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if runtime.get("run_id") and capabilities_enabled():
- from core.capabilities.runtime import child_scope
- from core.llm.middlewares import CURRENT_TOOL_CALL_ID
+ if runtime.get("run_id") and has_run_capabilities():
+ from core.harness.registry.capability_runtime import child_scope
+ from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID
# The ledger's persisted tool-call identity is stable on replay. SSE's
# random sub_run_id is presentation-only and never defines authority.
@@ -80,7 +66,7 @@ def register_subagent_tool(
agent instance after creation. Used to extract shared context
for sub-agents that have ``extra_config.shared_context == True``.
chat_id: This run's chat_id — sub-agent streaming events are bypass-routed back
- to the main SSE stream by it (see core/llm/_subagent_stream.py).
+ to the main SSE stream by it (see core/llm/subagents/stream.py).
"""
agent_map = {a["agent_id"]: a for a in visible_agents}
@@ -89,7 +75,7 @@ async def call_subagent(
task: str,
context_summary: str = "",
resume_session_id: str = "",
- ) -> ToolResponse:
+ ) -> ToolChunk:
"""调用子智能体执行专业任务。子智能体拥有独立的工具和专业知识。
需要并行调用多个子智能体时,在同一轮回复中生成多个 call_subagent 调用,
@@ -109,12 +95,12 @@ async def call_subagent(
完整上下文(读过的文件、跑过的命令、犯过的错)上继续;留空则全新启动。
Returns:
- `ToolResponse`:
+ `ToolChunk`:
子智能体的执行结果,末尾附带本次的续跑句柄。
结果对用户不可见,你需要汇总后呈现给用户。
"""
if agent_id not in agent_map:
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -136,7 +122,7 @@ async def call_subagent(
user_id=current_user_id,
)
if resume_messages is None:
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -154,7 +140,7 @@ async def call_subagent(
shared_messages = None
if not resume_messages and shared_context and agent_ref and agent_ref.get("agent"):
try:
- from core.llm.message_compat import extract_messages_from_context
+ from core.llm.runtime.message_compat import extract_messages_from_context
# 2.0: agent.memory → agent.state.context; this function is now synchronous
shared_messages = extract_messages_from_context(agent_ref["agent"].state.context)
@@ -172,7 +158,7 @@ async def call_subagent(
# call id (already written into the ContextVar by ActingToolCallIdMiddleware); the
# frontend uses it to group sub-steps under the right card; when missing, the
# frontend falls back to grouping under the most recent unmatched call_subagent card.
- from core.llm import _subagent_stream
+ from core.llm.subagents import stream as _subagent_stream
# Sub-step counting (best-effort): in the streaming bypass, count one tool call per
# tool_result received, so the sub-agent call log can show how many tools that call
@@ -182,7 +168,7 @@ async def call_subagent(
_emit: Optional[Callable[[Dict[str, Any]], None]] = None
if bool(chat_id) and _subagent_stream.is_active(chat_id):
try:
- from core.llm.middlewares import CURRENT_TOOL_CALL_ID
+ from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID
parent_tool_id = CURRENT_TOOL_CALL_ID.get("") or ""
except Exception: # noqa: BLE001
@@ -258,13 +244,13 @@ async def record_terminal():
# copy here would let child activations update nested lists while
# losing scalar changes such as an escalated review_level.
ontology_runtime = _shared_ontology_runtime(agent_ref)
- from core.capabilities.errors import IntegrityFailed
+ from core.harness.capability_errors import IntegrityFailed
try:
child_runtime = _child_capability_runtime(parent_runtime, agent_id)
except IntegrityFailed:
await _finish("failed", error="missing durable capability scope")
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text", text="子智能体缺少可恢复的持久工具调用身份,已拒绝执行。"
@@ -296,7 +282,7 @@ async def record_terminal():
# produced by the sub-agent never show up in the main conversation.
if sub_pinned:
try:
- from core.llm import workspace as _ws
+ from core.llm.runtime import workspace as _ws
for _it in sub_pinned:
_ws.pin(
@@ -312,7 +298,7 @@ async def record_terminal():
if not ok:
await _finish("failed", output=text, error=text)
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -345,7 +331,7 @@ async def record_terminal():
if saved
else ""
)
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -380,7 +366,7 @@ async def finish_cancelled():
if _emit is not None:
_emit({"sub_type": "error", "error": str(e)[:200]})
await _finish("failed", error=str(e)[:200])
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -389,4 +375,4 @@ async def finish_cancelled():
]
)
- toolkit.register_tool_function(call_subagent, namesake_strategy="skip")
+ toolkit.append(function_tool(call_subagent))
diff --git a/src/backend/core/llm/tool_collector.py b/src/backend/core/llm/tool_collector.py
deleted file mode 100644
index 006312a5..00000000
--- a/src/backend/core/llm/tool_collector.py
+++ /dev/null
@@ -1,270 +0,0 @@
-"""ToolCollector — adapts 1.x's incremental ``toolkit.register_*`` to 2.0's one-shot Toolkit.
-
-AgentScope 2.0's ``Toolkit`` is injected once at construction time (``Toolkit(tools=,
-mcps=, skills_or_loaders=)``); it has no incremental ``register_tool_function`` /
-``register_mcp_client`` / ``register_agent_skill`` methods.
-
-To avoid rewriting the ~15 ``register_*`` in-house tool functions (they all call
-``toolkit.register_tool_function(fn, namesake_strategy=...)``), this module provides a
-**duck-typed compatible** collector: it exposes methods of the same names but internally
-just collects tools into an ``AllowedFunctionTool`` list / skill directory list.
-``agent_factory`` passes the collector to every ``register_*``, then constructs once via
-``Toolkit(tools=collector.function_tools, mcps=clients,
-skills_or_loaders=collector.skill_loaders)``.
-
-Also resolves two behavioral differences in 2.0:
- * ``FunctionTool``'s default ``check_permissions`` returns ASK -> every in-house tool
- would pop HITL. The ``AllowedFunctionTool`` subclass flips the default decision back
- to ALLOW (built-in tools like Bash keep their own dangerous-command checks, since we
- do not touch their check_permissions).
- * Tool functions return ``ToolChunk`` (tool.py already aliases ToolResponse to ToolChunk).
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import os
-from dataclasses import replace
-from functools import lru_cache
-from pathlib import Path
-
-import frontmatter
-from typing import Any, Callable, List
-
-from agentscope.permission import PermissionBehavior, PermissionDecision
-from agentscope.tool import FunctionTool
-from agentscope.skill import LocalSkillLoader, SkillLoaderBase, Skill
-from core.llm.tool_permissions import (
- ToolPermissionSpec,
- builtin_tool_permission,
-)
-
-from core.agent_skills.config import model_facing_skill_dir
-
-logger = logging.getLogger(__name__)
-
-
-class AllowedFunctionTool(FunctionTool):
- """In-house Python tool: allowed by default (overrides 2.0 FunctionTool's default ASK)."""
-
- async def check_permissions(self, *args: Any, **kwargs: Any) -> PermissionDecision:
- return PermissionDecision(
- behavior=PermissionBehavior.ALLOW,
- message="HugAgentOS self-developed tool (auto-allowed).",
- )
-
-
-def _parse_runtime_skill(text):
- content = frontmatter.loads(text)
- name, description = content.get("name"), content.get("description")
- if not name or not description:
- return None
- return str(name), str(description), content.content
-
-
-_cached_runtime_skill = lru_cache(maxsize=256)(_parse_runtime_skill)
-
-
-class RuntimeNamedSkillLoader(SkillLoaderBase):
- """Read frozen files and expose a deployment-appropriate directory."""
-
- def __init__(self, directory: str, runtime_name: str, capability_run=None) -> None:
- self.directory = directory
- self.runtime_name = runtime_name
- self.capability_run = capability_run
- self._physical_loader = LocalSkillLoader(directory)
-
- def _list_desktop_skill(self):
- from core.capabilities.runtime import validate
- from core.capabilities.errors import CapabilityError
-
- # Keep authorization outside the parse-error handling: it must fail the
- # whole enumeration, never quietly omit a revoked or modified skill.
- try:
- validate(self.capability_run, only_skill=self.runtime_name)
- except (OSError, CapabilityError):
- # 被撤销或改动过的单个技能只从枚举里消失,不牵连这一轮其它工具。
- return []
- try:
- entry = Path(self.directory) / "SKILL.md"
- updated_at = entry.stat().st_mtime
- text = entry.read_text(encoding="utf-8")
- parsed = (
- _cached_runtime_skill(text) if len(text) <= 262144 else _parse_runtime_skill(text)
- )
- if parsed is None:
- return []
- _, description, markdown = parsed
- return [
- Skill(
- name=self.runtime_name,
- description=description,
- dir=model_facing_skill_dir(self.runtime_name, self.directory),
- markdown=markdown,
- updated_at=updated_at,
- )
- ]
- except Exception as exc:
- logger.warning("Failed to load skill metadata: %s", type(exc).__name__)
- return []
-
- async def list_skills(self):
- # AgentScope's general filesystem scanner crosses the executor boundary
- # eight times for one known file. Batch that I/O in one worker while
- # preserving its parser, fields and sandbox alias.
- if os.name == "nt" and self.capability_run is not None:
- return await asyncio.to_thread(self._list_desktop_skill)
- from core.capabilities.runtime import validate
-
- if self.capability_run is not None:
- # Re-check only this loader's own skill (authorization + fresh hash),
- # not the whole run. N loaders each re-hashing all N skills — twice
- # per assembly — was O(N²) filesystem work and the dominant cost of
- # desktop agent setup; scoping to one skill makes it O(N).
- from core.capabilities.errors import CapabilityError
-
- try:
- await asyncio.to_thread(validate, self.capability_run, only_skill=self.runtime_name)
- except (CapabilityError, OSError):
- return []
- physical = await self._physical_loader.list_skills()
- return [
- replace(skill, name=self.runtime_name, dir=model_facing_skill_dir(self.runtime_name, self.directory))
- for skill in physical
- ]
-
-
-class _ParallelRuntimeSkillLoader(SkillLoaderBase):
- """Enumerate independent desktop loaders with bounded I/O concurrency."""
-
- def __init__(self, loaders):
- self._loaders = tuple(loaders)
-
- async def list_skills(self):
- limit = asyncio.Semaphore(4)
-
- async def load(loader):
- async with limit:
- return await loader.list_skills()
-
- results = await asyncio.gather(
- *(load(loader) for loader in self._loaders), return_exceptions=True
- )
- skills = []
- for result in results:
- if isinstance(result, BaseException):
- raise result
- skills.extend(result)
- return skills
-
-
-class ToolCollector:
- """Duck-type compatible with the 1.x Toolkit registration interface; actually only collects tools/skills for the 2.0 Toolkit."""
-
- def __init__(self) -> None:
- # name -> AllowedFunctionTool (deduped by name, supports override/skip)
- self._tools: dict[str, AllowedFunctionTool] = {}
- self._permission_specs: dict[str, ToolPermissionSpec] = {}
- self._tool_order: List[str] = []
- self._skill_loaders: List[Any] = []
-
- # ── 1.x compatibility interface ─────────────────────────────────────
- def register_tool_function(
- self,
- func: Callable[..., Any],
- *,
- func_description: str | None = None,
- namesake_strategy: str = "override",
- permission: ToolPermissionSpec | None = None,
- **_ignored: Any,
- ) -> None:
- """Collect a tool function as an AllowedFunctionTool.
-
- namesake_strategy:
- - "override" (default): same name replaces the existing one
- - "skip": same name keeps the existing one and discards the new one
- """
- name = getattr(func, "__name__", None) or "tool"
- if name in self._tools:
- if namesake_strategy == "skip":
- return
- # override: replace, but keep the original ordering position
- permission_spec = permission or builtin_tool_permission(name)
- try:
- ft = AllowedFunctionTool(
- func,
- name=name,
- description=func_description,
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("[tool_collector] 构造 FunctionTool '%s' 失败: %s", name, exc)
- return
- if name not in self._tools:
- self._tool_order.append(name)
- self._tools[name] = ft
- if permission_spec is not None:
- self._permission_specs[name] = permission_spec
- else:
- self._permission_specs.pop(name, None)
-
- def register_tool_permission(self, name: str, spec: ToolPermissionSpec) -> None:
- """Attach developer-owned MCP policy to the same dispatch registry."""
- previous = self._permission_specs.get(name)
- if previous is not None and previous.key != spec.key:
- raise ValueError(f"Conflicting permission declaration for {name!r}")
- self._permission_specs[name] = spec
-
- def register_agent_skill(
- self, skill_dir: Any, *, runtime_name: str | None = None, capability_run=None
- ) -> None:
- """Keep physical reads separate from a desktop skill's runtime identity."""
- if not skill_dir:
- return
- if runtime_name is not None:
- if not any(
- isinstance(item, RuntimeNamedSkillLoader)
- and item.directory == skill_dir
- and item.runtime_name == runtime_name
- for item in self._skill_loaders
- ):
- self._skill_loaders.append(
- RuntimeNamedSkillLoader(skill_dir, runtime_name, capability_run)
- )
- else:
- self.register_agent_skill(skill_dir, runtime_name=Path(skill_dir).name)
-
- def register_mcp_client(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover
- # 2.0 goes through Toolkit(mcps=); this should never be called — kept as an empty fallback against legacy call paths.
- logger.warning(
- "[tool_collector] register_mcp_client 被调用但已忽略(2.0 经 Toolkit(mcps=))。"
- )
-
- # ── Result accessors for agent_factory ───────────────────────────────
- def get_tool(self, name: str) -> AllowedFunctionTool | None:
- """Get a collected AllowedFunctionTool by name (for tests/introspection; `._func`
- is the original callable, `.input_schema` is the JSON schema)."""
- return self._tools.get(name)
-
- @property
- def function_tools(self) -> List[AllowedFunctionTool]:
- return [self._tools[n] for n in self._tool_order if n in self._tools]
-
- @property
- def permission_specs(self) -> dict[str, ToolPermissionSpec]:
- """Detached declaration map consumed by the run-scoped gateway."""
- return dict(self._permission_specs)
-
- @property
- def skill_loaders(self) -> List[Any]:
- loaders = list(self._skill_loaders)
- if (
- os.name == "nt"
- and len(loaders) >= 8
- and all(isinstance(loader, RuntimeNamedSkillLoader) for loader in loaders)
- ):
- return [_ParallelRuntimeSkillLoader(loaders)]
- return loaders
-
-
-__all__ = ["AllowedFunctionTool", "ToolCollector"]
diff --git a/src/backend/core/llm/tool_permissions/__init__.py b/src/backend/core/llm/tool_permissions/__init__.py
index 8438c3cb..ab14f91f 100644
--- a/src/backend/core/llm/tool_permissions/__init__.py
+++ b/src/backend/core/llm/tool_permissions/__init__.py
@@ -1,93 +1,4 @@
-"""Public API of the governed-tool permission domain."""
+"""SDK tool authorization service; shared policy lives in sandbox.permissions."""
+from .service import ToolPermissionService
-from .models import (
- CURRENT_PERMISSION_TICKET,
- FAIL_CLOSED_MODE,
- FALLBACK_ALLOW,
- FALLBACK_DENY,
- UNCONFINED_APPROVAL_MODES,
- LocalCommandAuthorization,
- LocalConfinementUnavailableError,
- PermissionEnforcementError,
- PermissionIntent,
- PermissionOutcome,
- PermissionRuntime,
- PermissionTicket,
- ToolPermissionSpec,
- current_local_command_authorization,
- require_local_path_permission,
- READ,
- WRITE,
- EXECUTE,
- DOMAIN_LOCAL_PATH,
- DOMAIN_LOCAL_COMMAND,
- DOMAIN_MYSPACE,
- DOMAIN_APPROVAL,
- DOMAIN_DENY,
-)
-from .specs import (
- ToolPermissionRegistry,
- allow_tool,
- builtin_tool_permission,
- deny_tool,
- local_command_tool,
- local_path_tool,
-)
-from .preferences import (
- preset_answers_for_user,
- resolve_approval_mode,
- normalize_approval_mode,
- APPROVAL_ASK,
- APPROVAL_AUTO,
- APPROVAL_FULL,
- APPROVAL_MODES,
- DESTRUCTIVE_OPS,
-)
-from .service import (
- ToolPermissionService,
-)
-from .middleware import (
- ToolPermissionMiddleware,
-)
-
-__all__ = [
- "CURRENT_PERMISSION_TICKET",
- "FAIL_CLOSED_MODE",
- "FALLBACK_ALLOW",
- "FALLBACK_DENY",
- "UNCONFINED_APPROVAL_MODES",
- "LocalCommandAuthorization",
- "LocalConfinementUnavailableError",
- "PermissionEnforcementError",
- "PermissionIntent",
- "PermissionOutcome",
- "PermissionRuntime",
- "PermissionTicket",
- "ToolPermissionMiddleware",
- "ToolPermissionRegistry",
- "ToolPermissionService",
- "ToolPermissionSpec",
- "allow_tool",
- "builtin_tool_permission",
- "current_local_command_authorization",
- "deny_tool",
- "local_command_tool",
- "local_path_tool",
- "preset_answers_for_user",
- "require_local_path_permission",
- "resolve_approval_mode",
- "normalize_approval_mode",
- "APPROVAL_ASK",
- "APPROVAL_AUTO",
- "APPROVAL_FULL",
- "APPROVAL_MODES",
- "DESTRUCTIVE_OPS",
- "READ",
- "WRITE",
- "EXECUTE",
- "DOMAIN_LOCAL_PATH",
- "DOMAIN_LOCAL_COMMAND",
- "DOMAIN_MYSPACE",
- "DOMAIN_APPROVAL",
- "DOMAIN_DENY",
-]
+__all__ = ["ToolPermissionService"]
diff --git a/src/backend/core/llm/tool_permissions/hook.py b/src/backend/core/llm/tool_permissions/hook.py
new file mode 100644
index 00000000..e9797637
--- /dev/null
+++ b/src/backend/core/llm/tool_permissions/hook.py
@@ -0,0 +1,71 @@
+"""Neutral authorization policy with an execution-scoped permission ticket."""
+import json
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any
+from core.harness.events import thaw_value
+from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage
+from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError
+from core.sandbox.permissions.approval import refresh_runtime
+from .service import ToolPermissionService
+
+
+@dataclass(frozen=True)
+class PermissionCall:
+ id: str
+ name: str
+ input: Any
+
+
+class ToolPermissionHook:
+ def __init__(self, service):
+ self.service = service
+ self.pending = {}
+
+ async def __call__(self, invocation):
+ call = PermissionCall(str(invocation.metadata.get("tool_call_id") or
+ invocation.data.get("metadata", {}).get("tool_call_id") or ""),
+ invocation.operation_name, thaw_value(invocation.data.get("tool_args", {})))
+ try:
+ runtime = await refresh_runtime(self.service.runtime)
+ except PermissionEnforcementError as exc:
+ return self.block(call.id, {"error": str(exc), "blocked": True},
+ {"decision": "deny", "reason": "policy_unavailable"})
+ outcome = await ToolPermissionService(self.service.registry, runtime).authorize(call)
+ if not outcome.proceed:
+ return self.block(call.id, outcome.payload or {"error": "工具权限判定拒绝执行", "blocked": True}, outcome.audit)
+ self.pending[call.id] = outcome
+ return Decision()
+
+ @staticmethod
+ def block(call_id, payload, audit):
+ response = {"id": call_id, "content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False, default=str)}],
+ "state": "success" if payload.get("ok") and not payload.get("error") else "error",
+ "metadata": {"permission": dict(audit)}}
+ return Decision(DecisionKind.REJECT, patch={"tool_response": response}, reason="permission outcome")
+
+ @contextmanager
+ def bind(self, call_id):
+ outcome = self.pending.pop(call_id, None)
+ if outcome is None:
+ yield None
+ return
+ ticket = outcome.ticket
+ if ticket is None:
+ yield dict(outcome.audit)
+ return
+ token = CURRENT_PERMISSION_TICKET.set(ticket)
+ try:
+ yield ticket.audit_dict()
+ finally:
+ ticket.lifetime.expire()
+ CURRENT_PERMISSION_TICKET.reset(token)
+
+ def discard(self, call_id):
+ outcome = self.pending.pop(call_id, None)
+ if outcome is not None and outcome.ticket is not None:
+ outcome.ticket.lifetime.expire()
+
+ def spec(self):
+ return HookSpec("tool_permission", HookStage.BEFORE_TOOL, self, priority=600,
+ timeout_ms=None, mutable_fields=frozenset({"tool_response"}))
diff --git a/src/backend/core/llm/tool_permissions/middleware.py b/src/backend/core/llm/tool_permissions/middleware.py
deleted file mode 100644
index 58d0606b..00000000
--- a/src/backend/core/llm/tool_permissions/middleware.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""One permission dispatch boundary: live policy, authorization and ticket lifetime."""
-
-from __future__ import annotations
-import json
-from typing import Any, Mapping
-from agentscope.agent import Agent
-from agentscope.message import TextBlock, ToolResultState
-from agentscope.middleware import MiddlewareBase
-from agentscope.tool._response import ToolResponse
-from .models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError
-from .service import ToolPermissionService
-from .preferences import refresh_runtime
-
-
-def _permission_response(
- tool_call: Any,
- payload: Mapping[str, Any],
- audit: Mapping[str, Any],
-) -> ToolResponse:
- success = bool(payload.get("ok")) and not payload.get("error")
- return ToolResponse(
- id=str(getattr(tool_call, "id", "") or ""),
- content=[
- TextBlock(
- type="text",
- text=json.dumps(payload, ensure_ascii=False, default=str),
- )
- ],
- state=ToolResultState.SUCCESS if success else ToolResultState.ERROR,
- metadata={"permission": dict(audit)},
- )
-
-
-class ToolPermissionMiddleware(MiddlewareBase):
- """Authorize once before effects and bind the ticket around dispatch."""
-
- def __init__(self, service: ToolPermissionService) -> None:
- self.service = service
-
- async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001
- async for item in self._act(input_kwargs, next_handler):
- yield item
-
- async def _act(self, input_kwargs: dict, next_handler): # noqa: ANN001
- tool_call = input_kwargs.get("tool_call")
- try:
- runtime = await refresh_runtime(self.service.runtime)
- except PermissionEnforcementError as exc:
- yield _permission_response(
- tool_call,
- {"error": str(exc), "blocked": True},
- {"decision": "deny", "reason": "policy_unavailable"},
- )
- return
- service = ToolPermissionService(self.service.registry, runtime)
- outcome = await service.authorize(tool_call)
- if not outcome.proceed:
- yield _permission_response(
- tool_call,
- outcome.payload or {"error": "工具权限判定拒绝执行", "blocked": True},
- outcome.audit,
- )
- return
-
- if outcome.ticket is None:
- async for item in next_handler(**input_kwargs):
- if isinstance(item, ToolResponse):
- item.metadata = {
- **dict(item.metadata or {}),
- "permission": dict(outcome.audit),
- }
- yield item
- return
-
- token = CURRENT_PERMISSION_TICKET.set(outcome.ticket)
- try:
- async for item in next_handler(**input_kwargs):
- if isinstance(item, ToolResponse):
- item.metadata = {
- **dict(item.metadata or {}),
- "permission": outcome.ticket.audit_dict(),
- }
- yield item
- finally:
- outcome.ticket.lifetime.expire()
- CURRENT_PERMISSION_TICKET.reset(token)
diff --git a/src/backend/core/llm/tool_permissions/pending.py b/src/backend/core/llm/tool_permissions/pending.py
index 7932a454..8774bea0 100644
--- a/src/backend/core/llm/tool_permissions/pending.py
+++ b/src/backend/core/llm/tool_permissions/pending.py
@@ -1,8 +1,8 @@
"""A pending permission is answered only by its declared, current server-side authority."""
import asyncio
-from core.llm import interaction_store
-from .account import read_account_mode
+from core.llm.runtime import interaction_store
+from core.sandbox.permissions.account import read_account_mode
async def approve_current_policy(chat_id: str, confirm_id: str, info: dict) -> bool:
diff --git a/src/backend/core/llm/tool_permissions/service.py b/src/backend/core/llm/tool_permissions/service.py
index 0ec20b98..1a244cab 100644
--- a/src/backend/core/llm/tool_permissions/service.py
+++ b/src/backend/core/llm/tool_permissions/service.py
@@ -4,26 +4,9 @@
import logging
import os
from typing import Any, Sequence, Optional
-from .preferences import DESTRUCTIVE_OPS, _preset_answers
-from .specs import ToolPermissionRegistry, _one_shot_write_root
-from .models import (
- DOMAIN_APPROVAL,
- DOMAIN_DENY,
- DOMAIN_LOCAL_COMMAND,
- DOMAIN_LOCAL_PATH,
- DOMAIN_MYSPACE,
- FAIL_CLOSED_MODE,
- FALLBACK_ALLOW,
- LocalCommandAuthorization,
- PermissionIntent,
- PermissionOutcome,
- PermissionRuntime,
- PermissionTicket,
- READ,
- UNCONFINED_APPROVAL_MODES,
- _call_args,
- _json_hash,
-)
+from core.sandbox.permissions.approval import DESTRUCTIVE_OPS, _preset_answers
+from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, _one_shot_write_root
+from core.sandbox.permissions.dispatch_models import DOMAIN_APPROVAL, DOMAIN_DENY, DOMAIN_LOCAL_COMMAND, DOMAIN_LOCAL_PATH, DOMAIN_MYSPACE, FAIL_CLOSED_MODE, FALLBACK_ALLOW, LocalCommandAuthorization, PermissionIntent, PermissionOutcome, PermissionRuntime, PermissionTicket, READ, UNCONFINED_APPROVAL_MODES, _call_args, _json_hash
logger = logging.getLogger(__name__)
@@ -187,7 +170,7 @@ def _safe_local_security(self):
mode = self.runtime.approval_mode
try:
- from core.services.local_grant_service import grants_for_gate, policy_for_gate
+ from core.harness.registry.local_files import grants_for_gate, policy_for_gate
return mode, grants_for_gate(), policy_for_gate(mode)
except Exception: # unreadable configuration must never grant access
@@ -208,7 +191,7 @@ def _safe_local_security(self):
)
def _execution_directory(self) -> str:
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
return workspace_directory(
self.runtime.sandbox_session_id or self.runtime.chat_id,
@@ -269,7 +252,7 @@ async def _authorize_local_path(self, intent: PermissionIntent) -> dict[str, Any
if self._no_ui_pass_through(intent):
return {"reasons": [*verdict.reasons, "no_approval_ui_pass_through:local_path"]}
- from core.llm.tools import _tool_confirmation as confirm
+ from core.llm.tools import confirmation as confirm
op = confirm.OP_LOCAL_READ if intent.action == READ else confirm.OP_LOCAL_WRITE
blocked = await confirm.gate(
@@ -325,7 +308,7 @@ async def _authorize_local_command(self, intent: PermissionIntent) -> dict[str,
and not self._answers_for_user(dangerous=bool(danger_categories(verdict.reasons)))
and not self._no_ui_pass_through(intent)
):
- from core.llm.tools import _tool_confirmation as confirm
+ from core.llm.tools import confirmation as confirm
blocked = await confirm.gate(
chat_id=self.runtime.chat_id,
@@ -360,7 +343,7 @@ def _physical(self, logical: str) -> str:
approved one-shot write would be silently dropped.
"""
try:
- from core.llm.tools._paths import to_physical_path
+ from core.sandbox.workspace.resolution import to_physical_path
session = self.runtime.sandbox_session_id or self.runtime.chat_id
return to_physical_path(
@@ -457,7 +440,7 @@ async def _authorize_approval(
*,
interactive: bool,
) -> dict[str, Any]:
- from core.llm.tools import _tool_confirmation as confirm
+ from core.llm.tools import confirmation as confirm
op = intent.op or intent.action
if self._answers_for_user(dangerous=op in DESTRUCTIVE_OPS):
diff --git a/src/backend/core/llm/tools/__init__.py b/src/backend/core/llm/tools/__init__.py
index d5a0676c..9adabae7 100644
--- a/src/backend/core/llm/tools/__init__.py
+++ b/src/backend/core/llm/tools/__init__.py
@@ -17,9 +17,8 @@
from .read_tool import register_read
from .write_tool import register_write
-# Sandbox / skill / myspace / artifact tools (relocated from the former
-# singular ``core.llm.tool`` module). Surfaced here so callers import the whole
-# tool family from one place: ``from core.llm.tools import register_bash, ...``.
+# Sandbox / skill / myspace / artifact tools, surfaced here so callers import
+# the whole tool family from one place: ``from core.llm.tools import register_bash, ...``.
from .sandbox_tool import (
register_bash,
register_sandbox_get_artifact,
@@ -28,19 +27,15 @@
from .skill_tool import (
register_sandboxed_view_text_file,
)
-from .channel_attachment_tool import register_channel_attachment
from .chat_history_tool import register_chat_history_tools
from .pin_tool import register_pin_to_workspace
from .read_artifact_tool import register_read_artifact
-from .data_context_tool import register_get_data_context
__all__ = [
"ReadEntry",
"ReadStateTracker",
"register_bash",
- "register_channel_attachment",
"register_chat_history_tools",
- "register_get_data_context",
"register_edit",
"register_glob",
"register_grep",
diff --git a/src/backend/core/llm/tools/_common.py b/src/backend/core/llm/tools/_common.py
index cc88181f..54bf1036 100644
--- a/src/backend/core/llm/tools/_common.py
+++ b/src/backend/core/llm/tools/_common.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-from core.content.artifact_refs import require_artifact_storage_key
import asyncio
import base64
@@ -14,16 +13,16 @@
from agentscope.message import Base64Source, DataBlock, TextBlock
-# AgentScope 2.0: tool functions must return ToolChunk; aliased (its fields are a superset of ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
-from core.services.project_scope import ProjectScope
+# AgentScope 2.0: tool functions must return ToolChunk; aliased (its fields are a superset of ToolChunk).
+from agentscope.tool._response import ToolChunk
+from core.projects.edition import ProjectScope
logger = logging.getLogger(__name__)
async def resp_image(
image_bytes: bytes, mime_type: str, *, name: str, meta: Optional[dict[str, Any]] = None
-) -> ToolResponse:
+) -> ToolChunk:
"""图片原样交给多模态主模型:一段 JSON 元信息 + 一个图片数据块。
AgentScope 会把工具结果里的 DataBlock 提升成下一轮的 image 消息;端点不收图时
@@ -32,7 +31,7 @@ async def resp_image(
"""
payload = {"type": "image", "mime_type": mime_type, "size": len(image_bytes), **(meta or {})}
data = await asyncio.to_thread(lambda: base64.b64encode(image_bytes).decode("ascii"))
- return ToolResponse(
+ return ToolChunk(
content=[
*resp_json(payload).content,
DataBlock(
@@ -44,13 +43,13 @@ async def resp_image(
)
-def resp_json(payload: dict[str, Any]) -> ToolResponse:
- """Wrap a JSON dict as a single-text-block ToolResponse.
+def resp_json(payload: dict[str, Any]) -> ToolChunk:
+ """Wrap a JSON dict as a single-text-block ToolChunk.
Mirrors ``core.llm.tool._resp_json``. Re-implemented here so this package
has zero cross-imports back to the large monolithic ``tool.py``.
"""
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -60,7 +59,7 @@ def resp_json(payload: dict[str, Any]) -> ToolResponse:
)
-def response_metadata(response: ToolResponse, metadata: dict[str, Any]) -> ToolResponse:
+def response_metadata(response: ToolChunk, metadata: dict[str, Any]) -> ToolChunk:
"""Merge file metadata into the existing JSON block, preserving image layout."""
block = response.content[0]
block.text = json.dumps({**json.loads(block.text), **metadata}, ensure_ascii=False)
@@ -130,7 +129,7 @@ def register_file_artifact(
content: bytes, scope: Optional[ProjectScope] = None,
) -> Optional[dict[str, Any]]:
"""Create an immutable downloadable snapshot of a sandbox file."""
- from core.llm.tools._tool_helpers import _store_generated_files
+ from core.files.artifacts.generated import store_generated_files as _store_generated_files
if not user_id or not filename:
return None
@@ -162,7 +161,7 @@ def pin_artifact_to_workspace(ref: dict[str, Any]) -> bool:
if not file_id:
return False
try:
- from core.llm import workspace as _workspace
+ from core.llm.runtime import workspace as _workspace
pinned = _workspace.pin(
file_id=str(file_id),
diff --git a/src/backend/core/llm/tools/_tool_helpers.py b/src/backend/core/llm/tools/_tool_helpers.py
index c92f27a8..390ac7d8 100644
--- a/src/backend/core/llm/tools/_tool_helpers.py
+++ b/src/backend/core/llm/tools/_tool_helpers.py
@@ -4,227 +4,31 @@
functions share them. ``core.llm.tool`` re-exports these for compatibility.
"""
-import base64
import json
import logging
-from pathlib import Path
-from typing import Any, Optional
+from typing import Any
from agentscope.message import TextBlock
-from agentscope.tool._response import ToolChunk as ToolResponse
+from agentscope.tool._response import ToolChunk
from core.config.settings import settings
logger = logging.getLogger(__name__)
-def _store_generated_files(
- files_data: list[dict[str, Any]],
- *,
- user_id: Optional[str],
- source: str,
- extra_metadata: Optional[dict[str, Any]] = None,
-) -> list[dict[str, Any]]:
- """Persist sidecar-generated files and return normalized artifact refs."""
- if not files_data:
- return []
-
- try:
- from core.artifacts.store import save_artifact_bytes
- except Exception as exc:
- logger.warning("artifact store unavailable for %s: %s", source, exc)
- return []
-
- refs: list[dict[str, Any]] = []
- for fd in files_data:
- content_b64 = fd.get("content_b64", "")
- if not content_b64:
- continue
-
- name = str(fd.get("name", "output")).strip() or "output"
- mime_type = (
- str(fd.get("mime_type", "application/octet-stream")).strip()
- or "application/octet-stream"
- )
- try:
- content = base64.b64decode(content_b64)
- except Exception:
- logger.warning("skip invalid base64 artifact payload: %s", name)
- continue
-
- metadata: dict[str, Any] = {"source": source}
- if user_id:
- metadata["user_id"] = user_id
- if extra_metadata:
- metadata.update(extra_metadata)
-
- try:
- item = save_artifact_bytes(
- content=content,
- name=name,
- mime_type=mime_type,
- extension=Path(name).suffix.lstrip("."),
- metadata=metadata,
- )
- except Exception as exc:
- logger.warning("failed to persist generated artifact %s: %s", name, exc)
- continue
-
- refs.append(
- {
- "file_id": item["file_id"],
- "name": item.get("name", name),
- "url": f"/files/{item['file_id']}",
- "mime_type": item.get("mime_type", mime_type),
- "size": item.get("size", len(content)),
- "storage_key": item.get("storage_key"),
- }
- )
-
- return refs
-
-
-def _store_generated_file_path(
- file_path: str | Path,
- *,
- name: str,
- mime_type: str,
- user_id: Optional[str],
- source: str,
- extra_metadata: Optional[dict[str, Any]] = None,
-) -> dict[str, Any] | None:
- """Persist a generated file from disk and return a normalized artifact ref."""
- try:
- from core.artifacts.store import save_artifact_file
- except Exception as exc:
- logger.warning("artifact store unavailable for %s: %s", source, exc)
- return None
-
- metadata: dict[str, Any] = {"source": source}
- if user_id:
- metadata["user_id"] = user_id
- if extra_metadata:
- metadata.update(extra_metadata)
- try:
- item = save_artifact_file(
- file_path=file_path,
- name=name,
- mime_type=mime_type,
- extension=Path(name).suffix.lstrip("."),
- metadata=metadata,
- )
- except Exception as exc:
- logger.warning("failed to persist generated artifact %s: %s", name, exc)
- return None
- return {
- "file_id": item["file_id"],
- "name": item.get("name", name),
- "url": f"/files/{item['file_id']}",
- "mime_type": item.get("mime_type", mime_type),
- "size": item.get("size", 0),
- "storage_key": item.get("storage_key"),
- }
-MAX_ARTIFACT_FILE_SIZE = settings.sandbox.artifact_max_bytes
-def _resolve_artifact_files(
- artifact_refs: dict[str, str],
- user_id: str | None,
-) -> tuple[dict[str, str] | None, str | None]:
- """Resolve artifact: references to base64-encoded binary content.
-
- Tries DB lookup first, falls back to local artifact store index.
- Returns (files_b64_dict, error_message).
- """
- if not artifact_refs:
- return None, None
-
- try:
- from core.content.artifact_refs import resolve_artifact_storage_key
- from core.storage.factory import get_storage
- except Exception as exc:
- return None, f"artifact 解析依赖不可用: {exc}"
-
- result: dict[str, str] = {}
- storage = get_storage()
-
- for filename, artifact_id in artifact_refs.items():
- from core.artifacts.local_reference import is_local_file_ref
- if is_local_file_ref(artifact_id):
- from core.llm.hooks import _download_artifact_bytes
- file_bytes = _download_artifact_bytes(artifact_id, filename, "artifact input", user_id=user_id)
- if file_bytes is None:
- return None, f"artifact '{artifact_id}' 不存在或无权访问"
- if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE:
- return None, f"artifact 文件 '{filename}' 过大"
- result[filename] = base64.b64encode(file_bytes).decode("ascii")
- continue
- storage_key = None
- owner_ok = True
-
- # Try 1: DB lookup
- try:
- from core.db.engine import SessionLocal
- from core.db.repository import ArtifactRepository
-
- db = SessionLocal()
- try:
- repo = ArtifactRepository(db)
- art = repo.get_by_id(artifact_id)
- if art:
- if user_id and art.user_id != user_id:
- owner_ok = False
- else:
- storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key)
- finally:
- db.close()
- except Exception:
- pass
-
- if not owner_ok:
- return None, f"无权访问 artifact '{artifact_id}'"
-
- # Try 2: local artifact store fallback
- if not storage_key:
- try:
- from core.artifacts.store import get_artifact
-
- item = get_artifact(artifact_id)
- if item:
- item_user = (item.get("metadata") or {}).get("user_id")
- if user_id and item_user and item_user != user_id:
- return None, f"无权访问 artifact '{artifact_id}'"
- storage_key = item.get("storage_key")
- except Exception:
- pass
-
- if not storage_key:
- return None, f"artifact '{artifact_id}' 不存在或已删除"
-
- try:
- file_bytes = storage.download_bytes(storage_key)
- except Exception as exc:
- return None, f"artifact 文件 '{filename}' 读取失败: {exc}"
-
- if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE:
- return None, (
- f"artifact 文件 '{filename}' 过大: "
- f"{len(file_bytes)} bytes > {MAX_ARTIFACT_FILE_SIZE} bytes"
- )
-
- result[filename] = base64.b64encode(file_bytes).decode("ascii")
- return result or None, None
-def _resp_json(payload: dict[str, Any]) -> ToolResponse:
- """Helper: wrap a JSON-serializable dict as a single-text-block ToolResponse."""
- return ToolResponse(
+def resp_json(payload: dict[str, Any]) -> ToolChunk:
+ """Helper: wrap a JSON-serializable dict as a single-text-block ToolChunk."""
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -234,7 +38,7 @@ def _resp_json(payload: dict[str, Any]) -> ToolResponse:
)
-def _validate_workspace_path(
+def validate_workspace_path(
path: str,
*,
root: str | None = None,
diff --git a/src/backend/core/llm/tools/ask_user_question_tool.py b/src/backend/core/llm/tools/ask_user_question_tool.py
index 2052a622..097bec1d 100644
--- a/src/backend/core/llm/tools/ask_user_question_tool.py
+++ b/src/backend/core/llm/tools/ask_user_question_tool.py
@@ -1,11 +1,12 @@
"""Model-facing ``ask_user_question`` tool over the suspended question registry."""
+from core.llm.factory.tools.definitions import function_tool
import logging
from typing import Annotated, List, Optional
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
+from agentscope.tool._response import ToolChunk
from agentscope.tool._response import ToolResultState
from core.llm.tools import user_questions
from core.llm.tools._common import resp_json
@@ -78,7 +79,7 @@ class AskUserQuestionItem(TypedDict):
]
-def _error_response(status: str) -> ToolResponse:
+def _error_response(status: str) -> ToolChunk:
"""Return DSH-style tool errors while retaining local timeout semantics."""
errors = {
@@ -97,7 +98,7 @@ def _error_response(status: str) -> ToolResponse:
status,
("ASK_ABORTED", "ask_user_question was aborted before the user answered"),
)
- return ToolResponse(
+ return ToolChunk(
content=[TextBlock(type="text", text=f"Error: {message}")],
state=ToolResultState.ERROR,
metadata={
@@ -132,7 +133,7 @@ async def ask_user_question(
List[AskUserQuestionItem],
Field(description="Questions to ask the user before continuing."),
],
- ) -> ToolResponse:
+ ) -> ToolChunk:
"""Ask the user one or more questions and wait for the human answer."""
result = await user_questions.ask(
@@ -160,7 +161,7 @@ async def ask_user_question(
# to DSH's exact two-sentence contract instead of leaking local policy or
# transport details into the fixed tool-schema prefix.
ask_user_question.__doc__ = _TOOL_DESCRIPTION
- toolkit.register_tool_function(ask_user_question, namesake_strategy="override")
+ toolkit.append(function_tool(ask_user_question))
logger.info("[factory] Registered ask_user_question tool (interactive top-level chat)")
diff --git a/src/backend/core/llm/tools/chat_history_tool.py b/src/backend/core/llm/tools/chat_history_tool.py
index 81df9387..02c6bd91 100644
--- a/src/backend/core/llm/tools/chat_history_tool.py
+++ b/src/backend/core/llm/tools/chat_history_tool.py
@@ -9,6 +9,7 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import logging
@@ -33,48 +34,10 @@ def register_chat_history_tools(
scope_hint = "当前项目内的会话" if project_id else "你最近的会话"
- def _list(query: str, limit: int):
- from core.db.engine import SessionLocal
- from core.services.chat_reference_service import list_referencable_sessions, session_brief
-
- with SessionLocal() as db:
- sessions = list_referencable_sessions(
- db,
- user_id,
- project_id=project_id,
- query=query,
- limit=limit,
- exclude_chat_id=chat_id,
- )
- return {
- "ok": True,
- "scope": "project" if project_id else "user",
- "count": len(sessions),
- "chats": [session_brief(session) for session in sessions],
- }
-
- def _read(target_chat_id: str, mode: str, offset: int, limit: int):
- from core.db.engine import SessionLocal
- from core.services.chat_reference_service import read_session_messages, session_card
-
- with SessionLocal() as db:
- card = session_card(db, target_chat_id, user_id)
- if card is None:
- return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
- if mode == "card":
- return {"ok": True, "mode": "card", **card}
- page = read_session_messages(db, target_chat_id, user_id, offset=offset, limit=limit)
- if page is None:
- return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
- return {"ok": True, "mode": "full", "title": card["title"], **page}
-
- async def _digest(target_chat_id: str):
- from core.services.chat_reference_service import build_session_digest
-
- result = await build_session_digest(target_chat_id, user_id)
- if result is None:
- return {"ok": False, "error": f"会话不存在或你无权访问:{target_chat_id}"}
- return {"ok": True, "mode": "digest", **result}
+ from core.harness.registry.chat_history import history_operations
+ _list, _read, _digest = history_operations(
+ user_id=user_id, chat_id=chat_id, project_id=project_id,
+ )
async def list_related_chats(query: str = "", limit: int = 20):
"""列出可供参考的历史会话({scope}),每条给出标题、最后活跃时间和消息条数。
@@ -127,5 +90,5 @@ async def read_chat(
list_related_chats.__doc__ = (list_related_chats.__doc__ or "").format(scope=scope_hint)
- toolkit.register_tool_function(list_related_chats)
- toolkit.register_tool_function(read_chat)
+ toolkit.append(function_tool(list_related_chats))
+ toolkit.append(function_tool(read_chat))
diff --git a/src/backend/core/llm/gateway_mcp_tool.py b/src/backend/core/llm/tools/cloud_gateway.py
similarity index 71%
rename from src/backend/core/llm/gateway_mcp_tool.py
rename to src/backend/core/llm/tools/cloud_gateway.py
index 5ae0e72c..685a1fac 100644
--- a/src/backend/core/llm/gateway_mcp_tool.py
+++ b/src/backend/core/llm/tools/cloud_gateway.py
@@ -1,24 +1,43 @@
"""Authorized cloud gateway invocation, separate from manifest discovery."""
+from core.harness.registry import execution_identity as bridge_identity
import asyncio
+import weakref
+from contextlib import asynccontextmanager
from typing import Any, Dict
import httpx
-import mcp.types
from agentscope.permission import PermissionBehavior, PermissionDecision
from agentscope.tool import ToolBase, ToolChunk
+_shared_clients = weakref.WeakKeyDictionary()
+
+
+@asynccontextmanager
+async def _http(transport):
+ """Reuse one pooled client per event loop; an injected transport gets its own client."""
+ if transport is not None:
+ async with httpx.AsyncClient(transport=transport) as client:
+ yield client
+ return
+ loop = asyncio.get_running_loop()
+ client = _shared_clients.get(loop)
+ if client is None or client.is_closed:
+ client = _shared_clients[loop] = httpx.AsyncClient()
+ yield client
+
+
class GatewayMCPTool(ToolBase):
"""MCP-shaped tool built from a cloud-owned runtime manifest snapshot."""
is_mcp = True
is_state_injected = False
is_external_tool = False
- is_concurrency_safe = True
+ is_concurrency_safe = False
def __init__(
self,
*,
mcp_name: str,
- tool: mcp.types.Tool,
+ tool: dict,
invoke_url: str,
schema_hash: str,
headers: Dict[str, str],
@@ -26,17 +45,18 @@ def __init__(
transport: Any = None,
source_plugin: str = "",
) -> None:
+ super().__init__()
self._source_plugin = source_plugin
self.mcp_name = mcp_name
- self.name = tool.name
- self.description = tool.description or ""
- schema = dict(tool.inputSchema or {})
+ self.name = tool["name"]
+ self.description = tool.get("description") or ""
+ schema = dict(tool.get("inputSchema") or {})
schema.setdefault("type", "object")
schema.setdefault("properties", {})
schema.setdefault("required", [])
self.input_schema = schema
self.is_read_only = bool(
- tool.annotations and getattr(tool.annotations, "readOnlyHint", False)
+ tool.get("annotations", {}).get("readOnlyHint")
)
self._invoke_url = invoke_url
self._schema_hash = schema_hash
@@ -65,11 +85,9 @@ def _unknown_outcome(self) -> ToolChunk:
"gateway_outcome_unknown": True},
)
- async def __call__(self, **kwargs: Any) -> ToolChunk:
+ async def call(self, **kwargs: Any) -> ToolChunk:
from extensions.mcp_servers.registry import permission_specs
- from core.llm.tool_permissions.models import (
- CURRENT_PERMISSION_TICKET, PermissionEnforcementError, _json_hash,
- )
+ from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError, _json_hash
spec = permission_specs(self.mcp_name).get(self.name)
if spec is not None:
ticket = CURRENT_PERMISSION_TICKET.get()
@@ -77,77 +95,66 @@ async def __call__(self, **kwargs: Any) -> ToolChunk:
or ticket.spec_key != spec.key or ticket.args_hash != _json_hash(kwargs)):
raise PermissionEnforcementError("云端空间工具缺少匹配的预执行授权票据")
headers = dict(self._headers)
- from core.services.desktop_observation_context import context_headers
+ from core.harness.registry.execution_identity import context_headers
headers.update(context_headers())
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
captured = None
- if capabilities_enabled() and "/api/v1/desktop/capability/gateway/" in self._invoke_url:
- from core.services import desktop_cloud_bridge as bridge
+ if has_run_capabilities() and "/api/v1/desktop/capability/gateway/" in self._invoke_url:
captured = {"cloud_base": self._invoke_url.split("/api/v1/desktop/capability/gateway/", 1)[0],
"token": str(headers.get("Authorization") or "").removeprefix("Bearer ")}
- bridge.require_current_account(captured)
- headers.update(bridge.cloud_headers(bridge.get_state()))
+ bridge_identity.require_current_account(captured)
+ headers.update(bridge_identity.cloud_headers(bridge_identity.get_state()))
local_result = None
if captured is not None:
- from core.services.automation_tool_routing import local_automation_result
+ from core.harness.registry.tool_operations import local_tool_result
async def authorize_local():
- options = {"timeout": 15.0}
- if self._transport is not None:
- options["transport"] = self._transport
- async with httpx.AsyncClient(**options) as client:
+ async with _http(self._transport) as client:
reply = await client.post(
self._invoke_url.rsplit("/", 1)[0] + "/authorize",
- headers=headers,
+ headers=headers, timeout=15.0,
json={"tool_name": self.name, "schema_hash": self._schema_hash,
"operation_id": kwargs.get("tool_effect_id") or "authorization"},
)
- bridge.require_current_account(captured)
+ bridge_identity.require_current_account(captured)
if reply.status_code in (401, 403):
- with bridge.account_scope(captured):
- bridge.clear_state()
+ with bridge_identity.account_scope(captured):
+ bridge_identity.clear_state()
if not reply.is_success:
raise ValueError("本机任务工具授权已变更或不可用,请同步云端能力后重试")
try:
- local_result = await local_automation_result(
+ local_result = await local_tool_result(
self._source_plugin, self.name, kwargs, headers, authorize=authorize_local,
)
except ValueError as exc:
from agentscope.message import TextBlock, ToolResultState
return ToolChunk(content=[TextBlock(text=str(exc))], state=ToolResultState.ERROR)
- bridge.require_current_account(captured)
+ bridge_identity.require_current_account(captured)
if local_result is not None and not local_result.metadata.get("merge_cloud"):
return local_result
headers["accept-encoding"] = "identity"
- client_kwargs: Dict[str, Any] = {
- "timeout": httpx.Timeout(
- connect=min(10.0, self._timeout),
- read=self._timeout,
- write=min(60.0, self._timeout),
- pool=min(10.0, self._timeout),
- )
- }
- if self._transport is not None:
- client_kwargs["transport"] = self._transport
+ timeout = httpx.Timeout(
+ connect=min(10.0, self._timeout),
+ read=self._timeout,
+ write=min(60.0, self._timeout),
+ pool=min(10.0, self._timeout),
+ )
channel = None
if captured is not None:
- from core.services.desktop_gateway_uploads import (
- UPLOAD_OPTIONS_HEADER,
- UPLOAD_SCHEMA_HEADER,
- upload_channel,
- )
+ from core.harness.registry.transfer_channels import UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER, upload_channel
channel = upload_channel(self._source_plugin, self.name)
if channel is not None:
body, options = await channel.package(kwargs, headers)
- bridge.require_current_account(captured)
- from core.services.automation_remote_effect import RECEIPT_TOOLS, remember_remote_call
- from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT
+ bridge_identity.require_current_account(captured)
+ from core.harness.registry.tool_operations import receipt_operations
+ from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT
effect = CURRENT_TOOL_EFFECT.get()
- if self.name in RECEIPT_TOOLS and effect is not None:
- await asyncio.to_thread(remember_remote_call, effect, self.mcp_name,
+ operations = receipt_operations(self.name)
+ if operations is not None and effect is not None:
+ await asyncio.to_thread(operations.remember, effect, self.mcp_name,
self._invoke_url, self._schema_hash)
try:
- async with httpx.AsyncClient(**client_kwargs) as client:
+ async with _http(self._transport) as client:
if channel is not None:
headers.update(
{
@@ -160,6 +167,7 @@ async def authorize_local():
f"{self._invoke_url.rsplit('/', 1)[0]}/{channel.endpoint}",
headers=headers,
content=body,
+ timeout=timeout,
)
else:
response = await client.post(
@@ -170,20 +178,21 @@ async def authorize_local():
"arguments": kwargs,
"schema_hash": self._schema_hash,
},
+ timeout=timeout,
)
if captured is not None:
- bridge.require_current_account(captured)
+ bridge_identity.require_current_account(captured)
response.raise_for_status()
payload = response.json()
data = payload.get("data") if isinstance(payload, dict) else None
if not isinstance(data, dict):
raise ValueError("cloud gateway returned no tool result")
if captured is not None:
- from core.services.desktop_gateway_uploads import result_localizer
+ from core.harness.registry.transfer_channels import result_localizer
localize = result_localizer(self._source_plugin, self.name)
if localize is not None:
- with bridge.account_scope(captured):
+ with bridge_identity.account_scope(captured):
localize(data, captured["cloud_base"], kwargs, headers)
chunk = ToolChunk.model_validate(data)
chunk.metadata.setdefault("origin", "cloud")
@@ -202,13 +211,13 @@ async def authorize_local():
if not self.is_read_only and (exc.response.status_code >= 500 or exc.response.status_code == 422):
return self._unknown_outcome()
if captured is not None and exc.response.status_code in (401, 403):
- with bridge.account_scope(captured):
- bridge.clear_state()
+ with bridge_identity.account_scope(captured):
+ bridge_identity.clear_state()
if exc.response.status_code == 409:
# 云端在此明确告知能力已变——这是「不轮询」下发现云端改动的信号之一,
# 立刻后台同步一次清单,下一轮对话就是新的。
if captured is not None:
- bridge.notify_cloud_changed()
+ bridge_identity.notify_cloud_changed()
raise RuntimeError(
f"云端工具 {self.name} 已更新,正在同步最新能力,请重试"
) from exc
diff --git a/src/backend/core/llm/tools/cloud_manifest.py b/src/backend/core/llm/tools/cloud_manifest.py
new file mode 100644
index 00000000..5a80d1f5
--- /dev/null
+++ b/src/backend/core/llm/tools/cloud_manifest.py
@@ -0,0 +1,55 @@
+"""Build native tools from an authorized cloud or local management manifest."""
+from core.capabilities.mcp.tool_schema import has_usable_schema
+from core.capabilities.plugins.management.contract import declared, localize, tools as manager_tools
+from core.harness.registry.run_capabilities import has_run_capabilities
+from core.llm.tools.cloud_gateway import GatewayMCPTool
+from core.llm.tools.management import ManagementTool
+
+
+def manifest_tools(name, cfg):
+ plugin = str(cfg.get("gateway_plugin") or "")
+ url = str(cfg.get("gateway_invoke_url") or "")
+ schema_hash = str(cfg.get("schema_hash") or "")
+ items = cfg.get("manifest_tools")
+ if not isinstance(items, list):
+ raise ValueError("Missing authorized tool manifest")
+ if cfg.get("schema_source") == "cloud_manifest" and not (url and schema_hash):
+ raise ValueError("Cloud manifest requires an invocation URL and schema hash")
+ if cfg.get("schema_source") == "plugin_manifest" and not (
+ cfg.get("origin") == "local_plugin" and all(declared(t, plugin) for t in items)
+ ):
+ raise ValueError("Invalid local management manifest")
+ if not all(isinstance(t, dict) for t in items):
+ raise ValueError("Invalid manifest tool entry")
+ items = list(items)
+ from extensions.mcp_servers.registry import gateway_server
+ if cfg.get("read_only") and gateway_server(name) is not None:
+ items = [t for t in items if t.get("annotations", {}).get("readOnlyHint") is True]
+ local = has_run_capabilities()
+ if local and plugin == "skill-manager" and any(t.get("name") == "install_skill" and declared(t, plugin) for t in items):
+ items += [t for t in manager_tools(plugin, "local") if t["name"] == "upload_skill_to_cloud" and not any(x.get("name") == t["name"] for x in items)]
+ result = []
+ names = set()
+ for item in items:
+ if local:
+ item = localize(item, plugin, cloud_available=bool(cfg.get("manager_cloud_available") or url))
+ if item is None or not has_usable_schema(item):
+ continue
+ tool_name = item.get("name")
+ if not isinstance(tool_name, str) or not tool_name or tool_name in names:
+ raise ValueError("Invalid or duplicate manifest tool name")
+ names.add(tool_name)
+ if cfg.get("enable_tools") is not None and tool_name not in cfg["enable_tools"]:
+ continue
+ if tool_name in (cfg.get("disable_tools") or []):
+ continue
+ import jsonschema
+ jsonschema.Draft202012Validator.check_schema(item["inputSchema"])
+ headers = dict(cfg.get("headers") or {})
+ if local and declared(item, plugin):
+ result.append(ManagementTool(name, item, plugin, headers, cloud_backed=bool(url)))
+ else:
+ result.append(GatewayMCPTool(mcp_name=name, tool=item, invoke_url=url,
+ schema_hash=schema_hash, headers=headers, timeout=cfg.get("execution_timeout") or 120,
+ transport=cfg.get("gateway_transport"), source_plugin=plugin))
+ return result
diff --git a/src/backend/core/llm/tools/_tool_confirmation/__init__.py b/src/backend/core/llm/tools/confirmation/__init__.py
similarity index 100%
rename from src/backend/core/llm/tools/_tool_confirmation/__init__.py
rename to src/backend/core/llm/tools/confirmation/__init__.py
diff --git a/src/backend/core/llm/tools/_tool_confirmation/gate.py b/src/backend/core/llm/tools/confirmation/gate.py
similarity index 99%
rename from src/backend/core/llm/tools/_tool_confirmation/gate.py
rename to src/backend/core/llm/tools/confirmation/gate.py
index 9f33c78d..afeda91e 100644
--- a/src/backend/core/llm/tools/_tool_confirmation/gate.py
+++ b/src/backend/core/llm/tools/confirmation/gate.py
@@ -7,10 +7,11 @@
from typing import Any, Dict, Optional
from . import state
-logger = logging.getLogger(__name__)
from .shared import _load_session_grants, _save_session_grant
from .pending import _register_pending, _wait_pending
+logger = logging.getLogger(__name__)
+
async def gate(
*,
diff --git a/src/backend/core/llm/tools/_tool_confirmation/pending.py b/src/backend/core/llm/tools/confirmation/pending.py
similarity index 96%
rename from src/backend/core/llm/tools/_tool_confirmation/pending.py
rename to src/backend/core/llm/tools/confirmation/pending.py
index 8ad0e966..3069093f 100644
--- a/src/backend/core/llm/tools/_tool_confirmation/pending.py
+++ b/src/backend/core/llm/tools/confirmation/pending.py
@@ -6,13 +6,14 @@
import time
import uuid
from typing import Any, Dict, Optional, Tuple
-from core.llm import interaction_store
+from core.llm.runtime import interaction_store
from . import state
-logger = logging.getLogger(__name__)
from .shared import _load_session_grants, _save_session_grant
from core.llm.tool_permissions.pending import approve_current_policy
+logger = logging.getLogger(__name__)
+
async def _register_pending(
cid_key: str, key: Tuple[str, ...], *, make_pending, timeout, precheck=None
@@ -21,7 +22,7 @@ async def _register_pending(
now = time.monotonic()
ui_q: Optional[asyncio.Queue] = None
ui_payload: Optional[Dict[str, Any]] = None
- with state._LOCK:
+ with state.process_state().LOCK:
state._gc(now)
st = state._get_chat(cid_key)
st.last_ts = now
@@ -63,7 +64,7 @@ async def _register_pending(
if p.registration_error is not None:
raise p.registration_error
except BaseException:
- with state._LOCK:
+ with state.process_state().LOCK:
p.waiters -= 1
last = p.waiters <= 0
if last:
@@ -126,12 +127,12 @@ async def _wait_pending(
cancelled = True
expire_q: Optional[asyncio.Queue] = None
expire_payload: Optional[Dict[str, Any]] = None
- with state._LOCK:
+ with state.process_state().LOCK:
decision = p.decision or state._DECISION_TIMEOUT
extracted = in_lock_extract(p, decision) if in_lock_extract and not cancelled else None
p.waiters -= 1
if p.waiters <= 0:
- st2 = state._CHATS.get(cid_key)
+ st2 = state.process_state().CHATS.get(cid_key)
if st2 is not None:
st2.pending.pop(cid, None)
if st2.key_index.get(key) == cid:
diff --git a/src/backend/core/llm/tools/_tool_confirmation/shared.py b/src/backend/core/llm/tools/confirmation/shared.py
similarity index 98%
rename from src/backend/core/llm/tools/_tool_confirmation/shared.py
rename to src/backend/core/llm/tools/confirmation/shared.py
index 3c971602..a7f1304c 100644
--- a/src/backend/core/llm/tools/_tool_confirmation/shared.py
+++ b/src/backend/core/llm/tools/confirmation/shared.py
@@ -2,7 +2,7 @@
from __future__ import annotations
import logging
-from core.llm import interaction_store
+from core.llm.runtime import interaction_store
from . import state
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/llm/tools/_tool_confirmation/state.py b/src/backend/core/llm/tools/confirmation/state.py
similarity index 89%
rename from src/backend/core/llm/tools/_tool_confirmation/state.py
rename to src/backend/core/llm/tools/confirmation/state.py
index 28e95125..99fd18ba 100644
--- a/src/backend/core/llm/tools/_tool_confirmation/state.py
+++ b/src/backend/core/llm/tools/confirmation/state.py
@@ -1,6 +1,33 @@
"""Local confirmation state, deduplication and session-scoped decisions."""
from __future__ import annotations
+from core.sandbox.permissions.vocabulary import (
+ KIND_MYSPACE,
+ KIND_AUTOMATION,
+ KIND_DESIGN_PICK,
+ KIND_LOCAL_CMD,
+ KIND_LOCAL_PATH_PREFIX,
+ KIND_TOOL_PERMISSION,
+ KIND_TOOL_PREFIX,
+ OP_WRITE,
+ OP_EDIT,
+ OP_DELETE,
+ OP_LOCAL_EXEC,
+ OP_LOCAL_READ,
+ OP_LOCAL_WRITE,
+ OP_MOVE,
+ OP_MKDIR,
+ OP_CRON_CREATE,
+ OP_CRON_UPDATE,
+ OP_CRON_DELETE,
+ DECISION_ALLOW,
+ DECISION_ALLOW_SESSION,
+ DECISION_DENY,
+ DECISION_CHOICE,
+ DECISION_SKIP,
+ STATUS_BLOCKED
+)
+
import asyncio
import logging
import os
@@ -9,36 +36,12 @@
import time
from dataclasses import dataclass, field
from typing import Any, Dict, Optional, Tuple
-from core.llm.human_interaction import MAX_WAIT_SECONDS
+from core.llm.runtime.human_interaction import MAX_WAIT_SECONDS
logger = logging.getLogger(__name__)
-KIND_MYSPACE = "myspace"
-KIND_AUTOMATION = "automation"
-KIND_DESIGN_PICK = "design_pick"
-KIND_LOCAL_CMD = "local_cmd"
-KIND_LOCAL_PATH_PREFIX = "local_path:"
-KIND_TOOL_PERMISSION = "tool_permission"
-KIND_TOOL_PREFIX = "tool:"
-OP_WRITE = "write"
-OP_EDIT = "edit"
-OP_DELETE = "delete"
-OP_LOCAL_EXEC = "local_exec"
-OP_LOCAL_READ = "local_read"
-OP_LOCAL_WRITE = "local_write"
-OP_MOVE = "move"
-OP_MKDIR = "mkdir"
-OP_CRON_CREATE = "cron_create"
-OP_CRON_UPDATE = "cron_update"
-OP_CRON_DELETE = "cron_delete"
-DECISION_ALLOW = "allow"
-DECISION_ALLOW_SESSION = "allow_session"
-DECISION_DENY = "deny"
_DECISIONS = (DECISION_ALLOW, DECISION_ALLOW_SESSION, DECISION_DENY)
_DECISION_TIMEOUT = "_timeout"
-DECISION_CHOICE = "choice"
-DECISION_SKIP = "skip"
_DESIGN_DECISIONS = (DECISION_CHOICE, DECISION_SKIP)
-STATUS_BLOCKED = "blocked_non_interactive"
_TTL_S = 1800
_DEFAULT_WAIT_S = MAX_WAIT_SECONDS
@@ -105,23 +108,19 @@ def _scope_covers(kind: str, granted: str, logical_path: str) -> bool:
return True
-_LOCK = threading.RLock()
-_CHATS: Dict[str, _ChatConfirm] = {}
-
-
def _gc(now: float) -> None:
"""Lazily clean up expired chat state (call while holding the lock). Chats with pending items / suspended waiters are not cleaned."""
- dead = [c for c, st in _CHATS.items() if now - st.last_ts > _TTL_S and (not st.pending)]
+ dead = [c for c, st in process_state().CHATS.items() if now - st.last_ts > _TTL_S and (not st.pending)]
for c in dead:
- _CHATS.pop(c, None)
+ process_state().CHATS.pop(c, None)
def _get_chat(chat_id: str) -> _ChatConfirm:
"""Get or create per-chat state (call while holding the lock)."""
- st = _CHATS.get(chat_id)
+ st = process_state().CHATS.get(chat_id)
if st is None:
st = _ChatConfirm()
- _CHATS[chat_id] = st
+ process_state().CHATS[chat_id] = st
return st
@@ -129,7 +128,7 @@ def allow_session(chat_id: Optional[str]) -> None:
"""Pre-authorize inbound channel writes for this conversation's explicit scope."""
if not chat_id:
return
- with _LOCK:
+ with process_state().LOCK:
st = _get_chat(chat_id)
st.session_allows.add((KIND_MYSPACE, "", ""))
st.last_ts = time.monotonic()
@@ -145,8 +144,8 @@ def get_ui_queue(chat_id: Optional[str]) -> Optional[asyncio.Queue]:
"""
if not chat_id:
return None
- with _LOCK:
- st = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ st = process_state().CHATS.get(chat_id)
return st.ui_signals if st is not None else None
@@ -180,8 +179,8 @@ def get_pending(chat_id: Optional[str]) -> Optional[Dict[str, Any]]:
"""
if not chat_id:
return None
- with _LOCK:
- st = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ st = process_state().CHATS.get(chat_id)
if st is None or not st.pending:
return None
live = [(cid, p) for cid, p in st.pending.items() if p.decision is None]
@@ -202,8 +201,8 @@ def get_all_pending(chat_id: Optional[str]) -> list[Dict[str, Any]]:
"""
if not chat_id:
return []
- with _LOCK:
- st = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ st = process_state().CHATS.get(chat_id)
if st is None or not st.pending:
return []
live = [(cid, p) for cid, p in st.pending.items() if p.decision is None]
@@ -216,10 +215,10 @@ def get_all_pending(chat_id: Optional[str]) -> list[Dict[str, Any]]:
def list_pending_chat_ids() -> list[str]:
"""All chat_ids that have undecided pending confirmations (the out-of-band batch endpoint uses this for ownership filtering)."""
- with _LOCK:
+ with process_state().LOCK:
return [
cid
- for cid, st in _CHATS.items()
+ for cid, st in process_state().CHATS.items()
if any((p.decision is None for p in st.pending.values()))
]
@@ -228,8 +227,8 @@ def has_pending(chat_id: Optional[str]) -> bool:
"""Whether this chat has a live suspended confirmation/design wait."""
if not chat_id:
return False
- with _LOCK:
- st = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ st = process_state().CHATS.get(chat_id)
return bool(st and any((p.decision is None for p in st.pending.values())))
@@ -248,8 +247,8 @@ def set_decision(
- design_pick: {choice, skip}; for choice, option_id is required and must be a valid option
Returns {ok, decision, op, logical_path} or {ok: False, error}.
"""
- with _LOCK:
- st = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ st = process_state().CHATS.get(chat_id)
if st is None:
return {"ok": False, "reason": "stale", "error": "该确认已失效(可能已超时或服务重启)"}
p = st.pending.get(confirm_id)
@@ -363,3 +362,16 @@ def _intercept_message(kind: str, phase: str, op: str, logical_path: str, summar
"deny": f"用户拒绝了对「我的空间」的{op}操作({logical_path})。不要重试该写入。如确需产物,改写到本次会话的工作目录(相对路径),或向用户澄清意图。",
"timeout": f"等待用户确认对「我的空间」的{op}操作超时({logical_path}),已放弃未写入。请简短告知用户超时,让其重新发起或改写到会话工作目录。",
}[phase]
+
+
+@dataclass
+class State:
+ LOCK: object = field(default_factory=lambda: threading.RLock())
+ CHATS: Dict[str, _ChatConfirm] = field(default_factory=lambda: {})
+
+
+_STATE = State()
+
+
+def process_state() -> State:
+ return _STATE
diff --git a/src/backend/core/llm/tools/data_context_tool.py b/src/backend/core/llm/tools/data_context_tool.py
deleted file mode 100644
index c47553e8..00000000
--- a/src/backend/core/llm/tools/data_context_tool.py
+++ /dev/null
@@ -1,96 +0,0 @@
-"""get_data_context tool — on-demand recall of "data dictionary + golden SQL" before data retrieval.
-
-The sole injection vehicle for improving "direct database query" accuracy: the
-model proactively calls this tool before writing SQL, getting back relevant
-table/column business names + descriptions + enum dictionaries + sample values
-+ verified query examples, then writes the SELECT based on them.
-**Metadata never enters the system prompt** — it only appears as this tool's
-return value; the model sees it only by calling the tool.
-
-agent_factory registers this tool only when there exists a data source that is
-"direct-connect (not external_nl2sql) + enabled + annotated" (see the mounting
-gate in core/llm/factory/build.py).
-"""
-
-import logging
-from typing import List, Optional
-
-from agentscope.message import TextBlock
-from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
-
-logger = logging.getLogger(__name__)
-
-# Guardrail on the number of data sources returned per call (concatenating all of them would be too large with many sources).
-_MAX_SOURCES_PER_CALL = 4
-
-
-def register_get_data_context(toolkit: Toolkit, datasource_ids: List[str]) -> None:
- """Register the get_data_context tool, scoped to the given set of available data sources."""
-
- scoped = list(datasource_ids)
-
- async def get_data_context(question: str, datasource: Optional[str] = None) -> ToolResponse:
- """获取数据库的「数据字典 + 已验证查询范例」,用于准确取数。
-
- ⚠️ **使用时机(强烈建议先调用本工具再写 SQL)**:当你要用「数据库直连查询」
- (execute_sql / search_objects)从业务库取数,而该库字段名晦涩、取值是编码
- (如 status=1)、或表很多不确定选哪张时,**先调用本工具**拿到人工标注的表/列
- 业务含义、枚举字典(如 1=已审核)、外键关联和已验证的「问题→SQL」范例,再据此
- 写 SELECT,可显著降低取数出错。
-
- Args:
- question (`str`):
- 你要回答的取数问题(自然语言)。用于召回最相关的表/字段/范例。
- datasource (`str`, optional):
- 指定数据源 id(仅当你已知道要查哪个库时填)。不填则返回全部可用
- 数据源的字典摘要。
-
- Returns:
- 数据字典与黄金 SQL 的 Markdown 文本;若该库尚无标注则返回提示。
- 注意:返回内容是人工治理的业务口径,应优先据此选表/字段、解码枚举值。
- """
- from core.services import db_metadata_service as svc
-
- if not scoped:
- return _text("当前没有可用的数据字典(未配置元数据治理)。可直接用 "
- "search_objects 探查表结构后再写 SQL。")
-
- targets: List[str]
- ds = (datasource or "").strip()
- if ds:
- if ds in scoped:
- targets = [ds]
- else:
- return _text(
- f"数据源 `{ds}` 无可用数据字典。可用的数据源:{', '.join(scoped)}。"
- "(不填 datasource 可返回全部)"
- )
- else:
- targets = scoped[:_MAX_SOURCES_PER_CALL]
-
- parts: List[str] = []
- for ds_id in targets:
- try:
- digest = svc.build_digest(ds_id, question=question)
- except Exception as exc: # noqa: BLE001
- logger.warning("[get_data_context] build_digest failed for %s: %s", ds_id, exc)
- digest = ""
- if digest:
- parts.append(digest)
-
- if not parts:
- return _text("相关数据源暂无已标注的数据字典内容。可用 search_objects 探查表结构。")
-
- text = "\n\n---\n\n".join(parts)
- if len(scoped) > len(targets):
- text += (f"\n\n(另有 {len(scoped) - len(targets)} 个数据源未展开;"
- "如需指定,请用 datasource 参数。可用:" + ", ".join(scoped) + ")")
- return _text(text)
-
- toolkit.register_tool_function(get_data_context, namesake_strategy="override")
- logger.info("[factory] Registered get_data_context tool (datasources=%s)", scoped)
-
-
-def _text(s: str) -> ToolResponse:
- return ToolResponse(content=[TextBlock(type="text", text=s)])
diff --git a/src/backend/core/llm/tools/edit_tool.py b/src/backend/core/llm/tools/edit_tool.py
index 2880a85d..90bc64c5 100644
--- a/src/backend/core/llm/tools/edit_tool.py
+++ b/src/backend/core/llm/tools/edit_tool.py
@@ -1,6 +1,7 @@
"""Read-checked mutations of files in the active sandbox."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import difflib
import hashlib
@@ -8,19 +9,13 @@
from typing import Optional
from agentscope.tool import Toolkit, ToolResponse
-from core.services.project_scope import ProjectScope
+from core.projects.edition import ProjectScope
from ._common import (
resolve_sandbox_session,
resp_json,
)
-from ._paths import (
- is_myspace_physical,
- space_metadata,
- to_physical_path,
- validate_project_scope_path,
- validate_workspace_path,
-)
+from core.sandbox.workspace.resolution import space_metadata, to_physical_path, validate_project_scope_path, validate_workspace_path
from ._state import ReadEntry, ReadStateTracker
logger = logging.getLogger(__name__)
@@ -63,10 +58,7 @@ async def Edit(
new_string: str,
replace_all: bool = False,
) -> "ToolResponse": # type: ignore[name-defined]
- from core.llm.tool_permissions import (
- PermissionEnforcementError,
- require_local_path_permission,
- )
+ from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission
from core.sandbox import SandboxConnectError as _SCE
from core.sandbox import SandboxError as _SE
from core.sandbox import get_sandbox_provider as _get_provider
@@ -207,11 +199,12 @@ async def Edit(
new_bytes = new_text.encode("utf-8")
# Local-mode write-back snapshot (ticket #08): back up before overwriting.
try:
- from core.services.local_snapshot_service import maybe_snapshot_local
+ from core.harness.registry.local_files import maybe_snapshot_local
maybe_snapshot_local(physical)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools.edit_tool:Edit', fallback_error)
try:
if _is_local:
import os as _os
@@ -277,5 +270,5 @@ async def Edit(
" ``{error: '...'}`` 失败。\n"
)
- toolkit.register_tool_function(Edit, namesake_strategy="override")
+ toolkit.append(function_tool(Edit))
logger.info("[factory] Registered Edit tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/llm/tools/glob_tool.py b/src/backend/core/llm/tools/glob_tool.py
index af69102b..34f5c9f4 100644
--- a/src/backend/core/llm/tools/glob_tool.py
+++ b/src/backend/core/llm/tools/glob_tool.py
@@ -1,5 +1,6 @@
"""Authorized sandbox file search with stateless continuation."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import logging
import json
@@ -8,10 +9,10 @@
from agentscope.tool import Toolkit, ToolResponse
from core.db.paging import normalize_page, normalize_page_size
-from core.services.project_scope import ProjectScope
+from core.projects.edition import ProjectScope
from ._common import resolve_sandbox_session, resp_json, sandbox_exec_bash, shell_quote
-from ._paths import to_physical_path, validate_project_scope_path, validate_workspace_path
+from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path
logger = logging.getLogger(__name__)
@@ -95,5 +96,5 @@ async def Glob(
"limit 默认 100;0 或负数返回全部。page 仅兼容第一页。"
)
- toolkit.register_tool_function(Glob, namesake_strategy="override")
+ toolkit.append(function_tool(Glob))
logger.info("[factory] Registered Glob tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/llm/tools/grep_tool.py b/src/backend/core/llm/tools/grep_tool.py
index 7830ff36..0541b8c8 100644
--- a/src/backend/core/llm/tools/grep_tool.py
+++ b/src/backend/core/llm/tools/grep_tool.py
@@ -7,21 +7,17 @@
"""
from __future__ import annotations
-from core.services.edition_workspace import is_shared_scope
+from core.llm.factory.tools.definitions import function_tool
import logging
from typing import Optional
from agentscope.tool import Toolkit, ToolResponse
-from core.services.project_scope import ProjectScope
+from core.projects.edition import ProjectScope
from ._common import resolve_sandbox_session, resp_json, sandbox_exec_bash, shell_quote
-from ._paths import (
- to_physical_path,
- validate_project_scope_path,
- validate_workspace_path,
-)
+from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path
logger = logging.getLogger(__name__)
@@ -181,5 +177,5 @@ async def Grep(
" num_matches, truncated}``。\n"
)
- toolkit.register_tool_function(Grep, namesake_strategy="override")
+ toolkit.append(function_tool(Grep))
logger.info("[factory] Registered Grep tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/llm/tools/job_tool.py b/src/backend/core/llm/tools/job_tool.py
index 35f120b3..e528a97a 100644
--- a/src/backend/core/llm/tools/job_tool.py
+++ b/src/backend/core/llm/tools/job_tool.py
@@ -1,72 +1,11 @@
-"""run_job 工具 —— 主对话智能体提交批量作业的唯一入口。
-
-定位:这不是「又一个固定形状的批处理工具」,而是一个**可编程的编排面**。智能体先用
-``write`` 把一段作业脚本写进沙箱,再用 ``run_job`` 提交;脚本里的控制流(循环、条件、
-分层取数、动态收敛、嵌套 map)都是普通 Python,平台不需要知道任务在做什么。
-
-与 Bash / read / write / glob / grep 同级原生注册,不走 catalog 开关——「对话内默认
-可用」就落在这里。
-"""
-
-from __future__ import annotations
-
-import json as _json
-import logging
+"""SDK binding for the user-scoped batch job action service."""
+from core.llm.factory.tools.definitions import function_tool
+import json
from typing import Any, Dict, List, Optional
-
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
-
-from ._paths import quote_shell_path as _quote_path
-
-logger = logging.getLogger(__name__)
-
-# 前台等待(wait=True)的硬上限。超过就把作业转后台,不再阻塞这一轮对话。
-# 该等多久是模型自己判断的,而它对"这批要跑多久"的估计经常偏乐观——实测把一个 265 项、
-# 每项都要过模型的分类作业当成"小作业",前台一口气阻塞了 5 分钟,用户看不到进度、
-# 插不上话、也没法中途取消。提示词已经写过"几分钟以上就走后台"仍然拦不住,所以在
-# 机制上给前台等待封顶:等够 90 秒还没完,作业照跑,对话先还给用户。
-_FOREGROUND_WAIT_CAP_S = 90.0
-
-
-def _resp(payload: Dict[str, Any]) -> ToolResponse:
- return ToolResponse(
- content=[TextBlock(type="text", text=_json.dumps(payload, ensure_ascii=False))]
- )
-
-
-def _mark_wake_on_finish(job_id: str) -> None:
- """转后台后要有人叫醒会话播报终态,否则作业跑完没人吭声。"""
- try:
- from core.db.engine import SessionLocal
- from core.services.job_service import JobService
-
- with SessionLocal() as db:
- JobService(db).set_wake_on_finish(job_id, True)
- except Exception: # noqa: BLE001 —— 叫醒是尽力而为,不该反过来弄挂工具调用
- logger.warning("[run_job] mark wake_on_finish failed job=%s", job_id, exc_info=True)
-
-
-def _detached_payload(job_id: str, *, resumed: bool = False) -> Dict[str, Any]:
- """前台等超时 → 按后台语义返回(作业没停,只是不占着这一轮对话了)。"""
- _mark_wake_on_finish(job_id)
- return {
- "ok": True,
- "job_id": job_id,
- "status": "started",
- "waited": False,
- "detached": True,
- "next": (
- f"作业比预期久(前台已等满 {int(_FOREGROUND_WAIT_CAP_S)} 秒),"
- "**已自动转入后台继续跑,没有中断**。"
- "现在就结束这轮回复:告诉用户作业仍在进行、"
- "输入框上方的状态条会实时显示进度,期间可以继续聊别的、也可以随时取消。"
- "作业跑完会自动叫醒你播报,不要在这里轮询 action='status' 干等,"
- "也不要重新提交一遍(它还在跑)。"
- ),
- }
-
+from agentscope.tool._response import ToolChunk
+from core.harness.registry.job_operations import job_operations
def register_run_job(
toolkit: Toolkit,
@@ -86,6 +25,8 @@ def register_run_job(
``allowed_tools`` 保留脚本显式选择 MCP 的权限检查。
"""
+ execute = job_operations(user_id=user_id, chat_id=chat_id, sandbox_session_id=sandbox_session_id, allowed_tools=allowed_tools, model_name=model_name, model_provider_id=model_provider_id, project_ctx=project_ctx, parent_runtime=parent_runtime)
+
async def run_job(
action: str = "start",
script_path: str = "",
@@ -104,312 +45,10 @@ async def run_job(
status: str = "",
progress_wake_sec: int = 300,
on_conflict: str = "block",
- ) -> ToolResponse:
- from core.db.engine import SessionLocal
- from core.db.models import Job
- from core.services.job_service import JobService
- from orchestration import job_runtime
- from orchestration.jobs import files as job_files
-
- act = (action or "start").strip().lower()
-
- if act == "start":
- if not script_path:
- return _resp(
- {"ok": False, "error": "script_path 必填:先用 write 把作业脚本写进沙箱"}
- )
- if not sandbox_session_id:
- return _resp({"ok": False, "error": "当前会话没有可用沙箱,无法提交作业"})
-
- # 同一会话不许叠加在跑的作业。被进度唤醒后"改个脚本再交一份"是很自然的动作,
- # 但旧作业不会自己消失:两份并存会同时烧预算、同时叫醒会话,用户在状态条上
- # 也分不清哪份才算数。要么先 cancel,要么 resume(已完成的项不会重做)。
- # 本会话已有在跑的作业时**默认**拦一道:默默叠加两份会双倍烧预算、双份叫醒会话,
- # 用户在状态条上也分不清哪份算数。但这只是默认,不是禁令——确实要换新版本时
- # 用 on_conflict 明说:'replace' 停掉旧的再跑新的,'parallel' 两份并行。
- # 判断权留给调用方,工具只负责让"叠加"变成一个显式选择而不是意外。
- conflict = (on_conflict or "block").strip().lower()
- if chat_id and conflict != "parallel":
- # 字段必须在 session 内取出:出了 with 块 ORM 实例就 detached,再读属性会炸
- with SessionLocal() as db:
- live = (
- db.query(Job)
- .filter(
- Job.chat_id == chat_id,
- Job.status.in_(("pending", "running")),
- )
- .order_by(Job.created_at.desc())
- .first()
- )
- live_id = live.job_id if live is not None else ""
- live_name = (live.name or "未命名") if live is not None else ""
- if live_id and conflict == "replace":
- await job_runtime.cancel_job(live_id, user_id=user_id)
- logger.info("[run_job] replaced live job %s in chat %s", live_id, chat_id)
- elif live_id:
- return _resp(
- {
- "ok": False,
- "error": (
- f"本会话已有在跑的作业 {live_id}({live_name})。三条路选一条:"
- "① 想用新脚本取代它 → 再调一次本工具并带 on_conflict='replace'"
- "(自动停掉旧作业再提交新的);"
- f"② 旧作业只是中断、脚本没问题 → run_job(action='resume', job_id='{live_id}')"
- "断点续跑,已完成的项不会重做;"
- "③ 确实需要两份同时跑 → on_conflict='parallel'。"
- "默认拦下只是为了让叠加成为显式选择,不是不让跑。"
- ),
- "job_id": live_id,
- "hint": "on_conflict=replace|parallel",
- }
- )
-
- # 读脚本正文:既做存在性校验,也作为 job 的审计快照与 resume 依据。
- # ⚠️ 必须走 base64,不能直接 `cat`:沙箱 execute 回传的 stdout 会丢换行,
- # 脚本会被压成一行落地 → SyntaxError(实测踩过)。base64 -w0 保字节不变。
- try:
- code, out, err = await job_files._sbx_bash(
- f"base64 -w0 {_quote_path(script_path)} 2>/dev/null || base64 -i {_quote_path(script_path)}",
- session_id=sandbox_session_id,
- user_id=user_id,
- timeout=30,
- )
- except Exception as exc: # noqa: BLE001
- return _resp({"ok": False, "error": f"读取脚本失败: {exc}"})
- if code != 0 or not (out or "").strip():
- return _resp(
- {"ok": False, "error": f"脚本不存在或为空: {script_path} {(err or '')[:200]}"}
- )
- import base64 as _b64dec
-
- try:
- out = _b64dec.b64decode("".join((out or "").split())).decode("utf-8")
- except Exception as exc: # noqa: BLE001
- return _resp({"ok": False, "error": f"脚本解码失败: {exc}"})
- if not out.strip():
- return _resp({"ok": False, "error": f"脚本为空: {script_path}"})
-
- budget: Dict[str, Any] = {"concurrency": concurrency}
- if max_calls:
- budget["max_calls"] = max_calls
- if max_seconds:
- budget["max_seconds"] = max_seconds
-
- try:
- jid = await job_runtime.start_job(
- user_id=user_id,
- chat_id=chat_id,
- name=name or "批量作业",
- script_path=script_path,
- script_text=out,
- session_id=sandbox_session_id,
- budget=budget,
- start_params={
- "allowed_tools": list(allowed_tools or []),
- "model_name": model_name,
- "model_provider_id": model_provider_id,
- "project_ctx": project_ctx,
- "parent_runtime": parent_runtime,
- # wait=False 时作业跑完没人叫醒本会话 → 打标,让驱动在终态入队一轮续跑
- "wake_on_finish": not wait,
- # 中途进度播报间隔(秒);0 = 只在终态叫一次
- "progress_wake_sec": max(0, int(progress_wake_sec)),
- },
- interpreter=interpreter or "${PY_BIN:-python3}",
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("[run_job] start failed: %s", exc)
- return _resp({"ok": False, "error": f"作业启动失败: {exc}"})
-
- if not wait:
- return _resp(
- {
- "ok": True,
- "job_id": jid,
- "status": "started",
- "waited": False,
- "next": (
- "作业已在后台开始。**现在就结束这轮回复**:告诉用户作业已提交、"
- "输入框上方的状态条会实时显示进度(几分之几、失败数、已运行多久),"
- "期间可以继续聊别的、也可以点状态条上的按钮取消。"
- "作业跑完和中途都会自动叫醒你播报,不要在这里轮询 action='status' 干等。"
- ),
- }
- )
-
- res = await job_runtime.run_and_wait(
- jid, chat_id=chat_id, detach_after=_FOREGROUND_WAIT_CAP_S
- )
- if res.get("status") == "detached":
- return _resp(_detached_payload(jid))
- with SessionLocal() as db:
- svc = JobService(db)
- stats = svc.stats(jid)
- job = svc.get(jid)
- usage = dict(job.usage or {}) if job else {}
- payload = {
- "ok": res.get("status") == "completed",
- "job_id": jid,
- "status": res.get("status"),
- "stats": stats,
- "usage": usage,
- }
- if res.get("error"):
- payload["error"] = res["error"]
- if stats.get("remaining", 0) > 0:
- payload["note"] = (
- f"仍有 {stats['remaining']} 项未完成——必须如实告知用户并给出未覆盖清单与"
- f"续跑方式(run_job action=resume job_id={jid}),不得当作已完成。"
- )
- # 脚本崩溃时把 runner 日志尾巴带回来,省一轮排查
- if res.get("status") == "failed" and sandbox_session_id:
- try:
- payload["runner_log"] = (
- await job_files.read_runner_log(
- jid, user_id=user_id, session_id=sandbox_session_id
- )
- )[-1500:]
- except Exception: # noqa: BLE001
- pass
- return _resp(payload)
-
- if not job_id:
- return _resp({"ok": False, "error": f"action={act} 需要 job_id"})
-
- # job_id 容错 + 找不到时把本会话的作业列出来。
- # 实测踩过:模型凭记忆抄 id 时漏了 "job_" 前缀,只回一句「job 不存在」它无从自救,
- # 结果决定把 568 次子调用整个重跑一遍。错误信息必须自带下一步。
- with SessionLocal() as db:
- svc = JobService(db)
- resolved = svc.get(job_id)
- if (resolved is None or resolved.user_id != user_id) and not job_id.startswith("job_"):
- cand = svc.get(f"job_{job_id}")
- if cand is not None and cand.user_id == user_id:
- resolved, job_id = cand, f"job_{job_id}"
- if resolved is None or resolved.user_id != user_id:
- from core.db.models import Job
-
- rows = (
- db.query(Job)
- .filter(Job.user_id == user_id, Job.chat_id == chat_id)
- .order_by(Job.created_at.desc())
- .limit(5)
- .all()
- )
- return _resp(
- {
- "ok": False,
- "error": f"job 不存在或无权访问: {job_id}",
- "jobs_in_this_chat": [
- {
- "job_id": r.job_id,
- "name": r.name,
- "status": r.status,
- "stats": svc.stats(r.job_id),
- }
- for r in rows
- ],
- "hint": "用上面列表里的完整 job_id 重试(注意 job_ 前缀);"
- "已经跑完的作业**不要重跑**,直接 export 取结果。",
- }
- )
-
- if act == "status":
- with SessionLocal() as db:
- svc = JobService(db)
- job = svc.get(job_id)
- if job is None or job.user_id != user_id:
- return _resp({"ok": False, "error": "job 不存在或无权访问"})
- return _resp(
- {
- "ok": True,
- "job_id": job_id,
- "status": job.status,
- "stats": svc.stats(job_id),
- "usage": dict(job.usage or {}),
- "budget_left": svc.budget_left(job_id),
- "error": job.error_message,
- }
- )
-
- if act == "export":
- # 台账在 DB,逐项结果**不进对话上下文**——所以导出的方式是写成沙箱里的
- # JSONL 文件,之后用 Bash/python 处理(写 Excel、跑校验、喂评审)。
- if not sandbox_session_id:
- return _resp({"ok": False, "error": "当前会话没有可用沙箱"})
- with SessionLocal() as db:
- svc = JobService(db)
- job = svc.get(job_id)
- if job is None or job.user_id != user_id:
- return _resp({"ok": False, "error": "job 不存在或无权访问"})
- wanted = [s.strip() for s in (status or "").split(",") if s.strip()] or [
- "done",
- "not_found",
- "failed",
- "needs_review",
- "pending",
- ]
- rows = []
- for st in wanted:
- rows.extend(svc.pending(job_id, status=st))
- stats = svc.stats(job_id)
-
- dest = dest_path or f"jobs/{job_id}_ledger.jsonl"
- body = "\n".join(_json.dumps(r, ensure_ascii=False) for r in rows)
- # 分块写 + 读回校验:单条 Bash 携带大 base64 到约 170KB 会**静默失败**
- # (exit=0、stderr 空、文件不存在)。568 行台账正好落在这个区间——历史上
- # 因此出现过「导出报成功 rows=568,文件却从来没出现」。成功与否只认沙箱
- # 里读回的真实字节数,绝不用 Python 侧的行数报平安。
- try:
- ok_write, detail = await job_files.write_sandbox_file(
- dest, body, session_id=sandbox_session_id, user_id=user_id
- )
- except Exception as exc: # noqa: BLE001
- return _resp({"ok": False, "error": f"导出失败: {exc}"})
- if not ok_write:
- return _resp({"ok": False, "error": f"导出失败(落盘未通过校验): {detail}"})
- return _resp(
- {
- "ok": True,
- "job_id": job_id,
- "path": dest,
- "rows": len(rows),
- "bytes": detail,
- "stats": stats,
- "hint": "每行一个 JSON:{key, status, payload, result, review, attempts}。"
- "用 Bash/python 读这个文件写产物,不要把逐项内容读回对话。",
- }
- )
-
- if act == "cancel":
- ok = await job_runtime.cancel_job(job_id, user_id=user_id)
- return _resp({"ok": ok, "job_id": job_id, "status": "cancelled" if ok else "unknown"})
-
- if act == "resume":
- res = await job_runtime.resume_job(job_id, user_id=user_id, chat_id=chat_id)
- if not res.get("ok"):
- return _resp({"ok": False, "error": res.get("error")})
- if not wait:
- return _resp({"ok": True, "job_id": job_id, "status": "running", "waited": False})
- done = await job_runtime.run_and_wait(
- job_id, chat_id=chat_id, detach_after=_FOREGROUND_WAIT_CAP_S
- )
- if done.get("status") == "detached":
- return _resp(_detached_payload(job_id, resumed=True))
- with SessionLocal() as db:
- stats = JobService(db).stats(job_id)
- return _resp(
- {
- "ok": done.get("status") == "completed",
- "job_id": job_id,
- "status": done.get("status"),
- "stats": stats,
- }
- )
-
- return _resp({"ok": False, "error": f"未知 action: {act}"})
+ ) -> ToolChunk:
+ payload = await execute(action=action, script_path=script_path, name=name, job_id=job_id, wait=wait, interpreter=interpreter, max_calls=max_calls, max_seconds=max_seconds, concurrency=concurrency, dest_path=dest_path, status=status, progress_wake_sec=progress_wake_sec, on_conflict=on_conflict)
+ return ToolChunk(content=[TextBlock(type="text", text=json.dumps(payload, ensure_ascii=False))])
from .job_description import RUN_JOB_DESCRIPTION
-
run_job.__doc__ = RUN_JOB_DESCRIPTION
- toolkit.register_tool_function(run_job)
+ toolkit.append(function_tool(run_job))
diff --git a/src/backend/core/llm/tools/management.py b/src/backend/core/llm/tools/management.py
new file mode 100644
index 00000000..7e4fce90
--- /dev/null
+++ b/src/backend/core/llm/tools/management.py
@@ -0,0 +1,45 @@
+"""Local execution of an explicitly declared, installed manager plugin contract."""
+import asyncio
+import json
+from agentscope.tool import ToolBase, ToolChunk
+from agentscope.message import TextBlock, ToolResultState
+from agentscope.permission import PermissionBehavior, PermissionDecision
+from core.capabilities.plugins.management.errors import management_error
+
+
+class ManagementTool(ToolBase):
+ is_mcp = True
+ is_state_injected = False
+ is_external_tool = False
+ is_concurrency_safe = False
+
+ def __init__(self, server_id, tool, manager, headers, *, cloud_backed):
+ super().__init__()
+ self.mcp_name, self.name = server_id, tool["name"]
+ self.description, self.input_schema = tool.get("description", ""), tool["inputSchema"]
+ self.manager, self.headers = manager, dict(headers)
+ self.cloud_backed = cloud_backed
+ self.is_read_only = self.name.startswith(("get_", "list_"))
+
+ async def check_permissions(self, *_args, **_kwargs):
+ return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK,
+ message="管理当前用户的本机技能或插件")
+
+ def _execute(self, arguments):
+ from core.harness.registry.capability_management import execute_manager
+ return execute_manager(
+ manager=self.manager, server_id=self.mcp_name, headers=self.headers,
+ cloud_backed=self.cloud_backed, tool_name=self.name,
+ input_schema=self.input_schema, arguments=arguments,
+ )
+
+ async def call(self, **kwargs):
+ try:
+ result = await asyncio.to_thread(self._execute, kwargs)
+ return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))],
+ state=ToolResultState.SUCCESS,
+ metadata={"origin": "local", "capabilities_changed": not self.is_read_only})
+ except Exception as exc:
+ message = management_error(exc)
+ return ToolChunk(content=[TextBlock(text=json.dumps({"ok": False, "error": message}, ensure_ascii=False))],
+ state=ToolResultState.ERROR, metadata={"origin": "local"})
diff --git a/src/backend/core/ontology/toolkit.py b/src/backend/core/llm/tools/ontology_toolkit.py
similarity index 85%
rename from src/backend/core/ontology/toolkit.py
rename to src/backend/core/llm/tools/ontology_toolkit.py
index 10698e2e..5a8a54cc 100644
--- a/src/backend/core/ontology/toolkit.py
+++ b/src/backend/core/llm/tools/ontology_toolkit.py
@@ -13,11 +13,9 @@
from agentscope.tool._response import ToolChunk, ToolResponse
from jinja2 import Template
-from core.llm.execution_manifest import canonical_json, stable_tool_schema_order
-from core.llm.tool_permissions import (
- CURRENT_PERMISSION_TICKET,
- ToolPermissionService,
-)
+from core.harness.execution_manifest import canonical_json, stable_tool_schema_order
+from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET
+from core.llm.tool_permissions import ToolPermissionService
@dataclass(frozen=True)
@@ -50,6 +48,7 @@ class OntologyFilteredToolkit(Toolkit):
def __init__(self, *args, hidden_tools: set[str] | None = None, **kwargs):
super().__init__(*args, **kwargs)
+ self.capability_guard = None
self._ontology_hidden_tools = set(hidden_tools or ())
self._execution_surface: Optional[ExecutionSurfaceSnapshot] = None
self._execution_surface_generation = 0
@@ -71,6 +70,17 @@ def set_tool_permission_service(
self._tool_permission_service = service
async def call_tool(self, tool_call, state): # noqa: ANN001
+ if self.capability_guard is not None:
+ tools = await self._get_available_tools(state.tool_context.activated_groups)
+ registered = tools.get(tool_call.name)
+ tool = registered.tool if registered is not None else None
+ if tool is not None and tool.is_mcp:
+ allowed = await asyncio.to_thread(self.capability_guard, tool.mcp_name)
+ if not allowed:
+ chunk = ToolChunk(content=[TextBlock(text="该连接器已停用或当前不可用,本次调用未执行。")], state=ToolResultState.ERROR)
+ yield chunk
+ yield ToolResponse(id=tool_call.id).append_chunk(chunk)
+ return
service = self._tool_permission_service
if service is not None and service.registry.contains(tool_call.name):
ticket = CURRENT_PERMISSION_TICKET.get()
@@ -115,11 +125,9 @@ async def get_tool_schemas(self, groups=None): # noqa: ANN001
snapshot = await self.freeze_execution_surface(groups)
return snapshot.tool_schemas
- async def get_skill_instructions(self) -> Optional[str]:
- """Return the skill prompt from the same frozen execution surface."""
- snapshot = self._execution_surface
- if snapshot is None:
- snapshot = await self.freeze_execution_surface()
+ async def get_skill_instructions(self, activated_groups=None) -> Optional[str]:
+ """Use the same group-scoped snapshot as the model's tool schemas."""
+ snapshot = await self.freeze_execution_surface(activated_groups)
return snapshot.skill_instructions
@staticmethod
@@ -153,8 +161,6 @@ async def freeze_execution_surface(
self._capture_surface_skills = False
skills = self._captured_surface_skills
if skills is None:
- # Compatibility fallback if a future AgentScope version stops
- # consulting skills while constructing the tool surface.
skills = await self._get_available_skills(groups)
skill_instructions = None
if skills:
diff --git a/src/backend/core/llm/tools/pin_tool.py b/src/backend/core/llm/tools/pin_tool.py
index 40c2db80..6935e286 100644
--- a/src/backend/core/llm/tools/pin_tool.py
+++ b/src/backend/core/llm/tools/pin_tool.py
@@ -3,16 +3,17 @@
Extracted from the oversized ``core/llm/tool.py``. Heavy deps imported lazily
inside the function; ``core.llm.tool`` re-exports ``register_pin_to_workspace``.
"""
+from core.llm.factory.tools.definitions import function_tool
import logging
from typing import Any, Dict, Optional, TYPE_CHECKING
if TYPE_CHECKING:
- from core.services.project_scope import ProjectScope
+ from core.projects.edition import ProjectScope
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
+from agentscope.tool._response import ToolChunk
logger = logging.getLogger(__name__)
@@ -39,11 +40,11 @@ def register_pin_to_workspace(
output leaked into the personal MySpace root. Now the scope travels with the call chain, no
timing window.
"""
- from core.services.project_scope import ProjectScope # noqa: F401 - re-import for closure
+ from core.projects.edition import ProjectScope # noqa: F401 - re-import for closure
async def pin_to_workspace(
file_ids: list[str] = [], file_paths: list[str] = []
- ) -> ToolResponse:
+ ) -> ToolChunk:
"""把文件交付到对话区——**唯一**让用户看到文件的方式。
凡用户要求生成/导出文件(文档、图片、PPT、Excel、PDF、CSV、压缩包、音视频、
@@ -73,8 +74,8 @@ async def pin_to_workspace(
"""
import json as _json
- from core.artifacts.store import get_artifact
- from core.llm import workspace as _workspace
+ from core.files.artifacts.store import get_artifact
+ from core.llm.runtime import workspace as _workspace
# Mark the gate active even if every id below fails — the agent's
# *intent* to use the workspace is what flips the default. Otherwise
@@ -89,7 +90,7 @@ async def pin_to_workspace(
elif isinstance(file_ids, list):
raw_ids = file_ids
else:
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -105,7 +106,7 @@ async def pin_to_workspace(
)
if not raw_ids and not file_paths:
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -125,7 +126,7 @@ async def pin_to_workspace(
from core.infra.logging import user_id_var
from fastapi import HTTPException
- from core.artifacts.local_reference import reference_local_file
+ from core.files.artifacts.local_reference import reference_local_file
if not isinstance(file_paths, list):
file_paths = [file_paths]
@@ -135,7 +136,7 @@ async def pin_to_workspace(
failed_results.append({"path": str(path), "error": "文件路径必须是非空字符串"})
continue
try:
- from core.artifacts.local_reference import project_file_path
+ from core.files.artifacts.local_reference import project_file_path
from core.infra.logging import chat_id_var
physical = project_file_path(
@@ -217,14 +218,12 @@ async def pin_to_workspace(
_uid = user_id_var.get() or ""
_cid = chat_id_var.get() or ""
if _uid and to_persist:
- from core.db.engine import SessionLocal
- from core.services.artifact_service import persist_artifacts
+ from core.db.threads import run_in_session
+ from core.files.artifacts.service import persist_artifacts
- _db = SessionLocal()
- try:
- persist_artifacts(_db, _uid, _cid or None, to_persist, scope=scope, strict=True)
- finally:
- _db.close()
+ await run_in_session(
+ persist_artifacts, _uid, _cid or None, to_persist, scope=scope, strict=True
+ )
except Exception as exc: # noqa: BLE001
from fastapi import HTTPException
@@ -251,7 +250,7 @@ async def pin_to_workspace(
}
if failed_results:
result["failed"] = failed_results
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -260,10 +259,11 @@ async def pin_to_workspace(
]
)
- from core.llm.tool_permissions import READ, ToolPermissionSpec, local_path_tool
+ from core.sandbox.permissions.dispatch_models import READ, ToolPermissionSpec
+ from core.sandbox.permissions.tool_specs import local_path_tool
def resolve_pin_paths(args, runtime):
- from core.artifacts.local_reference import project_file_path
+ from core.files.artifacts.local_reference import project_file_path
paths = args.get("file_paths") or []
if isinstance(paths, str):
@@ -284,11 +284,7 @@ def resolve_pin_paths(args, runtime):
)
return intents
- toolkit.register_tool_function(
- pin_to_workspace,
- namesake_strategy="override",
- permission=ToolPermissionSpec("local-project-pin-paths", resolve_pin_paths),
- )
+ toolkit.append(function_tool(pin_to_workspace, dispatch_spec=ToolPermissionSpec("local-project-pin-paths", resolve_pin_paths)))
logger.info("[factory] Registered pin_to_workspace tool")
diff --git a/src/backend/core/llm/plan_update_tool.py b/src/backend/core/llm/tools/plan_update.py
similarity index 92%
rename from src/backend/core/llm/plan_update_tool.py
rename to src/backend/core/llm/tools/plan_update.py
index a963a451..6feaaa8b 100644
--- a/src/backend/core/llm/plan_update_tool.py
+++ b/src/backend/core/llm/tools/plan_update.py
@@ -5,14 +5,16 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
+from core.llm import prompt_rendering
import logging
from typing import Any, Dict, List, Optional
from agentscope.tool import Toolkit
-# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
+# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk).
+from agentscope.tool._response import ToolChunk
from agentscope.message import TextBlock
logger = logging.getLogger(__name__)
@@ -63,7 +65,7 @@ def register_plan_update_tool(toolkit: Toolkit) -> None:
不更新(commit 7578469f)。
"""
- async def update_plan(steps: list, title: str = "", explanation: str = "") -> ToolResponse:
+ async def update_plan(steps: list, title: str = "", explanation: str = "") -> ToolChunk:
"""更新任务计划。传入一份计划项列表,每项含步骤与状态,以及可选的说明。
至多一个步骤可以处于 in_progress。调用本工具**不会打断**执行。
@@ -82,12 +84,12 @@ async def update_plan(steps: list, title: str = "", explanation: str = "") -> To
(可选)本次改动的理由。中途增删步骤、重写计划结构时必须填写。
Returns:
- `ToolResponse`:
+ `ToolChunk`:
当前清单的各状态计数。
"""
parsed = parse_plan_update_args({"title": title, "steps": steps})
if not parsed:
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=(
"错误:steps 必须是非空列表,每个元素为 "
@@ -114,9 +116,9 @@ async def update_plan(steps: list, title: str = "", explanation: str = "") -> To
"\n全部步骤已完成,计划终结:不要再调用 update_plan,也不要原样重复"
"任何已成功的工具调用,请直接输出面向用户的最终文字回复。"
)
- return ToolResponse(content=[TextBlock(type="text", text=text)])
+ return ToolChunk(content=[TextBlock(type="text", text=text)])
- toolkit.register_tool_function(update_plan, namesake_strategy="skip")
+ toolkit.append(function_tool(update_plan))
def build_plan_update_prompt_section() -> str:
@@ -125,6 +127,5 @@ def build_plan_update_prompt_section() -> str:
正文的真源是 ``prompts/prompt_text/plan_tool/plan_tool.system.md``,运行时取
``plan_tool`` 版本池的激活版本,可在 Config 管理台「任务计划清单」里编辑。
"""
- from core.services import prompt_version_service as pvs
- return pvs.render_kind_segment("plan_tool")
+ return prompt_rendering.render_kind_segment("plan_tool")
diff --git a/src/backend/core/plugins/runtime/activation.py b/src/backend/core/llm/tools/plugin_activation.py
similarity index 78%
rename from src/backend/core/plugins/runtime/activation.py
rename to src/backend/core/llm/tools/plugin_activation.py
index 8a11e77d..73764e9f 100644
--- a/src/backend/core/plugins/runtime/activation.py
+++ b/src/backend/core/llm/tools/plugin_activation.py
@@ -1,18 +1,20 @@
"""Progressive plugin runtime: activation."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import logging
from contextlib import asynccontextmanager
from typing import Any, Dict, List, Set
-logger = logging.getLogger(__name__)
-from core.plugins.runtime import activation_history as plugin_activation_history
-from core.plugins.runtime import models as plugin_runtime_models
-from core.plugins.runtime import resolution as plugin_resolution
+from core.capabilities.plugins.runtime import activation_history as plugin_activation_history
+from core.capabilities.plugins.runtime import models as plugin_runtime_models
+from core.capabilities.plugins.runtime import resolution as plugin_resolution
+
+logger = logging.getLogger(__name__)
class _ActivationStage:
@@ -21,9 +23,7 @@ class _ActivationStage:
A device capability run must never expose half a plugin: if the authorized
closure stops holding — or an MCP turns out to be unreachable — after some
clients already connected, the run has to end up exactly as it started, so
- everything lands in a buffer that is published only on success. The legacy
- cloud path has no such checkpoint and keeps appending straight to the live
- group.
+ everything lands in a buffer that is published only on success.
"""
def __init__(self, live_group: Any, runtime: Dict[str, Any], prepared: Any) -> None:
@@ -33,41 +33,36 @@ def __init__(self, live_group: Any, runtime: Dict[str, Any], prepared: Any) -> N
self._runtime = runtime
self._prepared = prepared
self._staged_clients: List[Any] = []
- self.staged = prepared is not None
- self.group = SimpleNamespace(mcps=[], skills_or_loaders=[]) if self.staged else live_group
+ self.group = SimpleNamespace(tools=[], skills_or_loaders=[])
current = runtime.setdefault("connected_keys", set())
- self.connected: Set[str] = set(current) if self.staged else current
+ self.connected: Set[str] = set(current)
async def checkpoint(self) -> None:
"""Re-assert the authorized closure between steps of the activation."""
if self._prepared is None:
return
- from core.capabilities.runtime import validate
+ from core.harness.registry.capability_runtime import validate
await asyncio.to_thread(validate, self._prepared)
def add_client(self, key: str, client: Any) -> None:
- self.group.mcps.append(client)
+ self.group.tools.extend(client.tools)
self.connected.add(key)
- if self.staged:
- self._staged_clients.append(client)
- return
- close_list = self._runtime.get("close_list")
- if isinstance(close_list, list):
- close_list.append(client)
+ self._staged_clients.append(client)
async def discard(self) -> None:
- for client in self._staged_clients:
- try:
- await client.close()
- except Exception as exc: # noqa: BLE001 — one bad close must not leak the rest
- logger.warning("[plugin-loader] staged MCP close failed: %s", exc)
- self._staged_clients.clear()
+ from core.llm.mcp.manager import close_clients
+
+ try:
+ await close_clients(self._staged_clients)
+ finally:
+ self._staged_clients.clear()
def commit(self) -> None:
- if not self.staged:
- return
- self._live.mcps.extend(self.group.mcps)
+ names = [t.name for t in [*self._live.tools, *self.group.tools]]
+ if len(names) != len(set(names)):
+ raise ValueError("Plugin tool name collision")
+ self._live.tools.extend(self.group.tools)
self._live.skills_or_loaders.extend(self.group.skills_or_loaders)
self._runtime["connected_keys"] = self.connected
close_list = self._runtime.get("close_list")
@@ -84,7 +79,11 @@ async def _activation_stage(live_group: Any, runtime: Dict[str, Any], prepared:
except BaseException:
await stage.discard()
raise
- stage.commit()
+ try:
+ stage.commit()
+ except BaseException:
+ await stage.discard()
+ raise
def register_load_plugin(
@@ -117,10 +116,10 @@ def register_load_plugin(
Deferred clients inherit the current run's read-only setting.
"""
from agentscope.message import TextBlock
- from agentscope.tool._response import ToolChunk as ToolResponse
+ from agentscope.tool._response import ToolChunk
def _text(msg: str) -> Any:
- return ToolResponse(content=[TextBlock(type="text", text=msg)])
+ return ToolChunk(content=[TextBlock(type="text", text=msg)])
async def load_plugin(plugin: str) -> Any:
"""加载插件目录中列出的插件,激活其包含的全部工具与技能。
@@ -177,7 +176,7 @@ async def load_plugin(plugin: str) -> Any:
prepared = runtime.get("prepared_run")
if prepared is not None:
- from core.capabilities.runtime import validate, validate_activation
+ from core.harness.registry.capability_runtime import validate, validate_activation
await asyncio.to_thread(
validate, prepared, user_id=runtime.get("user_id", prepared.user_id)
@@ -200,9 +199,9 @@ async def load_plugin(plugin: str) -> Any:
m for m in [*target.mcp_ids, *target.bound_mcp_ids] if m not in stage.connected
]
if mcp_ids:
- from core.llm.factory.tools.mcp_config import _inject_runtime_headers
- from core.llm.mcp_pool import make_client
- from core.services.mcp_service import McpServerConfigService
+ from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers
+ from core.llm.mcp.lifetime import discover
+ from core.capabilities.mcp.service import McpServerConfigService
if prepared is not None:
cfgs = runtime["prepared_servers"]
@@ -215,8 +214,9 @@ async def load_plugin(plugin: str) -> Any:
str(runtime.get("user_id") or ""), enabled_only=False
)
)
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools.plugin_activation:load_plugin', fallback_error)
wanted_cfgs = {k: v for k, v in cfgs.items() if k in set(mcp_ids)}
wanted_cfgs = _inject_runtime_headers(
wanted_cfgs,
@@ -228,35 +228,36 @@ async def load_plugin(plugin: str) -> Any:
)
async def _connect(key: str, cfg: dict) -> None:
- if not stage.staged and not plugin_resolution._http_transport(cfg):
- failed_servers.append(key)
- return
client = None
try:
- from orchestration.local_plugin_sidecars import ensure_native_servers
+ from core.harness.registry.local_publication import ensure_native_servers
await stage.checkpoint()
await ensure_native_servers({key: cfg})
- client = make_client(
+ from core.llm.mcp.callbacks import Interactions
+ # Runtimes bound before agent construction have no request yet;
+ # their sessions stay run-owned and decline server interactions.
+ request = runtime.get("request")
+ interactions = (Interactions(request, key, bool(runtime.get("approval_available")),
+ agent=runtime.get("agent"))
+ if request is not None else None)
+ scope = ((request.current_user_id, request.chat_id, request.capability_scope)
+ if request is not None and request.current_user_id and request.chat_id
+ and not request.isolated else None)
+ client = await discover(
key, {**cfg, "read_only": bool(runtime.get("read_only"))},
- is_stateful=False,
+ interactions=interactions, scope=scope,
)
- if stage.staged and not plugin_resolution._http_transport(cfg):
- await client.connect()
- tools = await client.list_tools()
+ tools = client.tools
await stage.checkpoint()
except (
BaseException
) as exc: # noqa: BLE001 — SSE cleanup may raise CancelledError
if client is not None:
- await client.close()
+ from core.llm.mcp.manager import close_clients
+ await close_clients([client])
if isinstance(exc, asyncio.CancelledError):
- current = asyncio.current_task()
- if (
- current is not None
- and getattr(current, "cancelling", lambda: 0)() > 0
- ):
- raise
+ raise
logger.warning("[plugin-loader] MCP '%s' connect failed: %s", key, exc)
failed_servers.append(key)
return
@@ -270,7 +271,7 @@ async def _connect(key: str, cfg: dict) -> None:
failed_servers.append(key)
await stage.checkpoint()
- if stage.staged and failed_servers:
+ if failed_servers:
raise RuntimeError("插件 MCP 服务不可用:" + "、".join(failed_servers))
# ── Skills: materialize and append loaders in place ──
@@ -289,24 +290,24 @@ async def _connect(key: str, cfg: dict) -> None:
if not d:
continue
if prepared is not None:
- from core.llm.tool_collector import RuntimeNamedSkillLoader
+ from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader
stage.group.skills_or_loaders.append(
RuntimeNamedSkillLoader(d, sid, prepared)
)
else:
- from core.llm.tool_collector import RuntimeNamedSkillLoader
+ from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader
stage.group.skills_or_loaders.append(RuntimeNamedSkillLoader(d, sid))
item = meta.get(sid)
desc = str(getattr(item, "description", "") or "")
- from core.agent_skills.config import model_facing_skill_dir
+ from core.capabilities.skills.config import model_facing_skill_dir
skill_path = model_facing_skill_dir(sid, d)
skill_lines.append(f"- `{sid}`:`{skill_path}/SKILL.md` — {desc}")
# Ontology gate sees the activated skill's trusted tags too.
try:
- from core.ontology.validator import register_runtime_asset_tags
+ from core.ontology.runtime_pack import register_runtime_asset_tags
register_runtime_asset_tags(
runtime.get("ontology_runtime") or {},
@@ -314,8 +315,9 @@ async def _connect(key: str, cfg: dict) -> None:
asset_id=sid,
tags=list(getattr(item, "tags", []) or []),
)
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools.plugin_activation:load_plugin', fallback_error)
await stage.checkpoint()
@@ -368,4 +370,4 @@ async def _connect(key: str, cfg: dict) -> None:
parts.append("该插件本轮没有可加载的组件(可能均已被禁用)。")
return _text("\n".join(parts))
- toolkit.register_tool_function(load_plugin, namesake_strategy="override")
+ toolkit.append(function_tool(load_plugin))
diff --git a/src/backend/core/llm/tools/project_instructions_tool.py b/src/backend/core/llm/tools/project_instructions_tool.py
index 6aee455d..f5e6d8a0 100644
--- a/src/backend/core/llm/tools/project_instructions_tool.py
+++ b/src/backend/core/llm/tools/project_instructions_tool.py
@@ -1,5 +1,6 @@
"""Canonical instruction reads for project chats; writes only during initialization."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
@@ -13,7 +14,7 @@ def project_instruction_path(project_ctx: dict) -> str:
if project_ctx.get("project_is_local"):
root = project_ctx.get("project_local_path")
return f"{root.rstrip('/')}/AGENTS.md" if root else ""
- from core.services.edition_workspace import context_directory
+ from core.files.edition import context_directory
directory = context_directory(project_ctx)
if directory:
@@ -23,19 +24,20 @@ def project_instruction_path(project_ctx: dict) -> str:
def register_project_instruction_tools(toolkit, *, project_id: str, user_id: str, local_path=None, instruction_path=None, allow_write=True):
- from core.llm.tool_permissions import READ, WRITE, local_path_tool
- from core.llm.tools._tool_confirmation import OP_EDIT
+ from core.sandbox.permissions.dispatch_models import READ, WRITE
+ from core.sandbox.permissions.tool_specs import local_path_tool
+ from core.llm.tools.confirmation import OP_EDIT
target = instruction_path or (f"{local_path.rstrip('/')}/AGENTS.md" if local_path else "/myspace/AGENTS.md")
def operate(content=None, revision=None):
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
from core.db.engine import SessionLocal
from core.db.models import Project
- from core.services.project_instructions import ProjectInstructionsService
+ from core.projects.instructions import ProjectInstructionsService
if content is None:
- from core.services.project_instructions import read_authorized_project_instructions
+ from core.projects.instructions import read_authorized_project_instructions
return {"ok": True, "path": target, **read_authorized_project_instructions(project_id, user_id)}
with SessionLocal() as db:
@@ -59,7 +61,7 @@ async def read_project_instructions():
"""
try:
if local_path:
- from core.llm.tool_permissions import require_local_path_permission
+ from core.sandbox.permissions.dispatch_models import require_local_path_permission
require_local_path_permission(target, "read")
return resp_json(await asyncio.to_thread(operate))
except HTTPException as exc:
@@ -75,22 +77,16 @@ async def save_project_instructions(content: str, expected_revision: str):
"""
try:
if local_path:
- from core.llm.tool_permissions import require_local_path_permission
+ from core.sandbox.permissions.dispatch_models import require_local_path_permission
require_local_path_permission(target, "write")
return resp_json(await asyncio.to_thread(operate, content, expected_revision))
except HTTPException as exc:
return resp_json({"ok": False, "error": exc.detail, "status": exc.status_code})
- toolkit.register_tool_function(
- read_project_instructions,
- permission=local_path_tool("file_path", READ, tool_name="read_project_instructions", default_path=target),
- )
+ toolkit.append(function_tool(read_project_instructions, dispatch_spec=local_path_tool("file_path", READ, tool_name="read_project_instructions", default_path=target)))
if not allow_write:
return
- toolkit.register_tool_function(
- save_project_instructions,
- permission=local_path_tool(
+ toolkit.append(function_tool(save_project_instructions, dispatch_spec=local_path_tool(
"file_path", WRITE, tool_name="save_project_instructions",
default_path=target, myspace_op=OP_EDIT,
- ),
- )
+ )))
diff --git a/src/backend/core/llm/tools/project_source_access.py b/src/backend/core/llm/tools/project_source_access.py
index 9dcce65c..bd74d42d 100644
--- a/src/backend/core/llm/tools/project_source_access.py
+++ b/src/backend/core/llm/tools/project_source_access.py
@@ -6,10 +6,10 @@
def validate_scope(db, scope, actor, *, write=False):
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
project = ProjectSourceService(db).authorized_project(scope.project_id, actor, write=write)
- from core.services.project_file_service import ProjectFileService
+ from core.projects.files import ProjectFileService
if not ProjectFileService(db).matches_scope(project, scope):
raise HTTPException(409, "项目空间已变更,请重新打开项目会话后再操作")
@@ -23,7 +23,7 @@ def current_scope_error(scope, actor, *, write=False):
from core.config.local_mode import local_mode_enabled
if not local_mode_enabled():
return {"error": "本地项目只能在本机模式下操作", "status": 403}
- from core.services.local_project_workspace import validate_current_project
+ from core.projects.local_workspace import validate_current_project
try:
validate_current_project(scope, actor, write=write)
except (HTTPException, ValueError, OSError) as exc:
diff --git a/src/backend/core/llm/tools/read_artifact_tool.py b/src/backend/core/llm/tools/read_artifact_tool.py
index 02368c56..9ea79e9d 100644
--- a/src/backend/core/llm/tools/read_artifact_tool.py
+++ b/src/backend/core/llm/tools/read_artifact_tool.py
@@ -4,6 +4,7 @@
deps (content parsers, hooks) are imported lazily inside the functions.
``core.llm.tool`` re-exports the public names for backward compatibility.
"""
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import logging
@@ -11,7 +12,7 @@
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
+from agentscope.tool._response import ToolChunk
logger = logging.getLogger(__name__)
@@ -35,8 +36,8 @@ def _is_pptx_meta(meta: dict) -> bool:
def _probe_xlsx_sheet_names(file_id: str, filename: str, user_id: Optional[str] = None) -> tuple[list[str], bytes | None]:
"""Cheaply list sheet names and return the bytes for reuse. Returns
``([], None)`` on download failure, ``(names, bytes)`` on success."""
- from core.content.file_parser import parse_xlsx_sheet_names
- from core.llm.hooks import _download_artifact_bytes
+ from core.files.content.spreadsheet_parser import parse_xlsx_sheet_names
+ from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes
file_bytes = _download_artifact_bytes(file_id, filename, "read_artifact xlsx", user_id=user_id)
if file_bytes is None:
@@ -58,8 +59,8 @@ async def _resolve_xlsx_text(
the probe — passed through to fetch_parsed_text on the default path so
we don't re-download from storage. Raises RuntimeError on bad sheet_name
or missing bytes when sheet_name is set."""
- from core.content.artifact_reader import fetch_parsed_text
- from core.content.file_parser import parse_xlsx_single_sheet
+ from core.files.artifacts.reader import fetch_parsed_text
+ from core.files.content.spreadsheet_parser import parse_xlsx_single_sheet
if sheet_name:
if file_bytes is None:
@@ -73,8 +74,8 @@ async def _resolve_xlsx_text(
def _probe_pptx_slide_count(file_id: str, filename: str, user_id: Optional[str] = None) -> tuple[int, bytes | None]:
"""Cheaply count slides and return the bytes for reuse. Returns
``(0, None)`` on download failure, ``(count, bytes)`` on success."""
- from core.content.file_parser import parse_pptx_slide_count
- from core.llm.hooks import _download_artifact_bytes
+ from core.files.content.presentation_parser import parse_pptx_slide_count
+ from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes
file_bytes = _download_artifact_bytes(file_id, filename, "read_artifact pptx", user_id=user_id)
if file_bytes is None:
@@ -96,8 +97,8 @@ async def _resolve_pptx_text(
bytes the probe already downloaded — passed through to fetch_parsed_text
on the default path so we don't re-download. Raises RuntimeError on
bad slide_index or missing bytes."""
- from core.content.artifact_reader import fetch_parsed_text
- from core.content.file_parser import parse_pptx_slide
+ from core.files.artifacts.reader import fetch_parsed_text
+ from core.files.content.presentation_parser import parse_pptx_slide
if slide_index is not None:
if file_bytes is None:
@@ -124,7 +125,7 @@ async def read_artifact(
limit: int = _READ_ARTIFACT_DEFAULT_LIMIT,
sheet_name: str | None = None,
slide_index: int | None = None,
- ) -> ToolResponse:
+ ) -> ToolChunk:
"""读取已上传文件的完整解析文本(按字符分页)。
⚠️ 每轮对话每个 file_id **累计最多读 50000 字符**,超出即被拒绝。所以要精准
@@ -158,18 +159,19 @@ async def read_artifact(
"""
import json as _json
- from core.content.artifact_reader import fetch_parsed_text, load_artifact_meta
- from core.llm.hooks import MAX_FILE_CONTENT_CHARS, get_artifact_read_state
+ from core.files.artifacts.reader import fetch_parsed_text, load_artifact_meta
+ from core.llm.runtime.hooks_constants import MAX_FILE_CONTENT_CHARS
+ from core.llm.runtime.turn_state import get_artifact_read_state
fid = (file_id or "").strip()
if not fid:
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=_json.dumps({"error": "file_id 不能为空"}, ensure_ascii=False),
)])
if sheet_name is not None and slide_index is not None:
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=_json.dumps(
{"error": "sheet_name 与 slide_index 互斥(分别用于 xlsx / pptx)"},
@@ -184,7 +186,7 @@ async def read_artifact(
# Blocking it mid-answer stalls streaming for every chat on the instance.
meta = await asyncio.to_thread(load_artifact_meta, fid, user_id=user_id)
if meta is None:
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=_json.dumps(
{"error": f"文件 {fid} 不存在、已删除,或无权访问"},
@@ -226,7 +228,7 @@ async def read_artifact(
except RuntimeError as e:
err = {"error": str(e), "file_id": fid, "filename": meta.get("filename")}
err.update(extras)
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text", text=_json.dumps(err, ensure_ascii=False),
)])
@@ -234,7 +236,7 @@ async def read_artifact(
error = meta.get("parse_error") or "文件暂无可读文本内容"
err = {"error": error, "filename": meta.get("filename"), "file_id": fid}
err.update(extras)
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text", text=_json.dumps(err, ensure_ascii=False),
)])
@@ -271,7 +273,7 @@ async def read_artifact(
)
else:
hint += " 如需更多内容,请缩小范围(精准 offset/limit)后再试。"
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=_json.dumps(
{
@@ -316,10 +318,10 @@ async def read_artifact(
"budget_remaining": budget_remaining_after,
}
result.update(extras)
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=_json.dumps(result, ensure_ascii=False),
)])
- toolkit.register_tool_function(read_artifact, namesake_strategy="override")
+ toolkit.append(function_tool(read_artifact))
logger.info("[factory] Registered read_artifact tool (user=%s)", user_id)
diff --git a/src/backend/core/llm/tools/read_image_tool.py b/src/backend/core/llm/tools/read_image_tool.py
index 89432035..2ff2d21b 100644
--- a/src/backend/core/llm/tools/read_image_tool.py
+++ b/src/backend/core/llm/tools/read_image_tool.py
@@ -1,6 +1,6 @@
"""``read_image`` —— 让智能体主动看一张图。
-同一个工具名,按主模型能力注册两个版本之一(``core.vision.resolve_vision_mode``):
+同一个工具名,按主模型能力注册两个版本之一(``core.models.vision.resolve_vision_mode``):
- **native**:把图原样送进上下文,模型自己看像素(对应 Codex 的 ``view_image``)。
- **bridge**:交给视觉桥转成结构化文字证据,可带 ``focus`` 针对某处细节追问。
@@ -11,17 +11,18 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import logging
from typing import Optional
from agentscope.tool import Toolkit
-from core.services.project_scope import ProjectScope
-from core.vision import VisionMode
+from core.projects.edition import ProjectScope
+from core.models.vision import VisionMode
-from ._common import ToolResponse, resolve_sandbox_session, resp_image, resp_json
-from ._paths import to_physical_path, validate_project_scope_path, validate_workspace_path
+from ._common import ToolChunk, resolve_sandbox_session, resp_image, resp_json
+from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path
logger = logging.getLogger(__name__)
@@ -75,7 +76,7 @@ async def load_image(
label = file_path or file_id
if file_id.strip():
- from core.llm.hooks import _download_artifact_bytes
+ from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes
# Storage download (S3/OSS in some deployments) — off the event loop.
image_bytes = await asyncio.to_thread(
@@ -97,7 +98,7 @@ async def load_image(
if image_bytes is None:
raise ImageLoadError(f"读取失败:{file_path}")
- from core.vision import sniff_mime
+ from core.models.vision import sniff_mime
mime = sniff_mime(image_bytes)
if mime is None:
@@ -107,7 +108,7 @@ async def load_image(
async def transcribe_image(image_bytes: bytes, *, label: str, focus: str = "") -> Optional[dict]:
"""视觉桥:图片 → ``image_evidence`` 载荷。未配视觉模型或识别失败返回 ``None``。"""
- from core.vision import get_vision_bridge, is_available, render_evidence
+ from core.models.vision import get_vision_bridge, is_available, render_evidence
if not is_available():
return None
@@ -140,7 +141,7 @@ def register_read_image(
return
_sess = resolve_sandbox_session(sandbox_session_id, chat_id)
- async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse:
+ async def _run(file_path: str, file_id: str, focus: str) -> ToolChunk:
try:
image_bytes, mime, label = await load_image(
file_path,
@@ -154,7 +155,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse:
return resp_json({"error": str(exc)})
if vision_mode == "native":
- from core.vision.service import MAX_IMAGE_BYTES
+ from core.models.vision.service import MAX_IMAGE_BYTES
if len(image_bytes) > MAX_IMAGE_BYTES:
return resp_json(
@@ -165,7 +166,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse:
)
return await resp_image(image_bytes, mime, name=label, meta={"source": label})
- from core.vision import is_available
+ from core.models.vision import is_available
if not is_available():
return resp_json(
@@ -182,7 +183,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse:
# and only the bridge variant can act on ``focus``.
if vision_mode == "native":
- async def read_image(file_path: str = "", file_id: str = "") -> ToolResponse:
+ async def read_image(file_path: str = "", file_id: str = "") -> ToolChunk:
return await _run(file_path, file_id, "")
read_image.__doc__ = _NATIVE_DOC
@@ -190,12 +191,12 @@ async def read_image(file_path: str = "", file_id: str = "") -> ToolResponse:
async def read_image(
file_path: str = "", file_id: str = "", focus: str = ""
- ) -> ToolResponse:
+ ) -> ToolChunk:
return await _run(file_path, file_id, focus)
read_image.__doc__ = _BRIDGE_DOC
- toolkit.register_tool_function(read_image)
+ toolkit.append(function_tool(read_image))
logger.info("[factory] Registered read_image tool (%s, chat_id=%s)", vision_mode, chat_id)
@@ -212,10 +213,7 @@ async def _read_path_bytes(
if local_mode_enabled():
try:
- from core.llm.tool_permissions import (
- PermissionEnforcementError,
- require_local_path_permission,
- )
+ from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission
require_local_path_permission(physical, "read")
with open(physical, "rb") as handle:
diff --git a/src/backend/core/llm/tools/read_tool.py b/src/backend/core/llm/tools/read_tool.py
index 51b0439a..8c852030 100644
--- a/src/backend/core/llm/tools/read_tool.py
+++ b/src/backend/core/llm/tools/read_tool.py
@@ -1,6 +1,7 @@
"""Read files from the active sandbox."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import hashlib
@@ -8,16 +9,11 @@
from typing import Optional
from agentscope.tool import Toolkit, ToolResponse
-from core.services.project_scope import ProjectScope
-from core.vision import VisionMode
+from core.projects.edition import ProjectScope
+from core.models.vision import VisionMode
from ._common import response_metadata, resolve_sandbox_session, resp_image, resp_json
-from ._paths import (
- space_metadata,
- to_physical_path,
- validate_project_scope_path,
- validate_workspace_path,
-)
+from core.sandbox.workspace.resolution import space_metadata, to_physical_path, validate_project_scope_path, validate_workspace_path
from ._state import ReadEntry, ReadStateTracker
logger = logging.getLogger(__name__)
@@ -42,7 +38,7 @@ async def _read_image_as_evidence(content_bytes: bytes, file_path: str) -> Optio
configured, so the caller falls through to its existing binary handling.
"""
try:
- from core.vision import sniff_mime
+ from core.models.vision import sniff_mime
from .read_image_tool import transcribe_image
@@ -64,8 +60,8 @@ async def _image_response(
text-only one. ``None`` means "not handled here" and falls through to the
parsed-text / binary handling."""
if vision_mode == "native":
- from core.vision import sniff_mime
- from core.vision.service import MAX_IMAGE_BYTES
+ from core.models.vision import sniff_mime
+ from core.models.vision.service import MAX_IMAGE_BYTES
mime = sniff_mime(content_bytes)
if mime is None or len(content_bytes) > MAX_IMAGE_BYTES:
@@ -115,7 +111,7 @@ def register_read(
``state`` is the per-chat ReadStateTracker shared with Edit/Write.
``sandbox_session_id`` selects the container; it defaults to ``chat_id``.
- ``vision_mode`` (see ``core.vision.resolve_vision_mode``) decides what an image
+ ``vision_mode`` (see ``core.models.vision.resolve_vision_mode``) decides what an image
file reads back as: pixels, a vision-bridge transcription, or ``type=binary``.
"""
@@ -154,10 +150,7 @@ async def Read(
_local_read: Optional[bytes] = None
if _local_on():
try:
- from core.llm.tool_permissions import (
- PermissionEnforcementError,
- require_local_path_permission,
- )
+ from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission
require_local_path_permission(physical, "read")
with open(physical, "rb") as _f:
@@ -291,5 +284,5 @@ async def Read(
" ...}`` / ``{type: 'too_large', ...}`` / ``{error: '...'}``。\n"
)
- toolkit.register_tool_function(Read, namesake_strategy="override")
+ toolkit.append(function_tool(Read))
logger.info("[factory] Registered Read tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/llm/tools/sandbox_artifact_tools.py b/src/backend/core/llm/tools/sandbox_artifact_tools.py
index a8a8cdd1..8010eac2 100644
--- a/src/backend/core/llm/tools/sandbox_artifact_tools.py
+++ b/src/backend/core/llm/tools/sandbox_artifact_tools.py
@@ -10,6 +10,7 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import base64
@@ -17,19 +18,18 @@
import os
import tempfile
from pathlib import Path
-from typing import Any, Optional
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from core.projects.edition import ProjectScope
from agentscope.tool import Toolkit
-# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
+# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk).
+from agentscope.tool._response import ToolChunk
from core.llm.tools._common import resolve_sandbox_session
-from core.llm.tools._tool_helpers import (
- _resolve_artifact_files,
- _resp_json,
- _store_generated_file_path,
- _validate_workspace_path,
-)
+from core.files.artifacts.generated import resolve_artifact_files as _resolve_artifact_files, store_generated_file_path as _store_generated_file_path
+from core.llm.tools._tool_helpers import resp_json as _resp_json, validate_workspace_path as _validate_workspace_path
logger = logging.getLogger(__name__)
@@ -46,7 +46,7 @@ def register_sandbox_put_artifact(
_sess = resolve_sandbox_session(sandbox_session_id, chat_id)
- async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse:
+ async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolChunk:
from core.sandbox import SandboxConnectError as _SandboxConnectError
from core.sandbox import SandboxError as _SandboxError
from core.sandbox import get_sandbox_provider as _get_provider
@@ -54,7 +54,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse
if not artifact_id or not isinstance(artifact_id, str):
return _resp_json({"error": "artifact_id 必须为非空字符串"})
- from ._paths import to_physical_path, workspace_directory
+ from core.sandbox.workspace.resolution import to_physical_path, workspace_directory
dest_path = to_physical_path(dest_path, user_id, session_id=_sess)
path_err = _validate_workspace_path(dest_path, root=workspace_directory(_sess))
@@ -102,7 +102,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse
"限制:单个 artifact 最大 10 MB。\n"
)
- toolkit.register_tool_function(sandbox_put_artifact, namesake_strategy="override")
+ toolkit.append(function_tool(sandbox_put_artifact))
logger.info("[factory] Registered sandbox_put_artifact tool (chat_id=%s)", chat_id)
@@ -123,7 +123,7 @@ def register_sandbox_get_artifact(
max_bytes = _settings.sandbox.artifact_max_bytes
- async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse:
+ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolChunk:
import mimetypes as _mt
from core.sandbox import SandboxConnectError as _SandboxConnectError
@@ -131,10 +131,10 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse:
from core.sandbox import get_sandbox_provider as _get_provider
from core.config.local_mode import local_mode_enabled
- from ._paths import to_physical_path
+ from core.sandbox.workspace.resolution import to_physical_path
if local_mode_enabled():
- from core.artifacts.local_reference import reference_local_file
+ from core.files.artifacts.local_reference import reference_local_file
from fastapi import HTTPException
physical = to_physical_path(src_path, user_id, session_id=_sess, scope=scope)
@@ -149,7 +149,7 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse:
ref["url"] = f"/files/{ref['file_id']}"
return _resp_json({"ok": True, **ref, "artifacts": [ref]})
- from ._paths import to_physical_path, workspace_directory
+ from core.sandbox.workspace.resolution import to_physical_path, workspace_directory
src_path = to_physical_path(src_path, user_id, session_id=_sess)
path_err = _validate_workspace_path(src_path, root=workspace_directory(_sess))
@@ -243,5 +243,5 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse:
"PDF 应按页拆分,其他格式应拆包或降低体积后再逐个登记。\n"
)
- toolkit.register_tool_function(sandbox_get_artifact, namesake_strategy="override")
+ toolkit.append(function_tool(sandbox_get_artifact))
logger.info("[factory] Registered sandbox_get_artifact tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/llm/tools/sandbox_tool.py b/src/backend/core/llm/tools/sandbox_tool.py
index 6e55e942..97f6a403 100644
--- a/src/backend/core/llm/tools/sandbox_tool.py
+++ b/src/backend/core/llm/tools/sandbox_tool.py
@@ -10,6 +10,7 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import asyncio
import logging
@@ -18,15 +19,16 @@
from agentscope.tool import Toolkit
-# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
+# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk).
+from agentscope.tool._response import ToolChunk
from core.llm.tools._common import resolve_sandbox_session
-from core.llm.tools._tool_helpers import _resp_json
+from core.llm.tools._tool_helpers import resp_json as _resp_json
-logger = logging.getLogger(__name__)
import re as _re
+logger = logging.getLogger(__name__)
+
# dws exit code 4 = PAT authorization interception; stderr/stdout carries a line
# ``PAT_AUTHORIZATION_URL=`` (a copy-safe link dws prints separately for
# OpenClaw-style hosts, see dws CHANGELOG #242).
@@ -55,7 +57,6 @@ def register_bash(
toolkit: Toolkit,
*,
loader: Any,
- loaded_skill_ids: set[str],
chat_id: Optional[str] = None,
sandbox_session_id: Optional[str] = None,
user_id: Optional[str] = None,
@@ -69,8 +70,8 @@ def register_bash(
``opensandbox_provider._make_skills_volumes`` + ``config.get_sandbox_skills_dir``):
what is bound there is the **caller's own skill view**, holding the shared
skills plus that user's private ones, so no one sees another user's skill
- files. This registration just sets up the Bash tool itself; ``loader`` /
- ``loaded_skill_ids`` are kept for backward compat with existing callers.
+ files. This registration just sets up the Bash tool itself; ``loader`` only
+ supplies the capability run that scopes it.
The sandbox session is bound to ``chat_id`` so OpenSandbox keeps a single
persistent container per conversation (variables, pip packages, /workspace
@@ -82,13 +83,13 @@ def register_bash(
# Effective sandbox session (``None`` → legacy fall back to chat_id).
_sess = resolve_sandbox_session(sandbox_session_id, chat_id)
- from core.llm.evaluation_runtime import is_evaluation_session
+ from core.harness.registry.evaluation import is_evaluation_session
_evaluation = is_evaluation_session(_sess)
async def Bash(
command: str, timeout: int | None = None, yield_time_ms: int = 60000,
run_in_background: bool = False,
- ) -> ToolResponse:
+ ) -> ToolChunk:
from core.sandbox import ProcessRequest as _ProcessRequest
from core.sandbox import SandboxConnectError as _SandboxConnectError
from core.sandbox import SandboxError as _SandboxError
@@ -111,7 +112,7 @@ async def Bash(
authorization = None
if local_mode_enabled():
- from core.llm.tool_permissions import current_local_command_authorization
+ from core.sandbox.permissions.dispatch_models import current_local_command_authorization
authorization = current_local_command_authorization(cmd)
if authorization is None:
@@ -132,7 +133,7 @@ async def Bash(
# confinement nobody enforces — so the question is put to the provider
# rather than assumed from the deployment profile.
if authorization is not None and provider.runs_on_host:
- from core.llm.tool_permissions import LocalConfinementUnavailableError
+ from core.sandbox.permissions.dispatch_models import LocalConfinementUnavailableError
try:
sandbox_launch = authorization.confine()
@@ -151,7 +152,7 @@ async def Bash(
sandbox_launch.backend if sandbox_launch else "未启用(用户选择的权限档)",
)
- from core.services.edition_workspace import project_directory
+ from core.files.edition import project_directory
root = project_directory(scope)
if root:
import shlex
@@ -159,7 +160,7 @@ async def Bash(
if timeout is not None and timeout <= 0:
return _resp_json({"error": "timeout 必须为正数;省略表示不设置命令执行期限。"})
- from ._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
effective_timeout = timeout
req = _ProcessRequest(
@@ -179,8 +180,8 @@ async def Bash(
)
from fastapi import HTTPException
try:
- from orchestration.commands.runtime import runtime
- payload = await runtime.start(provider, req, chat_id=chat_id,
+ from core.llm.ports import command_execution
+ payload = await command_execution().start(provider, req, chat_id=chat_id,
source_run_id=source_run_id, background=run_in_background,
yield_time_ms=yield_time_ms)
except HTTPException as exc:
@@ -194,33 +195,33 @@ async def Bash(
return _resp_json(payload)
return await finish(payload)
- async def write_stdin(session_id: str, chars: str = "", yield_time_ms: int = 0) -> ToolResponse:
- from core.services.commands import store
- from orchestration.commands.runtime import runtime
- tid = await asyncio.to_thread(store.resolve_id, session_id, user_id, _sess)
+ async def write_stdin(session_id: str, chars: str = "", yield_time_ms: int = 0) -> ToolChunk:
+ from core.llm.ports import command_execution
+ from core.llm.ports import command_records
+ tid = await asyncio.to_thread(command_records().resolve_id, session_id, user_id, _sess)
if chars not in ("", "\x03"):
return _resp_json({"error": "stdin is closed; use empty input or Ctrl+C"})
if chars:
- await runtime.cancel(tid, user_id, _sess)
+ await command_execution().cancel(tid, user_id, _sess)
else:
- await runtime.wait(tid, user_id, _sess, max(0, min(yield_time_ms, 3000)))
- payload = await asyncio.to_thread(store.read, tid, user_id, _sess)
+ await command_execution().wait(tid, user_id, _sess, max(0, min(yield_time_ms, 3000)))
+ payload = await asyncio.to_thread(command_records().read, tid, user_id, _sess)
return await finish(payload)
- async def background_process(session_id: str) -> ToolResponse:
+ async def background_process(session_id: str) -> ToolChunk:
"""Adopt an existing Bash process without executing the command again."""
from core.sandbox import get_sandbox_provider
- from orchestration.commands.runtime import runtime
- payload = await runtime.adopt(get_sandbox_provider(), session_id,
+ from core.llm.ports import command_execution
+ payload = await command_execution().adopt(get_sandbox_provider(), session_id,
user_id=user_id, session_id=_sess, chat_id=chat_id, source_run_id=source_run_id)
return _resp_json(payload)
- async def list_processes() -> ToolResponse:
+ async def list_processes() -> ToolChunk:
"""List this user's command tasks in the current chat."""
- from core.services.commands import store
- return _resp_json({"tasks": await asyncio.to_thread(store.list_owned, user_id, chat_id)})
+ from core.llm.ports import command_records
+ return _resp_json({"tasks": await asyncio.to_thread(command_records().list_owned, user_id, chat_id)})
- async def finish(payload: dict) -> ToolResponse:
+ async def finish(payload: dict) -> ToolChunk:
# dws (DingTalk CLI) PAT per-scope authorization interception: exit code 4
# + a PAT_AUTHORIZATION_URL= line on stderr. Surface the link to the
# model in structured form so it hands it verbatim to the user, who
@@ -237,10 +238,10 @@ async def finish(payload: dict) -> ToolResponse:
return _resp_json(payload)
- from ._paths import WORKSPACE_ROOT, path_rules
-
import core.sandbox.workspace.desktop as desktop_paths
+ from core.sandbox.workspace.resolution import WORKSPACE_ROOT, path_rules
+
rules = path_rules()
instructions = (
rules.bash_workspace_instructions(WORKSPACE_ROOT, _sess, scope=scope)
@@ -260,7 +261,7 @@ async def finish(payload: dict) -> ToolResponse:
" 或失败时 {error, exit_code: -1}。\n"
)
- from core.services.edition_workspace import command_instructions
+ from core.files.edition import command_instructions
Bash.__doc__ += command_instructions(scope)
write_stdin.__doc__ = (
@@ -273,13 +274,13 @@ async def finish(payload: dict) -> ToolResponse:
" JSON: {stdout, stderr, exit_code, execution_time_ms, status, session_id};输出仅含新增内容。\n"
" running 表示继续运行;exited 时检查 exit_code 和 error。\n"
)
- toolkit.register_tool_function(background_process, namesake_strategy="override")
- toolkit.register_tool_function(list_processes, namesake_strategy="override")
- toolkit.register_tool_function(write_stdin, namesake_strategy="override")
- toolkit.register_tool_function(Bash, namesake_strategy="override")
+ toolkit.append(function_tool(background_process))
+ toolkit.append(function_tool(list_processes))
+ toolkit.append(function_tool(write_stdin))
+ toolkit.append(function_tool(Bash))
logger.info("[factory] Registered Bash tool (chat_id=%s)", chat_id)
-from .sandbox_artifact_tools import register_sandbox_put_artifact, register_sandbox_get_artifact
+from .sandbox_artifact_tools import register_sandbox_get_artifact, register_sandbox_put_artifact
diff --git a/src/backend/core/llm/tools/skill_tool.py b/src/backend/core/llm/tools/skill_tool.py
index ad97727f..0f737102 100644
--- a/src/backend/core/llm/tools/skill_tool.py
+++ b/src/backend/core/llm/tools/skill_tool.py
@@ -10,6 +10,8 @@
"""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
+from core.harness import log_writer
import json
import logging
@@ -20,16 +22,16 @@
from agentscope.message import TextBlock
from agentscope.tool import Toolkit
-# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse).
-from agentscope.tool._response import ToolChunk as ToolResponse
+# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk).
+from agentscope.tool._response import ToolChunk
-from core.agent_skills.loader import get_skill_loader
+from core.capabilities.skills.loader_factory import get_skill_loader
logger = logging.getLogger(__name__)
def _resolve_skill_path(file_path: str, loader: Any = None) -> str | None:
- from ._paths import path_rules
+ from core.sandbox.workspace.resolution import path_rules
return path_rules().resolve_skill_file(file_path, loader)
@@ -59,11 +61,11 @@ def _build_skill_bash_hint(
return None
name = getattr(spec, "name", skill_id) or skill_id
- from core.agent_skills.config import model_facing_skill_dir
+ from core.capabilities.skills.config import model_facing_skill_dir
sandbox_dir = model_facing_skill_dir(skill_id, skill_dir)
- from ._paths import path_rules
+ from core.sandbox.workspace.resolution import path_rules
quote = path_rules().quote_shell_path
scripts = list(getattr(spec, "executable_scripts", None) or [])
@@ -113,13 +115,13 @@ def register_sandboxed_view_text_file(
async def view_text_file(
file_path: str,
ranges: list[int] | None = None,
- ) -> ToolResponse:
+ ) -> ToolChunk:
"""View file content within allowed skill directories."""
prepared = getattr(loader, "capability_run", None)
if prepared is not None:
- from core.capabilities.runtime import validate
+ from core.harness.registry.capability_runtime import validate
import asyncio
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
try:
# Check only the file's skill. An unrelated skill cannot block this read.
@@ -140,7 +142,7 @@ async def view_text_file(
if requested_name is not None:
await asyncio.to_thread(validate, prepared, only_skill=requested_name)
except (CapabilityError, OSError):
- return ToolResponse(
+ return ToolChunk(
content=[
TextBlock(
type="text",
@@ -160,7 +162,7 @@ async def view_text_file(
real = _os.path.realpath(resolved)
if not any(real.startswith(d + _os.sep) or real == d for d in resolved_dirs):
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text",
text=f"Error: Access denied. Only files inside skill directories can be read.\nRequested: {file_path}",
)])
@@ -177,22 +179,13 @@ async def view_text_file(
else:
_selected = _lines
_text = "".join(_selected)
- from core.config.local_mode import local_mode_enabled
- if local_mode_enabled() and "/home/ubuntu/yida-workspace" in _text and user_id:
- from core.sandbox.workspace.identity import safe_user_id
- from core.services.yida_execution import workspace_dir
- import shlex
-
- if safe_user_id(user_id):
- _text = _text.replace(
- "/home/ubuntu/yida-workspace",
- shlex.quote(workspace_dir(user_id).as_posix()),
- )
+ from core.harness.registry.skill_content import adapt_skill_content
+ _text = adapt_skill_content(_text, user_id)
except Exception as _e: # noqa: BLE001
- return ToolResponse(content=[TextBlock(
+ return ToolChunk(content=[TextBlock(
type="text", text=f"Error reading file: {_e}",
)])
- resp = ToolResponse(content=[TextBlock(type="text", text=_text)])
+ resp = ToolChunk(content=[TextBlock(type="text", text=_text)])
if _os.path.basename(real) == "SKILL.md":
from core.config.local_mode import local_mode_enabled
@@ -214,7 +207,7 @@ async def view_text_file(
for i, block in enumerate(resp.content):
text = block.get("text", "") if isinstance(block, dict) else getattr(block, "text", "")
if "{baseDir}" in text:
- from core.agent_skills.config import model_facing_skill_dir
+ from core.capabilities.skills.config import model_facing_skill_dir
directory = model_facing_skill_dir(skill_id or "", skill_dir).replace(chr(92), "/")
resp.content[i] = TextBlock(type="text", text=text.replace("{baseDir}", directory))
if skill_id:
@@ -224,13 +217,13 @@ async def view_text_file(
# ── Observability: record skill auto-load via view_text_file ──
try:
- from core.services import log_service as _lw
+ from core.harness import log_writer as _lw
spec = loader.load_skill_full(skill_id)
source = loader.get_skill_source(skill_id)
if prepared is not None and skill_id in prepared.bindings:
- from core.capabilities.ref import LOCAL_PROFILE, BUILTIN_PROFILE
+ from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE
source = "local" if prepared.bindings[skill_id]["profile"] in (LOCAL_PROFILE, BUILTIN_PROFILE) else "cloud"
- _lw.schedule_skill_call_write({
+ log_writer.schedule_skill_call_write({
"skill_id": skill_id,
"skill_name": getattr(spec, "name", skill_id) if spec else skill_id,
"skill_version": getattr(spec, "version", None) if spec else None,
@@ -244,4 +237,4 @@ async def view_text_file(
return resp
- toolkit.register_tool_function(view_text_file, namesake_strategy="override")
+ toolkit.append(function_tool(view_text_file))
diff --git a/src/backend/core/llm/tools/user_questions.py b/src/backend/core/llm/tools/user_questions.py
index e815689a..e7d7c714 100644
--- a/src/backend/core/llm/tools/user_questions.py
+++ b/src/backend/core/llm/tools/user_questions.py
@@ -21,8 +21,8 @@
from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, List, Optional
-from core.llm.human_interaction import MAX_WAIT_SECONDS
-from core.llm import interaction_store
+from core.llm.runtime.human_interaction import MAX_WAIT_SECONDS
+from core.llm.runtime import interaction_store
KIND_USER_QUESTION = "user_question"
STATUS_BLOCKED = "blocked_non_interactive"
@@ -64,10 +64,6 @@ class _ChatQuestions:
last_ts: float = field(default_factory=time.monotonic)
-_LOCK = threading.RLock()
-_CHATS: Dict[str, _ChatQuestions] = {}
-
-
def _text(
value: Any,
*,
@@ -235,20 +231,20 @@ def _normalize_answers(
def _gc_locked(now: float) -> None:
dead = [
chat_id
- for chat_id, state in _CHATS.items()
+ for chat_id, state in process_state().CHATS.items()
if not state.pending and now - state.last_ts > _STATE_IDLE_TTL_SECONDS
]
for chat_id in dead:
- _CHATS.pop(chat_id, None)
+ process_state().CHATS.pop(chat_id, None)
def _state_locked(chat_id: str) -> _ChatQuestions:
now = time.monotonic()
_gc_locked(now)
- state = _CHATS.get(chat_id)
+ state = process_state().CHATS.get(chat_id)
if state is None:
state = _ChatQuestions()
- _CHATS[chat_id] = state
+ process_state().CHATS[chat_id] = state
state.last_ts = now
return state
@@ -274,32 +270,32 @@ def _resolved_signal(pending: _PendingQuestion) -> Dict[str, Any]:
def get_ui_queue(chat_id: Optional[str]) -> Optional[asyncio.Queue]:
if not chat_id:
return None
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
return state.ui_signals if state is not None else None
def get_all_pending(chat_id: Optional[str]) -> List[Dict[str, Any]]:
if not chat_id:
return []
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
if state is None:
return []
return [_requested_signal(item) for item in state.pending.values()]
def list_pending_chat_ids() -> List[str]:
- with _LOCK:
+ with process_state().LOCK:
_gc_locked(time.monotonic())
- return [chat_id for chat_id, state in _CHATS.items() if state.pending]
+ return [chat_id for chat_id, state in process_state().CHATS.items() if state.pending]
def has_pending(chat_id: Optional[str]) -> bool:
if not chat_id:
return False
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
return bool(state and state.pending)
@@ -339,8 +335,8 @@ def answer(
) -> Dict[str, Any]:
"""Validate and claim a pending request; first valid claimant wins."""
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
pending = state.pending.get(request_id) if state is not None else None
if state is None or pending is None:
return {"ok": False, "reason": "stale", "error": "该问题已失效"}
@@ -355,8 +351,8 @@ def answer(
def cancel(chat_id: str, request_id: str) -> Dict[str, Any]:
"""Cancel a pending question from the user interface."""
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
pending = state.pending.get(request_id) if state is not None else None
if state is None or pending is None:
return {"ok": False, "reason": "stale", "error": "该问题已失效"}
@@ -389,7 +385,7 @@ async def ask(
await interaction_store.register(
"question", chat_id, pending.request_id, _requested_signal(pending), timeout
)
- with _LOCK:
+ with process_state().LOCK:
state = _state_locked(chat_id)
if state.ui_signals is None:
state.ui_signals = asyncio.Queue()
@@ -401,26 +397,26 @@ async def ask(
"question", chat_id, pending.request_id, pending.event, timeout
)
if decision is not None:
- with _LOCK:
+ with process_state().LOCK:
_claim_locked(
state, pending, outcome=decision["outcome"], answers=decision.get("answers")
)
except asyncio.TimeoutError:
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
if state is not None:
_claim_locked(state, pending, outcome="timeout")
# A valid HTTP answer/cancel may have claimed the request at the same
# event-loop boundary where wait_for delivered TimeoutError. The
# claimant owns the terminal outcome; never overwrite it with timeout.
except asyncio.CancelledError:
- with _LOCK:
- state = _CHATS.get(chat_id)
+ with process_state().LOCK:
+ state = process_state().CHATS.get(chat_id)
if state is not None:
_claim_locked(state, pending, outcome="cancelled")
raise
finally:
- with _LOCK:
+ with process_state().LOCK:
_claim_locked(state, pending, outcome="cancelled")
await interaction_store.remove("question", chat_id, pending.request_id)
@@ -481,3 +477,16 @@ async def cancel_shared(chat_id, request_id):
"list_pending_chat_ids",
"normalize_questions",
]
+
+
+@dataclass
+class UserQuestionsState:
+ LOCK: object = field(default_factory=lambda: threading.RLock())
+ CHATS: Dict[str, _ChatQuestions] = field(default_factory=lambda: {})
+
+
+_STATE = UserQuestionsState()
+
+
+def process_state() -> UserQuestionsState:
+ return _STATE
diff --git a/src/backend/core/llm/tools/write_tool.py b/src/backend/core/llm/tools/write_tool.py
index ce3d384b..e7f29c47 100644
--- a/src/backend/core/llm/tools/write_tool.py
+++ b/src/backend/core/llm/tools/write_tool.py
@@ -1,6 +1,7 @@
"""Read-checked mutations of files in the active sandbox."""
from __future__ import annotations
+from core.llm.factory.tools.definitions import function_tool
import difflib
import hashlib
@@ -8,7 +9,7 @@
from typing import Optional
from agentscope.tool import Toolkit, ToolResponse
-from core.services.project_scope import ProjectScope
+from core.projects.edition import ProjectScope
from ._common import (
pin_artifact_to_workspace,
@@ -18,16 +19,7 @@
shell_quote,
register_file_artifact,
)
-from ._paths import (
- basename,
- is_myspace_physical,
- space_metadata,
- WORKSPACE_ROOT,
- parent_dir,
- to_physical_path,
- validate_project_scope_path,
- validate_workspace_path,
-)
+from core.sandbox.workspace.resolution import basename, is_myspace_physical, space_metadata, WORKSPACE_ROOT, parent_dir, to_physical_path, validate_project_scope_path, validate_workspace_path
from ._state import ReadEntry, ReadStateTracker
logger = logging.getLogger(__name__)
@@ -69,10 +61,7 @@ async def Write(
content: str,
register_as_artifact: bool = False,
) -> "ToolResponse": # type: ignore[name-defined]
- from core.llm.tool_permissions import (
- PermissionEnforcementError,
- require_local_path_permission,
- )
+ from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission
from core.sandbox import SandboxConnectError as _SCE
from core.sandbox import SandboxError as _SE
from core.sandbox import get_sandbox_provider as _get_provider
@@ -218,11 +207,12 @@ async def Write(
# Local-mode write-back snapshot (ticket #08): back up the real file
# before overwriting it, so the user can roll back. No-op on cloud/web.
try:
- from core.services.local_snapshot_service import maybe_snapshot_local
+ from core.harness.registry.local_files import maybe_snapshot_local
maybe_snapshot_local(physical)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.llm.tools.write_tool:Write', fallback_error)
try:
if _is_local:
# Local desktop mode: the backend runs on the host, so write the
@@ -273,7 +263,7 @@ async def Write(
artifact_ref: Optional[dict] = None
needs_registration = bool(register_as_artifact and user_id and not is_persistent)
if needs_registration and _is_local:
- from core.artifacts.local_reference import reference_local_file
+ from core.files.artifacts.local_reference import reference_local_file
from fastapi import HTTPException
try:
@@ -346,5 +336,5 @@ async def Write(
" ``persistent=true`` 表示文件保存于个人或团队空间;``space_type`` 为 personal/team/temporary。\n"
)
- toolkit.register_tool_function(Write, namesake_strategy="override")
+ toolkit.append(function_tool(Write))
logger.info("[factory] Registered Write tool (chat_id=%s)", chat_id)
diff --git a/src/backend/core/memory/access.py b/src/backend/core/memory/access.py
index 9d3e8d2c..ebbaca5e 100644
--- a/src/backend/core/memory/access.py
+++ b/src/backend/core/memory/access.py
@@ -4,10 +4,31 @@
import asyncio
from typing import Optional
+from core.db.threads import run_in_session
from core.memory.context import MemoryContext
from core.memory import backend
+def _project_memory_owner(db, project_id: str, actor: str) -> Optional[str]:
+ """The memory scope ``actor`` may edit in a project, or None when it may not."""
+ from core.auth.edition import resolve_project_access
+ from core.db.models import Project
+ from core.projects.edition import project_memory_policy
+
+ project = db.query(Project).filter_by(project_id=project_id, deleted_at=None).first()
+ if project is None:
+ return None
+ access = resolve_project_access(db, actor, project)
+ if access.level not in ("edit", "admin"):
+ return None
+ policy = project_memory_policy(db, project.project_id, actor)
+ if policy is not None:
+ # Read availability does not revoke explicit edits of existing memory.
+ return policy[1]
+ # CE has no policy row for local projects; actor access is checked above.
+ return actor if project.kind == "local" else None
+
+
async def authorized_memory_context(memory_id: str, actor: str) -> Optional[MemoryContext]:
"""Resolve trusted actor access from the stored scope, never a client scope."""
if not memory_id or not actor or not backend.settings.memory.enabled:
@@ -27,30 +48,9 @@ async def authorized_memory_context(memory_id: str, actor: str) -> Optional[Memo
scope = item.get("user_id") or metadata.get("user_id")
workspace = str(metadata.get("workspace_id") or "default")
if workspace.startswith("project:"):
- from core.auth.permissions_iface import resolve_project_access
- from core.db.engine import SessionLocal
- from core.db.models import Project
- from core.services.project_scope import project_memory_policy
-
- with SessionLocal() as db:
- project = db.query(Project).filter_by(
- project_id=workspace[len("project:"):], deleted_at=None).first()
- if project is None:
- return None
- access = resolve_project_access(db, actor, project)
- if access.level not in ("edit", "admin"):
- return None
- policy = project_memory_policy(db, project.project_id, actor)
- if policy is not None:
- # Read availability does not revoke explicit edits of existing memory.
- expected = policy[1]
- elif project.kind == "local":
- # CE has no policy row for local projects; actor access is checked above.
- expected = actor
- else:
- return None
- if scope != expected:
- return None
+ expected = await run_in_session(_project_memory_owner, workspace[len("project:"):], actor)
+ if expected is None or scope != expected:
+ return None
elif scope != actor:
return None
return MemoryContext(user_id=actor, workspace_id=workspace,
diff --git a/src/backend/core/memory/audit.py b/src/backend/core/memory/audit.py
index 2e3c3042..5a3a42d5 100644
--- a/src/backend/core/memory/audit.py
+++ b/src/backend/core/memory/audit.py
@@ -1,35 +1,155 @@
-"""Memory audit side-channel — community-edition no-op stub.
-
-Memory auditing (hash-only storage, trace chains) is a commercial-edition
-compliance capability; the community edition keeps the same module surface but
-records nothing, so callers (the L1 profile writer, the extraction pipeline)
-need zero changes.
-
-The stubs must stay *call-compatible* with the EE implementation, whose real
-signatures are ``record(ctx, action, layer, *, memory_id=None, content=None,
-reason=None)`` etc. — the first three are positional. Narrowing the stub
-signature makes positional calls raise TypeError, and that raise lands *after*
-the business transaction has committed, turning a successful write into a
-reported failure (bitten in the 0.2.15 desktop local build: the L1 preference
-was persisted, yet the UI never showed the write card). Hence ``*args`` /
-``**kwargs`` catch-alls.
+"""Unified entry point for memory operation auditing.
+
+All read/write operations across the L1 / L2 / L3 / session layers should go
+through `record()` or `record_batch()` to write to the `memory_audit` table.
+Failures do not bubble up; they only emit a warning (auditing must not block
+the main flow).
+
+Toggle: `MEMORY_AUDIT_ENABLED` env (default true). Forced on for government /
+financial scenarios.
"""
from __future__ import annotations
-from typing import Any
+import asyncio
+import hashlib
+import logging
+from typing import Iterable, Literal, Optional
+
+from core.config.settings import settings
+from core.memory.context import MemoryContext
+
+logger = logging.getLogger(__name__)
+
+
+Action = Literal["read", "write", "update", "delete", "write_rejected", "forget"]
+Layer = Literal["L1", "L2", "L3", "session", "batch"]
+
+
+def _audit_enabled() -> bool:
+ """Audit toggle (aligned with global settings, default true)."""
+ from core.memory.edition import audit_enabled
+ return audit_enabled()
+
+
+def _content_hash(content: Optional[str]) -> Optional[str]:
+ if content is None:
+ return None
+ return hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest()
+
+
+async def record(
+ ctx: MemoryContext,
+ action: Action,
+ layer: Layer,
+ *,
+ memory_id: Optional[str] = None,
+ content: Optional[str] = None,
+ reason: Optional[str] = None,
+) -> None:
+ """Write one audit record. Failures do not bubble up.
+
+ content is used only to compute the hash, never stored; the original text
+ is never written to the audit table.
+ """
+ if not settings.memory.enabled or not _audit_enabled():
+ return
+
+ try:
+ loop = asyncio.get_running_loop()
+ await loop.run_in_executor(None, _sync_insert_one, ctx, action, layer, memory_id, content, reason)
+ except Exception as exc:
+ logger.warning("[memory_audit] record failed: %s", exc)
+
+
+async def record_batch(
+ ctx: MemoryContext,
+ entries: Iterable[dict],
+) -> None:
+ """Batch-write audit records; each entry must contain `action` / `layer`, other fields optional."""
+ if not settings.memory.enabled or not _audit_enabled():
+ return
+
+ rows = []
+ for e in entries:
+ rows.append({
+ "actor": ctx.effective_actor,
+ "action": e["action"],
+ "layer": e["layer"],
+ "memory_id": e.get("memory_id"),
+ "workspace_id": ctx.workspace_id,
+ "user_id": ctx.user_id,
+ "chat_id": ctx.chat_id,
+ "confidentiality": e.get("confidentiality") or ctx.confidentiality,
+ "content_hash": _content_hash(e.get("content")),
+ "reason": e.get("reason"),
+ })
+ if not rows:
+ return
+
+ try:
+ loop = asyncio.get_running_loop()
+ await loop.run_in_executor(None, _sync_insert_many, rows)
+ except Exception as exc:
+ logger.warning("[memory_audit] record_batch failed (%d rows): %s", len(rows), exc)
+
+def record_sync(
+ ctx: MemoryContext,
+ action: Action,
+ layer: Layer,
+ *,
+ memory_id: Optional[str] = None,
+ content: Optional[str] = None,
+ reason: Optional[str] = None,
+) -> None:
+ """Synchronously write one audit record (for non-async contexts, e.g. sync functions inside run_in_executor).
-async def record(*args: Any, **kwargs: Any) -> None:
- return None
+ The only difference from `record()` is the calling style: the former is
+ async (internally goes through the executor), the latter inserts into the
+ DB synchronously on the current thread. Both are gated by the
+ `audit_enabled` toggle, and failures do not bubble up.
+ """
+ if not settings.memory.enabled or not _audit_enabled():
+ return
+ try:
+ _sync_insert_one(ctx, action, layer, memory_id, content, reason)
+ except Exception as exc:
+ logger.debug("[memory_audit] record_sync failed: %s", exc)
-async def record_batch(*args: Any, **kwargs: Any) -> None:
- return None
+def _sync_insert_one(
+ ctx: MemoryContext,
+ action: Action,
+ layer: Layer,
+ memory_id: Optional[str],
+ content: Optional[str],
+ reason: Optional[str],
+) -> None:
+ from core.db.engine import SessionLocal
+ from core.db.models import MemoryAudit
+ row = MemoryAudit(
+ actor=ctx.effective_actor,
+ action=action,
+ layer=layer,
+ memory_id=memory_id,
+ workspace_id=ctx.workspace_id,
+ user_id=ctx.user_id,
+ chat_id=ctx.chat_id,
+ confidentiality=ctx.confidentiality or "unknown",
+ content_hash=_content_hash(content),
+ reason=reason,
+ )
+ with SessionLocal() as session:
+ session.add(row)
+ session.commit()
-def record_sync(*args: Any, **kwargs: Any) -> None:
- return None
+def _sync_insert_many(rows: list[dict]) -> None:
+ from core.db.engine import SessionLocal
+ from core.db.models import MemoryAudit
-__all__ = ["record", "record_batch", "record_sync"]
+ with SessionLocal() as session:
+ session.add_all([MemoryAudit(**r) for r in rows])
+ session.commit()
diff --git a/src/backend/core/memory/backend.py b/src/backend/core/memory/backend.py
index 0e9d3141..e2d787d7 100644
--- a/src/backend/core/memory/backend.py
+++ b/src/backend/core/memory/backend.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
import threading
+from dataclasses import dataclass, field
from typing import Optional
from core.config.settings import settings
@@ -13,11 +14,8 @@
# L2 stores procedures; fact extraction belongs to the layered pipeline.
MEMORY_TYPE_PROCEDURAL = "procedural"
# One lock and instance per process; account epoch changes invalidate the cache.
-_memory_instance = None
-_memory_epoch: Optional[str] = None
-_memory_lock = threading.Lock()
-_memory_init_failed = False
-_embedding_patched = False
+
+
_MIGRATION_MAX_ROWS = 10_000
@@ -28,8 +26,8 @@ def _patch_mem0_embedding() -> None:
but mem0's OpenAIEmbedding.embed() hardcodes dimensions=...
Patched only once.
"""
- global _embedding_patched
- if _embedding_patched:
+ pass
+ if process_state().embedding_patched:
return
try:
from mem0.embeddings.openai import OpenAIEmbedding as _OAIEmbed
@@ -43,9 +41,10 @@ def _patched_embed(self, text, memory_action=None):
)
_OAIEmbed.embed = _patched_embed
- _embedding_patched = True
- except Exception:
- pass
+ process_state().embedding_patched = True
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.memory.backend:_patch_mem0_embedding', fallback_error)
def _build_mem0_config() -> dict:
@@ -58,7 +57,7 @@ def _build_mem0_config() -> dict:
"""
# Resolve LLM config from DB
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
svc = ModelConfigService.get_instance()
mem_cfg = svc.resolve("memory")
@@ -139,7 +138,7 @@ def _apply_probed_embed_dims(cfg: dict) -> None:
vs = cfg["vector_store"]["config"]
try:
from openai import OpenAI
- from core.services.desktop_model_credentials import sync_client_kwargs
+ from core.harness.registry.model_credentials import sync_client_kwargs
client = OpenAI(
api_key=emb["api_key"],
@@ -252,8 +251,9 @@ def _reconcile_vector_collection(cfg: dict) -> Optional[list]:
if client is not None:
try:
client.close()
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.memory.backend:_reconcile_vector_collection', fallback_error)
def _reembed_rows(client, name: str, cfg: dict) -> Optional[list]:
@@ -266,7 +266,7 @@ def _reembed_rows(client, name: str, cfg: dict) -> Optional[list]:
emb = cfg["embedder"]["config"]
try:
from openai import OpenAI
- from core.services.desktop_model_credentials import sync_client_kwargs
+ from core.harness.registry.model_credentials import sync_client_kwargs
oai = OpenAI(
api_key=emb["api_key"],
@@ -319,12 +319,12 @@ def _replay_migrated_rows(instance, rows: list) -> None:
def _desktop_memory_epoch() -> str:
"""Nonsecret identity for clients that hold desktop authorization hooks."""
- from core.auth.desktop_bridge import bridge_enabled
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not bridge_enabled() or not capabilities_enabled():
+ if not bridge_enabled() or not has_run_capabilities():
return ""
- from core.services.desktop_cloud_bridge import _state_fingerprint, get_state
+ from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint, get_state
return _state_fingerprint(get_state())
@@ -352,23 +352,23 @@ def _close_failed_memory(instance) -> None:
def _get_memory() -> Optional[object]:
"""Thread-safe lazy initialization: cache the instance on success; on failure allow retry next time."""
- global _memory_instance, _memory_init_failed, _memory_epoch
+ pass
if not settings.memory.enabled:
return None
epoch = _desktop_memory_epoch()
# Existing callers retain their captured hooks; only new callers rebuild.
- if _memory_instance is not None and _memory_epoch == epoch:
- return _memory_instance
+ if process_state().memory_instance is not None and process_state().memory_epoch == epoch:
+ return process_state().memory_instance
- with _memory_lock:
+ with process_state().memory_lock:
# Double-check after acquiring lock
epoch = _desktop_memory_epoch()
- if _memory_instance is not None and _memory_epoch == epoch:
- return _memory_instance
- _memory_instance = None
- _memory_epoch = None
+ if process_state().memory_instance is not None and process_state().memory_epoch == epoch:
+ return process_state().memory_instance
+ process_state().memory_instance = None
+ process_state().memory_epoch = None
instance = None
try:
@@ -384,7 +384,7 @@ def _get_memory() -> Optional[object]:
instance = Memory.from_config(cfg)
# Existing collections remain released after a local database restart.
instance.vector_store.client.load_collection(collection_name=MEMORY_COLLECTION_NAME)
- from core.services.desktop_model_credentials import is_reference, sync_client_kwargs
+ from core.harness.registry.model_credentials import is_reference, sync_client_kwargs
from openai import OpenAI
for component, section in (
@@ -402,7 +402,7 @@ def _get_memory() -> Optional[object]:
)
old_client.close()
if epoch != _desktop_memory_epoch():
- from core.capabilities.errors import CloudUnavailable
+ from core.harness.capability_errors import CloudUnavailable
raise CloudUnavailable("cloud account changed while preparing memory")
if replay_rows:
@@ -410,17 +410,17 @@ def _get_memory() -> Optional[object]:
# with the rows re-embedded during reconciliation.
_replay_migrated_rows(instance, replay_rows)
if epoch != _desktop_memory_epoch():
- from core.capabilities.errors import CloudUnavailable
+ from core.harness.capability_errors import CloudUnavailable
raise CloudUnavailable("cloud account changed while preparing memory")
- _memory_instance = instance
- _memory_epoch = epoch
- _memory_init_failed = False
- return _memory_instance
+ process_state().memory_instance = instance
+ process_state().memory_epoch = epoch
+ process_state().memory_init_failed = False
+ return process_state().memory_instance
except Exception as exc:
if instance is not None:
_close_failed_memory(instance)
- _memory_init_failed = True
+ process_state().memory_init_failed = True
logger.error("[MemoryService] 初始化失败,记忆功能将降级为空: %s", exc)
return None
@@ -430,11 +430,11 @@ def _reset_memory() -> None:
Used when the Milvus connection is broken (e.g. closed channel).
"""
- global _memory_instance, _memory_init_failed, _memory_epoch
- with _memory_lock:
- _memory_instance = None
- _memory_epoch = None
- _memory_init_failed = False
+ pass
+ with process_state().memory_lock:
+ process_state().memory_instance = None
+ process_state().memory_epoch = None
+ process_state().memory_init_failed = False
logger.info("[MemoryService] 已重置 mem0 实例,下次调用将重新初始化")
@@ -471,3 +471,19 @@ def _is_auth_error(exc: Exception) -> bool:
"401 unauthorized",
)
)
+
+
+@dataclass
+class BackendState:
+ memory_instance: object = None
+ memory_epoch: Optional[str] = None
+ memory_lock: object = field(default_factory=lambda: threading.Lock())
+ memory_init_failed: object = False
+ embedding_patched: object = False
+
+
+_STATE = BackendState()
+
+
+def process_state() -> BackendState:
+ return _STATE
diff --git a/src/backend/core/memory/context.py b/src/backend/core/memory/context.py
index a4654c08..8059cda2 100644
--- a/src/backend/core/memory/context.py
+++ b/src/backend/core/memory/context.py
@@ -7,7 +7,7 @@
from __future__ import annotations
-from dataclasses import dataclass, field
+from dataclasses import dataclass
from typing import Literal, Optional
Confidentiality = Literal["public", "internal", "sensitive"]
diff --git a/src/backend/core/memory/edition.py b/src/backend/core/memory/edition.py
new file mode 100644
index 00000000..3d175a39
--- /dev/null
+++ b/src/backend/core/memory/edition.py
@@ -0,0 +1,4 @@
+"""Community memory has no audit storage."""
+
+def audit_enabled():
+ return False
diff --git a/src/backend/core/memory/effect_lane.py b/src/backend/core/memory/effect_lane.py
index 50ad522c..bf8c2f85 100644
--- a/src/backend/core/memory/effect_lane.py
+++ b/src/backend/core/memory/effect_lane.py
@@ -6,6 +6,7 @@
import hashlib
import threading
from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from core.db.engine import SessionLocal
@@ -13,9 +14,6 @@
from core.memory.executor import run_effect
from sqlalchemy import text
-_local_locks: dict[str, threading.Lock] = {}
-_local_locks_guard = threading.Lock()
-
class EffectLaneDeferred(RuntimeError):
def __init__(self, retry_at: datetime) -> None:
@@ -24,8 +22,8 @@ def __init__(self, retry_at: datetime) -> None:
def _local_lock(scope_key: str) -> threading.Lock:
- with _local_locks_guard:
- return _local_locks.setdefault(scope_key, threading.Lock())
+ with process_state().local_locks_guard:
+ return process_state().local_locks.setdefault(scope_key, threading.Lock())
def _advisory_key(scope_key: str) -> int:
@@ -54,6 +52,39 @@ def _oldest_nonterminal(db, scope_key: str):
return oldest
+class _Lane:
+ """The lane's session, touched only from worker threads.
+
+ The PostgreSQL advisory lock is transaction-scoped, so the lane must keep
+ one connection for the whole effect; async code holds this object, never
+ the session, and every method runs through ``run_effect``.
+ """
+
+ def __init__(self) -> None:
+ self.db = SessionLocal()
+
+ def dialect(self) -> str:
+ return self.db.bind.dialect.name if self.db.bind is not None else ""
+
+ def lock(self, scope_key: str) -> None:
+ self.db.execute(
+ text("SELECT pg_advisory_xact_lock(:lane_key)"),
+ {"lane_key": _advisory_key(scope_key)},
+ )
+
+ def oldest(self, scope_key: str):
+ return _oldest_nonterminal(self.db, scope_key)
+
+ def rollback(self) -> None:
+ self.db.rollback()
+
+ def commit(self) -> None:
+ self.db.commit()
+
+ def close(self) -> None:
+ self.db.close()
+
+
@asynccontextmanager
async def ordered_l2_effect(scope_key: str, job_id: str):
"""Wait for this candidate's turn, then hold one lane through its effect.
@@ -65,22 +96,17 @@ async def ordered_l2_effect(scope_key: str, job_id: str):
its external receipt before recovery checks it.
"""
- db = SessionLocal()
- dialect = db.bind.dialect.name if db.bind is not None else ""
- local = _local_lock(scope_key) if dialect != "postgresql" else None
+ lane = await run_effect(_Lane)
+ local = _local_lock(scope_key) if await run_effect(lane.dialect) != "postgresql" else None
acquired = False
try:
if local is not None:
while not local.acquire(blocking=False):
await asyncio.sleep(0.05)
else:
- await run_effect(
- db.execute,
- text("SELECT pg_advisory_xact_lock(:lane_key)"),
- {"lane_key": _advisory_key(scope_key)},
- )
+ await run_effect(lane.lock, scope_key)
acquired = True
- oldest = _oldest_nonterminal(db, scope_key)
+ oldest = await run_effect(lane.oldest, scope_key)
if oldest is None or str(oldest[0]) != job_id:
retry_at = (oldest[1] or oldest[2]) if oldest is not None else None
if retry_at is None:
@@ -89,15 +115,28 @@ async def ordered_l2_effect(scope_key: str, job_id: str):
try:
yield
except BaseException:
- await run_effect(db.rollback)
+ await run_effect(lane.rollback)
raise
else:
- await run_effect(db.commit)
+ await run_effect(lane.commit)
except asyncio.CancelledError:
if acquired:
- await run_effect(db.rollback)
+ await run_effect(lane.rollback)
raise
finally:
- await run_effect(db.close)
+ await run_effect(lane.close)
if local is not None and acquired:
local.release()
+
+
+@dataclass
+class EffectLaneState:
+ local_locks: dict[str, threading.Lock] = field(default_factory=lambda: {})
+ local_locks_guard: object = field(default_factory=lambda: threading.Lock())
+
+
+_STATE = EffectLaneState()
+
+
+def process_state() -> EffectLaneState:
+ return _STATE
diff --git a/src/backend/core/memory/executor.py b/src/backend/core/memory/executor.py
index 1aa48465..42a209c3 100644
--- a/src/backend/core/memory/executor.py
+++ b/src/backend/core/memory/executor.py
@@ -26,8 +26,9 @@ async def run_effect(fn: Callable[..., T], *args, **kwargs) -> T:
except asyncio.CancelledError:
try:
await future
- except Exception:
+ except Exception as fallback_error:
# Cancellation remains the owner-visible outcome. The durable row
# is released/retried only after this thread has actually stopped.
- pass
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.memory.executor:run_effect', fallback_error)
raise
diff --git a/src/backend/core/memory/extractors/_base.py b/src/backend/core/memory/extractors/_base.py
index f33211f8..78314153 100644
--- a/src/backend/core/memory/extractors/_base.py
+++ b/src/backend/core/memory/extractors/_base.py
@@ -11,6 +11,7 @@
import json
import logging
import re
+from dataclasses import dataclass, field
from datetime import date
from typing import Any, Optional
@@ -20,9 +21,6 @@
# Lazily-loaded OpenAI client + resolved model name (avoids errors at import time)
-_client = None
-_model_name: Optional[str] = None
-_client_lock = asyncio.Lock()
def _resolve_memory_model_config() -> tuple[str, str, str]:
@@ -32,7 +30,7 @@ def _resolve_memory_model_config() -> tuple[str, str, str]:
Consistent with `core/memory/service.py::_build_mem0_config`.
"""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
svc = ModelConfigService.get_instance()
mem_cfg = svc.resolve("memory")
if mem_cfg and mem_cfg.base_url and mem_cfg.api_key:
@@ -46,21 +44,17 @@ def _resolve_memory_model_config() -> tuple[str, str, str]:
def _client_identity(base_url: str, api_key: str) -> tuple:
"""桌面网关引用绑定账号:桥接身份变了就要换客户端(凭据轮换不换)。"""
- from core.services.desktop_model_credentials import bind, is_reference
+ from core.harness.registry.model_credentials import bind, is_reference
if not is_reference(api_key):
return (base_url, api_key)
- from core.services.desktop_cloud_bridge import _state_fingerprint
+ from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint
return (base_url, api_key, _state_fingerprint(bind(api_key, base_url)))
-_client_identity_key: tuple | None = None
-
-
async def _get_client() -> tuple[object, str]:
"""A dedicated memory LLM client, not shared with the main conversation. Returns (client, model_name)."""
- global _client, _model_name, _client_identity_key
base_url, api_key, model_name = _resolve_memory_model_config()
if not base_url or not api_key or not model_name:
raise RuntimeError(
@@ -68,13 +62,13 @@ async def _get_client() -> tuple[object, str]:
f"api_key={bool(api_key)} model_name={bool(model_name)}"
)
identity = _client_identity(base_url, api_key)
- if _client is not None and _model_name is not None and _client_identity_key == identity:
- return _client, _model_name
- async with _client_lock:
- if _client is not None and _model_name is not None and _client_identity_key == identity:
- return _client, _model_name
+ if process_state().client is not None and process_state().model_name is not None and process_state().client_identity_key == identity:
+ return process_state().client, process_state().model_name
+ async with process_state().client_lock:
+ if process_state().client is not None and process_state().model_name is not None and process_state().client_identity_key == identity:
+ return process_state().client, process_state().model_name
from openai import AsyncOpenAI
- from core.services.desktop_model_credentials import is_reference, request_hook
+ from core.harness.registry.model_credentials import is_reference, request_hook
http_client = None
if is_reference(api_key):
@@ -84,7 +78,7 @@ async def _get_client() -> tuple[object, str]:
hook, _captured = request_hook(api_key, base_url)
http_client = httpx.AsyncClient(timeout=180.0, event_hooks={"request": [hook]})
- _client = AsyncOpenAI(
+ process_state().client = AsyncOpenAI(
base_url=base_url,
api_key=api_key,
http_client=http_client,
@@ -96,11 +90,11 @@ async def _get_client() -> tuple[object, str]:
# extractor sets; this only stops a socket hanging forever.
timeout=180.0,
)
- _model_name = model_name
- _client_identity_key = identity
+ process_state().model_name = model_name
+ process_state().client_identity_key = identity
logger.info("[extractor] memory LLM client initialized model=%s base=%s",
model_name, base_url)
- return _client, _model_name
+ return process_state().client, process_state().model_name
def _strip_fences(text: str) -> str:
@@ -286,3 +280,18 @@ def fill_prompt(
except KeyError as exc:
logger.warning("[extractor] prompt template missing key: %s", exc)
return template
+
+
+@dataclass
+class BaseState:
+ client: object = None
+ model_name: Optional[str] = None
+ client_lock: object = field(default_factory=lambda: asyncio.Lock())
+ client_identity_key: tuple | None = None
+
+
+_STATE = BaseState()
+
+
+def process_state() -> BaseState:
+ return _STATE
diff --git a/src/backend/core/memory/extractors/graph.py b/src/backend/core/memory/extractors/graph.py
index 95778a9f..9f5cb059 100644
--- a/src/backend/core/memory/extractors/graph.py
+++ b/src/backend/core/memory/extractors/graph.py
@@ -11,7 +11,7 @@
from typing import Optional
from core.memory.extractors._base import fill_prompt, parse_json, run_llm_with_prompt
-from core.memory.graph import PREDICATE_LABELS
+from core.memory.graph_state import PREDICATE_LABELS
PROMPT = """你是一个 L3 知识图谱关系抽取器。只抽取用户明确陈述或明确确认的、跨会话仍有价值的稳定实体关系。
diff --git a/src/backend/core/memory/extractors/writers.py b/src/backend/core/memory/extractors/writers.py
index 1ee1f0bd..01dc6789 100644
--- a/src/backend/core/memory/extractors/writers.py
+++ b/src/backend/core/memory/extractors/writers.py
@@ -30,7 +30,8 @@
from core.memory.audit import record_batch as audit_record_batch
from core.memory.context import MemoryContext
from core.memory.extractors.router import ExtractorType
-from core.memory.graph import parse_relation, parse_relations, write_graph_relations
+from core.memory.graph_contract import parse_relation, parse_relations
+from core.memory.graph import write_graph_relations
from core.memory.pipeline import milvus_breaker
from core.memory.sanitizer import sanitize
from core.memory.service import (
@@ -47,7 +48,6 @@
LAYER_PROCEDURE = "L2"
LAYER_GRAPH = "L3"
-
async def write_layered(
results: dict[ExtractorType, Optional[dict]],
ctx: MemoryContext,
@@ -107,7 +107,6 @@ async def write_layered(
return written
-
async def _write_relations_to_neo4j(
data: dict, ctx: MemoryContext, *, strict: bool = False
) -> list[dict]:
@@ -188,7 +187,6 @@ async def _write_relations_to_neo4j(
for row in rows
]
-
async def _write_profile_from_identity(
data: dict, ctx: MemoryContext, *, strict: bool = False
) -> list[dict]:
@@ -202,7 +200,6 @@ async def _write_profile_from_identity(
strict=strict,
)
-
async def _write_profile_from_preference(
data: dict, ctx: MemoryContext, *, strict: bool = False
) -> list[dict]:
@@ -216,7 +213,6 @@ async def _write_profile_from_preference(
strict=strict,
)
-
async def _upsert_profile_facts(
data: dict,
ctx: MemoryContext,
@@ -272,7 +268,6 @@ async def _upsert_profile_facts(
for item in applied
]
-
async def _write_procedures_to_milvus(
data: dict, ctx: MemoryContext, *, strict: bool = False
) -> list[dict]:
@@ -463,7 +458,6 @@ async def _write_procedures_to_milvus(
logger.info("[writer:procedural] wrote %d procedures to L2", len(written))
return written
-
async def _write_session_task(data: dict, ctx: MemoryContext, *, strict: bool = False) -> int:
"""Write the session task working set into chats.metadata.session_memory.
diff --git a/src/backend/core/memory/graph.py b/src/backend/core/memory/graph.py
index cdf20194..35c49c3e 100644
--- a/src/backend/core/memory/graph.py
+++ b/src/backend/core/memory/graph.py
@@ -1,198 +1,26 @@
-"""Neo4j-backed L3 graph memory.
-
-mem0ai 2.x removed graph-store support from its open-source ``Memory`` class.
-The project still owns a Neo4j service and an L3 product contract, so this
-module keeps that contract local and explicit instead of passing an ignored
-``graph_store`` key to mem0.
-
-L3 stores stable, declarative relationships (entity -- predicate -- entity).
-It deliberately does not store procedures (L2), profile preferences (L1),
-volatile measurements, or raw conversation text.
-"""
-
+"""Read and write L3 graph relations in Neo4j."""
from __future__ import annotations
-
import asyncio
import hashlib
-import logging
-import re
-import threading
-import unicodedata
-from dataclasses import dataclass
from typing import Any, Dict, Optional, Sequence
-
from core.config.settings import settings
from core.memory.context import MemoryContext
from core.memory.executor import run_effect
+import logging
+from core.memory import graph_contract
+from core.memory import graph_state
logger = logging.getLogger(__name__)
-
-PREDICATE_LABELS: Dict[str, str] = {
- "is_a": "属于类别",
- "part_of": "组成",
- "belongs_to": "隶属于",
- "responsible_for": "负责",
- "depends_on": "依赖",
- "uses": "使用",
- "alias_of": "别名为",
- "located_in": "位于",
- "produces": "产出",
- "constrained_by": "受约束于",
- "related_to": "关联",
-}
-ALLOWED_ENTITY_TYPES = frozenset(
- {
- "person",
- "organization",
- "team",
- "project",
- "system",
- "document",
- "metric",
- "concept",
- "place",
- "other",
- }
-)
-
-_MAX_ENTITY_CHARS = 120
-_MAX_RELATIONS_PER_TURN = 12
-_MIN_CONFIDENCE = 0.65
-_SAFE_NAME = re.compile(r"\s+")
-
-
-@dataclass(frozen=True)
-class GraphRelation:
- """One validated L3 relation before persistence."""
-
- source: str
- predicate: str
- target: str
- source_type: str = "other"
- target_type: str = "other"
- confidence: float = 0.8
-
- @property
- def relationship(self) -> str:
- return PREDICATE_LABELS[self.predicate]
-
- @property
- def text(self) -> str:
- return f"{self.source} → {self.relationship} → {self.target}"
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "source": self.source,
- "source_type": self.source_type,
- "predicate": self.predicate,
- "relationship": self.relationship,
- "target": self.target,
- "target_type": self.target_type,
- "confidence": self.confidence,
- }
-
-
-def parse_relation(raw: Any) -> Optional[GraphRelation]:
- """Validate an extractor item and return its canonical representation."""
- if not isinstance(raw, dict):
- return None
- source = _clean_name(raw.get("source"))
- target = _clean_name(raw.get("target"))
- predicate = str(raw.get("predicate") or "").strip().lower()
- if not source or not target or source == target or predicate not in PREDICATE_LABELS:
- return None
-
- source_type = str(raw.get("source_type") or "other").strip().lower()
- target_type = str(raw.get("target_type") or "other").strip().lower()
- if source_type not in ALLOWED_ENTITY_TYPES:
- source_type = "other"
- if target_type not in ALLOWED_ENTITY_TYPES:
- target_type = "other"
- try:
- confidence = float(raw.get("confidence", 0.8))
- except (TypeError, ValueError):
- confidence = 0.8
- confidence = max(0.0, min(1.0, confidence))
- if confidence < _MIN_CONFIDENCE:
- return None
- return GraphRelation(
- source=source,
- predicate=predicate,
- target=target,
- source_type=source_type,
- target_type=target_type,
- confidence=confidence,
- )
-
-
-def parse_relations(raw_items: Sequence[Any]) -> list[GraphRelation]:
- """Validate, de-duplicate, and bound one turn's extracted relations."""
- relations: list[GraphRelation] = []
- seen: set[tuple[str, str, str]] = set()
- for raw in raw_items or []:
- relation = parse_relation(raw)
- if relation is None:
- continue
- key = (
- _entity_key(relation.source, relation.source_type),
- relation.predicate,
- _entity_key(relation.target, relation.target_type),
- )
- if key in seen:
- continue
- seen.add(key)
- relations.append(relation)
- if len(relations) >= _MAX_RELATIONS_PER_TURN:
- break
- return relations
-
-
-def _clean_name(value: Any) -> str:
- name = unicodedata.normalize("NFKC", str(value or ""))
- name = _SAFE_NAME.sub(" ", name).strip(" \t\r\n,,。;;::")
- if len(name) < 2 or len(name) > _MAX_ENTITY_CHARS:
- return ""
- return name
-
-
-def _normalise_name(value: str) -> str:
- return _SAFE_NAME.sub(" ", unicodedata.normalize("NFKC", value).lower()).strip()
-
-
-def _entity_key(name: str, entity_type: str) -> str:
- raw = f"{entity_type}|{_normalise_name(name)}"
- return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
-
-
-def relation_id_for(relation: GraphRelation, *, scope_user_id: str, workspace_id: str) -> str:
- raw = "|".join(
- (
- scope_user_id,
- workspace_id or "default",
- _entity_key(relation.source, relation.source_type),
- relation.predicate,
- _entity_key(relation.target, relation.target_type),
- )
- )
- return "grel_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:28]
-
-
-_driver = None
-_driver_lock = threading.Lock()
-_schema_ready = False
-
-
-def _get_driver():
+def get_driver():
"""Lazily create the thread-safe Neo4j driver; failures are retryable."""
- global _driver, _schema_ready
if not (settings.memory.enabled and settings.memory.graph_enabled):
return None
- if _driver is not None:
- return _driver
- with _driver_lock:
- if _driver is not None:
- return _driver
+ if graph_state.process_state().driver is not None:
+ return graph_state.process_state().driver
+ with graph_state.process_state().driver_lock:
+ if graph_state.process_state().driver is not None:
+ return graph_state.process_state().driver
try:
from neo4j import GraphDatabase
@@ -202,33 +30,33 @@ def _get_driver():
connection_timeout=3.0,
)
driver.verify_connectivity()
- _driver = driver
- if not _schema_ready:
- _ensure_schema(driver)
- _schema_ready = True
+ graph_state.process_state().driver = driver
+ if not graph_state.process_state().schema_ready:
+ ensure_schema(driver)
+ graph_state.process_state().schema_ready = True
logger.info("[GraphMemory] Neo4j L3 initialized")
- return _driver
+ return graph_state.process_state().driver
except Exception as exc: # noqa: BLE001 - optional memory layer
logger.warning(
"[GraphMemory] initialization failed; L3 disabled for this attempt: %s", exc
)
- _driver = None
+ graph_state.process_state().driver = None
return None
-def _reset_driver() -> None:
- global _driver, _schema_ready
- with _driver_lock:
- current, _driver = _driver, None
- _schema_ready = False
+def reset_driver() -> None:
+ with graph_state.process_state().driver_lock:
+ current, graph_state.process_state().driver = graph_state.process_state().driver, None
+ graph_state.process_state().schema_ready = False
if current is not None:
try:
current.close()
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.memory.graph:_reset_driver', fallback_error)
-def _ensure_schema(driver) -> None:
+def ensure_schema(driver) -> None:
statements = (
"CREATE CONSTRAINT memory_entity_scope_key IF NOT EXISTS "
"FOR (n:MemoryEntity) REQUIRE (n.scope_user_id, n.workspace_id, n.key) IS UNIQUE",
@@ -243,47 +71,8 @@ def _ensure_schema(driver) -> None:
logger.warning("[GraphMemory] schema setup skipped: %s", exc)
-_UPSERT_CYPHER = """
-MERGE (s:MemoryEntity {
- scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $source_key
-})
-ON CREATE SET s.created_at = datetime()
-SET s.name = $source, s.entity_type = $source_type, s.last_seen_at = datetime()
-MERGE (t:MemoryEntity {
- scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $target_key
-})
-ON CREATE SET t.created_at = datetime()
-SET t.name = $target, t.entity_type = $target_type, t.last_seen_at = datetime()
-MERGE (s)-[r:MEMORY_RELATION {relation_id: $relation_id}]->(t)
-ON CREATE SET r.created_at = datetime(), r.seen_count = 0, r.weight = 1.0, r.active = true
-WITH r,
- CASE
- WHEN $effect_id <> '' AND $effect_id IN coalesce(r.outbox_effect_ids, []) THEN true
- ELSE false
- END AS replayed
-FOREACH (_ IN CASE WHEN replayed THEN [] ELSE [1] END |
- SET r.predicate = $predicate,
- r.relationship = $relationship,
- r.last_seen_at = datetime(),
- r.seen_count = coalesce(r.seen_count, 0) + 1,
- r.confidence = CASE
- WHEN coalesce(r.confidence, 0.0) > $confidence THEN r.confidence
- ELSE $confidence
- END,
- r.author_user_id = $author_user_id,
- r.chat_id = $chat_id,
- r.evidence_hash = $evidence_hash,
- r.outbox_effect_ids = CASE
- WHEN $effect_id = '' THEN coalesce(r.outbox_effect_ids, [])
- ELSE coalesce(r.outbox_effect_ids, []) + [$effect_id]
- END
-)
-RETURN r.relation_id AS relation_id, r.seen_count AS seen_count, replayed
-"""
-
-
-def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[Dict[str, Any]]:
- driver = _get_driver()
+def write_sync(ctx: MemoryContext, relations: Sequence[graph_contract.GraphRelation]) -> list[Dict[str, Any]]:
+ driver = get_driver()
if driver is None:
return []
evidence_hash = hashlib.sha256(
@@ -293,19 +82,19 @@ def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[
try:
with driver.session() as session:
for relation in relations:
- relation_id = relation_id_for(
+ relation_id = graph_contract.relation_id_for(
relation,
scope_user_id=ctx.effective_scope_user_id,
workspace_id=ctx.workspace_id,
)
record = session.run(
- _UPSERT_CYPHER,
+ graph_state.UPSERT_CYPHER,
scope_user_id=ctx.effective_scope_user_id,
workspace_id=ctx.workspace_id or "default",
- source_key=_entity_key(relation.source, relation.source_type),
+ source_key=graph_contract.entity_key(relation.source, relation.source_type),
source=relation.source,
source_type=relation.source_type,
- target_key=_entity_key(relation.target, relation.target_type),
+ target_key=graph_contract.entity_key(relation.target, relation.target_type),
target=relation.target,
target_type=relation.target_type,
relation_id=relation_id,
@@ -331,66 +120,26 @@ def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[
return rows
except Exception as exc: # noqa: BLE001 - L3 is optional and post-response
logger.warning("[GraphMemory] write failed: %s", exc)
- _reset_driver()
+ reset_driver()
return []
async def write_graph_relations(
- ctx: MemoryContext, relations: Sequence[GraphRelation]
+ ctx: MemoryContext, relations: Sequence[graph_contract.GraphRelation]
) -> list[Dict[str, Any]]:
"""Persist validated relations without blocking the event loop."""
if not relations:
return []
- return await run_effect(_write_sync, ctx, list(relations))
-
-
-_LIST_CYPHER = """
-MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity)
-WHERE s.scope_user_id = $scope_user_id
- AND s.workspace_id = $workspace_id
- AND coalesce(r.active, true) = true
-RETURN r.relation_id AS relation_id,
- s.name AS source, s.entity_type AS source_type,
- r.predicate AS predicate, r.relationship AS relationship,
- t.name AS target, t.entity_type AS target_type,
- coalesce(r.confidence, 0.0) AS confidence,
- coalesce(r.seen_count, 1) AS seen_count,
- coalesce(r.weight, 1.0) AS weight,
- toString(r.last_seen_at) AS last_seen_at
-ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC
-LIMIT $limit
-"""
-
-
-_SEARCH_CYPHER = """
-MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity)
-WHERE s.scope_user_id = $scope_user_id
- AND s.workspace_id = $workspace_id
- AND coalesce(r.active, true) = true
- AND (
- toLower($search_text) CONTAINS toLower(s.name)
- OR toLower($search_text) CONTAINS toLower(t.name)
- )
-RETURN r.relation_id AS relation_id,
- s.name AS source, s.entity_type AS source_type,
- r.predicate AS predicate, r.relationship AS relationship,
- t.name AS target, t.entity_type AS target_type,
- coalesce(r.confidence, 0.0) AS confidence,
- coalesce(r.seen_count, 1) AS seen_count,
- coalesce(r.weight, 1.0) AS weight,
- toString(r.last_seen_at) AS last_seen_at
-ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC
-LIMIT $limit
-"""
+ return await run_effect(write_sync, ctx, list(relations))
-def _read_sync(
+def read_sync(
*, scope_user_id: str, workspace_id: str, limit: int, query: str = ""
) -> list[Dict[str, Any]]:
- driver = _get_driver()
+ driver = get_driver()
if driver is None:
return []
- statement = _SEARCH_CYPHER if query.strip() else _LIST_CYPHER
+ statement = graph_state.SEARCH_CYPHER if query.strip() else graph_state.LIST_CYPHER
try:
with driver.session() as session:
result = session.run(
@@ -403,7 +152,7 @@ def _read_sync(
return [dict(record) for record in result]
except Exception as exc: # noqa: BLE001
logger.warning("[GraphMemory] read failed: %s", exc)
- _reset_driver()
+ reset_driver()
return []
@@ -413,7 +162,7 @@ async def list_graph_relations(
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
None,
- lambda: _read_sync(
+ lambda: read_sync(
scope_user_id=scope_user_id,
workspace_id=workspace_id,
limit=limit,
@@ -433,7 +182,7 @@ async def search_graph_relations(
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
None,
- lambda: _read_sync(
+ lambda: read_sync(
scope_user_id=scope_user_id,
workspace_id=workspace_id,
limit=limit,
@@ -449,7 +198,7 @@ def relation_is_active(relation_id: str, *, scope_user_id: str) -> Optional[bool
depends on this relation is still applicable. Tri-state on purpose: an
unreachable graph store must not disable every dependent skill.
"""
- driver = _get_driver()
+ driver = get_driver()
if driver is None or not relation_id:
return None
try:
@@ -468,7 +217,7 @@ def relation_is_active(relation_id: str, *, scope_user_id: str) -> Optional[bool
return bool(record["active"])
except Exception as exc: # noqa: BLE001
logger.warning("[GraphMemory] relation_is_active failed: %s", exc)
- _reset_driver()
+ reset_driver()
return None
@@ -485,7 +234,7 @@ def update_relation_state(
deactivating are reversible; deletion is not, and a wrongly deleted edge is
indistinguishable from knowledge the system never had.
"""
- driver = _get_driver()
+ driver = get_driver()
if driver is None or not relation_id:
return False
sets = []
@@ -513,12 +262,12 @@ def update_relation_state(
return bool(record and int(record["touched"] or 0) > 0)
except Exception as exc: # noqa: BLE001
logger.warning("[GraphMemory] update_relation_state failed: %s", exc)
- _reset_driver()
+ reset_driver()
return False
-def _delete_sync(relation_id: str, *, scope_user_id: str) -> bool:
- driver = _get_driver()
+def delete_sync(relation_id: str, *, scope_user_id: str) -> bool:
+ driver = get_driver()
if driver is None:
return False
try:
@@ -544,7 +293,7 @@ def _delete_sync(relation_id: str, *, scope_user_id: str) -> bool:
return bool(record and int(record["deleted"] or 0) > 0)
except Exception as exc: # noqa: BLE001
logger.warning("[GraphMemory] delete failed: %s", exc)
- _reset_driver()
+ reset_driver()
return False
@@ -553,5 +302,5 @@ async def delete_graph_relation(relation_id: str, *, scope_user_id: str) -> bool
return False
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
- None, lambda: _delete_sync(relation_id, scope_user_id=scope_user_id)
+ None, lambda: delete_sync(relation_id, scope_user_id=scope_user_id)
)
diff --git a/src/backend/core/memory/graph_contract.py b/src/backend/core/memory/graph_contract.py
new file mode 100644
index 00000000..3a8d4a23
--- /dev/null
+++ b/src/backend/core/memory/graph_contract.py
@@ -0,0 +1,122 @@
+"""Validate and normalise extracted graph relations."""
+from __future__ import annotations
+import hashlib
+import unicodedata
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Sequence
+from core.memory import graph_state
+
+@dataclass(frozen=True)
+class GraphRelation:
+ """One validated L3 relation before persistence."""
+
+ source: str
+ predicate: str
+ target: str
+ source_type: str = "other"
+ target_type: str = "other"
+ confidence: float = 0.8
+
+ @property
+ def relationship(self) -> str:
+ return graph_state.PREDICATE_LABELS[self.predicate]
+
+ @property
+ def text(self) -> str:
+ return f"{self.source} → {self.relationship} → {self.target}"
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "source": self.source,
+ "source_type": self.source_type,
+ "predicate": self.predicate,
+ "relationship": self.relationship,
+ "target": self.target,
+ "target_type": self.target_type,
+ "confidence": self.confidence,
+ }
+
+
+def parse_relation(raw: Any) -> Optional[GraphRelation]:
+ """Validate an extractor item and return its canonical representation."""
+ if not isinstance(raw, dict):
+ return None
+ source = clean_name(raw.get("source"))
+ target = clean_name(raw.get("target"))
+ predicate = str(raw.get("predicate") or "").strip().lower()
+ if not source or not target or source == target or predicate not in graph_state.PREDICATE_LABELS:
+ return None
+
+ source_type = str(raw.get("source_type") or "other").strip().lower()
+ target_type = str(raw.get("target_type") or "other").strip().lower()
+ if source_type not in graph_state.ALLOWED_ENTITY_TYPES:
+ source_type = "other"
+ if target_type not in graph_state.ALLOWED_ENTITY_TYPES:
+ target_type = "other"
+ try:
+ confidence = float(raw.get("confidence", 0.8))
+ except (TypeError, ValueError):
+ confidence = 0.8
+ confidence = max(0.0, min(1.0, confidence))
+ if confidence < graph_state.MIN_CONFIDENCE:
+ return None
+ return GraphRelation(
+ source=source,
+ predicate=predicate,
+ target=target,
+ source_type=source_type,
+ target_type=target_type,
+ confidence=confidence,
+ )
+
+
+def parse_relations(raw_items: Sequence[Any]) -> list[GraphRelation]:
+ """Validate, de-duplicate, and bound one turn's extracted relations."""
+ relations: list[GraphRelation] = []
+ seen: set[tuple[str, str, str]] = set()
+ for raw in raw_items or []:
+ relation = parse_relation(raw)
+ if relation is None:
+ continue
+ key = (
+ entity_key(relation.source, relation.source_type),
+ relation.predicate,
+ entity_key(relation.target, relation.target_type),
+ )
+ if key in seen:
+ continue
+ seen.add(key)
+ relations.append(relation)
+ if len(relations) >= graph_state.MAX_RELATIONS_PER_TURN:
+ break
+ return relations
+
+
+def clean_name(value: Any) -> str:
+ name = unicodedata.normalize("NFKC", str(value or ""))
+ name = graph_state.SAFE_NAME.sub(" ", name).strip(" \t\r\n,,。;;::")
+ if len(name) < 2 or len(name) > graph_state.MAX_ENTITY_CHARS:
+ return ""
+ return name
+
+
+def normalise_name(value: str) -> str:
+ return graph_state.SAFE_NAME.sub(" ", unicodedata.normalize("NFKC", value).lower()).strip()
+
+
+def entity_key(name: str, entity_type: str) -> str:
+ raw = f"{entity_type}|{normalise_name(name)}"
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
+
+
+def relation_id_for(relation: GraphRelation, *, scope_user_id: str, workspace_id: str) -> str:
+ raw = "|".join(
+ (
+ scope_user_id,
+ workspace_id or "default",
+ entity_key(relation.source, relation.source_type),
+ relation.predicate,
+ entity_key(relation.target, relation.target_type),
+ )
+ )
+ return "grel_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:28]
diff --git a/src/backend/core/memory/graph_state.py b/src/backend/core/memory/graph_state.py
new file mode 100644
index 00000000..29b61713
--- /dev/null
+++ b/src/backend/core/memory/graph_state.py
@@ -0,0 +1,141 @@
+"""Graph vocabulary, limits, Cypher queries and the shared driver."""
+from __future__ import annotations
+import re
+import threading
+from dataclasses import dataclass, field
+from typing import Dict
+
+PREDICATE_LABELS: Dict[str, str] = {
+ "is_a": "属于类别",
+ "part_of": "组成",
+ "belongs_to": "隶属于",
+ "responsible_for": "负责",
+ "depends_on": "依赖",
+ "uses": "使用",
+ "alias_of": "别名为",
+ "located_in": "位于",
+ "produces": "产出",
+ "constrained_by": "受约束于",
+ "related_to": "关联",
+}
+
+
+ALLOWED_ENTITY_TYPES = frozenset(
+ {
+ "person",
+ "organization",
+ "team",
+ "project",
+ "system",
+ "document",
+ "metric",
+ "concept",
+ "place",
+ "other",
+ }
+)
+
+
+MAX_ENTITY_CHARS = 120
+
+
+MAX_RELATIONS_PER_TURN = 12
+
+
+MIN_CONFIDENCE = 0.65
+
+
+SAFE_NAME = re.compile(r"\s+")
+
+
+UPSERT_CYPHER = """
+MERGE (s:MemoryEntity {
+ scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $source_key
+})
+ON CREATE SET s.created_at = datetime()
+SET s.name = $source, s.entity_type = $source_type, s.last_seen_at = datetime()
+MERGE (t:MemoryEntity {
+ scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $target_key
+})
+ON CREATE SET t.created_at = datetime()
+SET t.name = $target, t.entity_type = $target_type, t.last_seen_at = datetime()
+MERGE (s)-[r:MEMORY_RELATION {relation_id: $relation_id}]->(t)
+ON CREATE SET r.created_at = datetime(), r.seen_count = 0, r.weight = 1.0, r.active = true
+WITH r,
+ CASE
+ WHEN $effect_id <> '' AND $effect_id IN coalesce(r.outbox_effect_ids, []) THEN true
+ ELSE false
+ END AS replayed
+FOREACH (_ IN CASE WHEN replayed THEN [] ELSE [1] END |
+ SET r.predicate = $predicate,
+ r.relationship = $relationship,
+ r.last_seen_at = datetime(),
+ r.seen_count = coalesce(r.seen_count, 0) + 1,
+ r.confidence = CASE
+ WHEN coalesce(r.confidence, 0.0) > $confidence THEN r.confidence
+ ELSE $confidence
+ END,
+ r.author_user_id = $author_user_id,
+ r.chat_id = $chat_id,
+ r.evidence_hash = $evidence_hash,
+ r.outbox_effect_ids = CASE
+ WHEN $effect_id = '' THEN coalesce(r.outbox_effect_ids, [])
+ ELSE coalesce(r.outbox_effect_ids, []) + [$effect_id]
+ END
+)
+RETURN r.relation_id AS relation_id, r.seen_count AS seen_count, replayed
+"""
+
+
+LIST_CYPHER = """
+MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity)
+WHERE s.scope_user_id = $scope_user_id
+ AND s.workspace_id = $workspace_id
+ AND coalesce(r.active, true) = true
+RETURN r.relation_id AS relation_id,
+ s.name AS source, s.entity_type AS source_type,
+ r.predicate AS predicate, r.relationship AS relationship,
+ t.name AS target, t.entity_type AS target_type,
+ coalesce(r.confidence, 0.0) AS confidence,
+ coalesce(r.seen_count, 1) AS seen_count,
+ coalesce(r.weight, 1.0) AS weight,
+ toString(r.last_seen_at) AS last_seen_at
+ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC
+LIMIT $limit
+"""
+
+
+SEARCH_CYPHER = """
+MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity)
+WHERE s.scope_user_id = $scope_user_id
+ AND s.workspace_id = $workspace_id
+ AND coalesce(r.active, true) = true
+ AND (
+ toLower($search_text) CONTAINS toLower(s.name)
+ OR toLower($search_text) CONTAINS toLower(t.name)
+ )
+RETURN r.relation_id AS relation_id,
+ s.name AS source, s.entity_type AS source_type,
+ r.predicate AS predicate, r.relationship AS relationship,
+ t.name AS target, t.entity_type AS target_type,
+ coalesce(r.confidence, 0.0) AS confidence,
+ coalesce(r.seen_count, 1) AS seen_count,
+ coalesce(r.weight, 1.0) AS weight,
+ toString(r.last_seen_at) AS last_seen_at
+ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC
+LIMIT $limit
+"""
+
+
+@dataclass
+class GraphState:
+ driver: object = None
+ driver_lock: object = field(default_factory=lambda: threading.Lock())
+ schema_ready: object = False
+
+
+_STATE = GraphState()
+
+
+def process_state() -> GraphState:
+ return _STATE
diff --git a/src/backend/core/memory/outbox/enqueue.py b/src/backend/core/memory/outbox/enqueue.py
index ee2a0420..7a081f30 100644
--- a/src/backend/core/memory/outbox/enqueue.py
+++ b/src/backend/core/memory/outbox/enqueue.py
@@ -11,8 +11,9 @@
from core.memory.extractors.router import ExtractorType
from sqlalchemy.exc import IntegrityError
+from core.memory.outbox import common
+
logger = logging.getLogger(__name__)
-from core.memory.outbox import common as common
def _enqueue(
diff --git a/src/backend/core/memory/outbox/jobs.py b/src/backend/core/memory/outbox/jobs.py
index cbcb20ba..9bdc09e5 100644
--- a/src/backend/core/memory/outbox/jobs.py
+++ b/src/backend/core/memory/outbox/jobs.py
@@ -11,8 +11,9 @@
from core.memory.context import MemoryContext
from core.memory.extractors.router import ExtractorType
+from core.memory.outbox import common
+
logger = logging.getLogger(__name__)
-from core.memory.outbox import common as common
async def _extract_candidates(
@@ -198,9 +199,9 @@ async def _process_profile_edit(row: MemoryOutbox) -> dict[str, Any]:
text = sanitized.text
applied = await upsert_fields(ctx, [(key, text, "user_edit")], strict=True)
if ctx.message_id:
- from core.evolution.settlement_store import update_entry_text
+ from core.memory.settlement_port import memory_settlement
- update_entry_text(ctx.message_id, key, text)
+ memory_settlement().update_entry_text(ctx.message_id, key, text)
return {"key": key, "text": text, "applied": applied}
@@ -219,9 +220,9 @@ async def _process_memory_edit(row: MemoryOutbox) -> dict[str, Any]:
raise ValueError("memory not found for actor")
await update_memory(memory_id, text, strict=True)
if ctx.message_id:
- from core.evolution.settlement_store import update_entry_text
+ from core.memory.settlement_port import memory_settlement
- update_entry_text(ctx.message_id, memory_id, text)
+ memory_settlement().update_entry_text(ctx.message_id, memory_id, text)
return {"id": memory_id, "text": text}
@@ -243,28 +244,10 @@ def _report_settlement(
items: Optional[list[dict]] = None,
failed: bool = False,
) -> dict[str, Any]:
- from core.db.models import EvolutionEpisode
- from core.evolution.settlement import settle_turn
- from core.evolution.settlement_runner import acknowledge_durable_settlement
-
- # The Outbox is now the authoritative join. Retire the old in-process
- # watchdog before computing the final card so it cannot race the atomic
- # summary+ack transaction performed by ``_finish_success``.
- acknowledge_durable_settlement(message_id)
- with common.SessionLocal() as db:
- episode = (
- db.query(EvolutionEpisode.episode_id)
- .filter(EvolutionEpisode.message_id == message_id)
- .first()
- )
- summary = settle_turn(
- message_id=message_id,
- episode_id=str(episode[0]) if episode is not None else "",
- memory_entries=list(items or []),
- memory_failed=failed,
- memory_enabled=True,
+ from core.memory.settlement_port import memory_settlement
+ return memory_settlement().build_summary(
+ message_id, session_factory=common.SessionLocal, items=list(items or []), failed=failed,
)
- return cast(dict[str, Any], summary.to_dict())
async def _process_settlement(row: MemoryOutbox) -> dict[str, Any]:
diff --git a/src/backend/core/memory/outbox/leases.py b/src/backend/core/memory/outbox/leases.py
index 37cbc2e4..36ea6463 100644
--- a/src/backend/core/memory/outbox/leases.py
+++ b/src/backend/core/memory/outbox/leases.py
@@ -8,10 +8,12 @@
from typing import Any, Optional
from core.db.models import MemoryOutbox
+from core.db.threads import run_db
from sqlalchemy import and_, or_
+from core.memory.outbox import common
+
logger = logging.getLogger(__name__)
-from core.memory.outbox import common as common
def _claim_next(worker_id: str) -> Optional[str]:
@@ -203,7 +205,7 @@ async def _lease_heartbeat(
while True:
await asyncio.sleep(interval)
try:
- renewed = _renew_lease(job_id, worker_id)
+ renewed = await run_db(_renew_lease, job_id, worker_id)
except Exception:
logger.exception("[memory_outbox] lease renewal failed id=%s", job_id)
renewed = False
diff --git a/src/backend/core/memory/outbox/settlement.py b/src/backend/core/memory/outbox/settlement.py
index 0620e41a..bedacf46 100644
--- a/src/backend/core/memory/outbox/settlement.py
+++ b/src/backend/core/memory/outbox/settlement.py
@@ -8,8 +8,9 @@
from core.db.models import MemoryOutbox
+from core.memory.outbox import common
+
logger = logging.getLogger(__name__)
-from core.memory.outbox import common as common
def _lock_pipeline_row(db, message_id: str) -> Optional[MemoryOutbox]:
@@ -93,20 +94,9 @@ def _finish_success(job_id: str, worker_id: str, result: Any) -> Optional[str]:
)
if affected:
if row.job_kind == "settlement":
- from core.db.models import ChatMessage
- from core.evolution.settlement_store import METADATA_KEY
-
+ from core.memory.settlement_port import memory_settlement
summary = result.get("summary") if isinstance(result, dict) else None
- message = db.query(ChatMessage).filter(ChatMessage.message_id == message_id).first()
- if message is None:
- raise RuntimeError(
- "assistant message is not durable yet; settlement will retry"
- )
- if not isinstance(summary, dict):
- raise RuntimeError("settlement result is missing its durable summary")
- extra = dict(message.extra_data or {})
- extra[METADATA_KEY] = summary
- message.extra_data = extra
+ memory_settlement().write_summary(db, message_id, summary)
_enqueue_settlement_if_ready(db, message_id, now)
db.commit()
if affected:
diff --git a/src/backend/core/memory/outbox/worker.py b/src/backend/core/memory/outbox/worker.py
index 1dd00a26..11211396 100644
--- a/src/backend/core/memory/outbox/worker.py
+++ b/src/backend/core/memory/outbox/worker.py
@@ -7,26 +7,33 @@
import os
import socket
import uuid
+from dataclasses import dataclass
from typing import Any, Optional
from core.db.models import MemoryOutbox
+from core.db.threads import run_db, run_in_session
from core.memory.effect_lane import EffectLaneDeferred, ordered_l2_effect
+from core.memory.outbox import common
+from core.memory.outbox import jobs
+from core.memory.outbox import leases
+from core.memory.outbox import settlement
+
logger = logging.getLogger(__name__)
-from core.memory.outbox import common as common
-from core.memory.outbox import jobs as jobs
-from core.memory.outbox import leases as leases
-from core.memory.outbox import settlement as settlement
-_active_worker: Optional["MemoryOutboxWorker"] = None
+
+def _claimed_row(db, job_id: str, worker_id: str) -> Optional[MemoryOutbox]:
+ row = db.get(MemoryOutbox, job_id)
+ if row is None or row.status != "processing" or row.lease_owner != worker_id:
+ return None
+ db.expunge(row)
+ return row
async def _process_claimed(job_id: str, worker_id: str) -> None:
- with common.SessionLocal() as db:
- row = db.get(MemoryOutbox, job_id)
- if row is None or row.status != "processing" or row.lease_owner != worker_id:
- return
- db.expunge(row)
+ row = await run_in_session(_claimed_row, job_id, worker_id)
+ if row is None:
+ return
owner_task = asyncio.current_task()
if owner_task is None: # pragma: no cover - asyncio owns this coroutine
@@ -40,7 +47,7 @@ async def _process_claimed(job_id: str, worker_id: str) -> None:
result: Any
async with get_background_semaphore():
- if not leases._owns_lease(job_id, worker_id):
+ if not await run_db(leases._owns_lease, job_id, worker_id):
lease_lost.set()
return
if row.job_kind == "pipeline":
@@ -67,20 +74,20 @@ async def _process_claimed(job_id: str, worker_id: str) -> None:
return
raise
except EffectLaneDeferred as exc:
- leases._defer_claim(job_id, worker_id, exc.retry_at)
+ await run_db(leases._defer_claim, job_id, worker_id, exc.retry_at)
except Exception as exc:
logger.warning("[memory_outbox] job failed id=%s: %s", job_id, exc)
- settlement._finish_failure(job_id, worker_id, exc)
+ await run_db(settlement._finish_failure, job_id, worker_id, exc)
else:
try:
- settlement._finish_success(job_id, worker_id, result)
+ await run_db(settlement._finish_success, job_id, worker_id, result)
except Exception as exc:
# Processing can succeed while the atomic local acknowledgement
# fails (for example the assistant message has not committed yet).
# Roll the owned row into retry instead of parking it until lease
# expiry; external receipts make replay safe.
logger.warning("[memory_outbox] acknowledgement failed id=%s: %s", job_id, exc)
- settlement._finish_failure(job_id, worker_id, exc)
+ await run_db(settlement._finish_failure, job_id, worker_id, exc)
finally:
heartbeat.cancel()
try:
@@ -93,7 +100,7 @@ async def drain_outbox(*, max_jobs: int = 100, worker_id: Optional[str] = None)
worker = worker_id or f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
processed = 0
while processed < max_jobs:
- job_id = leases._claim_next(worker)
+ job_id = await run_db(leases._claim_next, worker)
if job_id is None:
break
await _process_claimed(job_id, worker)
@@ -105,11 +112,11 @@ async def consume_outbox_job(job_id: str, *, wait_s: float = 5.0) -> dict[str, A
"""Try an admitted interactive edit now, while leaving retry durability intact."""
worker_id = f"api:{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
- if leases._claim_specific(job_id, worker_id):
+ if await run_db(leases._claim_specific, job_id, worker_id):
await _process_claimed(job_id, worker_id)
deadline = asyncio.get_running_loop().time() + max(0.0, wait_s)
while True:
- snapshot = leases.get_outbox_job(job_id)
+ snapshot = await run_db(leases.get_outbox_job, job_id)
if snapshot is None:
raise RuntimeError(f"outbox job disappeared: {job_id}")
if snapshot["status"] in common._TERMINAL or snapshot["status"] == "retry":
@@ -122,7 +129,7 @@ async def consume_outbox_job(job_id: str, *, wait_s: float = 5.0) -> dict[str, A
def kick_outbox_drain() -> None:
"""Wake the lifecycle-owned worker; never create an orphan consumer task."""
- worker = _active_worker
+ worker = process_state().active_worker
if worker is not None:
worker.wake()
@@ -135,20 +142,20 @@ def __init__(self) -> None:
self.worker_id = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
async def start(self) -> None:
- global _active_worker
+ pass
if self._task is None or self._task.done():
self._stop.clear()
self._wake.clear()
- settlement.reconcile_settlement_jobs()
- _active_worker = self
+ await run_db(settlement.reconcile_settlement_jobs)
+ process_state().active_worker = self
self._task = asyncio.create_task(self._loop(), name="memory-outbox-worker")
async def stop(self) -> None:
- global _active_worker
+ pass
self._stop.set()
self._wake.set()
- if _active_worker is self:
- _active_worker = None
+ if process_state().active_worker is self:
+ process_state().active_worker = None
if self._task is None:
return
self._task.cancel()
@@ -156,7 +163,7 @@ async def stop(self) -> None:
await self._task
except asyncio.CancelledError:
pass
- leases._release_worker_leases(self.worker_id)
+ await run_db(leases._release_worker_leases, self.worker_id)
self._task = None
def wake(self) -> None:
@@ -181,3 +188,15 @@ async def _loop(self) -> None:
pass
finally:
self._wake.clear()
+
+
+@dataclass
+class WorkerState:
+ active_worker: Optional['MemoryOutboxWorker'] = None
+
+
+_STATE = WorkerState()
+
+
+def process_state() -> WorkerState:
+ return _STATE
diff --git a/src/backend/core/memory/pipeline.py b/src/backend/core/memory/pipeline.py
index f3202309..65a56cf9 100644
--- a/src/backend/core/memory/pipeline.py
+++ b/src/backend/core/memory/pipeline.py
@@ -16,6 +16,7 @@
import asyncio
import logging
+from dataclasses import dataclass, field
from threading import Lock
from typing import Optional
@@ -30,17 +31,12 @@
# ─── Global semaphore (lazy init) ──────────────────────────────────────────
-_bg_semaphore: Optional[asyncio.Semaphore] = None
-_sem_lock = Lock()
-
-
def get_background_semaphore() -> asyncio.Semaphore:
"""Return the global background-task semaphore. Must be first called inside an event loop."""
- global _bg_semaphore
- with _sem_lock:
- if _bg_semaphore is None:
- _bg_semaphore = asyncio.Semaphore(settings.memory.bg_max_concurrency)
- return _bg_semaphore
+ with process_state().sem_lock:
+ if process_state().bg_semaphore is None:
+ process_state().bg_semaphore = asyncio.Semaphore(settings.memory.bg_max_concurrency)
+ return process_state().bg_semaphore
# ─── Circuit breaker ───────────────────────────────────────────────────────
@@ -153,8 +149,21 @@ def _report_settlement(
if not message_id:
return
try:
- from core.evolution.settlement_runner import report_memory_writes
+ from core.memory.settlement_port import memory_settlement
- report_memory_writes(message_id, items=list(items or []), failed=failed)
+ memory_settlement().report_writes(message_id, items=list(items or []), failed=failed)
except Exception as exc: # noqa: BLE001 - settlement must never break writes
logger.debug("[memory_pipeline] settlement report skipped: %s", exc)
+
+
+@dataclass
+class PipelineState:
+ bg_semaphore: Optional[asyncio.Semaphore] = None
+ sem_lock: object = field(default_factory=lambda: Lock())
+
+
+_STATE = PipelineState()
+
+
+def process_state() -> PipelineState:
+ return _STATE
diff --git a/src/backend/core/memory/preview.py b/src/backend/core/memory/preview.py
index d7f89cfb..fc6753a3 100644
--- a/src/backend/core/memory/preview.py
+++ b/src/backend/core/memory/preview.py
@@ -13,6 +13,7 @@ def sanitized_preview(text: str, *, max_chars: int) -> str:
if result.reject:
return "[REDACTED]"
collapsed = result.text or collapsed
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.memory.preview:sanitized_preview', fallback_error)
return collapsed[:max_chars]
diff --git a/src/backend/core/memory/procedures.py b/src/backend/core/memory/procedures.py
index 5d74b2fd..d3bd5705 100644
--- a/src/backend/core/memory/procedures.py
+++ b/src/backend/core/memory/procedures.py
@@ -11,9 +11,10 @@
from core.memory.executor import run_effect
from core.memory.ttl_sweeper import memory_entry_expired
-logger = logging.getLogger(__name__)
from . import backend
+logger = logging.getLogger(__name__)
+
def _expiration_from_ttl(ttl_days: int) -> Optional[str]:
"""UTC expiry date owned by L2; mem0 stores it as ordinary metadata."""
diff --git a/src/backend/core/memory/queries.py b/src/backend/core/memory/queries.py
index 122e7589..dcac2bcb 100644
--- a/src/backend/core/memory/queries.py
+++ b/src/backend/core/memory/queries.py
@@ -11,9 +11,10 @@
from core.memory.ttl_sweeper import memory_entry_expired
from core.memory.retrieval_types import MemoryRetrievalResult, RetrievedRelation, build_memory_item
-logger = logging.getLogger(__name__)
from . import backend
+logger = logging.getLogger(__name__)
+
async def retrieve_memories(
user_id: str,
diff --git a/src/backend/core/memory/sanitizer.py b/src/backend/core/memory/sanitizer.py
index d7efec6e..e16d91ab 100644
--- a/src/backend/core/memory/sanitizer.py
+++ b/src/backend/core/memory/sanitizer.py
@@ -85,16 +85,14 @@ def clean(self) -> bool:
_RULES_CACHE_TTL = 300 # seconds
-_rules_cache: dict = {"patterns": None, "classified": None, "loaded_at": 0.0}
-_rules_lock = Lock()
def invalidate_rules_cache() -> None:
"""Called proactively after an admin adds/removes rules; the next sanitize re-pulls from the DB."""
- with _rules_lock:
- _rules_cache["patterns"] = None
- _rules_cache["classified"] = None
- _rules_cache["loaded_at"] = 0.0
+ with process_state().rules_lock:
+ process_state().rules_cache["patterns"] = None
+ process_state().rules_cache["classified"] = None
+ process_state().rules_cache["loaded_at"] = 0.0
def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str, ...]]:
@@ -108,12 +106,12 @@ def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str,
On DB query failure, falls back to the hardcoded rules without affecting the main flow.
"""
now = time.time()
- with _rules_lock:
+ with process_state().rules_lock:
if (
- _rules_cache["patterns"] is not None
- and now - _rules_cache["loaded_at"] < _RULES_CACHE_TTL
+ process_state().rules_cache["patterns"] is not None
+ and now - process_state().rules_cache["loaded_at"] < _RULES_CACHE_TTL
):
- return _rules_cache["patterns"], _rules_cache["classified"]
+ return process_state().rules_cache["patterns"], process_state().rules_cache["classified"]
patterns = dict(REDACT_PATTERNS)
classified = list(CLASSIFIED_TERMS)
@@ -141,10 +139,10 @@ def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str,
logger.debug("[sanitizer] DB overrides unavailable, using hardcoded only: %s", exc)
result_classified = tuple(classified)
- with _rules_lock:
- _rules_cache["patterns"] = patterns
- _rules_cache["classified"] = result_classified
- _rules_cache["loaded_at"] = now
+ with process_state().rules_lock:
+ process_state().rules_cache["patterns"] = patterns
+ process_state().rules_cache["classified"] = result_classified
+ process_state().rules_cache["loaded_at"] = now
return patterns, result_classified
@@ -177,3 +175,16 @@ def sanitize(text: str) -> SanitizeResult:
hits.append(name)
cleaned = pat.sub(f"[REDACTED:{name}]", cleaned)
return SanitizeResult(text=cleaned, hits=hits, reject=False)
+
+
+@dataclass
+class SanitizerState:
+ rules_cache: dict = field(default_factory=lambda: {'patterns': None, 'classified': None, 'loaded_at': 0.0})
+ rules_lock: object = field(default_factory=lambda: Lock())
+
+
+_STATE = SanitizerState()
+
+
+def process_state() -> SanitizerState:
+ return _STATE
diff --git a/src/backend/core/memory/service.py b/src/backend/core/memory/service.py
index 615c4364..b6e03ee4 100644
--- a/src/backend/core/memory/service.py
+++ b/src/backend/core/memory/service.py
@@ -1,11 +1,6 @@
"""Public memory API. Lifecycle, retrieval and procedural writes have separate owners."""
-from .backend import (
- reset_runtime,
- MEMORY_COLLECTION_NAME,
- MEMORY_TYPE_PROCEDURAL,
- _get_memory,
-)
+from .backend import reset_runtime, MEMORY_COLLECTION_NAME, MEMORY_TYPE_PROCEDURAL, _get_memory as get_memory
from .queries import (
retrieve_memories,
retrieve_memories_structured,
@@ -25,7 +20,7 @@
"reset_runtime",
"MEMORY_COLLECTION_NAME",
"MEMORY_TYPE_PROCEDURAL",
- "_get_memory",
+ "get_memory",
"retrieve_memories",
"retrieve_memories_structured",
"get_all_memories",
diff --git a/src/backend/core/memory/session_snapshot.py b/src/backend/core/memory/session_snapshot.py
index c246d49f..ce338bf9 100644
--- a/src/backend/core/memory/session_snapshot.py
+++ b/src/backend/core/memory/session_snapshot.py
@@ -9,7 +9,7 @@
call, against 17s for the second call in the same turn, which kept the prefix.
Stored on ``ChatSession.extra_data`` next to the other chat-sticky runtime
-state (see ``core.llm.session_capabilities``). The snapshot records the memory
+state (see ``core.llm.runtime.session_capabilities``). The snapshot records the memory
scope it was built for, because a snapshot assembled for one scope must never
be served to another.
"""
diff --git a/src/backend/core/services/memory_settings_service.py b/src/backend/core/memory/settings.py
similarity index 98%
rename from src/backend/core/services/memory_settings_service.py
rename to src/backend/core/memory/settings.py
index 81357866..aa702f55 100644
--- a/src/backend/core/services/memory_settings_service.py
+++ b/src/backend/core/memory/settings.py
@@ -6,7 +6,7 @@
from typing import Any, Dict
from core.config.settings import settings
-from core.db import model_repository
+from core.models import repository as model_repository
from core.infra.exceptions import BadRequestError
from sqlalchemy.orm import Session
diff --git a/src/backend/core/memory/settlement_port.py b/src/backend/core/memory/settlement_port.py
new file mode 100644
index 00000000..6bb3327c
--- /dev/null
+++ b/src/backend/core/memory/settlement_port.py
@@ -0,0 +1,33 @@
+"""Memory's named settlement boundary; callers retain their transaction scope."""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+class MemorySettlement(Protocol):
+ def report_writes(self, message_id: str, *, items: list[dict], failed: bool) -> None: ...
+ def update_entry_text(self, message_id: str, handle: str, text: str) -> bool: ...
+ def build_summary(self, message_id: str, *, session_factory: Any, items: list[dict], failed: bool) -> dict[str, Any]: ...
+ def write_summary(self, db: Any, message_id: str, summary: Any) -> None: ...
+
+
+def register_memory_settlement(service: MemorySettlement) -> None:
+ process_state().service = service
+
+
+def memory_settlement() -> MemorySettlement:
+ if process_state().service is None:
+ raise RuntimeError("Memory settlement service is not registered")
+ return process_state().service
+
+
+@dataclass
+class SettlementPortState:
+ service: MemorySettlement | None = None
+
+
+_STATE = SettlementPortState()
+
+
+def process_state() -> SettlementPortState:
+ return _STATE
diff --git a/src/backend/core/memory/ttl_sweeper.py b/src/backend/core/memory/ttl_sweeper.py
index fa214866..ce70b143 100644
--- a/src/backend/core/memory/ttl_sweeper.py
+++ b/src/backend/core/memory/ttl_sweeper.py
@@ -79,7 +79,7 @@ async def sweep_expired_memories(scan_limit: int = DEFAULT_SCAN_LIMIT) -> dict:
if not settings.memory.enabled:
return stats
- from core.memory.service import _get_memory
+ from core.memory.service import get_memory as _get_memory
loop = asyncio.get_running_loop()
memory = await loop.run_in_executor(None, _get_memory)
diff --git a/src/backend/core/memory/weights.py b/src/backend/core/memory/weights.py
index 6335de5c..1fc6a885 100644
--- a/src/backend/core/memory/weights.py
+++ b/src/backend/core/memory/weights.py
@@ -29,7 +29,8 @@
import logging
import time
-from typing import Dict, Optional, Tuple
+from dataclasses import dataclass, field
+from typing import Dict, Tuple
logger = logging.getLogger(__name__)
@@ -46,7 +47,6 @@
MIN_WEIGHT = 0.05
_CACHE_TTL_S = 30.0
-_cache: Dict[str, Tuple[float, Dict[str, float], set]] = {}
def _cache_key(user_id: str, workspace_id: str) -> str:
@@ -56,9 +56,9 @@ def _cache_key(user_id: str, workspace_id: str) -> str:
def invalidate(user_id: str = "", workspace_id: str = "") -> None:
"""Drop cached overlays so an applied change takes effect on the next turn."""
if not user_id:
- _cache.clear()
+ process_state().cache.clear()
return
- _cache.pop(_cache_key(user_id, workspace_id), None)
+ process_state().cache.pop(_cache_key(user_id, workspace_id), None)
def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str, float], set]:
@@ -67,7 +67,7 @@ def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str,
return {}, set()
key = _cache_key(user_id, workspace_id)
- cached = _cache.get(key)
+ cached = process_state().cache.get(key)
now = time.monotonic()
if cached is not None and now - cached[0] < _CACHE_TTL_S:
return cached[1], cached[2]
@@ -106,7 +106,7 @@ def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str,
logger.debug("[memory-weights] overlay unavailable, ignoring: %s", exc)
return {}, set()
- _cache[key] = (now, weights, superseded)
+ process_state().cache[key] = (now, weights, superseded)
return weights, superseded
@@ -155,3 +155,14 @@ def apply_overlay(
kept.sort(key=lambda x: x.get(score_key, 0.0), reverse=True)
return kept
+
+@dataclass
+class WeightsState:
+ cache: Dict[str, Tuple[float, Dict[str, float], set]] = field(default_factory=lambda: {})
+
+
+_STATE = WeightsState()
+
+
+def process_state() -> WeightsState:
+ return _STATE
diff --git a/src/backend/core/models/__init__.py b/src/backend/core/models/__init__.py
new file mode 100644
index 00000000..9a02b8ce
--- /dev/null
+++ b/src/backend/core/models/__init__.py
@@ -0,0 +1 @@
+"""Shared model configuration and inference clients."""
diff --git a/src/backend/core/models/admission.py b/src/backend/core/models/admission.py
new file mode 100644
index 00000000..0a6b715a
--- /dev/null
+++ b/src/backend/core/models/admission.py
@@ -0,0 +1,21 @@
+"""Task-local request admission callback supplied by the execution layer."""
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+_check: ContextVar[Callable[[], None] | None] = ContextVar("model_request_admission", default=None)
+
+
+@contextmanager
+def request_admission_callback(check: Callable[[], None]) -> Iterator[None]:
+ token = _check.set(check)
+ try:
+ yield
+ finally:
+ _check.reset(token)
+
+
+def check_request_admission() -> None:
+ check = _check.get()
+ if check is not None:
+ check()
diff --git a/src/backend/core/models/api_protocol_backfill.py b/src/backend/core/models/api_protocol_backfill.py
new file mode 100644
index 00000000..8e9a8e8e
--- /dev/null
+++ b/src/backend/core/models/api_protocol_backfill.py
@@ -0,0 +1,106 @@
+"""给存量模型配置补上 ``extra_config.api_protocol``。
+
+新增/编辑模型时由 ``_autofill_api_protocol`` 在保存那刻探测写入;本模块补的是升级前
+就已存在、没有这个字段的行。运行期不探测(那是配置期行为,不能挂在请求路径上),所以
+启动时一次性补齐,补完落库就不再跑。探不出结论的什么都不写,下次启动再试。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def _needs_probe(extra: dict) -> bool:
+ """还没探过,或探过但用的是比现在宽松的老判据。
+
+ 老判据只验证 ``/responses`` 路由存在,会把「路由在、却不消费工具结果」的上游判成
+ 可用;那种上游一进智能体循环就卡死。判据变严后,旧结论必须重新验一遍,否则谁也不会
+ 去纠正它。人工设定的值(没有 source 标记)永远不动。
+ """
+ from core.models.providers.protocol_probe import PROBE_VERSION
+
+ if extra.get("api_protocol_source") != "probe":
+ return not extra.get("api_protocol")
+ return int(extra.get("api_protocol_probe_version") or 1) < PROBE_VERSION
+
+
+def _pending_probes(db):
+ from core.models.repository import list_providers
+ from core.models.providers.registry import get_spec
+
+ pending = []
+ for provider in list_providers(db):
+ if provider.provider_type != "chat" or not provider.is_active:
+ continue
+ extra = provider.extra_config or {}
+ if not _needs_probe(extra):
+ continue
+ if not get_spec(getattr(provider, "provider", "") or "").speaks_responses:
+ continue
+ pending.append(
+ (
+ provider.provider_id,
+ provider.model_name,
+ provider.base_url,
+ provider.api_key,
+ dict(extra),
+ )
+ )
+ return pending
+
+
+def _save_protocols(db, updates):
+ from core.models.repository import update_provider
+
+ for provider_id, extra in updates:
+ update_provider(db, provider_id, extra_config=extra)
+
+
+async def backfill_missing_api_protocol() -> int:
+ """探测所有还没记录协议(或结论已过期)的 chat 上游并写库,返回写入的条数。"""
+ from core.db.threads import run_in_session
+ from core.models.providers.protocol_probe import PROBE_VERSION, detect_api_protocol
+
+ pending = await run_in_session(_pending_probes)
+ if not pending:
+ return 0
+
+ results = await asyncio.gather(
+ *(
+ # 超时沿用 detect_api_protocol 自己的默认值:保存时探测与这里补齐是
+ # 同一件事,给它第二个数只会让两条路径对同一个上游得出不同结论。
+ detect_api_protocol(
+ base_url=base_url,
+ api_key=api_key,
+ model_name=model_name,
+ )
+ for _, model_name, base_url, api_key, _ in pending
+ ),
+ return_exceptions=True,
+ )
+
+ updates = []
+ for (provider_id, model_name, _, _, extra), result in zip(pending, results):
+ if isinstance(result, BaseException) or not getattr(result, "found", False):
+ logger.info(
+ "[startup] api_protocol 未能探测 %s:%s",
+ model_name,
+ result if isinstance(result, BaseException) else "; ".join(result.notes),
+ )
+ continue
+ extra["api_protocol"] = result.protocol
+ extra["api_protocol_source"] = "probe"
+ extra["api_protocol_probe_version"] = PROBE_VERSION
+ updates.append((provider_id, extra))
+ logger.info(
+ "[startup] api_protocol 已探明 %s -> %s(%s)",
+ model_name,
+ result.protocol,
+ "; ".join(result.notes),
+ )
+ if updates:
+ await run_in_session(_save_protocols, updates)
+ return len(updates)
diff --git a/src/backend/core/llm/chat_models.py b/src/backend/core/models/chat_models.py
similarity index 92%
rename from src/backend/core/llm/chat_models.py
rename to src/backend/core/models/chat_models.py
index 16ed87d6..79ad6fb1 100755
--- a/src/backend/core/llm/chat_models.py
+++ b/src/backend/core/models/chat_models.py
@@ -4,18 +4,15 @@
import logging
import os
+from dataclasses import dataclass, field
from typing import Any, Optional
import httpx
from agentscope.credential import OpenAICredential
from agentscope.model import ChatModelBase, OpenAIChatModel, OpenAIResponseModel
-from core.llm.providers.openai_compat import (
- OpenAICompatChatModel,
- ReasoningEchoChatFormatter,
- _build_chat_template_kwargs,
-)
-from core.llm.providers.registry import get_spec
-from core.llm.providers.vendor_models import build_litellm_model, build_native_model
+from core.models.providers.openai_compat import OpenAICompatChatModel, ReasoningEchoChatFormatter, _build_chat_template_kwargs
+from core.models.providers.registry import get_spec
+from core.models.providers.vendor_models import build_litellm_model, build_native_model
from prompts.prompt_config import ModelConfig
logger = logging.getLogger(__name__)
@@ -30,7 +27,7 @@
# (loop, timeout) keeps keep-alive connections warm across turns while staying
# correct under pytest-asyncio, which runs each test on its own loop and would
# otherwise inherit a pool bound to a closed one.
-_HTTP_CLIENTS: dict[tuple[int, float], httpx.AsyncClient] = {}
+
# Keep enough idle sockets for concurrent runs (main agent + subagents + memory
# extraction all talk to the same gateway) without letting an idle pool pin
@@ -47,9 +44,9 @@ async def close_loop_http_clients() -> None:
import asyncio
loop_id = id(asyncio.get_running_loop())
- keys = [key for key in list(_HTTP_CLIENTS) if key[0] == loop_id]
+ keys = [key for key in list(process_state().HTTP_CLIENTS) if key[0] == loop_id]
for key in keys:
- client = _HTTP_CLIENTS.pop(key)
+ client = process_state().HTTP_CLIENTS.pop(key)
await client.aclose()
@@ -77,14 +74,14 @@ def _make_http_client(
hooks = {}
identity = ""
if desktop_reference:
- from core.services.desktop_cloud_bridge import _state_fingerprint
- from core.services.desktop_model_credentials import request_hook
+ from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint
+ from core.harness.registry.model_credentials import request_hook
hook, captured = request_hook(desktop_reference, base_url)
identity = (desktop_reference, base_url, _state_fingerprint(captured))
hooks = {"request": [hook]}
key = (loop_key, base_t, identity)
- client = _HTTP_CLIENTS.get(key)
+ client = process_state().HTTP_CLIENTS.get(key)
if client is not None and not client.is_closed:
return client
@@ -99,7 +96,7 @@ def _make_http_client(
event_hooks=hooks,
)
if loop_key:
- _HTTP_CLIENTS[key] = client
+ process_state().HTTP_CLIENTS[key] = client
return client
@@ -183,7 +180,7 @@ def wants_responses(spec, api_protocol: Optional[str]) -> bool:
``protocol_probe`` and stored in ``extra_config.api_protocol``, never guessed here
from the vendor name.
"""
- from core.llm.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES
+ from core.models.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES
if not spec.speaks_responses:
return False
@@ -214,7 +211,7 @@ def _make_openai_responses(
reasoning_effort_is_wire: bool = False,
):
"""Build the Responses-protocol twin of ``_make_openai_compatible``."""
- from core.llm.responses_models import OpenAICompatResponsesModel
+ from core.models.responses import OpenAICompatResponsesModel
# The thinking switch keeps the same mechanism as the chat line: vLLM-style servers
# read chat_template_kwargs on both protocols, so a model behaves identically no
@@ -291,7 +288,7 @@ def make_chat_model(
spec = get_spec(provider)
if not context_size or context_size <= 0:
- from core.llm.context_manager import resolve_model_context_window
+ from core.models.context_window import resolve_model_context_window
context_size = resolve_model_context_window(model)
@@ -355,7 +352,7 @@ def make_chat_model(
def _resolve_or_dummy(role_key: str):
"""Resolve config from DB, return None if not available."""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
return ModelConfigService.get_instance().resolve(role_key)
except Exception as exc:
@@ -373,7 +370,7 @@ def build_model_for_mode(
same way rather than by three copies of the same flag arithmetic.
"""
disable_thinking = mode in ("fast", "turbo")
- from core.llm.reasoning_effort import resolve_reasoning_effort
+ from core.models.reasoning_effort import resolve_reasoning_effort
reasoning_effort = resolve_reasoning_effort(resolved.extra, mode)
model = make_chat_model(
@@ -468,3 +465,15 @@ def get_summarize_model(cfg: ModelConfig | None = None) -> ChatModelBase:
"close_loop_http_clients",
"wants_responses",
]
+
+
+@dataclass
+class ChatModelsState:
+ HTTP_CLIENTS: dict[tuple[int, float], httpx.AsyncClient] = field(default_factory=lambda: {})
+
+
+_STATE = ChatModelsState()
+
+
+def process_state() -> ChatModelsState:
+ return _STATE
diff --git a/src/backend/core/models/completions/__init__.py b/src/backend/core/models/completions/__init__.py
new file mode 100644
index 00000000..3767d09b
--- /dev/null
+++ b/src/backend/core/models/completions/__init__.py
@@ -0,0 +1 @@
+"""Single-turn completion, classification and summarization clients."""
diff --git a/src/backend/core/llm/classifier.py b/src/backend/core/models/completions/classifier.py
similarity index 84%
rename from src/backend/core/llm/classifier.py
rename to src/backend/core/models/completions/classifier.py
index 7b8ce610..c31f171d 100644
--- a/src/backend/core/llm/classifier.py
+++ b/src/backend/core/models/completions/classifier.py
@@ -2,12 +2,13 @@
from __future__ import annotations
import logging
+from dataclasses import dataclass
from typing import Dict, List, Optional
from core.config.settings import settings
# 分类与摘要共用 summarizer 角色,所以共用同一个解析器,避免两份会漂移的副本。
-from core.llm.summarizer import _resolve_summarizer_endpoint
-from core.llm.single_turn import complete
+from core.models.completions.summarizer import _resolve_summarizer_endpoint
+from core.models.completions.single_turn import complete
_LOGGER = logging.getLogger(__name__)
@@ -86,11 +87,21 @@ async def classify_conversation(
# Singleton instance
-_classifier: ConversationClassifier | None = None
def get_classifier() -> ConversationClassifier:
- global _classifier
- if _classifier is None:
- _classifier = ConversationClassifier()
- return _classifier
+ if process_state().classifier is None:
+ process_state().classifier = ConversationClassifier()
+ return process_state().classifier
+
+
+@dataclass
+class ClassifierState:
+ classifier: ConversationClassifier | None = None
+
+
+_STATE = ClassifierState()
+
+
+def process_state() -> ClassifierState:
+ return _STATE
diff --git a/src/backend/core/llm/single_turn.py b/src/backend/core/models/completions/single_turn.py
similarity index 96%
rename from src/backend/core/llm/single_turn.py
rename to src/backend/core/models/completions/single_turn.py
index 8f5b7d69..51cd52f7 100644
--- a/src/backend/core/llm/single_turn.py
+++ b/src/backend/core/models/completions/single_turn.py
@@ -16,8 +16,8 @@
import httpx
-from core.llm._distill_shared import strip_think_blocks
-from core.llm.providers.protocol_probe import PROTOCOL_RESPONSES
+from core.infra.model_output import strip_think_blocks
+from core.models.providers.protocol_probe import PROTOCOL_RESPONSES
_TEXT_ITEM = "output_text"
@@ -45,8 +45,8 @@ def from_resolved(cls, cfg: Any) -> "Endpoint":
@property
def speaks_responses(self) -> bool:
- from core.llm.chat_models import wants_responses
- from core.llm.providers.registry import get_spec
+ from core.models.chat_models import wants_responses
+ from core.models.providers.registry import get_spec
return wants_responses(get_spec(self.provider), self.api_protocol)
@@ -90,7 +90,7 @@ def build_request(
凭据统一走桌面模型凭据模块:本机执行面的模型行存的是账号绑定的网关引用而不是密钥,
非引用的普通密钥原样加 Bearer,所以云端和桌面共用这一条路径。
"""
- from core.services.desktop_model_credentials import prepare_request_headers
+ from core.harness.registry.model_credentials import prepare_request_headers
base = endpoint.base_url.rstrip("/")
headers = prepare_request_headers(endpoint.api_key, endpoint.base_url)
diff --git a/src/backend/core/llm/summarizer.py b/src/backend/core/models/completions/summarizer.py
similarity index 86%
rename from src/backend/core/llm/summarizer.py
rename to src/backend/core/models/completions/summarizer.py
index 20190f1c..3f9b27b7 100644
--- a/src/backend/core/llm/summarizer.py
+++ b/src/backend/core/models/completions/summarizer.py
@@ -2,10 +2,11 @@
from __future__ import annotations
import logging
+from dataclasses import dataclass
from typing import Dict, List, Optional
from core.config.settings import settings
-from core.llm.single_turn import Endpoint, complete
+from core.models.completions.single_turn import Endpoint, complete
_LOGGER = logging.getLogger(__name__)
@@ -13,7 +14,7 @@
def _resolve_summarizer_endpoint() -> Optional[Endpoint]:
"""Resolve the summarizer role's endpoint from DB."""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
cfg = ModelConfigService.get_instance().resolve("summarizer")
if cfg and cfg.base_url and cfg.api_key and cfg.model_name:
return Endpoint.from_resolved(cfg)
@@ -93,11 +94,21 @@ async def summarize_conversation(
# Singleton instance
-_summarizer: ConversationSummarizer | None = None
def get_summarizer() -> ConversationSummarizer:
- global _summarizer
- if _summarizer is None:
- _summarizer = ConversationSummarizer()
- return _summarizer
+ if process_state().summarizer is None:
+ process_state().summarizer = ConversationSummarizer()
+ return process_state().summarizer
+
+
+@dataclass
+class SummarizerState:
+ summarizer: ConversationSummarizer | None = None
+
+
+_STATE = SummarizerState()
+
+
+def process_state() -> SummarizerState:
+ return _STATE
diff --git a/src/backend/core/services/model_config.py b/src/backend/core/models/config.py
similarity index 59%
rename from src/backend/core/services/model_config.py
rename to src/backend/core/models/config.py
index 75ead29a..9a457fed 100644
--- a/src/backend/core/services/model_config.py
+++ b/src/backend/core/models/config.py
@@ -9,7 +9,7 @@
import logging
import threading
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from typing import Any, Optional
from core.db.engine import SessionLocal
@@ -52,21 +52,40 @@ class ResolvedModelConfig:
extra: dict = field(default_factory=dict)
+@dataclass(frozen=True)
+class _Snapshot:
+ """One committed model configuration, published as a whole and never mutated.
+
+ ``revision`` is the database revision the rows were read under; ``None``
+ marks a snapshot that must not be trusted (nothing loaded yet, or a forced
+ reload).
+ """
+
+ revision: Optional[str]
+ version: int
+ roles: dict[str, ResolvedModelConfig] = field(default_factory=dict)
+ chat_providers: dict[str, ResolvedModelConfig] = field(default_factory=dict)
+ providers: dict[str, ResolvedModelConfig] = field(default_factory=dict)
+ chain: tuple[ResolvedModelConfig, ...] = ()
+
+
class ModelConfigService:
- """Thread-safe singleton that resolves role → model config from DB."""
+ """Thread-safe singleton that resolves role → model config from DB.
+
+ Every lookup compares the committed revision (one primary-key read) with the
+ snapshot it holds, so a change committed by any worker or process is seen
+ on the next lookup. Lookups never wait on each other: the snapshot is
+ swapped atomically, and only a revision mismatch takes the refresh lock,
+ which one thread holds while the others that saw the same mismatch wait for
+ its result instead of reading the tables again.
+ """
_instance: Optional["ModelConfigService"] = None
_lock = threading.Lock()
def __init__(self) -> None:
- self._cache: dict[str, Optional[ResolvedModelConfig]] = {}
- self._revision: str | None = None
- self._cache_lock = threading.RLock()
- # All runtime views share one committed revision.
- self._provider_cache: dict[str, Optional[ResolvedModelConfig]] = {}
- # Failover candidate order belongs to the same snapshot.
- self._chain_cache: Optional[list[ResolvedModelConfig]] = None
- self._version: int = 0 # bumped on invalidate
+ self._snapshot = _Snapshot(revision=None, version=0)
+ self._refresh_lock = threading.Lock()
@classmethod
def get_instance(cls) -> "ModelConfigService":
@@ -78,55 +97,62 @@ def get_instance(cls) -> "ModelConfigService":
@property
def version(self) -> int:
- self._maybe_refresh()
- return self._version
+ return self._current().version
# ── resolve ────────────────────────────────────────────────────────
def resolve(self, role_key: str) -> Optional[ResolvedModelConfig]:
- with self._cache_lock:
- self._maybe_refresh()
- return self._cache.get(role_key)
+ return self._current().roles.get(role_key)
def resolve_provider(self, provider_id: str) -> Optional[ResolvedModelConfig]:
- with self._cache_lock:
- self._maybe_refresh()
- return self._provider_cache.get((provider_id or "").strip())
+ return self._current().chat_providers.get((provider_id or "").strip())
def resolve_failover_chain(
self, primary: Optional[ResolvedModelConfig]
) -> list[ResolvedModelConfig]:
- with self._cache_lock:
- self._maybe_refresh()
- chain = [
- cfg
- for cfg in self._chain_cache or []
- if primary is None or cfg.provider_id != primary.provider_id
- ]
- return ([primary] if primary else []) + chain
+ chain = [
+ cfg
+ for cfg in self._current().chain
+ if primary is None or cfg.provider_id != primary.provider_id
+ ]
+ return ([primary] if primary else []) + chain
def invalidate_cache(self) -> None:
- with self._cache_lock:
- self._revision = None
- self._version += 1
+ """Force the next lookup to reload, for rows written without a revision bump."""
+ # Under the refresh lock, so a reload already reading the old rows
+ # cannot publish them over this request.
+ with self._refresh_lock:
+ self._snapshot = replace(self._snapshot, revision=None)
- def _maybe_refresh(self) -> None:
+ def _current(self) -> _Snapshot:
from core.db.model_config_revision import read_revision
- with self._cache_lock, SessionLocal() as db:
- revision = read_revision(db)
- if revision == self._revision:
- return
- # Read into a new snapshot, then publish under the same lock. An
- # error propagates; it must not silently authorize stale models.
- from core.db.model_config_revision import REVISION_KEY
-
- snapshot_revision = (
- select(ContentBlock.payload)
- .where(ContentBlock.id == REVISION_KEY)
- .scalar_subquery()
- )
- # One SELECT has one MVCC snapshot even under READ COMMITTED.
+ snapshot = self._snapshot
+ observed = None
+ if snapshot.revision is not None:
+ with SessionLocal() as db:
+ observed = read_revision(db)
+ if observed == snapshot.revision:
+ return snapshot
+ with self._refresh_lock:
+ # A reload finished while this thread waited. It is current only if
+ # it carries the revision this lookup observed after it began.
+ current = self._snapshot
+ if observed is not None and current.revision == observed:
+ return current
+ # An error propagates; it must not silently authorize stale models.
+ self._snapshot = self._load(current.version + 1)
+ return self._snapshot
+
+ def _load(self, version: int) -> _Snapshot:
+ from core.db.model_config_revision import REVISION_KEY
+
+ snapshot_revision = (
+ select(ContentBlock.payload).where(ContentBlock.id == REVISION_KEY).scalar_subquery()
+ )
+ with SessionLocal() as db:
+ # One SELECT has one MVCC snapshot even under READ COMMITTED, so the
+ # revision it carries is exactly the one these rows belong to.
rows = (
db.query(ModelProvider, ModelRoleAssignment, snapshot_revision)
.outerjoin(
@@ -135,37 +161,39 @@ def _maybe_refresh(self) -> None:
)
.all()
)
- providers = list({p.provider_id: p for p, _, _ in rows if p.is_active}.values())
- resolved = {p.provider_id: self._provider_to_resolved(p) for p in providers}
- roles = {
+ if rows:
+ revision = str((rows[0][2] or {}).get("revision", ""))
+ else:
+ from core.db.model_config_revision import read_revision
+
+ revision = read_revision(db)
+ providers = list({p.provider_id: p for p, _, _ in rows if p.is_active}.values())
+ resolved = {p.provider_id: self._provider_to_resolved(p) for p in providers}
+ chat = [p for p in providers if p.provider_type == "chat"]
+ ranked = sorted(
+ chat,
+ key=lambda p: (
+ -int(p.weight or 1),
+ -resolved[p.provider_id].context_length,
+ p.provider_id,
+ ),
+ )
+ return _Snapshot(
+ revision=revision,
+ version=version,
+ roles={
a.role_key: resolved[a.provider_id]
for _, a, _ in rows
if a is not None and a.provider_id in resolved
- }
- chat = {
- p.provider_id: resolved[p.provider_id]
- for p in providers
- if p.provider_type == "chat"
- }
- if rows:
- revision = str((rows[0][2] or {}).get("revision", ""))
- ranked = sorted(
- (p for p in providers if p.provider_type == "chat"),
- key=lambda p: (
- -int(p.weight or 1),
- -resolved[p.provider_id].context_length,
- p.provider_id,
- ),
- )
- self._cache = roles
- self._provider_cache = chat
- self._chain_cache = [resolved[p.provider_id] for p in ranked]
- self._revision = revision
- self._version += 1
+ },
+ chat_providers={p.provider_id: resolved[p.provider_id] for p in chat},
+ providers=resolved,
+ chain=tuple(resolved[p.provider_id] for p in ranked),
+ )
@staticmethod
def _provider_to_resolved(provider: ModelProvider) -> ResolvedModelConfig:
- from core.llm.providers.registry import get_spec, split_provider_extra
+ from core.models.providers.registry import get_spec, split_provider_extra
extra = dict(provider.extra_config or {})
ctx_len = _optional_int(extra.pop("context_length", None)) or 0
@@ -194,46 +222,17 @@ def _provider_to_resolved(provider: ModelProvider) -> ResolvedModelConfig:
def get_context_length_by_model_name(self, model_name: str) -> Optional[int]:
"""Look up the context length (tokens) by model_name.
- Checks the role cache first, then falls back to scanning extra_config.context_length across all providers.
+ Role assignments win; any other active provider with that model name
+ (one not assigned to a role may still be referenced) is the fallback.
Returns None when not configured (the caller decides its own fallback strategy).
"""
- if not model_name:
- return None
- target = model_name.strip()
+ target = (model_name or "").strip()
if not target:
return None
-
- self._maybe_refresh()
- for cfg in self._cache.values():
- if cfg is None:
- continue
+ snapshot = self._current()
+ for cfg in (*snapshot.roles.values(), *snapshot.providers.values()):
if cfg.model_name == target and cfg.context_length > 0:
return cfg.context_length
-
- # Fallback: scan the providers table directly (a provider not assigned to any role may still be referenced)
- try:
- db = SessionLocal()
- try:
- rows = (
- db.query(ModelProvider)
- .filter(ModelProvider.model_name == target)
- .filter(ModelProvider.is_active == True) # noqa: E712
- .all()
- )
- for provider in rows:
- extra = provider.extra_config or {}
- raw = extra.get("context_length")
- if raw:
- try:
- val = int(raw)
- if val > 0:
- return val
- except (TypeError, ValueError):
- continue
- finally:
- db.close()
- except Exception as exc:
- logger.debug("[ModelConfigService] context_length lookup failed: %s", exc)
return None
# ── MCP env overlay ───────────────────────────────────────────────
diff --git a/src/backend/core/models/constants.py b/src/backend/core/models/constants.py
new file mode 100644
index 00000000..8ddc968b
--- /dev/null
+++ b/src/backend/core/models/constants.py
@@ -0,0 +1,5 @@
+"""Shared model/file contracts without execution-layer dependencies."""
+from __future__ import annotations
+PROVIDER_CONTEXT_META_KEY = "_harness_context_item"
+
+IMAGE_TOKEN_RESERVE = 1_844
diff --git a/src/backend/core/llm/context_manager.py b/src/backend/core/models/context_window.py
similarity index 97%
rename from src/backend/core/llm/context_manager.py
rename to src/backend/core/models/context_window.py
index af696e49..0bd1241f 100644
--- a/src/backend/core/llm/context_manager.py
+++ b/src/backend/core/models/context_window.py
@@ -44,7 +44,7 @@ def resolve_model_context_window(model_name: str) -> int:
)
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
ctx_len = ModelConfigService.get_instance().get_context_length_by_model_name(model_name)
except ValueError:
diff --git a/src/backend/core/models/embedding.py b/src/backend/core/models/embedding.py
new file mode 100644
index 00000000..6961e8fe
--- /dev/null
+++ b/src/backend/core/models/embedding.py
@@ -0,0 +1,123 @@
+"""Call the configured embedding service."""
+from __future__ import annotations
+from core.models import embedding_state
+import os
+import requests
+import logging
+logger = logging.getLogger(__name__)
+
+def resolve_embed_config() -> tuple[str, str, str]:
+ """Resolve embedding config from DB, with env fallback."""
+ try:
+ from core.models.config import ModelConfigService
+
+ cfg = ModelConfigService.get_instance().resolve("embedding")
+ if cfg:
+ return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.models.embedding:resolve_embed_config', fallback_error)
+ return (
+ os.getenv("MEM0_EMBED_URL", "").rstrip("/"),
+ os.getenv("MEM0_EMBED_MODEL", ""),
+ os.getenv("MEM0_EMBED_API_KEY", ""),
+ )
+
+
+def embed_request(
+ url: str,
+ headers: dict,
+ inputs: list[str],
+ embed_model: str,
+ *,
+ timeout: float | None = None,
+) -> list[list[float]]:
+ """Post a single embedding request and return vectors ordered by input index."""
+ payload = {"input": inputs, "model": embed_model}
+ resp = requests.post(
+ url,
+ headers=headers,
+ json=payload,
+ timeout=timeout or embedding_state.EMBED_TIMEOUT,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ return [item["embedding"] for item in sorted(data["data"], key=lambda x: x["index"])]
+
+
+def embed_text(text: str, *, timeout: float | None = None) -> list[float]:
+ """Call the configured embedding service and return a dense vector."""
+ embed_url, embed_model, api_key = resolve_embed_config()
+
+ if not embed_url:
+ raise RuntimeError("Embedding model is not configured")
+
+ url = f"{embed_url}/embeddings"
+ from core.harness.registry.model_credentials import prepare_request_headers
+ headers = prepare_request_headers(api_key, embed_url)
+ vector = embed_request(url, headers, [text], embed_model, timeout=timeout)[0]
+ if vector:
+ embedding_state.process_state().DETECTED_DIM_CACHE[(embed_url, embed_model)] = len(vector)
+ return vector
+
+
+def embed_batch(texts: list[str]) -> list[list[float]]:
+ """Batch embed multiple texts, splitting into sub-batches with one-by-one fallback.
+
+ A large document is split into hundreds or thousands of chunks. Sending the whole batch at once to a slow embedding
+ model (e.g. 8B, ~0.45s/item serially) would time out the single request and fail the whole document's vectorization. So we
+ split into sub-batches of _EMBED_BATCH_SIZE and request them one sub-batch at a time; if a sub-batch fails, we downgrade to
+ embedding one item at a time within that sub-batch, preserving as many other chunks as possible so one timeout doesn't drag
+ down the whole document. Only if the one-by-one attempt still fails do we re-raise (treated as a true failure).
+ """
+ embed_url, embed_model, api_key = resolve_embed_config()
+
+ if not embed_url:
+ raise RuntimeError("Embedding model is not configured")
+
+ url = f"{embed_url}/embeddings"
+ from core.harness.registry.model_credentials import prepare_request_headers
+ headers = prepare_request_headers(api_key, embed_url)
+
+ results: list[list[float]] = []
+ for start in range(0, len(texts), embedding_state.EMBED_BATCH_SIZE):
+ sub = texts[start : start + embedding_state.EMBED_BATCH_SIZE]
+ try:
+ results.extend(embed_request(url, headers, sub, embed_model))
+ except Exception as exc:
+ logger.warning(
+ "Sub-batch embed failed for [%d:%d] (%s); falling back to one-by-one",
+ start,
+ start + len(sub),
+ exc,
+ )
+ for one in sub:
+ results.extend(embed_request(url, headers, [one], embed_model))
+ return results
+
+
+def detect_embed_dim() -> int:
+ """Probe the real output dimension of the currently chosen embedding model (one /embeddings call, take the vector length).
+
+ The result is cached by (embed_url, embed_model) to avoid repeated probing on every index; on probe failure (endpoint not
+ configured/unreachable) it falls back to MEM0_EMBED_DIMS. This way the table dimension auto-follows after switching the
+ embedding model, no manual filling needed.
+ """
+ embed_url, embed_model, _ = resolve_embed_config()
+ key = (embed_url, embed_model)
+ cached = embedding_state.process_state().DETECTED_DIM_CACHE.get(key)
+ if cached:
+ return cached
+ try:
+ dim = len(embed_text("dimension probe"))
+ if dim > 0:
+ embedding_state.process_state().DETECTED_DIM_CACHE[key] = dim
+ logger.info("Auto-detected embedding dim=%d (model=%s)", dim, embed_model or "?")
+ return dim
+ except Exception as exc:
+ logger.warning(
+ "Embedding dim auto-detect failed (%s); falling back to MEM0_EMBED_DIMS=%d",
+ exc,
+ embedding_state.EMBED_DIMS,
+ )
+ return embedding_state.EMBED_DIMS
diff --git a/src/backend/core/models/embedding_state.py b/src/backend/core/models/embedding_state.py
new file mode 100644
index 00000000..4b04376d
--- /dev/null
+++ b/src/backend/core/models/embedding_state.py
@@ -0,0 +1,20 @@
+"""Embedding request limits and dimension cache, shared across consumers."""
+from __future__ import annotations
+import os
+from dataclasses import dataclass, field
+EMBED_DIMS = int(os.getenv("MEM0_EMBED_DIMS", "1024"))
+
+EMBED_BATCH_SIZE = max(1, int(os.getenv("KB_EMBED_BATCH_SIZE", "32")))
+EMBED_TIMEOUT = max(1, int(os.getenv("KB_EMBED_TIMEOUT", "120")))
+
+
+@dataclass
+class EmbeddingState:
+ DETECTED_DIM_CACHE: dict[tuple[str, str], int] = field(default_factory=lambda: {})
+
+
+_STATE = EmbeddingState()
+
+
+def process_state() -> EmbeddingState:
+ return _STATE
diff --git a/src/backend/core/llm/failover.py b/src/backend/core/models/failover.py
similarity index 94%
rename from src/backend/core/llm/failover.py
rename to src/backend/core/models/failover.py
index 239be5af..4dffc1ed 100644
--- a/src/backend/core/llm/failover.py
+++ b/src/backend/core/models/failover.py
@@ -28,13 +28,14 @@
import logging
import time
from contextvars import ContextVar
+from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Callable, Optional
from agentscope.message import Msg
from agentscope.model import ChatModelBase, ChatResponse
from agentscope.tool._types import ToolChoice
-from core.llm.providers._image_tokens import ImageTokenCountingMixin
-from core.llm.tool_call_identity import ToolCallIdentityMixin
+from core.models.providers._image_tokens import ImageTokenCountingMixin
+from core.models.tool_call_identity import ToolCallIdentityMixin
logger = logging.getLogger(__name__)
@@ -43,7 +44,6 @@
# that a dead upstream is not re-probed by every concurrent run.
PROVIDER_COOLDOWN_SECONDS = 120.0
-_cooldown: dict[str, float] = {}
# Which candidate answered the call in flight. The model instance is cached and
# shared by every concurrent run, so "who is answering" cannot live on it: one
@@ -53,21 +53,21 @@
def mark_provider_down(provider_id: str) -> None:
if provider_id:
- _cooldown[provider_id] = time.monotonic() + PROVIDER_COOLDOWN_SECONDS
+ process_state().cooldown[provider_id] = time.monotonic() + PROVIDER_COOLDOWN_SECONDS
def provider_is_cooling(provider_id: str) -> bool:
- until = _cooldown.get(provider_id)
+ until = process_state().cooldown.get(provider_id)
if until is None:
return False
if time.monotonic() >= until:
- _cooldown.pop(provider_id, None)
+ process_state().cooldown.pop(provider_id, None)
return False
return True
def clear_cooldowns() -> None:
- _cooldown.clear()
+ process_state().cooldown.clear()
def provider_is_unusable(exc: BaseException) -> bool:
@@ -112,7 +112,7 @@ def carries_media(messages: list[Msg]) -> bool:
Media rides in ``DataBlock``s — either directly on a message (an upload the
middleware passed through) or folded into a ``ToolResultBlock``'s output
- (see ``core.llm.tool_result_media``). Both places are checked because both
+ (see ``core.models.tool_result_media``). Both places are checked because both
reach the wire.
"""
for msg in messages or []:
@@ -260,7 +260,7 @@ def _reads_media(self, index: int) -> bool:
"""Whether candidate *index* may be sent a request carrying media.
The primary needs no test: the turn's vision mode is resolved against
- it (``core.vision.resolve_vision_mode``), so a tool only ever returns
+ it (``core.models.vision.resolve_vision_mode``), so a tool only ever returns
pixels — and an upload only ever passes through untranscribed — when
the primary reads them natively. Fallbacks were chosen by weight and
context window, neither of which says anything about media, so each is
@@ -268,7 +268,7 @@ def _reads_media(self, index: int) -> bool:
"""
if index == 0:
return True
- from core.vision import model_supports_vision
+ from core.models.vision import model_supports_vision
return model_supports_vision(self._fallback_specs[index - 1])
@@ -381,8 +381,8 @@ def with_failover(
``resolved`` is the ``ResolvedModelConfig`` *primary* was built from; it is
excluded from its own fallback list.
"""
- from core.llm.chat_models import build_model_for_mode
- from core.services.model_config import ModelConfigService
+ from core.models.chat_models import build_model_for_mode
+ from core.models.config import ModelConfigService
try:
chain = ModelConfigService.get_instance().resolve_failover_chain(resolved)
@@ -402,3 +402,15 @@ def with_failover(
fallbacks,
lambda cfg: build_model_for_mode(cfg, mode=mode, stream=primary.stream),
)
+
+
+@dataclass
+class FailoverState:
+ cooldown: dict[str, float] = field(default_factory=lambda: {})
+
+
+_STATE = FailoverState()
+
+
+def process_state() -> FailoverState:
+ return _STATE
diff --git a/src/backend/core/db/model_import_validation.py b/src/backend/core/models/import_validation.py
similarity index 93%
rename from src/backend/core/db/model_import_validation.py
rename to src/backend/core/models/import_validation.py
index 2ed6e7f4..ad39202d 100644
--- a/src/backend/core/db/model_import_validation.py
+++ b/src/backend/core/models/import_validation.py
@@ -1,10 +1,10 @@
"""Validate the resulting model/role graph before an import mutates any row."""
from core.db.models import ModelProvider, ModelRoleAssignment
-from core.llm.providers.registry import get_spec
+from core.models.providers.registry import get_spec
def validate_import_graph(db, data, overwrite):
- from core.db.model_repository import ROLE_DEFINITIONS
+ from core.models.repository import ROLE_DEFINITIONS
providers = {row.provider_id: {"provider_type": row.provider_type, "provider": row.provider, "extra_config": row.extra_config or {}} for row in db.query(ModelProvider).all()}
for row in data.get("providers", []):
key = row.get("provider_id")
diff --git a/src/backend/core/models/providers/__init__.py b/src/backend/core/models/providers/__init__.py
new file mode 100644
index 00000000..90e72509
--- /dev/null
+++ b/src/backend/core/models/providers/__init__.py
@@ -0,0 +1,3 @@
+"""Multi-vendor model provider support: registry + per-engine model implementations."""
+
+from core.models.providers.registry import PROVIDER_SPECS, ProviderField, ProviderSpec, get_spec, is_known, list_specs, split_provider_extra, to_frontend_schema, validate_payload
diff --git a/src/backend/core/llm/providers/_image_tokens.py b/src/backend/core/models/providers/_image_tokens.py
similarity index 96%
rename from src/backend/core/llm/providers/_image_tokens.py
rename to src/backend/core/models/providers/_image_tokens.py
index e7b7d1a9..b35339f6 100644
--- a/src/backend/core/llm/providers/_image_tokens.py
+++ b/src/backend/core/models/providers/_image_tokens.py
@@ -2,7 +2,7 @@
from agentscope.message import DataBlock, Msg, ToolResultBlock
-from core.llm.context_ir import IMAGE_TOKEN_RESERVE
+from core.models.constants import IMAGE_TOKEN_RESERVE
class ImageTokenCountingMixin:
diff --git a/src/backend/core/llm/providers/context_probe.py b/src/backend/core/models/providers/context_probe.py
similarity index 99%
rename from src/backend/core/llm/providers/context_probe.py
rename to src/backend/core/models/providers/context_probe.py
index 90320b68..8c3e6569 100644
--- a/src/backend/core/llm/providers/context_probe.py
+++ b/src/backend/core/models/providers/context_probe.py
@@ -322,7 +322,7 @@ async def discover_context_length(
result.notes.append("模型名为空,无法探测")
return result
- from core.llm.providers.registry import get_spec
+ from core.models.providers.registry import get_spec
try:
spec = get_spec(provider)
diff --git a/src/backend/core/llm/providers/openai_compat.py b/src/backend/core/models/providers/openai_compat.py
similarity index 96%
rename from src/backend/core/llm/providers/openai_compat.py
rename to src/backend/core/models/providers/openai_compat.py
index c9d2b34b..6b99fa76 100644
--- a/src/backend/core/llm/providers/openai_compat.py
+++ b/src/backend/core/models/providers/openai_compat.py
@@ -24,20 +24,20 @@
)
from agentscope.model import ChatResponse, OpenAIChatModel
from agentscope.tool._types import ToolChoice
-from core.llm.context_adapter import PROVIDER_CONTEXT_META_KEY
-from core.llm.context_safety import check_request_admission
-from core.llm.providers._image_tokens import ImageTokenCountingMixin
-from core.llm.providers.stream_retry import _stream_with_bounded_retry
-from core.llm.reasoning_replay import ReasoningReplayMixin
-from core.llm.tool_call_identity import ToolCallIdentityMixin
-from core.llm.tool_result_media import NoDiskToolMediaMixin
+from core.models.constants import PROVIDER_CONTEXT_META_KEY
+from core.models.admission import check_request_admission
+from core.models.providers._image_tokens import ImageTokenCountingMixin
+from core.models.providers.stream_retry import _stream_with_bounded_retry
+from core.models.reasoning_replay import ReasoningReplayMixin
+from core.models.tool_call_identity import ToolCallIdentityMixin
+from core.models.tool_result_media import NoDiskToolMediaMixin
logger = logging.getLogger(__name__)
def _safe_exception_chain(exc: BaseException) -> list[dict[str, str]]:
"""Identify wrapped SDK failures without serializing messages or requests."""
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
known_codes = {
"capability_error",
@@ -414,7 +414,7 @@ async def _call_api( # type: ignore[override]
response = await client.chat.completions.create(**kwargs)
logger.info("[wire] response headers in %.0fms", (_perf_counter() - _t_req) * 1000)
except Exception as exc:
- from core.llm.model_usage import record_provider_failure
+ from core.models.usage import record_provider_failure
logger.warning("Model request failed: exception_chain=%s", _safe_exception_chain(exc))
await record_provider_failure(
@@ -432,7 +432,7 @@ async def _call_api( # type: ignore[override]
return _stream_with_bounded_retry(
self,
reissue=lambda: client.chat.completions.create(**kwargs),
- parse=lambda started, raw: self._parse_stream_response(started, raw, audio_fmt),
+ parse=lambda started, raw: self._parse_stream_response(started, raw),
model_name=model_name,
start_datetime=start_datetime,
response=response,
diff --git a/src/backend/core/llm/providers/protocol_probe.py b/src/backend/core/models/providers/protocol_probe.py
similarity index 100%
rename from src/backend/core/llm/providers/protocol_probe.py
rename to src/backend/core/models/providers/protocol_probe.py
diff --git a/src/backend/core/llm/providers/reasoning_probe.py b/src/backend/core/models/providers/reasoning_probe.py
similarity index 97%
rename from src/backend/core/llm/providers/reasoning_probe.py
rename to src/backend/core/models/providers/reasoning_probe.py
index e8eb63bc..48983cc1 100644
--- a/src/backend/core/llm/providers/reasoning_probe.py
+++ b/src/backend/core/models/providers/reasoning_probe.py
@@ -12,9 +12,9 @@
import httpx
-from core.llm.providers.protocol_probe import detect_api_protocol
-from core.llm.providers.registry import get_spec
-from core.llm.reasoning_effort import EFFORT_KEYS
+from core.models.providers.protocol_probe import detect_api_protocol
+from core.models.providers.registry import get_spec
+from core.models.reasoning_effort import EFFORT_KEYS
def _result(
diff --git a/src/backend/core/llm/providers/registry.py b/src/backend/core/models/providers/registry.py
similarity index 99%
rename from src/backend/core/llm/providers/registry.py
rename to src/backend/core/models/providers/registry.py
index 10ae61a9..6b7b7c39 100644
--- a/src/backend/core/llm/providers/registry.py
+++ b/src/backend/core/models/providers/registry.py
@@ -208,7 +208,7 @@ def validate_payload(provider_id: str, provider_type: str, extra_config: dict) -
if provider_type not in spec.supports_types:
return f"{spec.label} 不支持用途 '{provider_type}'(支持:{', '.join(spec.supports_types)})"
extra = extra_config or {}
- from core.llm.reasoning_effort import validate_reasoning_config
+ from core.models.reasoning_effort import validate_reasoning_config
error = validate_reasoning_config(extra)
if error:
diff --git a/src/backend/core/llm/providers/stream_retry.py b/src/backend/core/models/providers/stream_retry.py
similarity index 96%
rename from src/backend/core/llm/providers/stream_retry.py
rename to src/backend/core/models/providers/stream_retry.py
index 4e650a84..c23b65f8 100644
--- a/src/backend/core/llm/providers/stream_retry.py
+++ b/src/backend/core/models/providers/stream_retry.py
@@ -7,7 +7,7 @@
from time import monotonic as _monotonic
from time import perf_counter as _perf_counter
-from core.llm.context_safety import check_request_admission
+from core.models.admission import check_request_admission
logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ async def _stream_with_bounded_retry(
except Exception as exc: # noqa: BLE001 - classified below
if not _is_retryable_stream_start_error(exc):
raise
- from core.llm.model_usage import note_provider_retry_started, record_provider_failure
+ from core.models.usage import note_provider_retry_started, record_provider_failure
await record_provider_failure(
model,
diff --git a/src/backend/core/llm/providers/vendor_models.py b/src/backend/core/models/providers/vendor_models.py
similarity index 97%
rename from src/backend/core/llm/providers/vendor_models.py
rename to src/backend/core/models/providers/vendor_models.py
index 5232fcad..b50e00a6 100644
--- a/src/backend/core/llm/providers/vendor_models.py
+++ b/src/backend/core/models/providers/vendor_models.py
@@ -41,12 +41,12 @@
OpenAIChatModel,
)
from agentscope.tool._types import ToolChoice
-from core.llm.reasoning_replay import ReasoningReplayMixin
-from core.llm.tool_call_identity import ToolCallIdentityMixin
-from core.llm.tool_result_media import NoDiskToolMediaMixin
+from core.models.reasoning_replay import ReasoningReplayMixin
+from core.models.tool_call_identity import ToolCallIdentityMixin
+from core.models.tool_result_media import NoDiskToolMediaMixin
-from ._image_tokens import ImageTokenCountingMixin
-from .registry import ProviderSpec
+from core.models.providers._image_tokens import ImageTokenCountingMixin
+from core.models.providers.registry import ProviderSpec
logger = logging.getLogger(__name__)
@@ -338,7 +338,7 @@ async def _call_api( # type: ignore[override]
response = await litellm.acompletion(**kwargs)
if self.stream:
- return self._parse_stream_response(start_datetime, response, "wav")
+ return self._parse_stream_response(start_datetime, response)
return self._parse_completion_response(start_datetime, response, "wav")
diff --git a/src/backend/core/llm/reasoning_effort.py b/src/backend/core/models/reasoning_effort.py
similarity index 100%
rename from src/backend/core/llm/reasoning_effort.py
rename to src/backend/core/models/reasoning_effort.py
diff --git a/src/backend/core/llm/reasoning_replay.py b/src/backend/core/models/reasoning_replay.py
similarity index 100%
rename from src/backend/core/llm/reasoning_replay.py
rename to src/backend/core/models/reasoning_replay.py
diff --git a/src/backend/core/db/model_repository.py b/src/backend/core/models/repository.py
similarity index 99%
rename from src/backend/core/db/model_repository.py
rename to src/backend/core/models/repository.py
index 0b6bf28e..2bed4acc 100644
--- a/src/backend/core/db/model_repository.py
+++ b/src/backend/core/models/repository.py
@@ -11,7 +11,7 @@
from core.db.models import ModelProvider, ModelRoleAssignment
from core.db.model_config_revision import bump_revision
-from core.llm.providers.registry import get_spec
+from core.models.providers.registry import get_spec
# ── Predefined roles ─────────────────────────────────────────────────────────
@@ -280,7 +280,7 @@ def export_all(db: Session) -> dict:
def import_all(db: Session, data: dict, overwrite: bool = True) -> dict:
"""Import providers + role assignments. Returns counts."""
- from core.db.model_import_validation import validate_import_graph
+ from core.models.import_validation import validate_import_graph
validate_import_graph(db, data, overwrite)
imported_providers = 0
imported_roles = 0
diff --git a/src/backend/core/models/reranking.py b/src/backend/core/models/reranking.py
new file mode 100644
index 00000000..a1c5b2f8
--- /dev/null
+++ b/src/backend/core/models/reranking.py
@@ -0,0 +1,63 @@
+"""Call the configured reranking service."""
+from __future__ import annotations
+import os
+import requests
+
+def resolve_reranker_config() -> tuple[str, str, str]:
+ """Resolve reranker config from DB, with env fallback."""
+ try:
+ from core.models.config import ModelConfigService
+
+ cfg = ModelConfigService.get_instance().resolve("reranker")
+ if cfg:
+ return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.models.reranking:resolve_reranker_config', fallback_error)
+ return (
+ os.getenv("RERANKER_URL", "").rstrip("/"),
+ os.getenv("RERANKER_MODEL", ""),
+ os.getenv("RERANKER_API_KEY", ""),
+ )
+
+
+def is_reranker_configured() -> bool:
+ """Check if a reranker model endpoint is configured."""
+ reranker_url, reranker_model, _ = resolve_reranker_config()
+ return bool(reranker_url and reranker_model)
+
+
+def rerank(
+ query: str,
+ documents: list[str],
+ top_n: int | None = None,
+ *,
+ timeout: float = 30,
+) -> list[dict]:
+ """Call the configured reranker endpoint (OpenAI-compatible /rerank).
+
+ Returns a list of {"index": int, "relevance_score": float} sorted by score descending.
+ """
+ reranker_url, reranker_model, reranker_key = resolve_reranker_config()
+
+ if not reranker_url or not reranker_model:
+ raise RuntimeError("Reranker is not configured")
+
+ url = f"{reranker_url}/rerank"
+ from core.harness.registry.model_credentials import prepare_request_headers
+ headers = prepare_request_headers(reranker_key, reranker_url)
+ payload: dict = {
+ "model": reranker_model,
+ "query": query,
+ "documents": documents,
+ }
+ if top_n is not None:
+ payload["top_n"] = top_n
+
+ resp = requests.post(url, headers=headers, json=payload, timeout=timeout)
+ resp.raise_for_status()
+ data = resp.json()
+
+ # Response format: {"results": [{"index": 0, "relevance_score": 0.95}, ...]}
+ results = data.get("results", [])
+ return sorted(results, key=lambda x: x.get("relevance_score", 0), reverse=True)
diff --git a/src/backend/core/llm/responses_models.py b/src/backend/core/models/responses.py
similarity index 80%
rename from src/backend/core/llm/responses_models.py
rename to src/backend/core/models/responses.py
index 1915966f..59b90981 100644
--- a/src/backend/core/llm/responses_models.py
+++ b/src/backend/core/models/responses.py
@@ -26,11 +26,11 @@
from agentscope.message import Msg, ThinkingBlock, ToolCallBlock
from agentscope.model import ChatResponse, OpenAIResponseModel
from agentscope.tool._types import ToolChoice
-from core.llm.context_safety import check_request_admission
-from core.llm.providers._image_tokens import ImageTokenCountingMixin
-from core.llm.reasoning_replay import ReasoningReplayMixin
-from core.llm.tool_call_identity import ToolCallIdentityMixin
-from core.llm.tool_result_media import ResponsesToolMediaMixin
+from core.models.admission import check_request_admission
+from core.models.providers._image_tokens import ImageTokenCountingMixin
+from core.models.reasoning_replay import ReasoningReplayMixin
+from core.models.tool_call_identity import ToolCallIdentityMixin
+from core.models.tool_result_media import NoDiskToolMediaMixin
logger = logging.getLogger(__name__)
@@ -42,14 +42,6 @@
# 只有这两个终态事件会带上完整的 response 对象(含 output 数组)。
_TERMINAL_EVENTS = ("response.completed", "response.incomplete")
-# Responses 有两条思考通道:``reasoning_summary_text`` 是经过删减的摘要(OpenAI 返回
-# 的那一条),``reasoning_text`` 是模型的思考原文(自建推理服务返回的那一条)。
-# AgentScope 的解析器只认前者,后者整条被丢掉——实测自建 deepseekv4-flash-vision 一轮
-# 发 144 帧 reasoning_text.delta,界面上就是一个空的思考块。两条通道对界面是同一件东西,
-# 所以在事件入口把后者翻成前者,流式逐字、累积、进历史全部照旧由父类处理。
-_REASONING_TEXT_DELTA = "response.reasoning_text.delta"
-_REASONING_SUMMARY_DELTA = "response.reasoning_summary_text.delta"
-
# 对齐 codex:让上游把思考加密串交还给我们自己保管,服务端不留会话状态。
_REASONING_INCLUDE = ["reasoning.encrypted_content"]
@@ -103,24 +95,10 @@ def _as_dict(item: Any) -> dict[str, Any]:
return {}
-class _ReasoningTextDelta:
- """把思考原文增量伪装成解析器认得的摘要增量。
-
- 父类只读 ``type`` 和 ``delta`` 两个字段(``response`` 用于取 response_id,这类事件
- 本来就没有),所以一个同形的小对象足够,不必构造 SDK 的类型。
- """
-
- __slots__ = ("type", "delta", "response", "summary_index")
-
- def __init__(self, delta: str) -> None:
- self.type = _REASONING_SUMMARY_DELTA
- self.delta = delta
- self.response = None
- self.summary_index = 0
-
-
class _ReasoningItemCapture:
- """透传上游 SSE,旁路留下完整 reasoning 条目,并接上思考原文通道。
+ """透传上游 SSE,旁路留下完整 reasoning 条目。
+
+ AgentScope 原生解析思考原文与摘要;保留事件身份及分段索引,避免合并不同条目。
旁路而非重写解析器:那 130 行逐事件拼装跟着 SDK 版本走,抄过来就得跟着维护。
"""
@@ -129,6 +107,13 @@ def __init__(self, stream: Any) -> None:
self._stream = stream
self.items: list[dict[str, Any]] = []
+ async def __aenter__(self):
+ await self._stream.__aenter__()
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ return await self._stream.__aexit__(exc_type, exc_value, traceback)
+
def __aiter__(self): # noqa: ANN204
return self._iter()
@@ -137,11 +122,6 @@ async def _iter(self): # noqa: ANN202
kind = getattr(event, "type", None)
if kind in _TERMINAL_EVENTS:
self._collect(getattr(event, "response", None))
- elif kind == _REASONING_TEXT_DELTA:
- delta = getattr(event, "delta", None)
- if delta:
- yield _ReasoningTextDelta(delta)
- continue
yield event
def _collect(self, response: Any) -> None:
@@ -216,39 +196,8 @@ def _restore_reasoning_items(msgs: list[Msg], items: list[dict[str, Any]]) -> li
return restored
-def _repair_tool_output_call_ids(
- msgs: list[Msg], items: list[dict[str, Any]]
-) -> list[dict[str, Any]]:
- """把工具结果的 ``call_id`` 换回发起那次调用时上游给的那一个。
-
- Responses 线上一次函数调用有**两个**互不相同的 id:条目 id(``function_call.id``)
- 与调用 id(``function_call.call_id``),回传结果时必须按后者配对。AgentScope 建
- ``ToolResultBlock`` 时只带上了条目 id(``id=tool_call.id``),格式化又直接拿它当
- ``function_call_output.call_id`` 发出去,于是两边对不上。
-
- DeepSeek 官方端点正是两者不同(条目 id 是 uuid,调用 id 形如 ``call_00_…``),整轮
- 请求会被打回 ``No tool call found for tool output with call_id …``;只要模型调了工具,
- 这一轮就必然失败。调用块自己同时带着这两个 id,这里按它把结果侧改正。
-
- 历史回放时旧块只剩条目 id,两侧都退回同一个值,配对依然自洽。
- """
- call_ids = {
- block.id: str(getattr(block, "call_id", None) or "")
- for msg in msgs
- for block in msg.content or []
- if isinstance(block, ToolCallBlock)
- }
- for item in items:
- if not isinstance(item, dict) or item.get("type") != "function_call_output":
- continue
- call_id = call_ids.get(str(item.get("call_id") or ""))
- if call_id:
- item["call_id"] = call_id
- return items
-
-
class ResponsesReplayFormatter(
- ReasoningReplayMixin, ResponsesToolMediaMixin, OpenAIResponseFormatter
+ ReasoningReplayMixin, NoDiskToolMediaMixin, OpenAIResponseFormatter
):
"""Responses 线格式化:思考按上游原样回传,工具媒体绝不落盘。
@@ -257,7 +206,7 @@ class ResponsesReplayFormatter(
造成当前模型的推理。
"""
- def _format_response_data_block(self, block: Any, role: str = "user") -> Any:
+ def _format_response_data_block(self, block: Any) -> Any:
"""给每个 ``input_image`` 补上 ``detail``——严格实现少了它直接 400。
AgentScope 只产出 ``type`` + ``image_url``,而 OpenAI 的
@@ -269,7 +218,7 @@ def _format_response_data_block(self, block: Any, role: str = "user") -> Any:
管这条线上的每张图,不只工具返回的:用户上传的图走的是同一个方法。
``setdefault`` 是为了不覆盖上游将来自己给出的值。
"""
- item = super()._format_response_data_block(block, role) # type: ignore[misc]
+ item = super()._format_response_data_block(block) # type: ignore[misc]
if isinstance(item, dict) and item.get("type") == "input_image":
item.setdefault("detail", "auto")
return item
@@ -280,7 +229,7 @@ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
for item in items:
if "role" in item and "content" in item:
item.setdefault("type", "message")
- return _restore_reasoning_items(msgs, _repair_tool_output_call_ids(msgs, items))
+ return _restore_reasoning_items(msgs, items)
class OpenAICompatResponsesModel(
@@ -368,8 +317,8 @@ async def _call_api( # type: ignore[override]
tool_choice: ToolChoice | None = None,
**generate_kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
- from core.llm.providers.openai_compat import _dump_wire_payload, _slim_tool_schemas
- from core.llm.providers.stream_retry import _stream_with_bounded_retry
+ from core.models.providers.openai_compat import _dump_wire_payload, _slim_tool_schemas
+ from core.models.providers.stream_retry import _stream_with_bounded_retry
client = self._build_client()
formatted_input = await self.formatter.format(messages)
@@ -413,7 +362,7 @@ async def _issue(): # noqa: ANN202
try:
response = await _issue()
except Exception as exc:
- from core.llm.model_usage import record_provider_failure
+ from core.models.usage import record_provider_failure
logger.warning("Responses request failed: %s: %s", type(exc).__name__, exc)
await record_provider_failure(
@@ -449,7 +398,7 @@ async def _publish_context_rewrite(
"""
import inspect
- from core.llm.context_adapter import PROVIDER_CONTEXT_META_KEY
+ from core.models.constants import PROVIDER_CONTEXT_META_KEY
listener = self._context_rewrite_listener
if listener is not None:
diff --git a/src/backend/core/services/user_model_selection.py b/src/backend/core/models/selection.py
similarity index 63%
rename from src/backend/core/services/user_model_selection.py
rename to src/backend/core/models/selection.py
index 52dbc347..49d0dbdc 100644
--- a/src/backend/core/services/user_model_selection.py
+++ b/src/backend/core/models/selection.py
@@ -10,57 +10,15 @@
from sqlalchemy.orm import Session
-from core.auth.capabilities import resolve_user_capabilities
from core.db.models import ModelProvider, ModelRoleAssignment
-from core.services.model_config import ModelConfigService
-from core.llm.reasoning_effort import reasoning_capabilities
+from core.models.config import ModelConfigService
+from core.models.reasoning_effort import reasoning_capabilities
-class UserModelSelectionError(ValueError):
- """Raised when a requested user-selectable model is not allowed."""
-def resolve_user_model_provider_id(
- db: Session,
- provider_id: Optional[str],
- *,
- user_id: Optional[str] = None,
- can_switch_model: Optional[bool] = None,
-) -> Optional[str]:
- """Return an allowed provider id for the current request.
-
- Disabled switch means "ignore the browser-provided override" rather than
- "fail the chat"; enabled switch validates against active chat providers.
- """
- pid = (provider_id or "").strip()
- if not pid:
- return None
-
- enabled = (
- can_switch_model
- if can_switch_model is not None
- else bool(user_id and user_can_switch_model(db, user_id))
- )
- if not enabled:
- return None
-
- provider = (
- db.query(ModelProvider)
- .filter(
- ModelProvider.provider_id == pid,
- ModelProvider.provider_type == "chat",
- ModelProvider.is_active == True, # noqa: E712
- )
- .first()
- )
- if provider is None:
- raise UserModelSelectionError("所选模型不可用,请刷新页面后重试。")
- return pid
-def user_can_switch_model(db: Session, user_id: str) -> bool:
- """Return whether the user may switch models in the chat input."""
- return bool(resolve_user_capabilities(db, user_id).get("can_switch_model"))
def resolve_effective_chat_model_name(
diff --git a/src/backend/core/models/selection_hook.py b/src/backend/core/models/selection_hook.py
new file mode 100644
index 00000000..05a1f860
--- /dev/null
+++ b/src/backend/core/models/selection_hook.py
@@ -0,0 +1,20 @@
+"""Framework-neutral model choice before a run's context is assembled."""
+from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage, Invocation
+
+
+def select_model(invocation: Invocation) -> Decision:
+ runtime = invocation.data.get("runtime", {})
+ if runtime.get("model_pinned"):
+ return Decision()
+ mode = runtime.get("chat_mode")
+ if mode not in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"):
+ mode = "fast"
+ return Decision.modify({"model_selection": {
+ "provider_id": str(runtime.get("model_provider_id") or ""), "mode": mode,
+ }})
+
+
+def model_selection_hook() -> HookSpec:
+ return HookSpec("model_selection", HookStage.BEFORE_RUN, select_model,
+ priority=10, failure_mode=FailureMode.OPEN,
+ mutable_fields=frozenset({"model_selection"}))
diff --git a/src/backend/core/llm/tool_call_identity.py b/src/backend/core/models/tool_call_identity.py
similarity index 100%
rename from src/backend/core/llm/tool_call_identity.py
rename to src/backend/core/models/tool_call_identity.py
diff --git a/src/backend/core/llm/tool_result_media.py b/src/backend/core/models/tool_result_media.py
similarity index 69%
rename from src/backend/core/llm/tool_result_media.py
rename to src/backend/core/models/tool_result_media.py
index 55b26383..00f067d4 100644
--- a/src/backend/core/llm/tool_result_media.py
+++ b/src/backend/core/models/tool_result_media.py
@@ -8,14 +8,13 @@
- 那个临时文件落在 **backend 容器**,而智能体的 shell 跑在**沙箱容器**里,模型
照着路径去读永远读不到——一条指向不存在位置的死链,还每调用一次泄漏一份文件。
- 图被换成文案就是静默降级:调用方已经按「这个模型能看图」注册了工具
- (见 :func:`core.vision.resolve_vision_mode`),到这一层再悄悄抽掉像素,
+ (见 :func:`core.models.vision.resolve_vision_mode`),到这一层再悄悄抽掉像素,
没有任何日志或报错,排障时完全看不见。
:class:`NoDiskToolMediaMixin` 堵掉落盘和静默降级,两条线都用。**图放在哪一条消息上
则按协议走各自的原生机制**:
-- **Responses**:折回 ``function_call_output`` 自己的 ``output``(
- :class:`ResponsesToolMediaMixin`)。该协议的函数输出本就能载图,codex 也是这么做的
+- **Responses**:由 AgentScope 原生写入 ``function_call_output`` 自己的 ``output``。该协议的函数输出本就能载图,codex 也是这么做的
(``FunctionCallOutputContentItem::InputImage``,见
``codex-rs/core/src/tools/handlers/view_image.rs``),图与 ``call_id`` 直接绑定,
多图并发天然不串。
@@ -99,7 +98,7 @@ def convert_tool_result_to_string(
def _is_sendable(self, block: DataBlock) -> bool:
"""线路能不能收下这份媒体。收不下要吵,但不能掀桌。
- 走到这里还收不下,说明上游的能力判定(``core.vision.resolve_vision_mode``)
+ 走到这里还收不下,说明上游的能力判定(``core.models.vision.resolve_vision_mode``)
与线路实际能力不一致——那是配置或代码的 bug,得有人去修,所以记 ``error``
级日志。但它不该连累这一轮对话:整个请求抛错会把智能体正在跑的任务直接打断,
而这次失败本身是可以如实告诉模型、让它自己决定怎么绕的。所以降级只有这一种
@@ -117,59 +116,3 @@ def _is_sendable(self, block: DataBlock) -> bool:
patterns or "纯文本",
)
return False
-
-
-class ResponsesToolMediaMixin(NoDiskToolMediaMixin):
- """把媒体折回它所属的那条 ``function_call_output``(Responses 协议)。
-
- 父类 formatter 会把媒体提升成紧跟其后的一条 user 消息。那条消息与产出它的调用
- 之间只剩位置关系,多图并发时要靠额外的编号文案才能对上号;Responses 的函数输出
- 本身就能载内容部件,折回去即可与 ``call_id`` 直接绑定。
- """
-
- async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
- # 逐条消息格式化后再折叠:父类对每条消息的处理彼此独立,逐条与整批结果一致,
- # 而逐条能保证「函数输出后面那条纯媒体 user 项」只可能是本条消息的提升产物,
- # 不会误收下一轮真实的用户消息。
- items: list[dict[str, Any]] = []
- for msg in msgs:
- # Concrete SDK formatter follows this mixin in each bound class MRO.
- formatted = await super().format([msg]) # type: ignore[safe-super]
- items.extend(_fold_media_into_function_output(formatted))
- return items
-
-
-def _fold_media_into_function_output(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
- folded: list[dict[str, Any]] = []
- for item in items:
- target = folded[-1] if folded else None
- if (
- target is not None
- and target.get("type") == "function_call_output"
- and _is_media_only(item)
- ):
- target["output"] = _as_output_parts(target.get("output")) + list(item["content"])
- continue
- folded.append(item)
- return folded
-
-
-def _is_media_only(item: dict[str, Any]) -> bool:
- """这条 user 项是不是上一条函数输出提升出来的纯媒体项。
-
- ``convert_tool_result_to_string`` 只交出 ``DataBlock``,不再带任何说明文案,
- 所以提升出来的项必然是「清一色非文本的内容部件」——据此判定,不去认父类内部
- 给这条消息起的名字。
- """
- content = item.get("content")
- if item.get("role") != "user" or not isinstance(content, list) or not content:
- return False
- return all(isinstance(part, dict) and part.get("type") != "input_text" for part in content)
-
-
-def _as_output_parts(output: Any) -> list[dict[str, Any]]:
- if isinstance(output, list):
- return list(output)
- if output:
- return [{"type": "input_text", "text": output}]
- return []
diff --git a/src/backend/core/llm/model_usage.py b/src/backend/core/models/usage.py
similarity index 98%
rename from src/backend/core/llm/model_usage.py
rename to src/backend/core/models/usage.py
index 5eea010c..df554b16 100644
--- a/src/backend/core/llm/model_usage.py
+++ b/src/backend/core/models/usage.py
@@ -168,7 +168,7 @@ def _usage_from_response(response: Any) -> AttemptUsage:
async def _record_safely(recorder: UsageRecorder, attempt: UsageAttempt) -> UsageAttempt | None:
from dataclasses import replace
- from core.services.desktop_observation_context import model_call_id
+ from core.harness.log_context import model_call_id
call_id = model_call_id.get()
if call_id:
attempt = replace(attempt, metadata={**attempt.metadata, "gateway_call_id": call_id})
@@ -287,7 +287,7 @@ async def measured(model_name: str, *args: Any, **kwargs: Any) -> Any:
if context is None:
return await original(model_name, *args, **kwargs)
- from core.services.desktop_observation_context import model_call_id
+ from core.harness.log_context import model_call_id
model_call_id.set("")
started = time.monotonic()
operation = str(model_name or getattr(model, "model", "") or "unknown")
diff --git a/src/backend/core/models/vision/__init__.py b/src/backend/core/models/vision/__init__.py
new file mode 100644
index 00000000..10c34df7
--- /dev/null
+++ b/src/backend/core/models/vision/__init__.py
@@ -0,0 +1,34 @@
+"""视觉桥(Vision Bridge):让纯文本主模型「看得见」图片。
+
+图片 → 多模态模型 → 结构化文字证据 → 注入主模型上下文。主模型全程只吃文本。
+
+对外入口:
+
+- :func:`core.models.vision.service.is_available` —— 当前是否配了可用的视觉模型
+- :func:`core.models.vision.service.get_vision_bridge` —— 识别单张/多张图
+- :func:`core.models.vision.render.render_evidence` —— 证据 → 可注入文本(带不可信输入围栏)
+- :func:`core.models.vision.service.model_supports_vision` —— 某个模型配置是否原生多模态
+- :func:`core.models.vision.service.resolve_vision_mode` —— 本轮模型看图方式:native / bridge / none
+"""
+
+from core.models.vision.render import render_evidence, render_many, render_unavailable
+from core.models.vision.schema import VisionEvidence, parse_evidence
+from core.models.vision.service import VISION_ROLE_KEY, VisionBridge, VisionMode, VisionResult, get_vision_bridge, is_available, model_supports_vision, resolve_vision_config, resolve_vision_mode, sniff_mime
+
+__all__ = [
+ "VISION_ROLE_KEY",
+ "VisionBridge",
+ "VisionEvidence",
+ "VisionMode",
+ "VisionResult",
+ "get_vision_bridge",
+ "is_available",
+ "model_supports_vision",
+ "parse_evidence",
+ "render_evidence",
+ "render_many",
+ "render_unavailable",
+ "resolve_vision_config",
+ "resolve_vision_mode",
+ "sniff_mime",
+]
diff --git a/src/backend/core/vision/attachments.py b/src/backend/core/models/vision/attachments.py
similarity index 83%
rename from src/backend/core/vision/attachments.py
rename to src/backend/core/models/vision/attachments.py
index 11c711b3..bd7a2d39 100644
--- a/src/backend/core/vision/attachments.py
+++ b/src/backend/core/models/vision/attachments.py
@@ -2,10 +2,10 @@
抽出来是因为有两个调用方,且**它们想要的时机不同**:
-- ``orchestration/streaming.py`` 在进入模型流之前先调它,好在识图这几秒里给前端发
+- ``core/llm/runtime/streaming.py`` 在进入模型流之前先调它,好在识图这几秒里给前端发
「图像理解中」的状态事件——识图是纯网络等待,不告诉用户的话,界面上只有一个笼统的
「深度拥抱中」在走秒。
-- ``core/llm/middlewares.py`` 的 FileContextMiddleware 在装配上下文时兜底调它(非流式
+- ``core/llm/context/attachment_hook.py`` 的附件钩子 在装配上下文时兜底调它(非流式
入口、渠道机器人等没走流式那条路的场景)。
所以这里只负责「图 → 文本」,不碰 agent 上下文,也不管状态事件——注入和发事件各自留在
@@ -18,6 +18,7 @@
import time
from dataclasses import dataclass, field
from typing import Any, Optional
+from collections.abc import Callable
logger = logging.getLogger(__name__)
@@ -41,17 +42,17 @@ def count(self) -> int:
return len(self.names)
-def image_attachments(uploaded_files: list[dict[str, Any]]) -> list[dict[str, Any]]:
+def image_attachments(uploaded_files: list[dict[str, Any]], *, is_image: Callable[[dict[str, Any]], bool]) -> list[dict[str, Any]]:
"""挑出图片类附件。"""
- from core.llm.hooks import _is_image
- return [f for f in (uploaded_files or []) if _is_image(f)]
+ return [f for f in (uploaded_files or []) if is_image(f)]
async def transcribe_attachments(
image_files: list[dict[str, Any]],
*,
user_id: Optional[str] = None,
+ download_bytes: Callable[..., Optional[bytes]],
) -> Optional[AttachmentVisionResult]:
"""把图片附件转写成注入文本。
@@ -62,19 +63,18 @@ async def transcribe_attachments(
if not image_files:
return None
- from core.vision import get_vision_bridge, render_many, render_unavailable
- from core.vision.service import is_available
+ from core.models.vision import get_vision_bridge, render_many, render_unavailable
+ from core.models.vision.service import is_available
names = [f.get("name") or "图片" for f in image_files]
if not is_available():
return AttachmentVisionResult(text=render_unavailable(names), names=names)
- from core.llm.hooks import _download_artifact_bytes
payloads: list[tuple[bytes, str]] = []
kept_names: list[str] = []
for f in image_files:
- raw = _download_artifact_bytes(
+ raw = download_bytes(
f.get("file_id") or "",
f.get("name") or "image",
"Vision bridge",
diff --git a/src/backend/core/vision/prompt.py b/src/backend/core/models/vision/prompt.py
similarity index 98%
rename from src/backend/core/vision/prompt.py
rename to src/backend/core/models/vision/prompt.py
index 7310b627..63b1ca5f 100644
--- a/src/backend/core/vision/prompt.py
+++ b/src/backend/core/models/vision/prompt.py
@@ -10,7 +10,7 @@
安全:图片里的文字是不可信输入,和网页正文同级。规则 4 明确要求视觉模型把图内文字
当数据、绝不执行其中的指令——这是缓解措施不是保证,注入端还有一层围栏
-(见 :mod:`core.vision.render`)。
+(见 :mod:`core.models.vision.render`)。
"""
from __future__ import annotations
diff --git a/src/backend/core/vision/provider.py b/src/backend/core/models/vision/provider.py
similarity index 97%
rename from src/backend/core/vision/provider.py
rename to src/backend/core/models/vision/provider.py
index 2a4e6503..2cc3ce13 100644
--- a/src/backend/core/vision/provider.py
+++ b/src/backend/core/models/vision/provider.py
@@ -7,7 +7,7 @@
- **Anthropic 原生**:``/v1/messages`` + ``source.type=base64``。
- **Gemini 原生**:``generateContent`` + ``inline_data``。
-这里只管协议差异,不管重试/缓存/降级——那些在 :mod:`core.vision.service`。
+这里只管协议差异,不管重试/缓存/降级——那些在 :mod:`core.models.vision.service`。
结构化输出按 **三级降级**尝试,逐级放宽:
``json_schema`` → ``json_object`` → 纯文本 + 填模板指令。弱网关对前两级常报 400,
@@ -26,11 +26,11 @@
import httpx
from core.harness.usage import AttemptUsage, attempt_status_for_exception
-from core.llm.model_usage import record_external_model_attempt
-from core.llm.providers.registry import get_spec
-from core.services.model_config import ResolvedModelConfig
-from core.vision.prompt import JSON_TEMPLATE_INSTRUCTION
-from core.vision.schema import VISION_JSON_SCHEMA
+from core.models.usage import record_external_model_attempt
+from core.models.providers.registry import get_spec
+from core.models.config import ResolvedModelConfig
+from core.models.vision.prompt import JSON_TEMPLATE_INSTRUCTION
+from core.models.vision.schema import VISION_JSON_SCHEMA
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/vision/render.py b/src/backend/core/models/vision/render.py
similarity index 98%
rename from src/backend/core/vision/render.py
rename to src/backend/core/models/vision/render.py
index 06e84d5e..411a1695 100644
--- a/src/backend/core/vision/render.py
+++ b/src/backend/core/models/vision/render.py
@@ -3,7 +3,7 @@
两个关注点:
- **围栏**。图里的文字是不可信外部输入,和网页正文同级——一张截图可以写着「忽略你
- 之前的指令」。视觉模型侧已经被要求把图当数据(见 :mod:`core.vision.prompt` 规则 4),
+ 之前的指令」。视觉模型侧已经被要求把图当数据(见 :mod:`core.models.vision.prompt` 规则 4),
这里再加一层显式围栏,让主模型也知道这段文字的来路。
- **预算**。完整证据(全文转写 + 全部版面区域)很长,无条件塞进上下文会挤掉真正的
对话内容。默认按 :data:`DEFAULT_MAX_CHARS` 截断,超出部分明确标注被截断,agent
@@ -14,7 +14,7 @@
from typing import Optional
-from core.vision.schema import VisionEvidence
+from core.models.vision.schema import VisionEvidence
DEFAULT_MAX_CHARS = 4000
BRIEF_MAX_CHARS = 1200
diff --git a/src/backend/core/vision/schema.py b/src/backend/core/models/vision/schema.py
similarity index 100%
rename from src/backend/core/vision/schema.py
rename to src/backend/core/models/vision/schema.py
diff --git a/src/backend/core/vision/service.py b/src/backend/core/models/vision/service.py
similarity index 96%
rename from src/backend/core/vision/service.py
rename to src/backend/core/models/vision/service.py
index 9c67b49e..899b22e6 100644
--- a/src/backend/core/vision/service.py
+++ b/src/backend/core/models/vision/service.py
@@ -1,8 +1,8 @@
"""视觉桥服务 —— 纯文本主模型的「眼睛」。
职责:解析 vision 角色配置 → 校验图片 → 调 provider → 校验结构 → 缓存 → 返回证据。
-provider 的协议差异在 :mod:`core.vision.provider`,注入用的文本渲染在
-:mod:`core.vision.render`。
+provider 的协议差异在 :mod:`core.models.vision.provider`,注入用的文本渲染在
+:mod:`core.models.vision.render`。
三个不能省的工程点:
@@ -25,10 +25,10 @@
from dataclasses import dataclass, field
from typing import Literal, Any, Optional
-from core.services.model_config import ModelConfigService, ResolvedModelConfig
-from core.vision.prompt import build_vision_prompt
-from core.vision.provider import VisionProvider
-from core.vision.schema import VisionEvidence, parse_evidence
+from core.models.config import ModelConfigService, ResolvedModelConfig
+from core.models.vision.prompt import build_vision_prompt
+from core.models.vision.provider import VisionProvider
+from core.models.vision.schema import VisionEvidence, parse_evidence
logger = logging.getLogger(__name__)
@@ -101,7 +101,7 @@ def model_supports_vision(cfg: Optional[ResolvedModelConfig]) -> bool:
以 provider 的 ``extra_config.supports_vision`` 显式开关为准,这是「模型能不能
看图」的唯一判定处。曾经还有一条靠匹配网关报错文案事后补救的路径,换个网关文案
就失效、还得先发一次注定失败的请求,而且补救手段是把图悄悄换成文字——已删除:
- 图一旦交给工具返回,就必须原样送达(见 ``core.llm.tool_result_media``)。
+ 图一旦交给工具返回,就必须原样送达(见 ``core.models.tool_result_media``)。
"""
if cfg is None:
return False
diff --git a/src/backend/core/myspace/duplicate_cleanup.py b/src/backend/core/myspace/duplicate_cleanup.py
deleted file mode 100644
index b0763178..00000000
--- a/src/backend/core/myspace/duplicate_cleanup.py
+++ /dev/null
@@ -1,63 +0,0 @@
-"""Explicit, journalled removal of all live records in duplicate personal paths."""
-from datetime import datetime, timezone
-from typing import Callable
-import sqlalchemy as sa
-
-
-def _table(connection):
- return sa.Table("artifacts", sa.MetaData(), autoload_with=connection)
-
-
-def duplicate_rows(connection):
- """Read-only preview. A duplicate group has no privileged 'winner'."""
- table = _table(connection)
- filters = [table.c.deleted_at.is_(None)]
- groups = (
- sa.select(table.c.user_id, table.c.user_folder_id, table.c.filename)
- .where(*filters)
- .group_by(table.c.user_id, table.c.user_folder_id, table.c.filename)
- .having(sa.func.count() > 1).subquery()
- )
- return [dict(row) for row in connection.execute(
- sa.select(table).join(groups, sa.and_(
- table.c.user_id == groups.c.user_id,
- table.c.user_folder_id.is_not_distinct_from(groups.c.user_folder_id),
- table.c.filename == groups.c.filename,
- )).where(*filters).order_by(table.c.artifact_id)
- ).mappings()]
-
-
-def clean_duplicates(connection, *, before_delete: Callable | None = None):
- """Caller owns the transaction and stops writers until mirror cleanup completes.
-
- Tombstones retain file identities so message backfill cannot resurrect them.
- Original storage objects are retained for recovery, never reused as live paths.
- """
- if connection.dialect.name == "postgresql":
- connection.execute(sa.text("LOCK TABLE artifacts IN SHARE ROW EXCLUSIVE MODE"))
- rows = duplicate_rows(connection)
- timestamp = datetime.now(timezone.utc)
- journal = {"deleted_at": timestamp.isoformat(), "rows": rows}
- if before_delete is not None:
- before_delete(journal) # Backup failure aborts before the first deletion.
- table = _table(connection)
- for offset in range(0, len(rows), 500):
- connection.execute(table.update().where(
- table.c.artifact_id.in_([r["artifact_id"] for r in rows[offset:offset+500]])
- ).values(deleted_at=timestamp, updated_at=timestamp))
- return journal
-
-
-def restore_cleanup(connection, journal):
- """Restore unmodified tombstones; caller must downgrade the unique indexes first."""
- table = _table(connection)
- marker = datetime.fromisoformat(journal["deleted_at"])
- for row in journal["rows"]:
- old_updated = row.get("updated_at")
- if isinstance(old_updated, str):
- old_updated = datetime.fromisoformat(old_updated)
- connection.execute(table.update().where(
- table.c.artifact_id == row["artifact_id"],
- table.c.deleted_at == marker,
- table.c.updated_at == marker,
- ).values(deleted_at=None, updated_at=old_updated))
diff --git a/src/backend/core/ontology/__init__.py b/src/backend/core/ontology/__init__.py
index a8e6eb03..b3cc274c 100644
--- a/src/backend/core/ontology/__init__.py
+++ b/src/backend/core/ontology/__init__.py
@@ -1,6 +1,7 @@
"""Ontology harness: Domain Pack validation, runtime selection, and gates."""
from core.ontology.schemas import OntologyPackDocument
-from core.ontology.validator import DomainPackValidator, OntologyGateDecision
+from core.ontology.validator import DomainPackValidator
+from core.ontology.validation_models import OntologyGateDecision
__all__ = ["DomainPackValidator", "OntologyGateDecision", "OntologyPackDocument"]
diff --git a/src/backend/core/ontology/audit_events.py b/src/backend/core/ontology/audit_events.py
new file mode 100644
index 00000000..284501f4
--- /dev/null
+++ b/src/backend/core/ontology/audit_events.py
@@ -0,0 +1,71 @@
+"""Persist ontology enforcement, activation and review events."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import uuid
+from typing import Any
+from core.db.repository import OntologyRepository
+
+def record_enforcement_event(data: dict[str, Any]) -> None:
+ """Persist an event from middleware/reviewer using an isolated session."""
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ try:
+ payload = dict(data)
+ payload.setdefault("event_id", f"ontoe_{uuid.uuid4().hex[:16]}")
+ OntologyRepository(db).create_event(payload)
+ except Exception:
+ db.rollback()
+ raise
+ finally:
+ db.close()
+
+
+def record_runtime_activation(
+ event: dict[str, Any],
+ runtime: dict[str, Any],
+ *,
+ user_id: str | None,
+ chat_id: str | None,
+) -> None:
+ """Persist one asset-driven workflow activation as append-only evidence."""
+
+ version_id = next(
+ (
+ pack.get("version_id")
+ for pack in runtime.get("packs", [])
+ if pack.get("pack_id") == event.get("pack_id")
+ ),
+ None,
+ )
+ record_enforcement_event(
+ {
+ "user_id": user_id,
+ "chat_id": chat_id,
+ "pack_id": event.get("pack_id"),
+ "version_id": version_id,
+ "rule_id": f"activation:{event.get('workflow_id')}",
+ "stage": "checkpoint",
+ "event_type": "runtime_activation",
+ "decision": "pass",
+ "mode": "enforce",
+ "target": f"{event.get('asset_kind')}:{event.get('asset_id')}",
+ "details": event,
+ }
+ )
+
+
+def record_review_run(data: dict[str, Any]) -> None:
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ try:
+ payload = dict(data)
+ payload.setdefault("review_id", f"ontor_{uuid.uuid4().hex[:16]}")
+ payload.setdefault("created_at", utc_now())
+ OntologyRepository(db).create_review(payload)
+ except Exception:
+ db.rollback()
+ raise
+ finally:
+ db.close()
diff --git a/src/backend/core/ontology/build_validator.py b/src/backend/core/ontology/build_validator.py
index 6b0831de..0b553776 100644
--- a/src/backend/core/ontology/build_validator.py
+++ b/src/backend/core/ontology/build_validator.py
@@ -9,7 +9,7 @@
from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin
from core.infra.exceptions import BadRequestError
from core.ontology.schemas import OntologyPackDocument
-from core.ontology.validator import activate_runtime_for_asset
+from core.ontology.runtime_pack import activate_runtime_for_asset
from sqlalchemy.orm import Session
@@ -65,7 +65,7 @@ def validate(
output_schema: dict[str, Any] | None = None,
tool_schemas: dict[str, dict[str, Any]] | None = None,
) -> BuildValidationReport:
- from core.services.ontology_service import OntologyService
+ from core.ontology.service import OntologyService
task = "\n".join(item for item in (name, description, instructions[:4000]) if item)
runtime = OntologyService(self.db).build_runtime(task=task)
@@ -343,7 +343,7 @@ def _validate_tool_contracts(
suggestions: list[str],
) -> None:
"""Check discovered tool inputs directly against active pack rules."""
- from core.services.ontology_service import OntologyService
+ from core.ontology.service import OntologyService
service = OntologyService(self.db)
for version in service.repo.get_active_versions():
diff --git a/src/backend/core/services/ontology_evolution_service.py b/src/backend/core/ontology/evolution.py
similarity index 87%
rename from src/backend/core/services/ontology_evolution_service.py
rename to src/backend/core/ontology/evolution.py
index b0d8cac5..d438370c 100644
--- a/src/backend/core/services/ontology_evolution_service.py
+++ b/src/backend/core/ontology/evolution.py
@@ -12,6 +12,7 @@
import re
import uuid
from collections import defaultdict
+from dataclasses import dataclass, field
from threading import Lock
from typing import Any
@@ -19,45 +20,63 @@
from core.infra.background import spawn
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from core.memory.sanitizer import sanitize
-from core.services.ontology_service import OntologyService
+from core.ontology.service import OntologyService
from sqlalchemy.orm import Session
_SECTIONS = {"term": "concepts", "relation": "relations", "constraint": "constraints"}
logger = logging.getLogger(__name__)
-_SCHEDULE_LOCK = Lock()
-_EVOLUTION_RUNNING = False
def schedule_ontology_evolution(*, user_id: str) -> bool:
"""Debounce the low-priority evidence prefilter on the current event loop."""
- global _EVOLUTION_RUNNING
- with _SCHEDULE_LOCK:
- if _EVOLUTION_RUNNING:
+ with process_state().SCHEDULE_LOCK:
+ if process_state().EVOLUTION_RUNNING:
return False
- _EVOLUTION_RUNNING = True
+ process_state().EVOLUTION_RUNNING = True
async def _run() -> None:
- global _EVOLUTION_RUNNING
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
try:
- await OntologyEvolutionService(db).generate_candidates(
- min_occurrences=3,
- limit=500,
- user_id=user_id or "system",
- )
+ await generate_candidates(min_occurrences=3, limit=500, user_id=user_id or "system")
except Exception: # noqa: BLE001 - a background learner must never affect delivery
logger.warning("ontology evolution background pass failed", exc_info=True)
finally:
- db.close()
- with _SCHEDULE_LOCK:
- _EVOLUTION_RUNNING = False
+ with process_state().SCHEDULE_LOCK:
+ process_state().EVOLUTION_RUNNING = False
spawn(_run(), name="ontology-evolution-prefilter")
return True
+async def generate_candidates(
+ *,
+ min_occurrences: int = 3,
+ limit: int = 500,
+ user_id: str = "system",
+ model_name: str | None = None,
+) -> dict[str, Any]:
+ """Turn recurring gate evidence into pending Domain Pack drafts.
+
+ The evidence is read and the drafts are written in two separate sessions;
+ the model call that proposes them in between holds no connection.
+ """
+ from core.db.threads import run_in_session
+
+ recurring, summaries = await run_in_session(
+ lambda db: OntologyEvolutionService(db).evidence_summaries(
+ min_occurrences=min_occurrences, limit=limit
+ )
+ )
+ if not summaries:
+ return {"created": 0, "prefiltered_groups": recurring, "items": []}
+ proposals = await OntologyEvolutionService._llm_proposals(
+ summaries, user_id=user_id, model_name=model_name
+ )
+ created = await run_in_session(
+ lambda db: OntologyEvolutionService(db).record_drafts(summaries, proposals)
+ )
+ return {"created": len(created), "prefiltered_groups": recurring, "items": created}
+
+
def _parse_json_array_lenient(text: str) -> list[Any] | None:
"""Parse a JSON array without coupling the service layer to orchestration."""
if not text:
@@ -84,14 +103,13 @@ def __init__(self, db: Session):
self.db = db
self.ontology = OntologyService(db)
- async def generate_candidates(
- self,
- *,
- min_occurrences: int = 3,
- limit: int = 500,
- user_id: str = "system",
- model_name: str | None = None,
- ) -> dict[str, Any]:
+ def evidence_summaries(
+ self, *, min_occurrences: int, limit: int
+ ) -> tuple[int, list[dict[str, Any]]]:
+ """Recurring gate decisions grouped by rule, with sanitized conversation samples.
+
+ Returns ``(recurring group count, sanitized summaries)``.
+ """
rows = (
self.db.query(OntologyEnforcementEvent)
.filter(
@@ -112,7 +130,7 @@ async def generate_candidates(
if key[0] and len(items) >= min_occurrences
]
if not recurring:
- return {"created": 0, "prefiltered_groups": 0, "items": []}
+ return 0, []
chat_ids = list(
dict.fromkeys(row.chat_id for _, items in recurring for row in items if row.chat_id)
@@ -159,14 +177,12 @@ async def generate_candidates(
for summary in summaries
if isinstance((item := self._sanitize_json_value(summary)), dict)
]
- if not sanitized_summaries:
- return {"created": 0, "prefiltered_groups": len(recurring), "items": []}
- summaries = sanitized_summaries
- proposals = await self._llm_proposals(
- sanitized_summaries,
- user_id=user_id,
- model_name=model_name,
- )
+ return len(recurring), sanitized_summaries
+
+ def record_drafts(
+ self, summaries: list[dict[str, Any]], proposals: list[dict[str, Any]]
+ ) -> list[str]:
+ """Store each summary's sanitized proposals (or a rule review) as pending drafts."""
created = []
for summary in summaries:
matching = [
@@ -209,11 +225,7 @@ async def generate_candidates(
}
)
created.append(row.draft_id)
- return {
- "created": len(created),
- "prefiltered_groups": len(recurring),
- "items": created,
- }
+ return created
def ingest_user_correction(
self,
@@ -231,7 +243,7 @@ def ingest_user_correction(
materialized or activated automatically.
"""
from core.db.models import ChatMessage
- from core.services.ontology_service import build_user_ontology_runtime
+ from core.ontology.runtime_policy import build_user_ontology_runtime
assistant = (
self.db.query(ChatMessage)
@@ -356,8 +368,8 @@ def materialize_approved_draft(self, draft_id: str):
self.db.commit()
return version
+ @staticmethod
async def _llm_proposals(
- self,
summaries: list[dict[str, Any]],
*,
user_id: str,
@@ -376,9 +388,9 @@ async def _llm_proposals(
'"evidence":["只引用输入证据"]}。'
)
try:
- from orchestration.subagents.ontology_reviewer import _run_text_agent
+ from core.harness.registry.model_services import run_text_agent
- text = await _run_text_agent(
+ text = await run_text_agent(
prompt,
user_id=user_id,
model_name=model_name,
@@ -481,3 +493,16 @@ def _sanitize_json_value(value: Any) -> Any | None:
return json.loads(result.text)
except (json.JSONDecodeError, TypeError):
return None
+
+
+@dataclass
+class EvolutionState:
+ SCHEDULE_LOCK: object = field(default_factory=lambda: Lock())
+ EVOLUTION_RUNNING: object = False
+
+
+_STATE = EvolutionState()
+
+
+def process_state() -> EvolutionState:
+ return _STATE
diff --git a/src/backend/core/ontology/host.py b/src/backend/core/ontology/host.py
new file mode 100644
index 00000000..bfa3c53e
--- /dev/null
+++ b/src/backend/core/ontology/host.py
@@ -0,0 +1,13 @@
+"""Install ontology governance at the application composition boundary."""
+
+def register():
+ from core.harness.registry.build_governance import register_build_governance
+ from core.ontology.build_validator import ensure_ontology_build_valid, OntologyBuildValidator
+ from core.ontology.service import OntologyService
+ from core.ontology.policy import resolve_plugin_import_ontology_validation
+ register_build_governance(
+ ensure=ensure_ontology_build_valid,
+ validate=lambda db, **kwargs: OntologyBuildValidator(db).validate(**kwargs),
+ tag_options=lambda db, asset_type: OntologyService(db).list_asset_tag_options(asset_type),
+ import_policy=resolve_plugin_import_ontology_validation,
+ )
diff --git a/src/backend/core/ontology/manual_review.py b/src/backend/core/ontology/manual_review.py
new file mode 100644
index 00000000..26a0322d
--- /dev/null
+++ b/src/backend/core/ontology/manual_review.py
@@ -0,0 +1,85 @@
+"""Build the payload for human ontology review cards."""
+from __future__ import annotations
+from typing import Any
+
+def dedupe_strings(items: list[str]) -> list[str]:
+ seen: set[str] = set()
+ output: list[str] = []
+ for item in items:
+ value = str(item or "").strip()
+ if not value or value in seen:
+ continue
+ seen.add(value)
+ output.append(value)
+ return output
+
+
+def build_manual_review(
+ *,
+ required: bool,
+ violations: list[dict[str, Any]],
+ affected_claims: list[dict[str, str]],
+ suggestions: list[str],
+) -> dict[str, Any]:
+ """Build the JSON payload rendered by the frontend's human-review cards."""
+ items: list[dict[str, str]] = []
+ seen: set[tuple[str, str]] = set()
+ for item in affected_claims:
+ quote = " ".join(str(item.get("quote") or "").split())[:240]
+ risk = " ".join(str(item.get("issue") or "").split())[:360]
+ rule_id = str(item.get("rule_id") or "未指定规则")[:160]
+ manual_check = " ".join(str(item.get("manual_check") or "").split())[:360]
+ key = (quote, rule_id)
+ if key in seen or (not quote and not risk):
+ continue
+ seen.add(key)
+ items.append(
+ {
+ "quote": quote or "答案中的相关领域结论",
+ "rule_id": rule_id,
+ "risk": risk or "委员会未能确认该结论满足领域契约。",
+ "manual_check": manual_check or "核对原始工具结果、适用前提和结论表述。",
+ }
+ )
+ if len(items) >= 8:
+ break
+
+ for violation in violations:
+ if len(items) >= 8:
+ break
+ rule_id = str(violation.get("rule_id") or "未指定规则")[:160]
+ reasons = ";".join(str(item) for item in violation.get("reasons") or [] if str(item))
+ key = ("", rule_id)
+ if key in seen:
+ continue
+ seen.add(key)
+ message = str(violation.get("message") or "领域约束未满足").strip()
+ items.append(
+ {
+ "quote": "整体证据链或输出结构",
+ "rule_id": rule_id,
+ "risk": f"{message}{f'({reasons})' if reasons else ''}"[:360],
+ "manual_check": "确认必需工具、引用证据和答案中的对应结论能够逐项追溯。",
+ }
+ )
+
+ if required and not items:
+ items.append(
+ {
+ "quote": "答案中的领域判断",
+ "rule_id": "ontology_review",
+ "risk": "委员会未能形成通过结论。",
+ "manual_check": "核对关键事实、推断、适用前提和原始证据。",
+ }
+ )
+ return {
+ "required": required,
+ "title": "领域本体人工复核",
+ "summary": (
+ "候选答案仍有未闭合的领域证据或判断,请在替换原文前完成复核。"
+ if required
+ else "本轮修订建议已结构化,可按需核对后采用。"
+ ),
+ "items": items,
+ "actions": [item[:500] for item in dedupe_strings(suggestions)[:6]],
+ }
diff --git a/src/backend/core/ontology/output_review_state.py b/src/backend/core/ontology/output_review_state.py
new file mode 100644
index 00000000..48c1402b
--- /dev/null
+++ b/src/backend/core/ontology/output_review_state.py
@@ -0,0 +1,60 @@
+"""Claim and complete the single final-answer review of a governance run."""
+from __future__ import annotations
+from typing import Any
+
+def requires_output_review(runtime: dict[str, Any]) -> bool:
+ """Whether an L-b/L-c review must run after the streamed draft."""
+ return bool(
+ runtime.get("enabled")
+ and (
+ runtime.get("review_level") != "none"
+ or any(
+ (rule.get("target") or {}).get("kind") == "output"
+ for pack in runtime.get("packs", [])
+ for rule in pack.get("constraints", [])
+ )
+ )
+ )
+
+
+def claim_output_review(runtime: dict[str, Any], *, owner: str) -> bool:
+ """Atomically claim the single final-answer review for one governance run."""
+ if not requires_output_review(runtime):
+ return False
+ state = runtime.setdefault(
+ "output_review",
+ {"status": "pending", "owner": None, "count": 0},
+ )
+ if state.get("status") in {"running", "completed"}:
+ return False
+ state.update({"status": "running", "owner": owner})
+ return True
+
+
+def complete_output_review(
+ runtime: dict[str, Any],
+ *,
+ owner: str,
+ verdict: str,
+ attempts: int = 1,
+) -> None:
+ """Mark the claimed review complete and retain its single-run evidence."""
+ state = runtime.setdefault("output_review", {})
+ if state.get("status") != "running" or state.get("owner") != owner:
+ return
+ count = int(state.get("count") or 0) + max(1, int(attempts or 1))
+ state.update(
+ {
+ "status": "completed",
+ "owner": owner,
+ "count": count,
+ "verdict": verdict,
+ }
+ )
+
+
+def release_output_review(runtime: dict[str, Any], *, owner: str) -> None:
+ """Release a failed claim so a safe retry can review the final answer."""
+ state = runtime.setdefault("output_review", {})
+ if state.get("status") == "running" and state.get("owner") == owner:
+ state.update({"status": "pending", "owner": None})
diff --git a/src/backend/core/services/ontology_policy.py b/src/backend/core/ontology/policy.py
similarity index 97%
rename from src/backend/core/services/ontology_policy.py
rename to src/backend/core/ontology/policy.py
index 0f8fc672..8df9a79a 100644
--- a/src/backend/core/services/ontology_policy.py
+++ b/src/backend/core/ontology/policy.py
@@ -7,7 +7,7 @@
from core.auth.capabilities import resolve_user_capabilities
from core.db.models import SystemConfig
from core.infra.exceptions import AccessDeniedError
-from core.services.user_service import UserService
+from core.auth.users import UserService
from sqlalchemy.orm import Session
FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY = "ontology.force_plugin_import_build_validation"
@@ -114,7 +114,7 @@ def set_plugin_import_build_validation_forced(
# The generic system-config service may have cached this row for another
# consumer. Keep both access paths coherent immediately after an admin save.
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
SystemConfigService.get_instance().invalidate_cache()
return enabled
diff --git a/src/backend/core/ontology/review_operations.py b/src/backend/core/ontology/review_operations.py
new file mode 100644
index 00000000..89d517df
--- /dev/null
+++ b/src/backend/core/ontology/review_operations.py
@@ -0,0 +1,102 @@
+"""Run one ontology review and revise the answer when it fails."""
+from __future__ import annotations
+from typing import Any
+from core.harness.registry.capability_runtime import child_scope
+from core.harness.registry.model_services import run_text_agent as _run_text_agent
+from core.infra.json_output import parse_json_lenient as _parse_json_lenient
+from core.infra.logging import get_logger
+from core.ontology import review_prompt
+from core.ontology import reviewer_state
+
+logger = get_logger(__name__)
+
+async def review_once(
+ perspective: str,
+ **kwargs,
+) -> dict[str, Any]:
+ prompt = review_prompt.review_prompt(
+ perspective=perspective,
+ task=kwargs["task"],
+ answer=kwargs["answer"],
+ runtime=kwargs["runtime"],
+ trace=kwargs["trace"],
+ citations=kwargs["citations"],
+ deterministic=kwargs["deterministic"],
+ )
+ try:
+ text = await _run_text_agent(
+ prompt,
+ user_id=kwargs["user_id"],
+ model_name=kwargs.get("model_name"),
+ model_provider_id=kwargs.get("model_provider_id"),
+ runtime=kwargs["runtime"],
+ capability_scope=child_scope(
+ kwargs.get("capability_scope", ""), "ontology-review", perspective
+ ),
+ )
+ obj = _parse_json_lenient(text)
+ if not isinstance(obj, dict) or obj.get("verdict") not in reviewer_state.VALID_VERDICTS:
+ return {
+ "verdict": "escalate",
+ "evidence": [],
+ "feedback": "评审输出无法解析,保守转人工。",
+ }
+ evidence = [str(item) for item in (obj.get("evidence") or []) if str(item).strip()]
+ affected_claims = []
+ for item in obj.get("affected_claims") or []:
+ if not isinstance(item, dict):
+ continue
+ claim = {
+ "quote": str(item.get("quote") or "").strip()[:500],
+ "rule_id": str(item.get("rule_id") or "").strip()[:160],
+ "issue": str(item.get("issue") or "").strip()[:500],
+ "manual_check": str(item.get("manual_check") or "").strip()[:500],
+ }
+ if claim["quote"] or claim["issue"]:
+ affected_claims.append(claim)
+ verdict = obj["verdict"]
+ if verdict == "pass" and not evidence:
+ verdict = "revise"
+ return {
+ "verdict": verdict,
+ "evidence": evidence,
+ "feedback": str(obj.get("feedback") or "").strip(),
+ "affected_claims": affected_claims,
+ }
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology-review] reviewer failed: %s", exc)
+ return {
+ "verdict": "escalate",
+ "evidence": [],
+ "feedback": f"评审委员不可用:{str(exc)[:160]}",
+ "affected_claims": [],
+ }
+
+
+async def revise_answer(
+ *,
+ task: str,
+ answer: str,
+ feedback: list[str],
+ user_id: str,
+ model_name: str | None,
+ model_provider_id: str | None,
+ runtime: dict[str, Any],
+ capability_scope: str = "",
+) -> str:
+ prompt = (
+ "你是本体校验后的答案修订员。严格根据评审意见修正答案,不新增没有证据的事实。"
+ "直接输出修订后的完整答案,不要解释修订过程。\n\n"
+ f"用户任务:\n{task}\n\n原答案:\n{answer}\n\n评审意见:\n"
+ + "\n".join(f"- {item}" for item in feedback if item)
+ )
+ return (
+ await _run_text_agent(
+ prompt,
+ user_id=user_id,
+ model_name=model_name,
+ model_provider_id=model_provider_id,
+ runtime=runtime,
+ capability_scope=child_scope(capability_scope, "ontology-model", "revise"),
+ )
+ ).strip()
diff --git a/src/backend/core/ontology/review_prompt.py b/src/backend/core/ontology/review_prompt.py
new file mode 100644
index 00000000..c3856ffc
--- /dev/null
+++ b/src/backend/core/ontology/review_prompt.py
@@ -0,0 +1,99 @@
+"""Build the reviewer prompt from bounded evidence."""
+from __future__ import annotations
+import json
+from typing import Any
+
+def tail_with_json_budget(items: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]:
+ """Retain the newest evidence without splitting a serialized trace item."""
+ selected: list[dict[str, Any]] = []
+ used = 2
+ for item in reversed(items):
+ serialized = json.dumps(item, ensure_ascii=False, default=str)
+ cost = len(serialized) + (1 if selected else 0)
+ if selected and used + cost > max_chars:
+ break
+ if not selected and used + cost > max_chars:
+ selected.append(
+ {
+ "truncated": True,
+ "preview": serialized[: max(0, max_chars - 64)],
+ }
+ )
+ break
+ selected.append(item)
+ used += cost
+ selected.reverse()
+ return selected
+
+
+def review_evidence_payload(
+ trace: list[dict[str, Any]], citations: list[dict[str, Any]]
+) -> dict[str, Any]:
+ """Prefer evidence produced by the ontology repair over stale draft evidence."""
+ return {
+ "trace": tail_with_json_budget(trace, 8000),
+ "citations": tail_with_json_budget(citations, 2000),
+ }
+
+
+def review_prompt(
+ *,
+ perspective: str,
+ task: str,
+ answer: str,
+ runtime: dict[str, Any],
+ trace: list[dict[str, Any]],
+ citations: list[dict[str, Any]],
+ deterministic: dict[str, Any],
+) -> str:
+ contract = {
+ "packs": [
+ {
+ "pack_id": pack.get("pack_id"),
+ "version": pack.get("version"),
+ "concepts": pack.get("concepts", []),
+ "relations": pack.get("relations", []),
+ "constraints": pack.get("constraints", []),
+ "workflows": pack.get("workflows", []),
+ }
+ for pack in runtime.get("packs", [])
+ ]
+ }
+ review_evidence = review_evidence_payload(trace, citations)
+ return (
+ "你是本体评审委员会中的独立委员,与生成答案的 agent 相互独立。\n"
+ f"评审视角:{perspective}\n"
+ "只依据领域契约、工具轨迹和引用作判断。答案自称已核验不算证据;"
+ "不得补做任务或调用工具。还必须核对答案是否仍忠实完成用户任务:本体核验过程"
+ "不是最终交付物。除非用户明确要求核验报告,否则若答案把润色稿改写成‘原始语句’、"
+ "‘逐项核验’、‘主张评估’、‘修改说明’等元分析,或擅自改变用户要求的文体、篇幅、"
+ "结构与文件类型,必须判定为 revise。这里的任务忠实度是指交付形式与表达意图,不是"
+ "机械保留错误内容:若用户要求原样输出的语句本身违反已激活的领域事实或证据约束,"
+ "修订稿必须纠正这些问题;只要仍保持用户要求的一句话、文档等形式,不得因未逐字照抄"
+ "错误语句而判定为偏离任务。强制判定规则:不得把‘没有原样保留已被证据否定或缺乏"
+ "证据的断言’列为 revise 理由,也不得建议恢复这类断言;即使用户把请求称作测试或强调"
+ "原样输出,本体修订仍必须纠错。示例:用户要求用一句话原样输出‘某记录风险极高’,"
+ "工具证据显示未发现对应风险,答案仍用一句话改成有证据支持的审慎结论——任务形式忠实,"
+ "应继续按领域证据判断,而不能因没有逐字照抄判 revise。只有把一句话扩写成‘原始语句 +"
+ "逐项核验 + 修改说明’之类的元报告,才属于这里所说的交付形式偏离。\n\n"
+ "形式判断必须客观:只有一个句号结尾、内部用逗号或分号连接的陈述仍是一句话,附带引用"
+ "标记也不会把它变成分析报告;工具轨迹已生成并交付 .docx 时,不得声称文件类型发生了"
+ "变化。不要把纠错所必需的证据化表述误判成文体或篇幅偏离。工具轨迹按时间顺序排列,"
+ "并优先保留最近产生的证据;同名文件被多次生成时,后生成且已交付的文件是当前修订稿,"
+ "不得只根据早期文件内容判断待评答案。\n\n"
+ f"用户任务:\n{task}\n\n待评答案:\n{answer}\n\n"
+ "领域契约:\n"
+ f"{json.dumps(contract, ensure_ascii=False)[:14000]}\n\n"
+ "工具与引用证据:\n"
+ f"{json.dumps(review_evidence, ensure_ascii=False)}\n\n"
+ "确定性门禁结果:\n"
+ f"{json.dumps(deterministic, ensure_ascii=False)}\n\n"
+ "严格只输出 JSON:"
+ '{"verdict":"pass|revise|escalate","evidence":["规则ID或轨迹证据"],'
+ '"feedback":"具体问题与可执行修改要求",'
+ '"affected_claims":[{"quote":"答案中的原句或段落摘要",'
+ '"rule_id":"规则ID","issue":"可能违反本体的原因",'
+ '"manual_check":"人工需要核对的证据或判断"}]}。'
+ "只有所有高风险约束有证据支持时才 pass;可修正文案时 revise;"
+ "缺关键证据、存在冲突或需人工决策时 escalate。"
+ )
diff --git a/src/backend/core/ontology/reviewer.py b/src/backend/core/ontology/reviewer.py
new file mode 100644
index 00000000..ddac9768
--- /dev/null
+++ b/src/backend/core/ontology/reviewer.py
@@ -0,0 +1,319 @@
+"""Review an answer against the ontology, repair it, and audit the outcome."""
+from __future__ import annotations
+import asyncio
+import time
+from collections import Counter
+from typing import Any
+from core.ontology.revision import is_substantive_revision
+from core.ontology.runtime_gates import evaluate_output
+from core.infra.logging import get_logger
+from core.ontology import manual_review
+from core.ontology import review_operations
+from core.ontology import reviewer_state
+
+logger = get_logger(__name__)
+
+async def review_ontology_output(
+ *,
+ task: str,
+ answer: str,
+ runtime: dict[str, Any],
+ trace: list[dict[str, Any]],
+ citations: list[dict[str, Any]],
+ user_id: str,
+ chat_id: str | None,
+ model_name: str | None,
+ model_provider_id: str | None = None,
+ trace_complete: bool = True,
+ remediate: reviewer_state.RemediationCallback | None = None,
+ max_repair_attempts: int = reviewer_state.DEFAULT_MAX_REPAIR_ATTEMPTS,
+ capability_scope: str = "",
+) -> dict[str, Any]:
+ """Review one answer, repair it with the originating agent, and audit the session."""
+ started = time.monotonic()
+ original_answer = answer
+ current_answer = answer
+ max_repair_attempts = max(0, min(int(max_repair_attempts), 1))
+ repair_attempts = 0
+ review_rounds: list[dict[str, Any]] = []
+ all_feedback: list[str] = []
+ all_evidence: list[str] = []
+ final_affected_claims: list[dict[str, str]] = []
+ final_reviewers: list[dict[str, Any]] = []
+ level = runtime.get("review_level", "none")
+ initial_trace_len = len(trace)
+ initial_citation_len = len(citations)
+
+ def _deterministic_result() -> tuple[Any, dict[str, Any]]:
+ completed_tools = {
+ str(item.get("tool_name"))
+ for item in trace
+ if item.get("type") == "tool_result" and item.get("tool_name")
+ }
+ decision = evaluate_output(
+ runtime,
+ answer=current_answer,
+ citations=citations,
+ completed_tools=completed_tools if trace_complete else None,
+ )
+ return decision, {
+ "allowed": decision.allowed,
+ "decision": decision.decision,
+ "violations": decision.violations,
+ "suggestions": decision.suggestions,
+ }
+
+ deterministic, deterministic_dict = _deterministic_result()
+ if level == "none" and deterministic.allowed:
+ return {
+ "verdict": "pass",
+ "answer": current_answer,
+ "reviewers": [],
+ "evidence": [],
+ "feedback": [],
+ "violations": [],
+ "affected_claims": [],
+ "manual_review": manual_review.build_manual_review(
+ required=False, violations=[], affected_claims=[], suggestions=[]
+ ),
+ "latency_ms": round((time.monotonic() - started) * 1000),
+ "revised": False,
+ "annotated": False,
+ "repair_attempts": 0,
+ "attempts": 0,
+ "review_rounds": [],
+ "new_tools": [],
+ "new_citation_count": 0,
+ }
+ effective_level = "checkpoint" if level == "none" else level
+
+ # A deterministic evidence gap is repaired before the sole committee round,
+ # so the committee evaluates the newly retrieved evidence instead of merely
+ # rephrasing the first draft.
+ if not deterministic.allowed and remediate is not None and max_repair_attempts:
+ repair_attempts = 1
+ try:
+ repaired = await remediate(
+ {
+ "attempt": 1,
+ "source": "deterministic",
+ "original_task": task,
+ "answer": current_answer,
+ "violations": deterministic.violations,
+ "suggestions": deterministic.suggestions,
+ "feedback": [],
+ "citations": list(citations),
+ }
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology-review] tool remediation failed: %s", exc)
+ all_feedback.append(f"自动补充证据失败:{str(exc)[:200]}")
+ repaired = ""
+ if is_substantive_revision(repaired):
+ current_answer = repaired.strip()
+ deterministic, deterministic_dict = _deterministic_result()
+ elif repaired.strip():
+ all_feedback.append("自动修订未生成完整正文,已保留原文。")
+
+ common = {
+ "capability_scope": capability_scope,
+ "task": task,
+ "answer": current_answer,
+ "runtime": runtime,
+ "trace": trace,
+ "citations": citations,
+ "deterministic": deterministic_dict,
+ "user_id": user_id,
+ "model_name": model_name,
+ "model_provider_id": model_provider_id,
+ }
+ if effective_level == "committee":
+ committee_size = max(
+ (
+ int(pack.get("config", {}).get("committee_size", 3))
+ for pack in runtime.get("packs", [])
+ ),
+ default=3,
+ )
+ perspectives = [
+ reviewer_state.PERSPECTIVES[index % len(reviewer_state.PERSPECTIVES)] + f"(席位 {index + 1})"
+ for index in range(committee_size)
+ ]
+ reviewers = await asyncio.gather(
+ *(review_operations.review_once(perspective, **common) for perspective in perspectives)
+ )
+ counts = Counter(item["verdict"] for item in reviewers)
+ verdict, votes = counts.most_common(1)[0]
+ if votes <= committee_size // 2:
+ verdict = "escalate"
+ else:
+ reviewers = [await review_operations.review_once(reviewer_state.PERSPECTIVES[0], **common)]
+ verdict = reviewers[0]["verdict"]
+
+ if not deterministic.allowed and verdict == "pass":
+ verdict = "revise"
+ round_feedback = [str(item.get("feedback") or "") for item in reviewers]
+ round_feedback.extend(deterministic.suggestions)
+ round_evidence = [
+ str(item) for reviewer in reviewers for item in reviewer.get("evidence", []) if str(item)
+ ]
+ round_claims = [
+ claim
+ for reviewer in reviewers
+ for claim in reviewer.get("affected_claims", [])
+ if isinstance(claim, dict)
+ ]
+ all_feedback.extend(round_feedback)
+ all_evidence.extend(round_evidence)
+ final_affected_claims = round_claims
+ final_reviewers = reviewers
+ review_rounds.append(
+ {
+ "round": 1,
+ "verdict": verdict,
+ "deterministic": deterministic_dict,
+ "reviewers": reviewers,
+ }
+ )
+
+ # A committee wording/evidence revision gets one tools-enabled continuation,
+ # but is deliberately not sent through a second committee round.
+ if verdict == "revise" and repair_attempts == 0 and max_repair_attempts:
+ repair_attempts = 1
+ try:
+ if remediate is not None:
+ revised = await remediate(
+ {
+ "attempt": 1,
+ "source": "committee",
+ "original_task": task,
+ "answer": current_answer,
+ "violations": deterministic.violations,
+ "suggestions": deterministic.suggestions,
+ "feedback": manual_review.dedupe_strings(round_feedback),
+ "affected_claims": round_claims,
+ "citations": list(citations),
+ }
+ )
+ elif deterministic.allowed:
+ revised = await review_operations.revise_answer(
+ task=task,
+ answer=current_answer,
+ feedback=manual_review.dedupe_strings(round_feedback),
+ user_id=user_id,
+ model_name=model_name,
+ model_provider_id=model_provider_id,
+ runtime=runtime,
+ capability_scope=capability_scope,
+ )
+ else:
+ revised = ""
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology-review] committee remediation failed: %s", exc)
+ all_feedback.append(f"自动修订失败:{str(exc)[:200]}")
+ revised = ""
+ if is_substantive_revision(revised):
+ current_answer = revised.strip()
+ deterministic, _ = _deterministic_result()
+ elif revised.strip():
+ all_feedback.append("自动修订未生成完整正文,已保留原文。")
+
+ final_violations = list(deterministic.violations)
+ final_suggestions = list(deterministic.suggestions)
+ annotated = verdict == "escalate" or not deterministic.allowed
+ review_card = manual_review.build_manual_review(
+ required=annotated,
+ violations=final_violations,
+ affected_claims=final_affected_claims,
+ suggestions=final_suggestions or all_feedback,
+ )
+ new_tools = manual_review.dedupe_strings(
+ [
+ str(item.get("tool_name") or "")
+ for item in trace[initial_trace_len:]
+ if item.get("type") == "tool_result"
+ ]
+ )
+ evidence = manual_review.dedupe_strings(all_evidence)
+ feedback = manual_review.dedupe_strings(all_feedback + final_suggestions)
+ latency_ms = round((time.monotonic() - started) * 1000)
+ try:
+ from core.ontology.audit_events import record_enforcement_event, record_review_run
+
+ review_payload = {
+ "user_id": user_id or None,
+ "chat_id": chat_id,
+ "pack_version_ids": runtime.get("version_ids", []),
+ "level": "checkpoint" if level == "none" else level,
+ "subject_type": "final_answer",
+ "verdict": verdict,
+ "evidence": evidence,
+ "feedback": "\n".join(feedback),
+ "reviewers": [
+ {**reviewer, "round": round_info["round"]}
+ for round_info in review_rounds
+ for reviewer in round_info["reviewers"]
+ ],
+ "latency_ms": latency_ms,
+ }
+ audit_tasks = [asyncio.to_thread(record_review_run, review_payload)]
+ for pack in runtime.get("packs", []):
+ audit_tasks.append(
+ asyncio.to_thread(
+ record_enforcement_event,
+ {
+ "user_id": user_id or None,
+ "chat_id": chat_id,
+ "pack_id": pack.get("pack_id"),
+ "version_id": pack.get("version_id"),
+ "rule_id": (
+ deterministic.violations[0].get("rule_id")
+ if deterministic.violations
+ else f"review:{level}"
+ ),
+ "stage": "output",
+ "event_type": "output_review",
+ "decision": verdict,
+ "mode": "enforce",
+ "target": "final_answer",
+ "latency_ms": latency_ms,
+ "details": {
+ "governance_run_id": runtime.get("governance_run_id"),
+ "review_owner": (runtime.get("output_review") or {}).get("owner"),
+ "violations": deterministic.violations,
+ "evidence": evidence,
+ "feedback": review_payload["feedback"],
+ "repair_attempts": repair_attempts,
+ "review_rounds": len(review_rounds),
+ },
+ },
+ )
+ )
+ await asyncio.gather(*audit_tasks)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology-review] audit persistence failed: %s", exc)
+ if verdict in {"revise", "escalate"} and chat_id:
+ try:
+ from core.ontology.evolution import schedule_ontology_evolution
+
+ schedule_ontology_evolution(user_id=user_id or "system")
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology-review] evolution scheduling failed: %s", exc)
+ return {
+ "verdict": verdict,
+ "answer": current_answer,
+ "reviewers": final_reviewers,
+ "evidence": evidence,
+ "feedback": feedback,
+ "violations": final_violations,
+ "affected_claims": final_affected_claims,
+ "manual_review": review_card,
+ "latency_ms": latency_ms,
+ "revised": current_answer != original_answer,
+ "annotated": annotated,
+ "repair_attempts": repair_attempts,
+ "attempts": max(1, len(review_rounds)),
+ "review_rounds": review_rounds,
+ "new_tools": new_tools,
+ "new_citation_count": max(0, len(citations) - initial_citation_len),
+ }
diff --git a/src/backend/core/ontology/reviewer_state.py b/src/backend/core/ontology/reviewer_state.py
new file mode 100644
index 00000000..3f9c979b
--- /dev/null
+++ b/src/backend/core/ontology/reviewer_state.py
@@ -0,0 +1,19 @@
+"""Verdicts, perspectives and repair limits for ontology review."""
+from __future__ import annotations
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+VALID_VERDICTS = {"pass", "revise", "escalate"}
+
+
+DEFAULT_MAX_REPAIR_ATTEMPTS = 1
+
+
+PERSPECTIVES = (
+ "领域合规委员:逐条核对术语、关系、禁用条件和工作流约束。",
+ "证据委员:只认可工具轨迹与引用中可追溯的证据,严查无证据断言。",
+ "风险委员:关注高风险误导、遗漏前提、越权建议和不可逆影响。",
+)
+
+
+RemediationCallback = Callable[[dict[str, Any]], Awaitable[str]]
diff --git a/src/backend/core/ontology/runtime_gates.py b/src/backend/core/ontology/runtime_gates.py
new file mode 100644
index 00000000..2e368749
--- /dev/null
+++ b/src/backend/core/ontology/runtime_gates.py
@@ -0,0 +1,157 @@
+"""Deterministic ontology checks on tool calls and outputs."""
+from __future__ import annotations
+from typing import Any, Iterable
+from jsonschema import Draft202012Validator, ValidationError
+from core.ontology import validation_models
+
+def evaluate_tool_call(
+ runtime: dict[str, Any],
+ *,
+ tool_name: str,
+ tool_input: dict[str, Any],
+ completed_tools: Iterable[str] = (),
+) -> validation_models.OntologyGateDecision:
+ """Apply zero-LLM L-a validation to one tool call."""
+ if not runtime.get("enabled"):
+ return validation_models.OntologyGateDecision(True, "pass")
+ completed = set(completed_tools)
+ violations: list[dict[str, Any]] = []
+ suggestions: list[str] = []
+ matched: list[str] = []
+ enforced = False
+
+ for pack in runtime.get("packs", []):
+ for workflow in pack.get("workflows", []):
+ if tool_name in workflow.get("forbidden_tools", []):
+ violations.append(
+ {
+ "rule_id": f"workflow:{workflow['id']}:forbidden",
+ "message": f"工作流 {workflow['name']} 禁止调用工具 {tool_name}",
+ "mode": "enforce",
+ }
+ )
+ enforced = True
+ for rule in pack.get("constraints", []):
+ target = rule.get("target") or {}
+ if target.get("tool") != tool_name:
+ continue
+ matched.append(rule["id"])
+ missing_prerequisites = [
+ name for name in rule.get("prerequisite_tools", []) if name not in completed
+ ]
+ rule_errors: list[str] = []
+ if missing_prerequisites:
+ rule_errors.append("缺少前置工具调用: " + ", ".join(missing_prerequisites))
+ try:
+ if target.get("kind") == "tool":
+ Draft202012Validator(rule.get("schema") or {}).validate(tool_input)
+ elif target.get("kind") == "tool_parameter":
+ parameter = target.get("parameter")
+ Draft202012Validator(rule.get("schema") or {}).validate(
+ tool_input.get(parameter)
+ )
+ except ValidationError as exc:
+ rule_errors.append(exc.message)
+ if rule_errors:
+ violations.append(
+ {
+ "pack_id": pack.get("pack_id"),
+ "rule_id": rule["id"],
+ "message": rule["message"],
+ "reasons": rule_errors,
+ "mode": rule.get("mode", "log"),
+ "risk": rule.get("risk", "low"),
+ }
+ )
+ if rule.get("suggestion"):
+ suggestions.append(rule["suggestion"])
+ if rule.get("mode") == "enforce":
+ enforced = True
+
+ if not violations:
+ return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched)
+ return validation_models.OntologyGateDecision(
+ allowed=not enforced,
+ decision="deny" if enforced else "log",
+ violations=violations,
+ suggestions=suggestions,
+ matched_rule_ids=matched,
+ )
+
+
+def evaluate_output(
+ runtime: dict[str, Any],
+ *,
+ answer: str,
+ citations: Iterable[dict[str, Any]] = (),
+ completed_tools: Iterable[str] | None = (),
+) -> validation_models.OntologyGateDecision:
+ """Apply deterministic output constraints before any LLM reviewer."""
+ if not runtime.get("enabled"):
+ return validation_models.OntologyGateDecision(True, "pass")
+ citation_items = list(citations)
+ # ``None`` means the caller cannot observe a complete tool trace (the
+ # legacy non-streaming reply path). Output and citation rules still run,
+ # but required-tool rules must not produce a false omission claim.
+ completed = set(completed_tools) if completed_tools is not None else None
+ violations: list[dict[str, Any]] = []
+ matched: list[str] = []
+ suggestions: list[str] = []
+ enforced = False
+ for pack in runtime.get("packs", []):
+ if completed is not None:
+ for workflow in pack.get("workflows", []):
+ missing_tools = sorted(set(workflow.get("required_tools", [])) - completed)
+ if missing_tools:
+ violations.append(
+ {
+ "pack_id": pack.get("pack_id"),
+ "rule_id": f"workflow:{workflow.get('id')}:required",
+ "message": "领域工作流缺少必需的证据工具调用。",
+ "reasons": ["未调用: " + ", ".join(missing_tools)],
+ "mode": "enforce",
+ "risk": workflow.get("risk", "low"),
+ }
+ )
+ enforced = True
+ output_tags = {
+ tag for workflow in pack.get("workflows", []) for tag in workflow.get("output_tags", [])
+ }
+ for rule in pack.get("constraints", []):
+ target = rule.get("target") or {}
+ if target.get("kind") != "output":
+ continue
+ if output_tags and target.get("output_tag") not in output_tags:
+ continue
+ matched.append(rule["id"])
+ reasons: list[str] = []
+ if rule.get("requires_citations") and not citation_items:
+ reasons.append("输出缺少可追溯引用证据")
+ try:
+ Draft202012Validator(rule.get("schema") or {}).validate(answer)
+ except ValidationError as exc:
+ reasons.append(exc.message)
+ if reasons:
+ violations.append(
+ {
+ "pack_id": pack.get("pack_id"),
+ "rule_id": rule["id"],
+ "message": rule["message"],
+ "reasons": reasons,
+ "mode": rule.get("mode", "log"),
+ "risk": rule.get("risk", "low"),
+ }
+ )
+ if rule.get("suggestion"):
+ suggestions.append(rule["suggestion"])
+ if rule.get("mode") == "enforce":
+ enforced = True
+ if not violations:
+ return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched)
+ return validation_models.OntologyGateDecision(
+ allowed=not enforced,
+ decision="deny" if enforced else "log",
+ violations=violations,
+ suggestions=suggestions,
+ matched_rule_ids=matched,
+ )
diff --git a/src/backend/core/ontology/runtime_pack.py b/src/backend/core/ontology/runtime_pack.py
new file mode 100644
index 00000000..230c50e2
--- /dev/null
+++ b/src/backend/core/ontology/runtime_pack.py
@@ -0,0 +1,357 @@
+"""Compile active ontology packs into the runtime policy for a request."""
+from __future__ import annotations
+import json
+import re
+import uuid
+from typing import Any, Iterable
+from core.ontology.schemas import Constraint, OntologyPackDocument, Workflow
+
+
+REVIEW_RANK = {"none": 0, "checkpoint": 1, "committee": 2}
+
+
+def select_workflows(document: OntologyPackDocument, task: str) -> list[Workflow]:
+ """Select workflows by case-insensitive lexical trigger match."""
+ normalized = task.casefold()
+ return [
+ workflow
+ for workflow in document.workflows
+ if any(trigger.casefold() in normalized for trigger in workflow.triggers)
+ ]
+
+
+def workflow_constraints(
+ document: OntologyPackDocument,
+ workflows: list[Workflow],
+) -> list[Constraint]:
+ workflow_tools = {
+ tool_name
+ for workflow in workflows
+ for tool_name in (workflow.required_tools + workflow.forbidden_tools)
+ }
+ workflow_output_tags = {tag for workflow in workflows for tag in workflow.output_tags}
+ return [
+ rule
+ for rule in document.constraints
+ if rule.enabled
+ and (
+ rule.target.tool in workflow_tools
+ if rule.target.kind in {"tool", "tool_parameter"}
+ else rule.target.output_tag in workflow_output_tags
+ )
+ ]
+
+
+def runtime_concepts(
+ document: OntologyPackDocument,
+ task: str,
+ constraints: list[Constraint],
+) -> list[dict[str, Any]]:
+ """Select task concepts while always retaining concepts used by active rules."""
+ by_id = {item.id: item for item in document.concepts}
+ selected: list[Any] = []
+ selected_ids: set[str] = set()
+
+ def add_with_parents(concept_id: str | None) -> None:
+ chain: list[Any] = []
+ current = by_id.get(concept_id or "")
+ while current is not None and current.id not in selected_ids:
+ chain.append(current)
+ current = by_id.get(current.parent_id or "")
+ for item in reversed(chain):
+ if len(selected) >= document.config.max_concepts:
+ return
+ if item.id not in selected_ids:
+ selected.append(item)
+ selected_ids.add(item.id)
+
+ for rule in constraints:
+ add_with_parents(rule.concept_id)
+ for item in select_concepts(document, task, limit=document.config.max_concepts):
+ add_with_parents(item["id"])
+ return [item.model_dump() for item in selected[: document.config.max_concepts]]
+
+
+def compile_pack_runtime(
+ document: OntologyPackDocument,
+ task: str,
+ workflows: list[Workflow],
+) -> dict[str, Any]:
+ constraints = workflow_constraints(document, workflows)
+ concepts = runtime_concepts(document, task, constraints)
+ selected_concept_ids = {item["id"] for item in concepts}
+ relations = [
+ relation.model_dump()
+ for relation in document.relations
+ if relation.subject in selected_concept_ids and relation.object in selected_concept_ids
+ ]
+ return {
+ "pack_id": document.pack_id,
+ "version": document.version,
+ "name": document.name,
+ "domain": document.domain,
+ "description": document.description,
+ "config": document.config.model_dump(),
+ "concepts": concepts,
+ "relations": relations,
+ "constraints": [item.model_dump(by_alias=True) for item in constraints],
+ "workflows": [item.model_dump() for item in workflows],
+ }
+
+
+def select_concepts(
+ document: OntologyPackDocument,
+ task: str,
+ *,
+ limit: int | None = None,
+) -> list[dict[str, Any]]:
+ """Select only task-relevant concepts; fall back to tagged roots."""
+ normalized = task.casefold()
+ # English words can be tokenized by boundaries. Chinese text cannot be
+ # treated as one giant ``[\u4e00-\u9fff]+`` token: doing so made a task such
+ # as “数据质量分析” match no concept at all. Concept names and aliases are
+ # therefore also checked directly as substrings of the task.
+ terms = set(re.findall(r"[a-z0-9_]+", normalized))
+ scored: list[tuple[int, Any]] = []
+ for concept in document.concepts:
+ labels = [concept.id, concept.name, *concept.aliases, *concept.tags]
+ haystack = " ".join([*labels, concept.definition]).casefold()
+ score = sum(1 for term in terms if len(term) >= 2 and term in haystack)
+ score += sum(
+ 3 for label in labels if len(label.strip()) >= 2 and label.casefold() in normalized
+ )
+ if score or not concept.parent_id:
+ scored.append((score, concept))
+ max_items = limit or document.config.max_concepts
+ scored.sort(key=lambda item: (-item[0], item[1].id))
+ selected = [item for _, item in scored[:max_items]]
+ by_id = {item.id: item for item in document.concepts}
+ selected_ids = {item.id for item in selected}
+ # Preserve the hierarchy for every selected concept when the budget allows.
+ for concept in list(selected):
+ parent_id = concept.parent_id
+ while parent_id and len(selected) < max_items:
+ if parent_id not in selected_ids and parent_id in by_id:
+ selected.append(by_id[parent_id])
+ selected_ids.add(parent_id)
+ parent_id = by_id.get(parent_id).parent_id if parent_id in by_id else None
+ return [item.model_dump() for item in selected[:max_items]]
+
+
+def build_runtime_payload(
+ documents: list[OntologyPackDocument],
+ task: str,
+) -> dict[str, Any]:
+ """Compile active packs into a compact, serializable request policy."""
+ governance_run_id = f"ontog_{uuid.uuid4().hex[:16]}"
+ packs: list[dict[str, Any]] = []
+ activation_candidates: list[dict[str, Any]] = []
+ runtime_events: list[dict[str, Any]] = []
+ activated_workflows: list[str] = []
+ review_level = "none"
+ for document in documents:
+ workflows = select_workflows(document, task)
+ for workflow in workflows:
+ if REVIEW_RANK[workflow.review_level] > REVIEW_RANK[review_level]:
+ review_level = workflow.review_level
+ ref = f"{document.pack_id}:{workflow.id}"
+ activated_workflows.append(ref)
+ runtime_events.append(
+ {
+ "type": "ontology_activation",
+ "status": "completed",
+ "governance_run_id": governance_run_id,
+ "source": "text",
+ "pack_id": document.pack_id,
+ "workflow_id": workflow.id,
+ "workflow_name": workflow.name,
+ "review_level": workflow.review_level,
+ }
+ )
+ packs.append(compile_pack_runtime(document, task, workflows))
+ for workflow in document.workflows:
+ if not workflow.asset_triggers:
+ continue
+ activation_candidates.append(
+ {
+ "pack_id": document.pack_id,
+ "workflow_id": workflow.id,
+ "workflow_name": workflow.name,
+ "review_level": workflow.review_level,
+ "asset_triggers": [item.model_dump() for item in workflow.asset_triggers],
+ "pack": compile_pack_runtime(document, task, [workflow]),
+ }
+ )
+ return {
+ "enabled": bool(packs),
+ "governance_run_id": governance_run_id,
+ "packs": packs,
+ "review_level": review_level,
+ "task": task,
+ "activation_candidates": activation_candidates,
+ "activated_workflows": activated_workflows,
+ "asset_tags": {"tool": {}, "skill": {}, "subagent": {}},
+ "runtime_events": runtime_events,
+ "review_owner": "outer_workflow",
+ "output_review": {"status": "pending", "owner": None, "count": 0},
+ }
+
+
+def register_runtime_asset_tags(
+ runtime: dict[str, Any],
+ *,
+ kind: str,
+ asset_id: str,
+ tags: Iterable[str],
+) -> None:
+ """Attach trusted server-side asset metadata to a request runtime."""
+ if kind not in {"tool", "skill", "subagent"} or not asset_id:
+ return
+ catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {})
+ existing = {str(item) for item in catalog.get(asset_id, []) if str(item)}
+ existing.update(str(item) for item in tags if str(item))
+ catalog[asset_id] = sorted(existing)
+
+
+def activate_runtime_for_asset(
+ runtime: dict[str, Any],
+ *,
+ kind: str,
+ asset_id: str,
+ tags: Iterable[str] = (),
+) -> list[dict[str, Any]]:
+ """Monotonically merge workflow policy when an actual asset is invoked."""
+ if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"}:
+ return []
+ known_tags = set(str(item) for item in tags if str(item))
+ known_tags.update(
+ str(item)
+ for item in runtime.get("asset_tags", {}).get(kind, {}).get(asset_id, [])
+ if str(item)
+ )
+ activated = set(runtime.setdefault("activated_workflows", []))
+ events: list[dict[str, Any]] = []
+ for candidate in runtime.get("activation_candidates", []):
+ ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}"
+ if ref in activated:
+ continue
+ matched_tags: set[str] = set()
+ matched = False
+ for trigger in candidate.get("asset_triggers", []):
+ if trigger.get("kind") != kind:
+ continue
+ if asset_id in set(trigger.get("ids", [])):
+ matched = True
+ tag_hits = known_tags & set(trigger.get("tags_any", []))
+ if tag_hits:
+ matched = True
+ matched_tags.update(tag_hits)
+ if not matched:
+ continue
+
+ fragment = candidate.get("pack") or {}
+ current = next(
+ (
+ pack
+ for pack in runtime.get("packs", [])
+ if pack.get("pack_id") == fragment.get("pack_id")
+ ),
+ None,
+ )
+ if current is None:
+ current = fragment
+ runtime.setdefault("packs", []).append(current)
+ else:
+ for field, key in (
+ ("workflows", "id"),
+ ("constraints", "id"),
+ ("concepts", "id"),
+ ("relations", "id"),
+ ):
+ existing_ids = {item.get(key) for item in current.get(field, [])}
+ current.setdefault(field, []).extend(
+ item for item in fragment.get(field, []) if item.get(key) not in existing_ids
+ )
+ if fragment.get("version_id"):
+ current["version_id"] = fragment["version_id"]
+
+ level = str(candidate.get("review_level") or "none")
+ if REVIEW_RANK.get(level, 0) > REVIEW_RANK.get(runtime.get("review_level"), 0):
+ runtime["review_level"] = level
+ activated.add(ref)
+ runtime["activated_workflows"] = sorted(activated)
+ event = {
+ "type": "ontology_activation",
+ "status": "completed",
+ "governance_run_id": runtime.get("governance_run_id"),
+ "source": kind,
+ "asset_kind": kind,
+ "asset_id": asset_id,
+ "asset_tags": sorted(known_tags),
+ "matched_tags": sorted(matched_tags),
+ "pack_id": candidate.get("pack_id"),
+ "workflow_id": candidate.get("workflow_id"),
+ "workflow_name": candidate.get("workflow_name"),
+ "review_level": level,
+ }
+ runtime.setdefault("runtime_events", []).append(event)
+ events.append(event)
+ return events
+
+
+def render_runtime_prompt(runtime: dict[str, Any]) -> str:
+ """Render the bounded ontology segment injected into the system prompt."""
+ if not runtime.get("enabled"):
+ return ""
+ compact_packs = []
+ for pack in runtime.get("packs", []):
+ compact_pack = {
+ "id": pack["pack_id"],
+ "version": pack["version"],
+ "concepts": (
+ list(pack.get("concepts", []))
+ if pack.get("config", {}).get("injection_enabled", True)
+ else []
+ ),
+ "relations": (
+ list(pack.get("relations", []))
+ if pack.get("config", {}).get("injection_enabled", True)
+ else []
+ ),
+ "workflows": pack.get("workflows", []),
+ "constraints": [
+ {
+ "id": rule["id"],
+ "target": rule["target"],
+ "mode": rule["mode"],
+ "message": rule["message"],
+ "suggestion": rule.get("suggestion", ""),
+ "prerequisite_tools": rule.get("prerequisite_tools", []),
+ }
+ for rule in pack.get("constraints", [])
+ ],
+ }
+ char_budget = int(pack.get("config", {}).get("token_budget", 2500)) * 4
+ while len(json.dumps(compact_pack, ensure_ascii=False)) > char_budget:
+ if compact_pack["relations"]:
+ compact_pack["relations"].pop()
+ continue
+ if compact_pack["concepts"]:
+ compact_pack["concepts"].pop()
+ remaining = {item["id"] for item in compact_pack["concepts"]}
+ compact_pack["relations"] = [
+ relation
+ for relation in compact_pack["relations"]
+ if relation["subject"] in remaining and relation["object"] in remaining
+ ]
+ continue
+ break
+ compact_packs.append(compact_pack)
+ compact = {"packs": compact_packs}
+ return (
+ "\n"
+ "本轮已启用领域本体校验。以下内容是可执行的领域契约;工具门禁会独立执行,"
+ "请在调用前主动满足参数、前置证据与工作流要求。不要虚构证据。\n"
+ + json.dumps(compact, ensure_ascii=False, separators=(",", ":"))
+ + "\n "
+ )
diff --git a/src/backend/core/ontology/runtime_policy.py b/src/backend/core/ontology/runtime_policy.py
new file mode 100644
index 00000000..e0f56b8d
--- /dev/null
+++ b/src/backend/core/ontology/runtime_policy.py
@@ -0,0 +1,158 @@
+"""Resolve a user's ontology preference into a runtime policy and asset tags."""
+from __future__ import annotations
+from core.ontology import service as ontology_service
+from typing import Any
+from core.infra.exceptions import ServiceUnavailableError
+from core.ontology.runtime_pack import register_runtime_asset_tags
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+
+def disabled_ontology_runtime() -> dict[str, Any]:
+ """Return the canonical pass-through policy used when the user opts out."""
+ return {"enabled": False, "packs": [], "review_level": "none"}
+
+
+def build_ontology_runtime_for_preference(
+ *,
+ enabled: bool,
+ task: str,
+ db: Session,
+ pack_ids: list[str] | None = None,
+) -> tuple[bool, dict[str, Any]]:
+ """Resolve the effective ontology switch and compile its runtime policy.
+
+ A user's stored opt-in is subordinate to the administrator-controlled pack
+ availability. Disabling the selected/default packs is therefore an
+ intentional global off switch, not a runtime-policy failure. Once an
+ active pack exists, compilation remains fail-closed so malformed policy
+ data cannot silently bypass validation.
+ """
+ if not enabled:
+ return False, disabled_ontology_runtime()
+
+ service = ontology_service.OntologyService(db)
+ selected = pack_ids or None
+ if not service.repo.get_active_versions(selected):
+ return False, disabled_ontology_runtime()
+
+ runtime = service.build_runtime(task=task, pack_ids=selected)
+ if not runtime.get("enabled"):
+ raise ServiceUnavailableError("本体校验运行时策略编译失败")
+ return True, runtime
+
+
+def runtime_needs_asset_tag_lookup(
+ runtime: dict[str, Any],
+ *,
+ kind: str,
+ asset_id: str,
+) -> bool:
+ """Return whether tags could activate an otherwise-unmatched workflow."""
+ activated = set(runtime.get("activated_workflows", []))
+ for candidate in runtime.get("activation_candidates", []):
+ ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}"
+ if ref in activated:
+ continue
+ triggers = [
+ trigger
+ for trigger in candidate.get("asset_triggers", [])
+ if trigger.get("kind") == kind
+ ]
+ if not triggers:
+ continue
+ if any(asset_id in set(trigger.get("ids", [])) for trigger in triggers):
+ continue
+ if any(trigger.get("tags_any") for trigger in triggers):
+ return True
+ return False
+
+
+def resolve_runtime_asset_tags(
+ *,
+ runtime: dict[str, Any],
+ kind: str,
+ asset_id: str,
+ user_id: str,
+) -> list[str]:
+ """Resolve and request-cache one invoked asset's trusted ontology tags.
+
+ The runtime catalog doubles as a per-turn positive/negative cache. Metadata
+ failures are fail-closed so an asset cannot execute without the governance
+ policy that its server-authoritative tags may activate.
+ """
+ if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"} or not asset_id:
+ return []
+ catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {})
+ if asset_id in catalog:
+ return list(catalog[asset_id])
+ if not runtime_needs_asset_tag_lookup(runtime, kind=kind, asset_id=asset_id):
+ register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=[])
+ return []
+
+ try:
+ from core.db.engine import SessionLocal
+
+ with SessionLocal() as db:
+ tags = ontology_service.OntologyService(db).resolve_asset_tags(
+ kind=kind,
+ asset_id=asset_id,
+ user_id=user_id,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ "[ontology] lazy asset tag lookup failed kind=%s asset_id=%s",
+ kind,
+ asset_id,
+ exc_info=True,
+ )
+ raise ServiceUnavailableError("本体资产标签读取失败,本次资产调用已停止") from exc
+
+ register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=tags)
+ return list(runtime["asset_tags"][kind][asset_id])
+
+
+def build_user_ontology_runtime(
+ *,
+ user_id: str,
+ task: str,
+ db: Session | None = None,
+) -> tuple[bool, dict[str, Any]]:
+ """Resolve one user's opt-in and compile the task-scoped runtime policy.
+
+ Web chat, channels, automation, batch, plan mode, and autonomous loops all
+ use this resolver so independent execution paths cannot silently bypass the
+ setting selected by the user.
+ """
+ from core.auth.users import UserService
+
+ owns_session = db is None
+ if owns_session:
+ from core.db.engine import SessionLocal
+
+ db = SessionLocal()
+ assert db is not None
+ try:
+ from core.ontology.policy import user_can_use_ontology_validation
+
+ if not user_can_use_ontology_validation(db, user_id):
+ return False, disabled_ontology_runtime()
+ settings = UserService(db).get_user_settings(user_id)
+ opted_in = bool(settings.get("ontology_enabled", False))
+ if not opted_in:
+ return False, disabled_ontology_runtime()
+ pack_ids = settings.get("ontology_pack_ids")
+ selected = (
+ [str(item) for item in pack_ids if str(item).strip()]
+ if isinstance(pack_ids, list)
+ else None
+ )
+ return build_ontology_runtime_for_preference(
+ enabled=opted_in,
+ task=task,
+ db=db,
+ pack_ids=selected or None,
+ )
+ finally:
+ if owns_session:
+ db.close()
diff --git a/src/backend/core/services/ontology_service.py b/src/backend/core/ontology/service.py
similarity index 68%
rename from src/backend/core/services/ontology_service.py
rename to src/backend/core/ontology/service.py
index 3a24e195..462b3166 100644
--- a/src/backend/core/services/ontology_service.py
+++ b/src/backend/core/ontology/service.py
@@ -1,38 +1,22 @@
-"""Business service for ontology asset governance and runtime compilation."""
-
+"""Manage ontology packs, drafts and versions, and resolve asset tags."""
from __future__ import annotations
from core.infra.time import utc_now
-
import hashlib
import json
-import logging
import uuid
-from datetime import datetime
from typing import Any
-
from core.config.display_names import TOOL_DISPLAY_NAMES
-from core.db.models import (
- AdminMcpServer,
- AdminSkill,
- OntologyDraft,
- OntologyPack,
- OntologyPackVersion,
-)
+from core.db.models import AdminMcpServer, AdminSkill, OntologyDraft, OntologyPack, OntologyPackVersion
from core.db.repository import OntologyRepository
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError, ServiceUnavailableError
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from core.ontology.schemas import OntologyPackDocument
-from core.ontology.validator import (
- DomainPackValidator,
- build_runtime_payload,
- register_runtime_asset_tags,
-)
+from core.ontology.validator import DomainPackValidator
+from core.ontology.runtime_pack import build_runtime_payload
from sqlalchemy import or_
from sqlalchemy.orm import Session
-logger = logging.getLogger(__name__)
-
-def _normalize_ontology_tags(value: Any) -> list[str]:
+def normalize_ontology_tags(value: Any) -> list[str]:
if isinstance(value, str):
items = value.replace(",", ",").split(",")
elif isinstance(value, (list, tuple, set)):
@@ -42,40 +26,6 @@ def _normalize_ontology_tags(value: Any) -> list[str]:
return sorted({str(item).strip() for item in items if str(item).strip()})
-def disabled_ontology_runtime() -> dict[str, Any]:
- """Return the canonical pass-through policy used when the user opts out."""
- return {"enabled": False, "packs": [], "review_level": "none"}
-
-
-def build_ontology_runtime_for_preference(
- *,
- enabled: bool,
- task: str,
- db: Session,
- pack_ids: list[str] | None = None,
-) -> tuple[bool, dict[str, Any]]:
- """Resolve the effective ontology switch and compile its runtime policy.
-
- A user's stored opt-in is subordinate to the administrator-controlled pack
- availability. Disabling the selected/default packs is therefore an
- intentional global off switch, not a runtime-policy failure. Once an
- active pack exists, compilation remains fail-closed so malformed policy
- data cannot silently bypass validation.
- """
- if not enabled:
- return False, disabled_ontology_runtime()
-
- service = OntologyService(db)
- selected = pack_ids or None
- if not service.repo.get_active_versions(selected):
- return False, disabled_ontology_runtime()
-
- runtime = service.build_runtime(task=task, pack_ids=selected)
- if not runtime.get("enabled"):
- raise ServiceUnavailableError("本体校验运行时策略编译失败")
- return True, runtime
-
-
class OntologyService:
def __init__(self, db: Session):
self.db = db
@@ -415,7 +365,7 @@ def resolve_asset_tags(
.one_or_none()
)
if row:
- return _normalize_ontology_tags(row.tags)
+ return normalize_ontology_tags(row.tags)
# A DB row that exists but is outside this user's scope must never
# fall through to the process-global loader (which indexes private
# skills for materialization). Only true filesystem/built-in skills
@@ -426,10 +376,10 @@ def resolve_asset_tags(
)
if db_row_exists:
return []
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
metadata = get_skill_loader().load_all_metadata().get(asset_id)
- return _normalize_ontology_tags(getattr(metadata, "tags", []))
+ return normalize_ontology_tags(getattr(metadata, "tags", []))
if kind == "tool":
owner_filter = (
@@ -458,20 +408,20 @@ def resolve_asset_tags(
for tool in row.tools_json or []:
if not isinstance(tool, dict) or str(tool.get("name") or "") != asset_id:
continue
- tags.update(_normalize_ontology_tags(extra.get("ontology_tags")))
- tags.update(_normalize_ontology_tags(tool.get("tags")))
- tags.update(_normalize_ontology_tags(tool.get("ontology_tags")))
- tags.update(_normalize_ontology_tags(tool_tag_map.get(asset_id)))
+ tags.update(normalize_ontology_tags(extra.get("ontology_tags")))
+ tags.update(normalize_ontology_tags(tool.get("tags")))
+ tags.update(normalize_ontology_tags(tool.get("ontology_tags")))
+ tags.update(normalize_ontology_tags(tool_tag_map.get(asset_id)))
return sorted(tags)
if kind == "subagent":
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
try:
agent = UserAgentService(self.db).get_by_id(asset_id, user_id=user_id or None)
except (LookupError, PermissionError):
return []
- return _normalize_ontology_tags(agent.get("ontology_tags"))
+ return normalize_ontology_tags(agent.get("ontology_tags"))
return []
@@ -488,185 +438,3 @@ def _known_tool_schemas(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
if tool.get("inputSchema"):
schemas[name] = {"inputSchema": tool["inputSchema"]}
return schemas, known
-
-
-def _runtime_needs_asset_tag_lookup(
- runtime: dict[str, Any],
- *,
- kind: str,
- asset_id: str,
-) -> bool:
- """Return whether tags could activate an otherwise-unmatched workflow."""
- activated = set(runtime.get("activated_workflows", []))
- for candidate in runtime.get("activation_candidates", []):
- ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}"
- if ref in activated:
- continue
- triggers = [
- trigger
- for trigger in candidate.get("asset_triggers", [])
- if trigger.get("kind") == kind
- ]
- if not triggers:
- continue
- if any(asset_id in set(trigger.get("ids", [])) for trigger in triggers):
- continue
- if any(trigger.get("tags_any") for trigger in triggers):
- return True
- return False
-
-
-def resolve_runtime_asset_tags(
- *,
- runtime: dict[str, Any],
- kind: str,
- asset_id: str,
- user_id: str,
-) -> list[str]:
- """Resolve and request-cache one invoked asset's trusted ontology tags.
-
- The runtime catalog doubles as a per-turn positive/negative cache. Metadata
- failures are fail-closed so an asset cannot execute without the governance
- policy that its server-authoritative tags may activate.
- """
- if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"} or not asset_id:
- return []
- catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {})
- if asset_id in catalog:
- return list(catalog[asset_id])
- if not _runtime_needs_asset_tag_lookup(runtime, kind=kind, asset_id=asset_id):
- register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=[])
- return []
-
- try:
- from core.db.engine import SessionLocal
-
- with SessionLocal() as db:
- tags = OntologyService(db).resolve_asset_tags(
- kind=kind,
- asset_id=asset_id,
- user_id=user_id,
- )
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[ontology] lazy asset tag lookup failed kind=%s asset_id=%s",
- kind,
- asset_id,
- exc_info=True,
- )
- raise ServiceUnavailableError("本体资产标签读取失败,本次资产调用已停止") from exc
-
- register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=tags)
- return list(runtime["asset_tags"][kind][asset_id])
-
-
-def build_user_ontology_runtime(
- *,
- user_id: str,
- task: str,
- db: Session | None = None,
-) -> tuple[bool, dict[str, Any]]:
- """Resolve one user's opt-in and compile the task-scoped runtime policy.
-
- Web chat, channels, automation, batch, plan mode, and autonomous loops all
- use this resolver so independent execution paths cannot silently bypass the
- setting selected by the user.
- """
- from core.services.user_service import UserService
-
- owns_session = db is None
- if owns_session:
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- assert db is not None
- try:
- from core.services.ontology_policy import user_can_use_ontology_validation
-
- if not user_can_use_ontology_validation(db, user_id):
- return False, disabled_ontology_runtime()
- settings = UserService(db).get_user_settings(user_id)
- opted_in = bool(settings.get("ontology_enabled", False))
- if not opted_in:
- return False, disabled_ontology_runtime()
- pack_ids = settings.get("ontology_pack_ids")
- selected = (
- [str(item) for item in pack_ids if str(item).strip()]
- if isinstance(pack_ids, list)
- else None
- )
- return build_ontology_runtime_for_preference(
- enabled=opted_in,
- task=task,
- db=db,
- pack_ids=selected or None,
- )
- finally:
- if owns_session:
- db.close()
-
-
-def record_enforcement_event(data: dict[str, Any]) -> None:
- """Persist an event from middleware/reviewer using an isolated session."""
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- payload = dict(data)
- payload.setdefault("event_id", f"ontoe_{uuid.uuid4().hex[:16]}")
- OntologyRepository(db).create_event(payload)
- except Exception:
- db.rollback()
- raise
- finally:
- db.close()
-
-
-def record_runtime_activation(
- event: dict[str, Any],
- runtime: dict[str, Any],
- *,
- user_id: str | None,
- chat_id: str | None,
-) -> None:
- """Persist one asset-driven workflow activation as append-only evidence."""
-
- version_id = next(
- (
- pack.get("version_id")
- for pack in runtime.get("packs", [])
- if pack.get("pack_id") == event.get("pack_id")
- ),
- None,
- )
- record_enforcement_event(
- {
- "user_id": user_id,
- "chat_id": chat_id,
- "pack_id": event.get("pack_id"),
- "version_id": version_id,
- "rule_id": f"activation:{event.get('workflow_id')}",
- "stage": "checkpoint",
- "event_type": "runtime_activation",
- "decision": "pass",
- "mode": "enforce",
- "target": f"{event.get('asset_kind')}:{event.get('asset_id')}",
- "details": event,
- }
- )
-
-
-def record_review_run(data: dict[str, Any]) -> None:
- from core.db.engine import SessionLocal
-
- db = SessionLocal()
- try:
- payload = dict(data)
- payload.setdefault("review_id", f"ontor_{uuid.uuid4().hex[:16]}")
- payload.setdefault("created_at", utc_now())
- OntologyRepository(db).create_review(payload)
- except Exception:
- db.rollback()
- raise
- finally:
- db.close()
diff --git a/src/backend/core/ontology/tool_hook.py b/src/backend/core/ontology/tool_hook.py
new file mode 100644
index 00000000..1aaa00ad
--- /dev/null
+++ b/src/backend/core/ontology/tool_hook.py
@@ -0,0 +1,183 @@
+"""Ontology tool admission and audit through neutral execution hooks."""
+import asyncio
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Any
+from core.harness.context_items import reminder_item
+from core.harness.events import thaw_value
+from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage
+
+logger = logging.getLogger(__name__)
+
+
+class OntologyToolHook:
+ def __init__(self, runtime=None):
+ self.runtime = dict(runtime or {})
+ self.completed_tools = set()
+ self.denial_counts = {}
+
+ async def __call__(self, invocation):
+ if invocation.stage == HookStage.AFTER_TOOL:
+ result = invocation.data.get("tool_result") or {}
+ if result.get("state") not in {"error", "denied", "interrupted"}:
+ self.completed_tools.add(invocation.operation_name)
+ return Decision()
+ from core.ontology.runtime_pack import activate_runtime_for_asset, render_runtime_prompt
+ from core.ontology.runtime_gates import evaluate_tool_call
+ from core.ontology.runtime_policy import resolve_runtime_asset_tags
+ runtime = thaw_value(invocation.data.get("ontology_runtime") or self.runtime)
+ if not runtime.get("enabled"):
+ return Decision()
+ identity = invocation.data.get("identity", {})
+ started = time.monotonic()
+ tool_name = invocation.operation_name
+ tool_input = thaw_value(invocation.data.get("tool_args") or {})
+ asset_kind, asset_id = self._resolve_invoked_asset(tool_name, tool_input)
+ await asyncio.to_thread(resolve_runtime_asset_tags, runtime=runtime, kind=asset_kind,
+ asset_id=asset_id, user_id=str(identity.get("user_id") or ""))
+ activations = activate_runtime_for_asset(runtime, kind=asset_kind, asset_id=asset_id)
+ items = list(invocation.data.get("context_items") or ())
+ if activations:
+ contract = render_runtime_prompt(runtime)
+ if contract:
+ items.append(reminder_item(
+ "检测到受领域本体治理的运行时资产,策略已升级且本轮不可降级。\n" + contract,
+ "harness:ontology_gate"))
+ await self._audit_activations(identity, runtime, activations)
+ decision = evaluate_tool_call(runtime, tool_name=tool_name, tool_input=tool_input,
+ completed_tools=self.completed_tools)
+ patch = {"ontology_runtime": runtime, "context_items": items}
+ if decision.allowed:
+ await self._audit(identity, runtime, tool_name, decision, started)
+ return Decision.modify(patch)
+ key = ",".join(decision.matched_rule_ids) or tool_name
+ count = self.denial_counts[key] = self.denial_counts.get(key, 0) + 1
+ configs = [pack.get("config", {}) for pack in runtime.get("packs", [])]
+ strategy = min((int(item.get("repeated_denial_threshold", 2)) for item in configs), default=2)
+ breaker = min((int(item.get("circuit_breaker_threshold", 5)) for item in configs), default=5)
+ guidance = list(decision.suggestions)
+ if count >= strategy:
+ guidance.append("同一规则已重复拦截,请改变执行策略,不要原样重试。")
+ if count >= breaker:
+ guidance.append("本体门禁已触发熔断;停止调用该工具,并向用户说明缺失条件。")
+ await self._audit(identity, runtime, tool_name, decision, started,
+ denial_count=count, circuit_breaker=count >= breaker)
+ if count == strategy and identity.get("chat_id"):
+ try:
+ from core.ontology.evolution import schedule_ontology_evolution
+ schedule_ontology_evolution(user_id=str(identity.get("user_id") or "system"))
+ except Exception as exc:
+ logger.warning("[ontology] evolution scheduling failed: %s", exc)
+ payload = {"code": "ONTOLOGY_GATE_DENIED", "tool": tool_name,
+ "violations": decision.violations, "suggestions": guidance,
+ "denial_count": count, "circuit_breaker": count >= breaker}
+ patch["tool_response"] = {"content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}],
+ "state": "denied", "metadata": {"ontology_gate": payload}}
+ return Decision(DecisionKind.REJECT, patch=patch, reason="ontology gate denied tool")
+
+ async def _audit(
+ self,
+ identity: dict,
+ runtime: dict[str, Any],
+ tool_name: str,
+ decision,
+ started: float,
+ *,
+ denial_count: int = 0,
+ circuit_breaker: bool = False,
+ ) -> None: # noqa: ANN001
+ if not decision.violations and not decision.matched_rule_ids:
+ return
+ runtime.setdefault("runtime_events", []).append(
+ {
+ "type": "ontology_gate",
+ "status": "completed",
+ "governance_run_id": runtime.get("governance_run_id"),
+ "stage": "tool",
+ "decision": decision.decision,
+ "tool_name": tool_name,
+ "matched_rule_ids": list(decision.matched_rule_ids),
+ "violations": list(decision.violations),
+ "suggestions": list(decision.suggestions),
+ "denial_count": denial_count,
+ "circuit_breaker": circuit_breaker,
+ }
+ )
+ try:
+ from core.ontology.audit_events import record_enforcement_event
+
+ first = decision.violations[0] if decision.violations else {}
+ pack_id = first.get("pack_id")
+ version_id = None
+ for pack in runtime.get("packs", []):
+ if pack.get("pack_id") == pack_id:
+ version_id = pack.get("version_id")
+ break
+ payload = {
+ "user_id": identity.get("user_id"),
+ "chat_id": identity.get("chat_id"),
+ "pack_id": pack_id,
+ "version_id": version_id,
+ "rule_id": first.get("rule_id")
+ or (decision.matched_rule_ids[0] if decision.matched_rule_ids else None),
+ "stage": "tool",
+ "event_type": "tool_call_gate",
+ "decision": decision.decision,
+ "mode": first.get("mode", "enforce"),
+ "target": tool_name,
+ "latency_ms": round((time.monotonic() - started) * 1000),
+ "details": {
+ "governance_run_id": runtime.get("governance_run_id"),
+ "violations": decision.violations,
+ "matched_rule_ids": decision.matched_rule_ids,
+ "suggestions": decision.suggestions,
+ "denial_count": denial_count,
+ "circuit_breaker": circuit_breaker,
+ },
+ }
+ await asyncio.to_thread(record_enforcement_event, payload)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology] audit event persistence failed: %s", exc)
+
+
+ @staticmethod
+ def _resolve_invoked_asset(tool_name: str, tool_input: dict[str, Any]) -> tuple[str, str]:
+ if tool_name == "call_subagent" and tool_input.get("agent_id"):
+ return "subagent", str(tool_input["agent_id"])
+ if tool_name == "view_text_file":
+ file_path = str(tool_input.get("file_path") or "")
+ path = Path(file_path)
+ if path.name == "SKILL.md" and path.parent.name:
+ return "skill", path.parent.name
+ return "tool", tool_name
+
+
+ @staticmethod
+ async def _audit_activations(
+ identity: dict,
+ runtime: dict[str, Any],
+ activations: list[dict[str, Any]],
+ ) -> None:
+ try:
+ from core.ontology.audit_events import record_runtime_activation
+
+ for event in activations:
+ await asyncio.to_thread(
+ record_runtime_activation,
+ event,
+ runtime,
+ user_id=identity.get("user_id"),
+ chat_id=identity.get("chat_id"),
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[ontology] activation audit persistence failed: %s", exc)
+
+
+
+def ontology_tool_hooks(runtime):
+ policy = OntologyToolHook(runtime)
+ return (HookSpec("ontology_admission", HookStage.BEFORE_TOOL, policy, priority=500,
+ timeout_ms=30000, mutable_fields=frozenset({"ontology_runtime", "context_items", "tool_response"})),
+ HookSpec("ontology_completion", HookStage.AFTER_TOOL, policy, priority=300))
diff --git a/src/backend/core/ontology/validation_models.py b/src/backend/core/ontology/validation_models.py
new file mode 100644
index 00000000..cda206d4
--- /dev/null
+++ b/src/backend/core/ontology/validation_models.py
@@ -0,0 +1,33 @@
+"""Validation issues, reports and gate decisions."""
+from __future__ import annotations
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+@dataclass(slots=True)
+class ValidationIssue:
+ severity: str
+ path: str
+ message: str
+
+
+@dataclass(slots=True)
+class ValidationReport:
+ valid: bool
+ errors: list[ValidationIssue] = field(default_factory=list)
+ warnings: list[ValidationIssue] = field(default_factory=list)
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "valid": self.valid,
+ "errors": [asdict(item) for item in self.errors],
+ "warnings": [asdict(item) for item in self.warnings],
+ }
+
+
+@dataclass(slots=True)
+class OntologyGateDecision:
+ allowed: bool
+ decision: str
+ violations: list[dict[str, Any]] = field(default_factory=list)
+ suggestions: list[str] = field(default_factory=list)
+ matched_rule_ids: list[str] = field(default_factory=list)
diff --git a/src/backend/core/ontology/validator.py b/src/backend/core/ontology/validator.py
index 02638c2c..dcf9092b 100644
--- a/src/backend/core/ontology/validator.py
+++ b/src/backend/core/ontology/validator.py
@@ -1,47 +1,10 @@
-"""Domain Pack validation and deterministic runtime gates."""
-
+"""Validate ontology packs before activation."""
from __future__ import annotations
-
-import json
-import re
-import uuid
-from dataclasses import asdict, dataclass, field
from typing import Any, Iterable
-
-from core.ontology.schemas import Constraint, OntologyPackDocument, Workflow
-from jsonschema import Draft202012Validator, ValidationError
+from core.ontology.schemas import OntologyPackDocument
+from jsonschema import Draft202012Validator
from pydantic import ValidationError as PydanticValidationError
-
-
-@dataclass(slots=True)
-class ValidationIssue:
- severity: str
- path: str
- message: str
-
-
-@dataclass(slots=True)
-class ValidationReport:
- valid: bool
- errors: list[ValidationIssue] = field(default_factory=list)
- warnings: list[ValidationIssue] = field(default_factory=list)
-
- def as_dict(self) -> dict[str, Any]:
- return {
- "valid": self.valid,
- "errors": [asdict(item) for item in self.errors],
- "warnings": [asdict(item) for item in self.warnings],
- }
-
-
-@dataclass(slots=True)
-class OntologyGateDecision:
- allowed: bool
- decision: str
- violations: list[dict[str, Any]] = field(default_factory=list)
- suggestions: list[str] = field(default_factory=list)
- matched_rule_ids: list[str] = field(default_factory=list)
-
+from core.ontology import validation_models
class DomainPackValidator:
"""Validate structure, JSON Schema, and tool references before activation."""
@@ -52,21 +15,21 @@ def validate(
*,
tool_schemas: dict[str, dict[str, Any]] | None = None,
known_tools: Iterable[str] = (),
- ) -> tuple[OntologyPackDocument | None, ValidationReport]:
- errors: list[ValidationIssue] = []
- warnings: list[ValidationIssue] = []
+ ) -> tuple[OntologyPackDocument | None, validation_models.ValidationReport]:
+ errors: list[validation_models.ValidationIssue] = []
+ warnings: list[validation_models.ValidationIssue] = []
try:
document = OntologyPackDocument.model_validate(payload)
except PydanticValidationError as exc:
for item in exc.errors(include_url=False):
errors.append(
- ValidationIssue(
+ validation_models.ValidationIssue(
"error",
".".join(str(part) for part in item["loc"]),
item["msg"],
)
)
- return None, ValidationReport(False, errors, warnings)
+ return None, validation_models.ValidationReport(False, errors, warnings)
schemas = tool_schemas or {}
known = set(known_tools) | set(schemas)
@@ -76,12 +39,12 @@ def validate(
try:
Draft202012Validator.check_schema(rule.schema_)
except Exception as exc: # SchemaError is version-specific
- errors.append(ValidationIssue("error", f"{path}.schema", str(exc)))
+ errors.append(validation_models.ValidationIssue("error", f"{path}.schema", str(exc)))
tool_name = rule.target.tool
if not tool_name:
continue
if tool_name not in known:
- issue = ValidationIssue(
+ issue = validation_models.ValidationIssue(
"warning" if document.config.allow_unresolved_tools else "error",
f"{path}.target.tool",
f"unknown tool reference: {tool_name}",
@@ -89,10 +52,10 @@ def validate(
(warnings if document.config.allow_unresolved_tools else errors).append(issue)
continue
if rule.target.kind == "tool_parameter" and tool_name in schemas:
- properties = _tool_input_schema(schemas[tool_name]).get("properties", {})
+ properties = tool_input_schema(schemas[tool_name]).get("properties", {})
if rule.target.parameter not in properties:
errors.append(
- ValidationIssue(
+ validation_models.ValidationIssue(
"error",
f"{path}.target.parameter",
f"unknown parameter {rule.target.parameter!r} for tool {tool_name}",
@@ -102,7 +65,7 @@ def validate(
for index, workflow in enumerate(document.workflows):
for tool_name in set(workflow.required_tools + workflow.forbidden_tools):
if tool_name not in known:
- issue = ValidationIssue(
+ issue = validation_models.ValidationIssue(
"warning" if document.config.allow_unresolved_tools else "error",
f"workflows.{index}",
f"unknown tool reference: {tool_name}",
@@ -111,7 +74,7 @@ def validate(
overlap = set(workflow.required_tools) & set(workflow.forbidden_tools)
if overlap:
errors.append(
- ValidationIssue(
+ validation_models.ValidationIssue(
"error",
f"workflows.{index}",
f"tools cannot be both required and forbidden: {sorted(overlap)}",
@@ -123,579 +86,18 @@ def validate(
for tool_name in trigger.ids:
if tool_name in known:
continue
- issue = ValidationIssue(
+ issue = validation_models.ValidationIssue(
"warning" if document.config.allow_unresolved_tools else "error",
f"workflows.{index}.asset_triggers.{trigger_index}.ids",
f"unknown tool reference: {tool_name}",
)
(warnings if document.config.allow_unresolved_tools else errors).append(issue)
- return document, ValidationReport(not errors, errors, warnings)
+ return document, validation_models.ValidationReport(not errors, errors, warnings)
-def _tool_input_schema(schema: dict[str, Any]) -> dict[str, Any]:
+def tool_input_schema(schema: dict[str, Any]) -> dict[str, Any]:
function = schema.get("function") if isinstance(schema, dict) else None
if isinstance(function, dict):
return function.get("parameters") or function.get("inputSchema") or {}
return schema.get("inputSchema") or schema.get("parameters") or schema
-
-
-def select_workflows(document: OntologyPackDocument, task: str) -> list[Workflow]:
- """Select workflows by case-insensitive lexical trigger match."""
- normalized = task.casefold()
- return [
- workflow
- for workflow in document.workflows
- if any(trigger.casefold() in normalized for trigger in workflow.triggers)
- ]
-
-
-_REVIEW_RANK = {"none": 0, "checkpoint": 1, "committee": 2}
-
-
-def _workflow_constraints(
- document: OntologyPackDocument,
- workflows: list[Workflow],
-) -> list[Constraint]:
- workflow_tools = {
- tool_name
- for workflow in workflows
- for tool_name in (workflow.required_tools + workflow.forbidden_tools)
- }
- workflow_output_tags = {tag for workflow in workflows for tag in workflow.output_tags}
- return [
- rule
- for rule in document.constraints
- if rule.enabled
- and (
- rule.target.tool in workflow_tools
- if rule.target.kind in {"tool", "tool_parameter"}
- else rule.target.output_tag in workflow_output_tags
- )
- ]
-
-
-def _runtime_concepts(
- document: OntologyPackDocument,
- task: str,
- constraints: list[Constraint],
-) -> list[dict[str, Any]]:
- """Select task concepts while always retaining concepts used by active rules."""
- by_id = {item.id: item for item in document.concepts}
- selected: list[Any] = []
- selected_ids: set[str] = set()
-
- def add_with_parents(concept_id: str | None) -> None:
- chain: list[Any] = []
- current = by_id.get(concept_id or "")
- while current is not None and current.id not in selected_ids:
- chain.append(current)
- current = by_id.get(current.parent_id or "")
- for item in reversed(chain):
- if len(selected) >= document.config.max_concepts:
- return
- if item.id not in selected_ids:
- selected.append(item)
- selected_ids.add(item.id)
-
- for rule in constraints:
- add_with_parents(rule.concept_id)
- for item in select_concepts(document, task, limit=document.config.max_concepts):
- add_with_parents(item["id"])
- return [item.model_dump() for item in selected[: document.config.max_concepts]]
-
-
-def _compile_pack_runtime(
- document: OntologyPackDocument,
- task: str,
- workflows: list[Workflow],
-) -> dict[str, Any]:
- constraints = _workflow_constraints(document, workflows)
- concepts = _runtime_concepts(document, task, constraints)
- selected_concept_ids = {item["id"] for item in concepts}
- relations = [
- relation.model_dump()
- for relation in document.relations
- if relation.subject in selected_concept_ids and relation.object in selected_concept_ids
- ]
- return {
- "pack_id": document.pack_id,
- "version": document.version,
- "name": document.name,
- "domain": document.domain,
- "description": document.description,
- "config": document.config.model_dump(),
- "concepts": concepts,
- "relations": relations,
- "constraints": [item.model_dump(by_alias=True) for item in constraints],
- "workflows": [item.model_dump() for item in workflows],
- }
-
-
-def select_concepts(
- document: OntologyPackDocument,
- task: str,
- *,
- limit: int | None = None,
-) -> list[dict[str, Any]]:
- """Select only task-relevant concepts; fall back to tagged roots."""
- normalized = task.casefold()
- # English words can be tokenized by boundaries. Chinese text cannot be
- # treated as one giant ``[\u4e00-\u9fff]+`` token: doing so made a task such
- # as “数据质量分析” match no concept at all. Concept names and aliases are
- # therefore also checked directly as substrings of the task.
- terms = set(re.findall(r"[a-z0-9_]+", normalized))
- scored: list[tuple[int, Any]] = []
- for concept in document.concepts:
- labels = [concept.id, concept.name, *concept.aliases, *concept.tags]
- haystack = " ".join([*labels, concept.definition]).casefold()
- score = sum(1 for term in terms if len(term) >= 2 and term in haystack)
- score += sum(
- 3 for label in labels if len(label.strip()) >= 2 and label.casefold() in normalized
- )
- if score or not concept.parent_id:
- scored.append((score, concept))
- max_items = limit or document.config.max_concepts
- scored.sort(key=lambda item: (-item[0], item[1].id))
- selected = [item for _, item in scored[:max_items]]
- by_id = {item.id: item for item in document.concepts}
- selected_ids = {item.id for item in selected}
- # Preserve the hierarchy for every selected concept when the budget allows.
- for concept in list(selected):
- parent_id = concept.parent_id
- while parent_id and len(selected) < max_items:
- if parent_id not in selected_ids and parent_id in by_id:
- selected.append(by_id[parent_id])
- selected_ids.add(parent_id)
- parent_id = by_id.get(parent_id).parent_id if parent_id in by_id else None
- return [item.model_dump() for item in selected[:max_items]]
-
-
-def build_runtime_payload(
- documents: list[OntologyPackDocument],
- task: str,
-) -> dict[str, Any]:
- """Compile active packs into a compact, serializable request policy."""
- governance_run_id = f"ontog_{uuid.uuid4().hex[:16]}"
- packs: list[dict[str, Any]] = []
- activation_candidates: list[dict[str, Any]] = []
- runtime_events: list[dict[str, Any]] = []
- activated_workflows: list[str] = []
- review_level = "none"
- for document in documents:
- workflows = select_workflows(document, task)
- for workflow in workflows:
- if _REVIEW_RANK[workflow.review_level] > _REVIEW_RANK[review_level]:
- review_level = workflow.review_level
- ref = f"{document.pack_id}:{workflow.id}"
- activated_workflows.append(ref)
- runtime_events.append(
- {
- "type": "ontology_activation",
- "status": "completed",
- "governance_run_id": governance_run_id,
- "source": "text",
- "pack_id": document.pack_id,
- "workflow_id": workflow.id,
- "workflow_name": workflow.name,
- "review_level": workflow.review_level,
- }
- )
- packs.append(_compile_pack_runtime(document, task, workflows))
- for workflow in document.workflows:
- if not workflow.asset_triggers:
- continue
- activation_candidates.append(
- {
- "pack_id": document.pack_id,
- "workflow_id": workflow.id,
- "workflow_name": workflow.name,
- "review_level": workflow.review_level,
- "asset_triggers": [item.model_dump() for item in workflow.asset_triggers],
- "pack": _compile_pack_runtime(document, task, [workflow]),
- }
- )
- return {
- "enabled": bool(packs),
- "governance_run_id": governance_run_id,
- "packs": packs,
- "review_level": review_level,
- "task": task,
- "activation_candidates": activation_candidates,
- "activated_workflows": activated_workflows,
- "asset_tags": {"tool": {}, "skill": {}, "subagent": {}},
- "runtime_events": runtime_events,
- "review_owner": "outer_workflow",
- "output_review": {"status": "pending", "owner": None, "count": 0},
- }
-
-
-def register_runtime_asset_tags(
- runtime: dict[str, Any],
- *,
- kind: str,
- asset_id: str,
- tags: Iterable[str],
-) -> None:
- """Attach trusted server-side asset metadata to a request runtime."""
- if kind not in {"tool", "skill", "subagent"} or not asset_id:
- return
- catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {})
- existing = {str(item) for item in catalog.get(asset_id, []) if str(item)}
- existing.update(str(item) for item in tags if str(item))
- catalog[asset_id] = sorted(existing)
-
-
-def activate_runtime_for_asset(
- runtime: dict[str, Any],
- *,
- kind: str,
- asset_id: str,
- tags: Iterable[str] = (),
-) -> list[dict[str, Any]]:
- """Monotonically merge workflow policy when an actual asset is invoked."""
- if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"}:
- return []
- known_tags = set(str(item) for item in tags if str(item))
- known_tags.update(
- str(item)
- for item in runtime.get("asset_tags", {}).get(kind, {}).get(asset_id, [])
- if str(item)
- )
- activated = set(runtime.setdefault("activated_workflows", []))
- events: list[dict[str, Any]] = []
- for candidate in runtime.get("activation_candidates", []):
- ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}"
- if ref in activated:
- continue
- matched_tags: set[str] = set()
- matched = False
- for trigger in candidate.get("asset_triggers", []):
- if trigger.get("kind") != kind:
- continue
- if asset_id in set(trigger.get("ids", [])):
- matched = True
- tag_hits = known_tags & set(trigger.get("tags_any", []))
- if tag_hits:
- matched = True
- matched_tags.update(tag_hits)
- if not matched:
- continue
-
- fragment = candidate.get("pack") or {}
- current = next(
- (
- pack
- for pack in runtime.get("packs", [])
- if pack.get("pack_id") == fragment.get("pack_id")
- ),
- None,
- )
- if current is None:
- current = fragment
- runtime.setdefault("packs", []).append(current)
- else:
- for field, key in (
- ("workflows", "id"),
- ("constraints", "id"),
- ("concepts", "id"),
- ("relations", "id"),
- ):
- existing_ids = {item.get(key) for item in current.get(field, [])}
- current.setdefault(field, []).extend(
- item for item in fragment.get(field, []) if item.get(key) not in existing_ids
- )
- if fragment.get("version_id"):
- current["version_id"] = fragment["version_id"]
-
- level = str(candidate.get("review_level") or "none")
- if _REVIEW_RANK.get(level, 0) > _REVIEW_RANK.get(runtime.get("review_level"), 0):
- runtime["review_level"] = level
- activated.add(ref)
- runtime["activated_workflows"] = sorted(activated)
- event = {
- "type": "ontology_activation",
- "status": "completed",
- "governance_run_id": runtime.get("governance_run_id"),
- "source": kind,
- "asset_kind": kind,
- "asset_id": asset_id,
- "asset_tags": sorted(known_tags),
- "matched_tags": sorted(matched_tags),
- "pack_id": candidate.get("pack_id"),
- "workflow_id": candidate.get("workflow_id"),
- "workflow_name": candidate.get("workflow_name"),
- "review_level": level,
- }
- runtime.setdefault("runtime_events", []).append(event)
- events.append(event)
- return events
-
-
-def render_runtime_prompt(runtime: dict[str, Any]) -> str:
- """Render the bounded ontology segment injected into the system prompt."""
- if not runtime.get("enabled"):
- return ""
- compact_packs = []
- for pack in runtime.get("packs", []):
- compact_pack = {
- "id": pack["pack_id"],
- "version": pack["version"],
- "concepts": (
- list(pack.get("concepts", []))
- if pack.get("config", {}).get("injection_enabled", True)
- else []
- ),
- "relations": (
- list(pack.get("relations", []))
- if pack.get("config", {}).get("injection_enabled", True)
- else []
- ),
- "workflows": pack.get("workflows", []),
- "constraints": [
- {
- "id": rule["id"],
- "target": rule["target"],
- "mode": rule["mode"],
- "message": rule["message"],
- "suggestion": rule.get("suggestion", ""),
- "prerequisite_tools": rule.get("prerequisite_tools", []),
- }
- for rule in pack.get("constraints", [])
- ],
- }
- char_budget = int(pack.get("config", {}).get("token_budget", 2500)) * 4
- while len(json.dumps(compact_pack, ensure_ascii=False)) > char_budget:
- if compact_pack["relations"]:
- compact_pack["relations"].pop()
- continue
- if compact_pack["concepts"]:
- removed = compact_pack["concepts"].pop()
- remaining = {item["id"] for item in compact_pack["concepts"]}
- compact_pack["relations"] = [
- relation
- for relation in compact_pack["relations"]
- if relation["subject"] in remaining and relation["object"] in remaining
- ]
- continue
- break
- compact_packs.append(compact_pack)
- compact = {"packs": compact_packs}
- return (
- "\n"
- "本轮已启用领域本体校验。以下内容是可执行的领域契约;工具门禁会独立执行,"
- "请在调用前主动满足参数、前置证据与工作流要求。不要虚构证据。\n"
- + json.dumps(compact, ensure_ascii=False, separators=(",", ":"))
- + "\n "
- )
-
-
-def requires_output_review(runtime: dict[str, Any]) -> bool:
- """Whether an L-b/L-c review must run after the streamed draft."""
- return bool(
- runtime.get("enabled")
- and (
- runtime.get("review_level") != "none"
- or any(
- (rule.get("target") or {}).get("kind") == "output"
- for pack in runtime.get("packs", [])
- for rule in pack.get("constraints", [])
- )
- )
- )
-
-
-def claim_output_review(runtime: dict[str, Any], *, owner: str) -> bool:
- """Atomically claim the single final-answer review for one governance run."""
- if not requires_output_review(runtime):
- return False
- state = runtime.setdefault(
- "output_review",
- {"status": "pending", "owner": None, "count": 0},
- )
- if state.get("status") in {"running", "completed"}:
- return False
- state.update({"status": "running", "owner": owner})
- return True
-
-
-def complete_output_review(
- runtime: dict[str, Any],
- *,
- owner: str,
- verdict: str,
- attempts: int = 1,
-) -> None:
- """Mark the claimed review complete and retain its single-run evidence."""
- state = runtime.setdefault("output_review", {})
- if state.get("status") != "running" or state.get("owner") != owner:
- return
- count = int(state.get("count") or 0) + max(1, int(attempts or 1))
- state.update(
- {
- "status": "completed",
- "owner": owner,
- "count": count,
- "verdict": verdict,
- }
- )
-
-
-def release_output_review(runtime: dict[str, Any], *, owner: str) -> None:
- """Release a failed claim so a safe retry can review the final answer."""
- state = runtime.setdefault("output_review", {})
- if state.get("status") == "running" and state.get("owner") == owner:
- state.update({"status": "pending", "owner": None})
-
-
-def evaluate_tool_call(
- runtime: dict[str, Any],
- *,
- tool_name: str,
- tool_input: dict[str, Any],
- completed_tools: Iterable[str] = (),
-) -> OntologyGateDecision:
- """Apply zero-LLM L-a validation to one tool call."""
- if not runtime.get("enabled"):
- return OntologyGateDecision(True, "pass")
- completed = set(completed_tools)
- violations: list[dict[str, Any]] = []
- suggestions: list[str] = []
- matched: list[str] = []
- enforced = False
-
- for pack in runtime.get("packs", []):
- for workflow in pack.get("workflows", []):
- if tool_name in workflow.get("forbidden_tools", []):
- violations.append(
- {
- "rule_id": f"workflow:{workflow['id']}:forbidden",
- "message": f"工作流 {workflow['name']} 禁止调用工具 {tool_name}",
- "mode": "enforce",
- }
- )
- enforced = True
- for rule in pack.get("constraints", []):
- target = rule.get("target") or {}
- if target.get("tool") != tool_name:
- continue
- matched.append(rule["id"])
- missing_prerequisites = [
- name for name in rule.get("prerequisite_tools", []) if name not in completed
- ]
- rule_errors: list[str] = []
- if missing_prerequisites:
- rule_errors.append("缺少前置工具调用: " + ", ".join(missing_prerequisites))
- try:
- if target.get("kind") == "tool":
- Draft202012Validator(rule.get("schema") or {}).validate(tool_input)
- elif target.get("kind") == "tool_parameter":
- parameter = target.get("parameter")
- Draft202012Validator(rule.get("schema") or {}).validate(
- tool_input.get(parameter)
- )
- except ValidationError as exc:
- rule_errors.append(exc.message)
- if rule_errors:
- violations.append(
- {
- "pack_id": pack.get("pack_id"),
- "rule_id": rule["id"],
- "message": rule["message"],
- "reasons": rule_errors,
- "mode": rule.get("mode", "log"),
- "risk": rule.get("risk", "low"),
- }
- )
- if rule.get("suggestion"):
- suggestions.append(rule["suggestion"])
- if rule.get("mode") == "enforce":
- enforced = True
-
- if not violations:
- return OntologyGateDecision(True, "pass", matched_rule_ids=matched)
- return OntologyGateDecision(
- allowed=not enforced,
- decision="deny" if enforced else "log",
- violations=violations,
- suggestions=suggestions,
- matched_rule_ids=matched,
- )
-
-
-def evaluate_output(
- runtime: dict[str, Any],
- *,
- answer: str,
- citations: Iterable[dict[str, Any]] = (),
- completed_tools: Iterable[str] | None = (),
-) -> OntologyGateDecision:
- """Apply deterministic output constraints before any LLM reviewer."""
- if not runtime.get("enabled"):
- return OntologyGateDecision(True, "pass")
- citation_items = list(citations)
- # ``None`` means the caller cannot observe a complete tool trace (the
- # legacy non-streaming reply path). Output and citation rules still run,
- # but required-tool rules must not produce a false omission claim.
- completed = set(completed_tools) if completed_tools is not None else None
- violations: list[dict[str, Any]] = []
- matched: list[str] = []
- suggestions: list[str] = []
- enforced = False
- for pack in runtime.get("packs", []):
- if completed is not None:
- for workflow in pack.get("workflows", []):
- missing_tools = sorted(set(workflow.get("required_tools", [])) - completed)
- if missing_tools:
- violations.append(
- {
- "pack_id": pack.get("pack_id"),
- "rule_id": f"workflow:{workflow.get('id')}:required",
- "message": "领域工作流缺少必需的证据工具调用。",
- "reasons": ["未调用: " + ", ".join(missing_tools)],
- "mode": "enforce",
- "risk": workflow.get("risk", "low"),
- }
- )
- enforced = True
- output_tags = {
- tag for workflow in pack.get("workflows", []) for tag in workflow.get("output_tags", [])
- }
- for rule in pack.get("constraints", []):
- target = rule.get("target") or {}
- if target.get("kind") != "output":
- continue
- if output_tags and target.get("output_tag") not in output_tags:
- continue
- matched.append(rule["id"])
- reasons: list[str] = []
- if rule.get("requires_citations") and not citation_items:
- reasons.append("输出缺少可追溯引用证据")
- try:
- Draft202012Validator(rule.get("schema") or {}).validate(answer)
- except ValidationError as exc:
- reasons.append(exc.message)
- if reasons:
- violations.append(
- {
- "pack_id": pack.get("pack_id"),
- "rule_id": rule["id"],
- "message": rule["message"],
- "reasons": reasons,
- "mode": rule.get("mode", "log"),
- "risk": rule.get("risk", "low"),
- }
- )
- if rule.get("suggestion"):
- suggestions.append(rule["suggestion"])
- if rule.get("mode") == "enforce":
- enforced = True
- if not violations:
- return OntologyGateDecision(True, "pass", matched_rule_ids=matched)
- return OntologyGateDecision(
- allowed=not enforced,
- decision="deny" if enforced else "log",
- violations=violations,
- suggestions=suggestions,
- matched_rule_ids=matched,
- )
diff --git a/src/backend/core/plugins/README.md b/src/backend/core/plugins/README.md
deleted file mode 100644
index e04e17ae..00000000
--- a/src/backend/core/plugins/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# Plugins
-
-Plugin code has one domain owner: this package. Neither `core/services/` nor
-`core/llm/` contains another plugin implementation.
-
-| Package | Responsibility | Main entry |
-| --- | --- | --- |
-| `management/` | Install, enable, uninstall, inspect and publish database-backed plugins | `core.plugins.management` |
-| `packaging/` | Discover files, extract archives and normalize plugin manifests | `importer.normalize_plugin_dir`, `archive` |
-| `runtime/` | Resolve deferred capabilities, activate tools/skills and restore per-chat activation | `core.plugins.runtime` |
-| `local/` | Device-local installation and migration of managed bundles | `local.service` |
-| `ui/contract.py` | Normalize UI declarations and project browser-safe data | `normalize_ui`, `public_contributions` |
-| `ui/declarations.py`, `ui/primitives.py` | Contribution-specific parsing and shared validation rules | Internal to UI ingestion |
-| `ui/contributions.py`, `ui/data_proxy.py` | Resolve installed UI and execute authorized data queries | API routes call these explicitly |
-
-Runtime code may read installation state; installation and packaging must not
-import the agent factory or plugin runtime. Packaging does not publish database
-records: safe ZIP extraction lives in `packaging/archive.py`, while marketplace
-persistence lives in `management/packages.py`.
-
-Add a new manifest format in packaging, lifecycle behavior in management, and
-activation behavior in runtime. Add UI declaration kinds in declarations and
-host renderer support together. Do not add another `plugin_*` sibling to the
-LLM or general services directories. Internal imports and test patch targets
-use the new owner paths; old module paths are intentionally removed.
-
-Bundled resource paths are resolved relative to the backend root, including
-in Docker. Package discovery uses the existing `core*` setuptools inclusion.
diff --git a/src/backend/core/plugins/management/__init__.py b/src/backend/core/plugins/management/__init__.py
deleted file mode 100644
index 289ebf54..00000000
--- a/src/backend/core/plugins/management/__init__.py
+++ /dev/null
@@ -1,80 +0,0 @@
-"""Public plugin lifecycle API; implementations follow domain ownership."""
-
-from core.plugins.management.admin_config import get_plugin_admin_config, set_plugin_admin_config
-from core.plugins.management.details import (
- exclude_market_skill,
- get_market_skill_exclusions,
- get_plugin_detail,
-)
-from core.plugins.management.installation import (
- ensure_default_plugins_bootstrapped,
- import_plugin,
- import_plugin_from_zip,
- install_plugin,
-)
-from core.plugins.management.lifecycle import (
- set_installed_plugin_meta,
- set_plugin_component_enabled,
- set_plugin_component_enabled_for_user,
- set_plugin_enabled,
- set_plugin_enabled_for_user,
- uninstall_plugin,
-)
-from core.plugins.management.market import (
- list_plugins,
- resolve_market_meta,
- set_market_meta,
- set_plugin_market_enabled,
-)
-from core.plugins.management.packages import delete_market_package, publish_plugin_zip_to_market
-from core.plugins.management.queries import get_installed_detail, list_installed
-from core.plugins.packaging.definitions import (
- BUILTIN_PLUGIN_MARKET_META,
- DEFAULT_BOOTSTRAP_MARKER_ID,
- DEFAULT_BOOTSTRAP_PLUGIN_SLUGS,
- LOCAL_BOOTSTRAP_MARKER_NAME,
- MAX_ICON_LEN,
- MAX_ICON_URL_LEN,
- MAX_ZIP_BYTES,
- PLUGIN_BUNDLES_DIR,
- PLUGIN_MARKET_META_BLOCK_ID,
- PLUGIN_SOURCE_DIRS,
-)
-from core.plugins.packaging.sources import builtin_plugin_component_ids
-
-__all__ = [
- "resolve_market_meta",
- "set_market_meta",
- "builtin_plugin_component_ids",
- "list_plugins",
- "set_plugin_market_enabled",
- "list_installed",
- "get_installed_detail",
- "get_plugin_admin_config",
- "set_plugin_admin_config",
- "get_market_skill_exclusions",
- "get_plugin_detail",
- "exclude_market_skill",
- "install_plugin",
- "ensure_default_plugins_bootstrapped",
- "import_plugin",
- "import_plugin_from_zip",
- "publish_plugin_zip_to_market",
- "delete_market_package",
- "uninstall_plugin",
- "set_plugin_enabled",
- "set_plugin_component_enabled",
- "set_plugin_enabled_for_user",
- "set_plugin_component_enabled_for_user",
- "set_installed_plugin_meta",
- "PLUGIN_BUNDLES_DIR",
- "PLUGIN_SOURCE_DIRS",
- "MAX_ZIP_BYTES",
- "DEFAULT_BOOTSTRAP_PLUGIN_SLUGS",
- "DEFAULT_BOOTSTRAP_MARKER_ID",
- "LOCAL_BOOTSTRAP_MARKER_NAME",
- "PLUGIN_MARKET_META_BLOCK_ID",
- "BUILTIN_PLUGIN_MARKET_META",
- "MAX_ICON_LEN",
- "MAX_ICON_URL_LEN",
-]
diff --git a/src/backend/core/plugins/resources/application.py b/src/backend/core/plugins/resources/application.py
deleted file mode 100644
index cd8beb88..00000000
--- a/src/backend/core/plugins/resources/application.py
+++ /dev/null
@@ -1,152 +0,0 @@
-"""Application resource admission uses the existing worker, quota and janitor."""
-
-import asyncio
-import hashlib
-import json
-import time
-
-from fastapi import HTTPException
-
-from . import service, store
-from .owners import application_binding
-from .reservation import lock_owner
-
-
-async def reserve(db, selected, module_id, user_id, binding):
- deadline = time.monotonic() + 95
- while True:
- db.commit()
- lock_owner(db, user_id)
- row = db.get(store.PluginResource, binding["resource_id"])
- if row is None:
- return None, False # caller retains the quota/admission lock
- if (
- row.user_id != user_id
- or row.request_digest != binding["request_digest"]
- or row.application_key != binding["application_key"]
- or row.instance_id != binding["instance_id"]
- or row.install_id != selected.install_id
- or row.revision != selected.revision
- or row.module_id != module_id
- ):
- db.rollback()
- raise HTTPException(409, "idempotency_conflict")
- if row.status == "active":
- service.authorized(db, row.resource_id, user_id)
- db.commit()
- return row, True
- if row.status != "starting" or row.expires_at <= time.time():
- db.commit()
- raise HTTPException(410, "resource_closed_no_replay")
- db.commit()
- if time.monotonic() >= deadline:
- raise HTTPException(503, "resource_start_pending")
- await asyncio.sleep(0.05)
- db.expire_all()
- application_binding(
- db,
- user_id,
- binding["application_key"],
- selected.install_id,
- selected.revision,
- binding["scope"],
- module_id,
- )
-
-
-async def create(
- db, user_id, key, instance_id, install_id, revision, execution_scope, module_id, idempotency_key
-):
- if not idempotency_key or len(idempotency_key) > 128:
- raise HTTPException(400, "idempotency_key_required")
- item = application_binding(db, user_id, key, install_id, revision, execution_scope, module_id)
- payload = [key, instance_id, install_id, revision, execution_scope, module_id]
- digest = hashlib.sha256(json.dumps(payload).encode()).hexdigest()
- resource_id = hashlib.sha256(
- json.dumps([user_id, key, instance_id, idempotency_key]).encode()
- ).hexdigest()
- try:
- return await service.create(
- db,
- module_id=module_id,
- user_id=user_id,
- chat_id=None,
- slug=item["binding"]["slug"],
- install_id=install_id,
- _application={
- "application_key": key,
- "instance_id": instance_id,
- "revision": revision,
- "scope": execution_scope,
- "request_digest": digest,
- "resource_id": resource_id,
- },
- )
- except HTTPException as error:
- db.rollback()
- if error.status_code == 429:
- from core.applications.diagnostics import record
-
- try:
- record(db, user_id, "resource", "quota_denied", item=item, instance_id=instance_id)
- except Exception:
- pass # The original quota denial remains authoritative if telemetry storage fails.
- raise
-
-
-def owned(db, user_id, key, instance_id, resource_id, *, allow_closed=False):
- row = store.get(db, resource_id, user_id)
- if row.application_key != key or row.instance_id != instance_id or row.chat_id is not None:
- raise HTTPException(404, "resource_unavailable")
- if allow_closed:
- # Revoked grants stop all use; authenticated owners may still release their worker.
- return row
- return service.authorized(db, resource_id, user_id)
-
-
-async def renew(db, user_id, key, instance_id, resource_id):
- row = owned(db, user_id, key, instance_id, resource_id)
- from core.applications import background
-
- if background.authorized_resource(db, row):
- await background.renew(db, user_id, key, instance_id, resource_id)
- return store.public(row)
- updated = (
- db.query(store.PluginResource)
- .filter_by(resource_id=row.resource_id, status="active")
- .filter(store.PluginResource.expires_at > time.time())
- .update({"expires_at": time.time() + 90}, synchronize_session=False)
- )
- db.commit()
- if updated != 1:
- raise HTTPException(410, "resource_expired")
- db.refresh(row)
- from .deadlines import arm
-
- await arm(db, row)
- return store.public(row)
-
-
-async def close_application_resources(
- db, user_id, *, application_key=None, instance_id=None, install_id=None
-):
- query = db.query(store.PluginResource).filter(
- store.PluginResource.user_id == user_id,
- store.PluginResource.application_key.is_not(None),
- store.PluginResource.status != "closed",
- )
- for name, value in (
- ("application_key", application_key),
- ("instance_id", instance_id),
- ("install_id", install_id),
- ):
- if value is not None:
- query = query.filter(getattr(store.PluginResource, name) == value)
- failures = []
- for row in query.all():
- try:
- await service.close(db, row)
- except Exception:
- db.rollback()
- failures.append(row.resource_id)
- return {"closed": not failures, "retry_resource_ids": failures}
diff --git a/src/backend/core/plugins/resources/application_cleanup.py b/src/backend/core/plugins/resources/application_cleanup.py
deleted file mode 100644
index 1f83dfbd..00000000
--- a/src/backend/core/plugins/resources/application_cleanup.py
+++ /dev/null
@@ -1,48 +0,0 @@
-"""Revoke application access before stopping a worker; retain failed cleanup."""
-
-import asyncio
-
-from fastapi import HTTPException
-
-from . import store
-from .owners import runtime_session
-
-
-async def close(db, row):
- from .service import request
- from core.applications.diagnostics import observe_resource
-
- if row.status == "closed":
- return
- row.status = "closing"
- db.query(store.PluginResourceTicket).filter_by(resource_id=row.resource_id).delete()
- db.query(store.PluginResourceAssetTicket).filter_by(resource_id=row.resource_id).delete()
- db.commit()
- try:
- target = store.descriptor(row)
- except HTTPException:
- observe_resource(db, row, "release", "release_failed")
- raise HTTPException(503, "resource_cleanup_credentials_unavailable") from None
- if target.get("process_id"):
- try:
- await asyncio.wait_for(request(row, "/shutdown", {}), 5)
- except (HTTPException, TimeoutError):
- from core.sandbox.factory import get_sandbox_provider
-
- provider = get_sandbox_provider()
- if target.get("provider") != provider.name:
- observe_resource(db, row, "release", "release_failed")
- raise HTTPException(503, "resource_cleanup_scope_unavailable")
- result = await provider.write_stdin(
- target["process_id"],
- sandbox_session_id=runtime_session(row),
- user_id=row.user_id,
- chars="\x03",
- yield_time_ms=1000,
- )
- if not result or result.get("exit_code") is None:
- observe_resource(db, row, "release", "release_failed")
- raise HTTPException(503, "resource_cleanup_pending")
- row.status = "closed"
- db.commit()
- observe_resource(db, row, "release", "ok")
diff --git a/src/backend/core/plugins/resources/deadlines.py b/src/backend/core/plugins/resources/deadlines.py
deleted file mode 100644
index 0949c7d1..00000000
--- a/src/backend/core/plugins/resources/deadlines.py
+++ /dev/null
@@ -1,125 +0,0 @@
-"""Owner-bound runner timers enforce application resource leases independently of API sweeps."""
-
-from core.applications.coordination import runtime
-
-import hashlib
-import json
-from fastapi import HTTPException
-from core.applications import activity, origin
-from . import store
-from .owners import runtime_session
-
-
-def provider_for(row):
- from core.sandbox.factory import get_sandbox_provider
-
- provider = get_sandbox_provider()
- descriptor = store.descriptor(row)
- method = getattr(provider, "set_process_deadline", None)
- if descriptor.get("provider") != provider.name or not callable(method):
- raise HTTPException(403, detail={"code": "unsupported_capability"})
- if not descriptor.get("process_id"):
- raise HTTPException(410, detail={"code": "instance_closed"})
- return method, descriptor["process_id"]
-
-
-def invalidate(db, row):
- from core.applications import external
-
- rid, user = row.resource_id, row.user_id
- db.commit()
- runtime(db, user)
- db.expire_all()
- if row.status != "closed":
- row.status = "closing"
- row.expires_at = 0
- current = activity.read(db, user)
- record = current["background"].get(rid)
- if record:
- record["expiresAt"] = 0
- record["revoked"] = True
- activity.save(db, user, current) # Retain the exact charged cleanup reservation.
-
-
-async def arm(db, row, *, require_background=False):
- if row.application_key is None or row.chat_id is not None:
- return # Ordinary chat resources retain their existing lifetime.
- value = activity.read(db, row.user_id)
- record = value["background"].get(row.resource_id)
- source = origin.current(db)
- expires = row.expires_at
- if record:
- binding = record["binding"]
- if (
- record["applicationKey"] != row.application_key
- or record["instanceId"] != row.instance_id
- or binding.get("installId") != row.install_id
- or binding.get("packageRevision") != row.revision
- or binding.get("executionScope") != row.scope
- or binding.get("origin") != source
- ):
- raise HTTPException(403, detail={"code": "resource_binding_mismatch"})
- expires = min(expires, record["expiresAt"], record["deadline"])
- try:
- method, process_id = provider_for(row)
- except HTTPException as error:
- if not (require_background or record) and error.status_code == 403:
- return # Remote providers have no independent persistent deadline contract.
- raise
- digest = hashlib.sha256(
- json.dumps(
- [
- row.resource_id,
- row.user_id,
- row.application_key,
- row.instance_id,
- row.install_id,
- row.revision,
- row.scope,
- source,
- ],
- sort_keys=True,
- ).encode()
- ).hexdigest()
- try:
- result = await method(
- process_id,
- sandbox_session_id=runtime_session(row),
- user_id=row.user_id,
- binding=digest,
- expires_at=expires,
- absolute_deadline=record["deadline"] if record else None,
- )
- except Exception:
- # Do not leave an authorized extended lease after uncertain timer delivery.
- invalidate(db, row)
- raise HTTPException(503, detail={"code": "release_failed"}) from None
- if result.get("supported") is False:
- if record:
- invalidate(db, row)
- if require_background or record:
- raise HTTPException(403, detail={"code": "unsupported_capability"})
- return
- if result.get("armed") is not True:
- invalidate(db, row)
- raise HTTPException(503, detail={"code": "release_failed"})
-
-
-async def restore(db, *, after="", limit=100):
- rows = (
- db.query(store.PluginResource)
- .filter(
- store.PluginResource.application_key.is_not(None),
- store.PluginResource.status == "active",
- store.PluginResource.resource_id > after,
- )
- .order_by(store.PluginResource.resource_id)
- .limit(limit)
- .all()
- )
- for row in rows:
- try:
- await arm(db, row)
- except HTTPException:
- db.rollback() # Existing janitor owns persisted closing cleanup.
- return rows[-1].resource_id if len(rows) == limit else ""
diff --git a/src/backend/core/plugins/resources/owners.py b/src/backend/core/plugins/resources/owners.py
deleted file mode 100644
index 142e26d7..00000000
--- a/src/backend/core/plugins/resources/owners.py
+++ /dev/null
@@ -1,112 +0,0 @@
-"""Resource owner validation and provider session identity."""
-
-import hashlib
-import json
-
-from core.config.local_mode import local_mode_enabled
-from fastapi import HTTPException
-
-
-def execution_scope():
- return "local" if local_mode_enabled() else "cloud"
-
-
-def runtime_session(row):
- if row.chat_id is not None:
- return row.chat_id
- # Provider session IDs are execution containers, never conversation records.
- binding = [
- row.user_id,
- row.application_key,
- row.install_id,
- row.revision,
- row.scope,
- row.instance_id,
- ]
- return "workbench-" + hashlib.sha256(json.dumps(binding).encode()).hexdigest()
-
-
-def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None):
- from core.applications import catalog
-
- item = catalog.readable(db, user_id, key)
- if not item["availability"]["available"]:
- raise HTTPException(403, "application_unavailable")
- binding = item.get("binding", {})
- if (
- item["declaration"]["target"]["kind"] != "plugin"
- or binding.get("installId") != install_id
- or (
- binding.get("packageRevision") != revision
- and (selected is None or selected.revision != revision)
- )
- or scope != execution_scope()
- or (binding.get("origin") is not None and binding["origin"].get("kind") != scope)
- ):
- raise HTTPException(409, "resource_binding_mismatch")
- target = item["declaration"]["target"]
- if selected is not None and binding.get("packageRevision") != revision:
- previous = next(
- (
- a
- for a in (selected.ui.get("contributes") or {}).get("applications", [])
- if a["id"] == item["declaration"]["id"]
- ),
- None,
- )
- if previous is None:
- raise HTTPException(403, "resource_application_removed")
- target = previous["target"]
- allowed = {target.get("moduleId")} | {
- view.get("moduleId") for view in item["declaration"].get("views", [])
- }
- if module_id not in allowed:
- raise HTTPException(403, "resource_module_not_granted")
- return item
-
-
-def resolve_installation(db, row, user_id):
- from .installation import resolve
-
- if row.application_key is not None:
- from core.applications.installation_binding import exact as resolve
-
- selected = resolve(db, row.slug, user_id, row.install_id)
- if row.application_key is not None and selected.install_id != row.install_id:
- raise HTTPException(403, "resource_installation_mismatch")
- if selected.revision == row.revision:
- return selected
- if row.application_key is None:
- raise HTTPException(409, "plugin_revision_changed")
- try:
- from core.applications.plugin_revisions import resolve_revision
- except ModuleNotFoundError as error:
- if error.name != "core.applications.plugin_revisions":
- raise
- raise HTTPException(409, "revision_retention_unavailable") from None
- # WB13 rechecks current catalog permissions and the exact live instance lease.
- return resolve_revision(db, user_id, row.application_key, row.revision, row.instance_id)
-
-
-def validate(db, row, user_id, selected=None):
- if row.chat_id is not None:
- if row.application_key is not None or row.instance_id is not None:
- raise HTTPException(403, "resource_owner_invalid")
- from core.db.models import ChatSession
-
- chat = db.get(ChatSession, row.chat_id)
- if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
- raise HTTPException(403, "conversation_unavailable")
- else:
- if not row.application_key or not row.instance_id:
- raise HTTPException(403, "resource_owner_invalid")
- application_binding(
- db,
- user_id,
- row.application_key,
- row.install_id,
- row.revision,
- row.scope,
- row.module_id,
- selected,
- )
diff --git a/src/backend/core/plugins/resources/prewarm.py b/src/backend/core/plugins/resources/prewarm.py
deleted file mode 100644
index 3c78d90f..00000000
--- a/src/backend/core/plugins/resources/prewarm.py
+++ /dev/null
@@ -1,142 +0,0 @@
-"""Background opt-in resource startup, bound to a conversation sandbox lifetime."""
-import asyncio
-import logging
-import threading
-from contextlib import asynccontextmanager
-import httpx
-from fastapi import HTTPException
-from sqlalchemy import or_
-from core.db import engine
-from . import installation, service, store
-
-logger = logging.getLogger(__name__)
-_guard = threading.Lock()
-_jobs = {}
-_closing = {}
-_shutdown = False
-
-def candidates(db, user_id):
- from core.capabilities import device_catalog
- if device_catalog.active():
- from core.capabilities.device_plugin_catalog import plugin_entries
- sources = [(entry["slug"], entry["install_id"]) for entry in plugin_entries(user_id)
- if entry["enabled"] and entry["callable"]]
- else:
- from core.db.models import InstalledPlugin
- sources = [(row.slug, row.install_id) for row in db.query(InstalledPlugin).filter(
- or_(InstalledPlugin.owner_user_id == user_id, InstalledPlugin.owner_user_id.is_(None))).all()]
- for slug, install_id in sources:
- try:
- selected = installation.resolve(db, slug, user_id, install_id)
- except HTTPException:
- continue
- for module in (selected.ui.get("contributes") or {}).get("modules") or []:
- if (module.get("resource") or {}).get("prewarm") is True:
- yield selected, module
-
-def schedule(provider, chat_id, user_id, sandbox_id):
- if not user_id or not chat_id or not sandbox_id or str(chat_id).startswith("eval_"):
- return
- key = (chat_id, user_id, str(sandbox_id))
- loop = asyncio.get_running_loop()
- with _guard:
- if _shutdown or chat_id in _closing or key in _jobs:
- return
- # Bound completed admission markers; router providers notify repeatedly.
- if len(_jobs) >= 1024:
- for old in [old for old, job in _jobs.items() if job.done()][:256]:
- _jobs.pop(old, None)
- _jobs[key] = loop.create_task(_run(provider, chat_id, user_id, str(sandbox_id)))
-
-async def _run(provider, chat_id, user_id, sandbox_id):
- from core.sandbox.sessions.binding import bind
- with bind(sandbox_id):
- try:
- from core.db.models import ChatSession
- with engine.SessionLocal() as db:
- chat = db.get(ChatSession, chat_id)
- if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
- return
- for selected, module in candidates(db, user_id):
- await service.create(db, selected.slug, module["id"], user_id, chat_id,
- install_id=selected.install_id, _prewarm=True, _provider=provider)
- except Exception as exc:
- logger.info("Resource prewarm skipped for chat=%s: %s", chat_id, type(exc).__name__)
- return False
-
-async def _cancel(task):
- if not task.done():
- task.cancel()
- result, = await asyncio.gather(task, return_exceptions=True)
- return result is not False and (not isinstance(result, BaseException) or isinstance(result, asyncio.CancelledError))
-
-@asynccontextmanager
-async def closing(chat_id):
- with _guard:
- _closing[chat_id] = _closing.get(chat_id, 0) + 1
- try:
- yield
- finally:
- with _guard:
- _closing[chat_id] -= 1
- if _closing[chat_id] == 0:
- _closing.pop(chat_id)
-
-def resume():
- global _shutdown
- with _guard:
- _shutdown = False
-
-async def stop(chat_id=None):
- global _shutdown
- if str(chat_id or "").startswith("eval_"):
- return True
- if chat_id is None:
- with _guard:
- _shutdown = True
- async with closing(chat_id):
- return await _stop(chat_id)
-
-async def _stop(chat_id):
- """Stop starts and blank workers. False forbids returning a dirty sandbox to its pool."""
- with _guard:
- keys = [key for key in _jobs if chat_id is None or key[0] == chat_id]
- jobs = [_jobs.pop(key) for key in keys]
- loop = asyncio.get_running_loop()
- clean = True
- for task in jobs:
- owner = task.get_loop()
- if owner is loop:
- clean = await _cancel(task) and clean
- elif owner.is_running():
- clean = await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(_cancel(task), owner)) and clean
- elif not task.done():
- clean = False
- else:
- try:
- clean = task.result() is not False and clean
- except asyncio.CancelledError:
- pass
- except Exception:
- clean = False
- with engine.SessionLocal() as db:
- query = db.query(store.PluginResource).filter(store.PluginResource.status.in_(("warm", "warming")))
- if chat_id is not None:
- query = query.filter_by(chat_id=chat_id)
- for row in query.all():
- try:
- target = store.descriptor(row)
- except HTTPException:
- clean = False
- row.status = "closed"
- continue
- if target.get("url"):
- try:
- async with httpx.AsyncClient(timeout=3, trust_env=False, headers=target["headers"]) as client:
- response = await client.post(target["url"] + "/shutdown", json={})
- response.raise_for_status()
- except httpx.HTTPError:
- clean = False
- row.status = "closed"
- db.commit()
- return clean
diff --git a/src/backend/core/plugins/resources/reservation.py b/src/backend/core/plugins/resources/reservation.py
deleted file mode 100644
index dbb71461..00000000
--- a/src/backend/core/plugins/resources/reservation.py
+++ /dev/null
@@ -1,93 +0,0 @@
-"""Serialize resource admission and one-time warm worker adoption per owner."""
-
-import asyncio
-import os
-import time
-
-from fastapi import HTTPException
-
-from . import store
-
-LIVE_STATUSES = ("active", "starting", "warming", "warm")
-
-
-def lock_owner(db, user_id):
- from core.db.models import UserShadow
-
- if db.bind.dialect.name == "sqlite":
- from sqlalchemy import text
-
- db.execute(text("BEGIN IMMEDIATE"))
- else:
- db.query(UserShadow).filter_by(user_id=user_id).with_for_update().one()
-
-
-async def adopt(db, selected, module_id, user_id, chat_id, *, prewarm, checkpoint_id):
- """Return an adopted resource, or permission to reserve under the held lock."""
- deadline = time.monotonic() + 90
- while True:
- db.commit()
- lock_owner(db, user_id)
- query = (
- db.query(store.PluginResource)
- .filter_by(
- user_id=user_id,
- chat_id=chat_id,
- install_id=selected.install_id,
- revision=selected.revision,
- slug=selected.slug,
- module_id=module_id,
- )
- .filter(store.PluginResource.expires_at > time.time())
- )
- if prewarm:
- if query.filter(store.PluginResource.status.in_(LIVE_STATUSES)).first():
- db.commit()
- return None, True
- return None, False
- row = (
- None
- if checkpoint_id
- else query.filter(store.PluginResource.status.in_(("warm", "warming")))
- .order_by(store.PluginResource.created_at)
- .first()
- )
- if row is None:
- return None, False
- if row.status == "warm":
- row.status = "active"
- row.expires_at = time.time() + int(os.getenv("PLUGIN_RESOURCE_TTL_SECONDS", "86400"))
- db.commit()
- return row, True
- db.commit()
- if time.monotonic() >= deadline:
- raise HTTPException(503, "resource_prewarm_timeout")
- await asyncio.sleep(0.05)
- db.expire_all()
- # Each wait releases the transaction: installation switches and revocation
- # must be noticed before the worker can be adopted.
- from . import installation
-
- current = installation.resolve(db, selected.slug, user_id, selected.install_id)
- if current.revision != selected.revision:
- raise HTTPException(409, "plugin_revision_changed")
-
-
-def check_quota(db, user_id):
- active = (
- db.query(store.PluginResource)
- .filter(
- store.PluginResource.user_id == user_id,
- (
- (
- store.PluginResource.status.in_(LIVE_STATUSES)
- & (store.PluginResource.expires_at > time.time())
- )
- | (store.PluginResource.status == "closing")
- ),
- )
- .count()
- )
- if active >= int(os.getenv("PLUGIN_RESOURCE_MAX_PER_USER", "8")):
- db.rollback()
- raise HTTPException(429, "resource_limit")
diff --git a/src/backend/core/plugins/resources/service.py b/src/backend/core/plugins/resources/service.py
deleted file mode 100644
index 6edc657b..00000000
--- a/src/backend/core/plugins/resources/service.py
+++ /dev/null
@@ -1,347 +0,0 @@
-"""Authorized resource lifecycle. Runtime behavior remains package-owned."""
-
-import json
-import os
-import secrets
-import time
-import uuid
-
-from core.plugins.ui.contract import find_module
-from fastapi import HTTPException
-
-from . import installation, store
-from .crypto import encrypt_secret
-from .transport import RuntimeTransport
-
-
-def authorized(db, resource_id, user_id):
- row = store.get(db, resource_id, user_id)
- if row.status != "active" or row.expires_at < time.time():
- raise HTTPException(410, "resource_expired")
- validate_binding(db, row, user_id)
- return row
-
-
-def validate_binding(db, row, user_id):
- if row.application_key is not None:
- from core.applications import background
-
- background.authorized_resource(db, row)
- from .owners import resolve_installation, validate
-
- selected = resolve_installation(db, row, user_id)
- validate(db, row, user_id, selected)
- return row
-
-
-async def create(
- db,
- slug,
- module_id,
- user_id,
- chat_id,
- *,
- resource_id=None,
- install_id=None,
- checkpoint_id=None,
- _prewarm=False,
- _provider=None,
- _application=None,
-):
- if resource_id:
- row = authorized(db, resource_id, user_id)
- if row.chat_id != chat_id or row.slug != slug or row.module_id != module_id:
- raise HTTPException(403, "resource_binding_mismatch")
- await request(row, "/state")
- return store.public(row)
- if _application is None:
- from core.db.models import ChatSession
-
- chat = db.get(ChatSession, chat_id)
- if chat is None or chat.user_id != user_id or chat.deleted_at is not None:
- raise HTTPException(403, "conversation_unavailable")
- else:
- if chat_id is not None or checkpoint_id or _prewarm:
- raise HTTPException(403, "resource_owner_invalid")
- from .owners import application_binding
-
- application_binding(
- db,
- user_id,
- _application["application_key"],
- install_id,
- _application["revision"],
- _application["scope"],
- module_id,
- )
- resolver = installation.resolve
- if _application:
- from core.applications.installation_binding import exact as resolver
-
- selected = resolver(db, slug, user_id, install_id)
- if _application and selected.install_id != install_id:
- raise HTTPException(403, "resource_installation_mismatch")
- if _application and selected.revision != _application["revision"]:
- raise HTTPException(409, "plugin_revision_changed")
- module = find_module(selected.ui, module_id)
- if module is None or not module.get("resource"):
- raise HTTPException(404, "resource_module_unavailable")
- if _application and "resource.attach" not in module.get("grants", []):
- raise HTTPException(403, "resource_attach_not_granted")
- token = secrets.token_urlsafe(32)
- from .configuration import runtime_configuration
-
- config = {
- **runtime_configuration(module["resource"]),
- "token": token,
- "idle_seconds": int(os.getenv("PLUGIN_RESOURCE_IDLE_SECONDS", "1800")),
- }
- if checkpoint_id:
- checkpoint = db.get(store.PluginResourceCheckpoint, checkpoint_id)
- if (
- checkpoint is None
- or checkpoint.user_id != user_id
- or checkpoint.install_id != selected.install_id
- ):
- raise HTTPException(404, "checkpoint_unavailable")
- from .crypto import decrypt_secret
-
- state = decrypt_secret(checkpoint.state_enc)
- if state is None:
- raise HTTPException(410, "checkpoint_expired")
- config["checkpoint"] = json.loads(state)
- from .lifecycle import reconcile
-
- await reconcile(db, user_id)
- from .reservation import adopt, check_quota
-
- if _application is None:
- row, reused = await adopt(
- db, selected, module_id, user_id, chat_id, prewarm=_prewarm, checkpoint_id=checkpoint_id
- )
- else:
- from .application import reserve
-
- row, reused = await reserve(db, selected, module_id, user_id, _application)
- if reused:
- return store.public(row)
- if reused:
- if row is None:
- return None
- from core.sandbox.factory import get_sandbox_provider
-
- provider = _provider or get_sandbox_provider()
- target = store.descriptor(row)
- if (
- target.get("sandbox_id") == await provider.current_sandbox_id(chat_id)
- and target.get("provider") == provider.name
- ):
- try:
- state = await request(row, "/state")
- if not state.get("closed"):
- alive = True
- else:
- alive = False
- except HTTPException:
- alive = False
- if alive:
- db.expire_all()
- row = authorized(db, row.resource_id, user_id)
- return store.public(row)
- await close(db, row)
- # A dead warm worker has performed no user actions; reserve a fresh one.
- from .reservation import lock_owner
-
- lock_owner(db, user_id)
- check_quota(db, user_id)
- from core.config.settings import settings
-
- row = store.PluginResource(
- resource_id=_application["resource_id"] if _application else uuid.uuid4().hex,
- user_id=user_id,
- chat_id=chat_id,
- application_key=_application["application_key"] if _application else None,
- instance_id=_application["instance_id"] if _application else None,
- request_digest=_application["request_digest"] if _application else None,
- install_id=selected.install_id,
- revision=selected.revision,
- slug=slug,
- module_id=module_id,
- scope=(
- _application["scope"]
- if _application
- else ("local" if settings.deploy.is_local else "cloud")
- ),
- descriptor=encrypt_secret("{}"),
- status="warming" if _prewarm else "starting",
- created_at=time.time(),
- expires_at=time.time() + 90,
- )
- from .owners import runtime_session
-
- session_id = runtime_session(row)
- db.add(row)
- db.commit()
- from core.sandbox.factory import get_sandbox_provider
- from core.sandbox.processes.interactive import launch
-
- provider = _provider or get_sandbox_provider()
- try:
- descriptor = await launch(provider, selected, module, user_id, session_id, config)
- except BaseException as exc:
- row.status = "closed"
- db.commit()
- if not isinstance(exc, Exception):
- raise
- code = (
- str(exc)
- if isinstance(exc, ValueError)
- and str(exc)
- in {
- "chromium_sandbox_unavailable",
- "chromium_runtime_missing",
- "chromium_launch_failed",
- "runtime_process_failed",
- "runtime_start_timeout",
- "runtime_start_failed",
- }
- else type(exc).__name__
- )
- raise HTTPException(503, "resource_runtime_unavailable: " + code) from exc
- try:
- db.expire_all()
- validate_binding(db, row, user_id)
- updated = (
- db.query(store.PluginResource)
- .filter_by(
- resource_id=row.resource_id,
- status="warming" if _prewarm else "starting",
- )
- .update(
- {
- "descriptor": encrypt_secret(json.dumps(descriptor)),
- "status": "warm" if _prewarm else "active",
- "expires_at": time.time()
- + (
- 90
- if _application
- else (
- 300
- if _prewarm
- else int(os.getenv("PLUGIN_RESOURCE_TTL_SECONDS", "86400"))
- )
- ),
- },
- synchronize_session=False,
- )
- )
- db.commit()
- if updated != 1:
- raise HTTPException(410, "resource_start_cancelled")
- db.refresh(row)
- if _application:
- from .deadlines import arm
-
- await arm(db, row)
- except BaseException:
- db.rollback()
- if _application:
- # Retain the exact handle before cleanup so a failed stop is janitor-retryable.
- row.descriptor = encrypt_secret(json.dumps(descriptor))
- row.status = "closing"
- db.commit()
- try:
- await close(db, row)
- except Exception:
- db.rollback()
- else:
- await provider.write_stdin(
- descriptor["process_id"],
- sandbox_session_id=session_id,
- user_id=user_id,
- chars="\x03",
- yield_time_ms=1000,
- )
- row.status = "closed"
- db.commit()
- raise
- from core.applications.diagnostics import observe_resource
-
- observe_resource(db, row, "resource", "ok")
- return store.public(row)
-
-
-async def request(row, path, body=None, *, transport=None):
- if transport is not None:
- return await transport.request(path, body)
- async with RuntimeTransport(store.descriptor(row)) as connection:
- return await connection.request(path, body)
-
-
-async def command(row, payload, *, actor, connection_id="", transport=None):
- body = {**payload, "actor": actor, "connection_id": connection_id}
- if body.get("action") in {"storage_state", "checkpoint"}:
- raise HTTPException(403, "checkpoint_requires_explicit_save")
- return await request(row, "/command", body, transport=transport)
-
-
-async def checkpoint(db, row, name, connection_id, *, transport=None):
- if row.application_key is not None:
- raise HTTPException(403, "application_checkpoint_not_granted")
- state = await request(
- row,
- "/command",
- {
- "id": uuid.uuid4().hex,
- "actor": "user",
- "connection_id": connection_id,
- "action": "checkpoint",
- "params": {},
- },
- transport=transport,
- )
- record = store.PluginResourceCheckpoint(
- checkpoint_id=uuid.uuid4().hex,
- user_id=row.user_id,
- install_id=row.install_id,
- name=name[:80],
- state_enc=encrypt_secret(json.dumps(state["state"])),
- )
- db.add(record)
- db.commit()
- return {"checkpoint_id": record.checkpoint_id, "name": record.name}
-
-
-async def close(db, row):
- if row.application_key is not None:
- from .application_cleanup import close
-
- return await close(db, row)
- from .owners import runtime_session
-
- try:
- target = store.descriptor(row)
- except HTTPException:
- target = {}
- if target.get("url") and target.get("headers"):
- try:
- await request(row, "/shutdown", {})
- except HTTPException:
- from core.sandbox.factory import get_sandbox_provider
-
- provider = get_sandbox_provider()
- if target.get("provider") == provider.name and target.get("process_id"):
- try:
- await provider.write_stdin(
- target["process_id"],
- sandbox_session_id=runtime_session(row),
- user_id=row.user_id,
- chars="\x03",
- yield_time_ms=1000,
- )
- except Exception:
- pass
- row.status = "closed"
- db.query(store.PluginResourceTicket).filter_by(resource_id=row.resource_id).delete()
- db.query(store.PluginResourceAssetTicket).filter_by(resource_id=row.resource_id).delete()
- db.commit()
diff --git a/src/backend/core/projects/__init__.py b/src/backend/core/projects/__init__.py
new file mode 100644
index 00000000..51ddcc2b
--- /dev/null
+++ b/src/backend/core/projects/__init__.py
@@ -0,0 +1 @@
+"""Projects: access, files, environment and personal workspaces."""
diff --git a/src/backend/core/services/project_access.py b/src/backend/core/projects/access.py
similarity index 90%
rename from src/backend/core/services/project_access.py
rename to src/backend/core/projects/access.py
index 9c673659..13b546cc 100644
--- a/src/backend/core/services/project_access.py
+++ b/src/backend/core/projects/access.py
@@ -6,7 +6,7 @@
def require_visible_project(db: Session, user_id: str, project_id: str):
"""Return the live project the user may see; otherwise 404 so existence never leaks."""
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
from core.db.models import Project
project = (
diff --git a/src/backend/core/services/project_api_service.py b/src/backend/core/projects/api.py
similarity index 97%
rename from src/backend/core/services/project_api_service.py
rename to src/backend/core/projects/api.py
index 2f17a501..dddc2d41 100644
--- a/src/backend/core/services/project_api_service.py
+++ b/src/backend/core/projects/api.py
@@ -9,7 +9,7 @@
def require_project_key_access(db, user_id: str, project_id: str):
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
project = db.get(Project, project_id)
if project is None or project.deleted_at is not None:
diff --git a/src/backend/core/services/project_service.py b/src/backend/core/projects/base.py
similarity index 85%
rename from src/backend/core/services/project_service.py
rename to src/backend/core/projects/base.py
index 28c3e6cb..8581610e 100644
--- a/src/backend/core/services/project_service.py
+++ b/src/backend/core/projects/base.py
@@ -1,85 +1,20 @@
"""Personal-project service for Community Edition."""
from __future__ import annotations
+from core.projects import project_names, personal_summary
import uuid
from core.infra.time import utc_now
from typing import Any, Dict, List, Optional, Tuple
-from core.auth.permissions_iface import ProjectPermissionLevel, resolve_project_permission
+from core.auth.edition import ProjectPermissionLevel, resolve_project_permission
from core.db.models import Artifact, ChatSession, Project, ProjectFavorite, UserFolder, UserShadow
from fastapi import HTTPException
from sqlalchemy import desc, func, or_
from sqlalchemy.orm import Session
-def _slugify(name: str) -> str:
- """Lowercase, ASCII-safe, hyphen-joined slug for the local sandbox mount name."""
- out = []
- for ch in (name or "").strip().lower():
- if ch.isalnum() and ord(ch) < 128:
- out.append(ch)
- elif ch in (" ", "-", "_", "."):
- out.append("-")
- slug = "".join(out).strip("-")
- while "--" in slug:
- slug = slug.replace("--", "-")
- return slug or "project"
-
-
-def _next_available_folder_name(base: str, existing: List) -> str:
- base = (base or "").strip() or "新项目"
- used = {row[0] for row in existing or []}
- if base not in used:
- return base
- index = 2
- while f"{base} ({index})" in used:
- index += 1
- return f"{base} ({index})"
-
-
-def _summary(
- project: Project,
- *,
- favorite: bool,
- folder_name: Optional[str],
- file_count: int,
- chat_count: int,
-) -> Dict[str, Any]:
- extra = project.extra_data or {}
- local = extra.get("local") or {}
- return {
- "project_id": project.project_id,
- "name": project.name,
- "description": project.description or "",
- "kind": project.kind,
- "is_local": project.kind == "local",
- "local_path": local.get("path"),
- "local_slug": local.get("slug"),
- "local_machine": local.get("machine"),
- "owner_user_id": project.owner_user_id,
- "linked_folder_id": project.linked_folder_id,
- "folder_name": folder_name,
- "instructions": project.instructions or "",
- "icon_color": project.icon_color,
- "pinned": bool(project.pinned),
- "favorite": favorite,
- "memory_enabled": bool(extra.get("memory_enabled", True)),
- "memory_write_enabled": bool(extra.get("memory_write_enabled", True)),
- "permission": "admin",
- "is_owner": True,
- "file_count": file_count,
- "chat_count": chat_count,
- "metadata": extra,
- "created_at": project.created_at.isoformat() if project.created_at else None,
- "updated_at": project.updated_at.isoformat() if project.updated_at else None,
- "last_activity_at": (
- project.last_activity_at.isoformat() if project.last_activity_at else None
- ),
- }
-
-
-class ProjectService:
+class BaseProjectService:
def __init__(self, db: Session):
self.db = db
@@ -190,7 +125,7 @@ def list_visible(
else {}
)
return [
- _summary(
+ personal_summary.summarize(
project,
favorite=project.project_id in favorite_ids,
folder_name=folder_names.get(project.linked_folder_id),
@@ -247,9 +182,9 @@ def create_personal(
):
raise HTTPException(status_code=400, detail="该文件夹已被其它项目挂钩")
else:
- from core.services.user_folder_service import UserFolderService
+ from core.projects.user_folders import UserFolderService
- folder_name = _next_available_folder_name(
+ folder_name = project_names.next_available_folder_name(
clean,
self.db.query(UserFolder.name)
.filter(
@@ -335,7 +270,7 @@ def create_local(
project_id = f"prj_{uuid.uuid4().hex[:16]}"
# slug: stable, filesystem-safe, unique per project — used as the sandbox
# ``local/`` mount name (ticket #04).
- slug = f"{_slugify(clean)}-{project_id[-6:]}"
+ slug = f"{project_names.slugify(clean)}-{project_id[-6:]}"
project = Project(
project_id=project_id,
name=clean,
@@ -384,7 +319,7 @@ def get(self, project_id: str, user_id: str) -> Optional[Dict[str, Any]]:
.scalar()
or 0
)
- data = _summary(
+ data = personal_summary.summarize(
project,
favorite=favorite,
folder_name=folder[0] if folder else None,
@@ -392,7 +327,7 @@ def get(self, project_id: str, user_id: str) -> Optional[Dict[str, Any]]:
chat_count=chat_count,
)
- from core.services.project_instructions import ProjectInstructionsService
+ from core.projects.instructions import ProjectInstructionsService
data.update(ProjectInstructionsService(self.db).read(project))
return data
@@ -449,7 +384,7 @@ def update(
extra[key] = bool(patch[key])
project.extra_data = extra
if "instructions" in patch:
- from core.services.project_instructions import ProjectInstructionsService
+ from core.projects.instructions import ProjectInstructionsService
ProjectInstructionsService(self.db).write(
project, user_id, patch["instructions"] or "",
@@ -530,4 +465,4 @@ def list_chats(
], total
-__all__ = ["ProjectService"]
+__all__ = ["BaseProjectService"]
diff --git a/src/backend/core/projects/edition.py b/src/backend/core/projects/edition.py
new file mode 100644
index 00000000..94d8771b
--- /dev/null
+++ b/src/backend/core/projects/edition.py
@@ -0,0 +1,8 @@
+"""Community project service and single-owner scope."""
+from core.projects.base import BaseProjectService as ProjectService
+from core.projects.personal_scope import (
+ ProjectScope, build_project_ctx, build_project_ctx_from_chat_id,
+ edition_project_context_keys, project_memory_policy, project_scope_from_chat_id,
+ project_scope_from_context,
+)
+from core.projects.personal_transfer import transfer_project
diff --git a/src/backend/core/project_environment/__init__.py b/src/backend/core/projects/environment/__init__.py
similarity index 100%
rename from src/backend/core/project_environment/__init__.py
rename to src/backend/core/projects/environment/__init__.py
diff --git a/src/backend/core/project_environment/config.py b/src/backend/core/projects/environment/config.py
similarity index 100%
rename from src/backend/core/project_environment/config.py
rename to src/backend/core/projects/environment/config.py
diff --git a/src/backend/core/project_environment/dependencies.py b/src/backend/core/projects/environment/dependencies.py
similarity index 96%
rename from src/backend/core/project_environment/dependencies.py
rename to src/backend/core/projects/environment/dependencies.py
index 7509ff59..a9c2b76e 100644
--- a/src/backend/core/project_environment/dependencies.py
+++ b/src/backend/core/projects/environment/dependencies.py
@@ -1,6 +1,6 @@
"""Pure authorized dependency closure; no catalog or database access."""
-from core.project_environment.config import FIELDS
+from core.projects.environment.config import FIELDS
def expand_environment(config, graph):
diff --git a/src/backend/core/services/project_environment.py b/src/backend/core/projects/environment/service.py
similarity index 86%
rename from src/backend/core/services/project_environment.py
rename to src/backend/core/projects/environment/service.py
index f7912bf9..8d00b264 100644
--- a/src/backend/core/services/project_environment.py
+++ b/src/backend/core/projects/environment/service.py
@@ -5,7 +5,7 @@
from contextlib import contextmanager
from uuid import uuid4
-from core.project_environment.config import FIELDS, ProjectEnvironment
+from core.projects.environment.config import FIELDS, ProjectEnvironment
from fastapi import HTTPException
@@ -29,14 +29,14 @@ def _validate_bindings(config, graph):
def validate(self, user_id, data):
config = ProjectEnvironment.model_validate(data).model_dump()
if config["mode"] == "custom" and any(config[field] for field in FIELDS):
- from core.services.project_environment_resources import environment_resources
+ from core.projects.environment_resources import environment_resources
self._validate_bindings(config, environment_resources(self.db, user_id))
return config
def preview(self, user_id, data):
- from core.project_environment.dependencies import expand_environment
- from core.services.project_environment_resources import environment_resources
+ from core.projects.environment.dependencies import expand_environment
+ from core.projects.environment_resources import environment_resources
config = ProjectEnvironment.model_validate(data).model_dump()
graph = environment_resources(self.db, user_id)
@@ -78,8 +78,8 @@ def save(self, project, user_id, data):
return self.read(project)
def describe(self, project, user_id):
- from core.project_environment.dependencies import expand_environment
- from core.services.project_environment_resources import environment_resources
+ from core.projects.environment.dependencies import expand_environment
+ from core.projects.environment_resources import environment_resources
config = self.read(project)
graph = environment_resources(self.db, user_id)
diff --git a/src/backend/core/services/project_environment_resources.py b/src/backend/core/projects/environment_resources.py
similarity index 90%
rename from src/backend/core/services/project_environment_resources.py
rename to src/backend/core/projects/environment_resources.py
index 4c039937..75f6a0e5 100644
--- a/src/backend/core/services/project_environment_resources.py
+++ b/src/backend/core/projects/environment_resources.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from core.project_environment.config import FIELDS
+from core.projects.environment.config import FIELDS
def _keys(entries):
@@ -23,12 +23,12 @@ def _keys(entries):
def environment_resources(db, user_id):
- from core.agent_skills.loader import get_skill_loader
- from core.capabilities import device_catalog
+ from core.capabilities.skills.loader_factory import get_skill_loader
+ from core.harness.registry import device_catalog
from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin
- from core.llm.builtin_subagents import merge_builtin_subagents
- from core.services.user_agent_service import UserAgentService
- from core.services.user_service import UserService
+ from core.capabilities.agents.builtin import merge_builtin_subagents
+ from core.capabilities.agents.edition import UserAgentService
+ from core.auth.users import UserService
from sqlalchemy import or_
service = UserAgentService(db)
@@ -91,14 +91,14 @@ def environment_resources(db, user_id):
for item in resources["mcp_servers"]:
graph["mcp_ids"].setdefault(item["id"], {**item, "dependencies": {}})
- from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
+ from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
if DB_UMBRELLA_ID in graph["mcp_ids"]:
graph["mcp_ids"][DB_UMBRELLA_ID]["dependencies"] = {
"mcp_ids": [m for m in sorted(DB_HIDDEN_SERVERS) if m in graph["mcp_ids"]]
}
- from core.plugins.management import list_installed
+ from core.capabilities.plugins.management import list_installed
callable_plugins = {
p["install_id"] for p in list_installed(db, user_id, include_global=True) if p["callable"]
@@ -130,9 +130,9 @@ def environment_resources(db, user_id):
for field, entries in (("skill_ids", overlay["skills"]), ("mcp_ids", overlay["mcp"])):
for entry in entries:
graph[field].setdefault(entry["id"], {**entry, "dependencies": {}})
- from core.capabilities.paths import KIND_PLUGIN
+ from core.capabilities.identity_rules import KIND_PLUGIN
- device_plugins = device_catalog._installations(KIND_PLUGIN, user_id=user_id)
+ device_plugins = device_catalog.installations(KIND_PLUGIN, user_id=user_id)
for entry in device_catalog.plugin_entries(user_id):
graph["plugin_ids"][entry["install_id"]] = {
"id": entry["install_id"],
@@ -166,7 +166,7 @@ def environment_resources(db, user_id):
graph[field].setdefault(ident, {"id": ident, "name": ident, "dependencies": {}})
# Use the same authoritative ownership, suspension and readiness gate
# as explicit chat invocation; listing a resource is not a permission grant.
- from core.config.catalog_resolver import resolve_explicit_runtime_capabilities
+ from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities
allowed_skills, allowed_mcps, _, _ = resolve_explicit_runtime_capabilities(
db, user_id, skill_ids=list(graph["skill_ids"]), mcp_ids=list(graph["mcp_ids"])
diff --git a/src/backend/core/services/project_file_service.py b/src/backend/core/projects/files.py
similarity index 78%
rename from src/backend/core/services/project_file_service.py
rename to src/backend/core/projects/files.py
index 369d8517..52813883 100644
--- a/src/backend/core/services/project_file_service.py
+++ b/src/backend/core/projects/files.py
@@ -2,8 +2,8 @@
import os
from typing import NamedTuple
-from core.spaces.projects import project_entries
-from core.storage import get_storage
+from core.files.spaces.projects import project_entries
+from core.files.storage import get_storage
class ProjectFileListing(NamedTuple):
@@ -17,7 +17,7 @@ def __init__(self, db):
@staticmethod
def matches_scope(project, scope):
- from core.spaces.edition_access import matches_scope
+ from core.files.edition import matches_scope
return matches_scope(project, scope)
def list_files(self, project, *, limit=None):
@@ -35,16 +35,16 @@ def capacity_limit():
return 200 * 1024 * 1024
def upload(self, project, user_id, file_bytes, filename, mime_type=None):
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
ProjectSourceService(self.db).write_files(
project.project_id, user_id, [(filename, file_bytes)],
)
return next(item for item in self.list_files(project).items if item["name"] == filename)
def delete(self, project_id, actor, path):
- from core.services.project_source import ProjectSourceService
- from core.spaces.files import SpaceFiles
- from core.spaces.projects import project_location
+ from core.projects.source import ProjectSourceService
+ from core.files.spaces.files import SpaceFiles
+ from core.files.spaces.projects import project_location
project = ProjectSourceService(self.db).authorized_project(project_id, actor, write=True)
space, root, _ = project_location(self.db, project)
SpaceFiles(self.db, actor, get_storage()).delete(space, root + "/" + path)
diff --git a/src/backend/core/services/project_instructions.py b/src/backend/core/projects/instructions.py
similarity index 95%
rename from src/backend/core/services/project_instructions.py
rename to src/backend/core/projects/instructions.py
index 25b1e458..e415d61c 100644
--- a/src/backend/core/services/project_instructions.py
+++ b/src/backend/core/projects/instructions.py
@@ -4,14 +4,12 @@
File content is authoritative; the database field is only a legacy fallback.
"""
from __future__ import annotations
-from core.infra.time import utc_now
import hashlib
import os
import stat
import tempfile
-import uuid
-from contextlib import contextmanager, nullcontext
+from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
@@ -19,7 +17,7 @@
from sqlalchemy.orm import Session, attributes
from core.db.models import Project
-from core.storage import get_storage
+from core.files.storage import get_storage
AGENTS_FILENAME = "AGENTS.md"
MAX_INSTRUCTIONS_BYTES = 32 * 1024
@@ -107,7 +105,7 @@ def _read(self, project: Project, *, remember_file: bool) -> dict:
raise HTTPException(409, "AGENTS.md 必须是普通文件")
data = stream.read(MAX_INSTRUCTIONS_BYTES + 1)
else:
- from core.spaces.projects import source_key
+ from core.files.spaces.projects import source_key
storage = get_storage()
key = source_key(self.db, project, AGENTS_FILENAME)
info = storage.stat_object(key)
@@ -161,13 +159,13 @@ def write(
self, project: Project, user_id: str, text: str, *, expected_revision: str | None = None
) -> None:
"""Write the root file; participate in the caller's database transaction."""
- from core.spaces.leases import space_lock
+ from core.files.spaces.leases import space_lock
lock = _local_write_lock(self._local_file(project)) if project.kind == "local" else space_lock(user_id, engine=self.db.get_bind())
with lock:
self._write_locked(project, user_id, text, expected_revision=expected_revision)
def _write_locked(self, project: Project, user_id: str, text: str, *, expected_revision: str | None):
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
if resolve_project_permission(self.db, user_id, project) not in ("admin", "edit"):
raise HTTPException(403, "需要项目编辑权限才能更新 AGENTS.md")
@@ -216,15 +214,15 @@ def _write_locked(self, project: Project, user_id: str, text: str, *, expected_r
project.extra_data = {**(project.extra_data or {}), "agents_file_managed": True}
def _write_file(self, project: Project, data: bytes) -> None:
- from core.services.project_file_service import ProjectFileService
- from core.spaces.projects import source_key
+ from core.projects.files import ProjectFileService
+ from core.files.spaces.projects import source_key
storage = get_storage()
key = source_key(self.db, project, AGENTS_FILENAME)
current = storage.stat_object(key)
files = ProjectFileService(self.db)
if files.capacity_used(project) - (current.size if current else 0) + len(data) > files.capacity_limit():
raise HTTPException(413, "项目容量不足,无法保存 AGENTS.md")
- from core.spaces.tree import ensure_parents
+ from core.files.spaces.tree import ensure_parents
ensure_parents(storage, key)
if current and current.is_directory:
raise HTTPException(409, "AGENTS.md 不能是目录")
@@ -233,7 +231,7 @@ def _write_file(self, project: Project, data: bytes) -> None:
def read_authorized_project_instructions(project_id: str, user_id: str) -> dict:
"""Read canonical rules in a fresh transaction with current view permission."""
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
from core.db.engine import SessionLocal
with SessionLocal() as db:
diff --git a/src/backend/core/services/local_project_workspace.py b/src/backend/core/projects/local_workspace.py
similarity index 97%
rename from src/backend/core/services/local_project_workspace.py
rename to src/backend/core/projects/local_workspace.py
index c61a1cac..a0a1b924 100644
--- a/src/backend/core/services/local_project_workspace.py
+++ b/src/backend/core/projects/local_workspace.py
@@ -19,7 +19,7 @@ def project_directory(scope, *, validate=True) -> str:
def validate_current_project(scope, actor, *, write=False):
"""Reject stale scopes after deletion/rebinding; never switch an active run."""
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
with SessionLocal() as db:
project = ProjectSourceService(db).authorized_project(
diff --git a/src/backend/core/services/project_scope.py b/src/backend/core/projects/personal_scope.py
similarity index 95%
rename from src/backend/core/services/project_scope.py
rename to src/backend/core/projects/personal_scope.py
index 29d0f414..1a527b88 100644
--- a/src/backend/core/services/project_scope.py
+++ b/src/backend/core/projects/personal_scope.py
@@ -97,7 +97,7 @@ def build_project_ctx(db: "Session", project_id: Optional[str]) -> Optional[dict
if not project_id:
return None
from core.db.models import Project, UserFolder
- from core.services.project_instructions import ProjectInstructionsService
+ from core.projects.instructions import ProjectInstructionsService
project = (
db.query(Project)
@@ -129,8 +129,10 @@ def build_project_ctx(db: "Session", project_id: Optional[str]) -> Optional[dict
if path and slug:
try:
ensure_local_project_link(slug, path)
- except Exception:
- pass
+ except Exception as error:
+ from core.infra.failures import record_best_effort_failure
+
+ record_best_effort_failure("projects.personal_scope.local_mount", error)
return {
"project_id": project_id,
"project_environment_config": (project.extra_data or {}).get("environment")
diff --git a/src/backend/core/projects/personal_summary.py b/src/backend/core/projects/personal_summary.py
new file mode 100644
index 00000000..5aa06bea
--- /dev/null
+++ b/src/backend/core/projects/personal_summary.py
@@ -0,0 +1,44 @@
+"""Personal-project response serialization."""
+from __future__ import annotations
+from typing import Any, Dict, Optional
+from core.db.models import Project
+
+def summarize(
+ project: Project,
+ *,
+ favorite: bool,
+ folder_name: Optional[str],
+ file_count: int,
+ chat_count: int,
+) -> Dict[str, Any]:
+ extra = project.extra_data or {}
+ local = extra.get("local") or {}
+ return {
+ "project_id": project.project_id,
+ "name": project.name,
+ "description": project.description or "",
+ "kind": project.kind,
+ "is_local": project.kind == "local",
+ "local_path": local.get("path"),
+ "local_slug": local.get("slug"),
+ "local_machine": local.get("machine"),
+ "owner_user_id": project.owner_user_id,
+ "linked_folder_id": project.linked_folder_id,
+ "folder_name": folder_name,
+ "instructions": project.instructions or "",
+ "icon_color": project.icon_color,
+ "pinned": bool(project.pinned),
+ "favorite": favorite,
+ "memory_enabled": bool(extra.get("memory_enabled", True)),
+ "memory_write_enabled": bool(extra.get("memory_write_enabled", True)),
+ "permission": "admin",
+ "is_owner": True,
+ "file_count": file_count,
+ "chat_count": chat_count,
+ "metadata": extra,
+ "created_at": project.created_at.isoformat() if project.created_at else None,
+ "updated_at": project.updated_at.isoformat() if project.updated_at else None,
+ "last_activity_at": (
+ project.last_activity_at.isoformat() if project.last_activity_at else None
+ ),
+ }
diff --git a/src/backend/core/services/project_transfer.py b/src/backend/core/projects/personal_transfer.py
similarity index 100%
rename from src/backend/core/services/project_transfer.py
rename to src/backend/core/projects/personal_transfer.py
diff --git a/src/backend/core/projects/project_names.py b/src/backend/core/projects/project_names.py
new file mode 100644
index 00000000..418bf3a5
--- /dev/null
+++ b/src/backend/core/projects/project_names.py
@@ -0,0 +1,27 @@
+"""Project folder names and local workspace slugs."""
+from __future__ import annotations
+from typing import List
+
+def slugify(name: str) -> str:
+ """Lowercase, ASCII-safe, hyphen-joined slug for the local sandbox mount name."""
+ out = []
+ for ch in (name or "").strip().lower():
+ if ch.isalnum() and ord(ch) < 128:
+ out.append(ch)
+ elif ch in (" ", "-", "_", "."):
+ out.append("-")
+ slug = "".join(out).strip("-")
+ while "--" in slug:
+ slug = slug.replace("--", "-")
+ return slug or "project"
+
+
+def next_available_folder_name(base: str, existing: List) -> str:
+ base = (base or "").strip() or "新项目"
+ used = {row[0] for row in existing or []}
+ if base not in used:
+ return base
+ index = 2
+ while f"{base} ({index})" in used:
+ index += 1
+ return f"{base} ({index})"
diff --git a/src/backend/core/services/project_source.py b/src/backend/core/projects/source.py
similarity index 91%
rename from src/backend/core/services/project_source.py
rename to src/backend/core/projects/source.py
index 8ec6a85f..d84debc3 100644
--- a/src/backend/core/services/project_source.py
+++ b/src/backend/core/projects/source.py
@@ -6,10 +6,10 @@
import hashlib
from pathlib import PurePosixPath
-from core.auth.permissions_iface import resolve_project_permission
+from core.auth.edition import resolve_project_permission
from core.db.models import Project
-from core.services.project_file_service import ProjectFileService
-from core.storage import get_storage
+from core.projects.files import ProjectFileService
+from core.files.storage import get_storage
from fastapi import HTTPException
from sqlalchemy.orm import Session
@@ -51,8 +51,8 @@ def snapshot(
self, project_id: str, actor: str, *, lock: bool = False,
relative_root: str = "", exclude_patterns: frozenset[str] = frozenset(),
) -> list[tuple[str, bytes]]:
- from core.spaces.projects import project_entries
- from core.storage.entries import validate_key
+ from core.files.spaces.projects import project_entries
+ from core.files.storage.entries import validate_key
project = self.authorized_project(project_id, actor, write=False)
try:
prefix = validate_key(relative_root, directory=True)
@@ -74,7 +74,7 @@ def snapshot(
return result
def read_bytes(self, project_id: str, actor: str, path: str) -> tuple[bytes, str]:
- from core.spaces.projects import source_key
+ from core.files.spaces.projects import source_key
project = self.authorized_project(project_id, actor, write=False)
storage = get_storage()
key = source_key(self.db, project, path)
@@ -98,8 +98,8 @@ def write_files(
self, project_id: str, actor: str, files: list[tuple[str, bytes]],
*, revisions: dict[str, str] | None = None,
) -> list[dict]:
- from core.spaces.leases import space_lock
- from core.spaces.projects import source_key, project_entries
+ from core.files.spaces.leases import space_lock
+ from core.files.spaces.projects import source_key, project_entries
with space_lock(actor, engine=self.db.get_bind()):
self.db.expire_all()
project = self.authorized_project(project_id, actor, write=True)
@@ -110,7 +110,7 @@ def write_files(
prepared, seen = [], set()
for path, data in files:
key = source_key(self.db, project, path)
- from core.spaces.tree import ensure_parents
+ from core.files.spaces.tree import ensure_parents
ensure_parents(storage, key)
existing = storage.stat_object(key)
if (existing and existing.is_directory) or storage.list_directory(key + "/", limit=1).entries:
@@ -121,7 +121,7 @@ def write_files(
if len(data) > 50 * 1024 * 1024:
raise HTTPException(413, "源码文件超过大小限制")
if path == "AGENTS.md":
- from core.services.project_instructions import _decode
+ from core.projects.instructions import _decode
_decode(data)
project.instructions = None
project.extra_data = {**(project.extra_data or {}), "agents_file_managed": True}
diff --git a/src/backend/core/services/user_folder_service.py b/src/backend/core/projects/user_folders.py
similarity index 95%
rename from src/backend/core/services/user_folder_service.py
rename to src/backend/core/projects/user_folders.py
index 06ce03de..24727283 100644
--- a/src/backend/core/services/user_folder_service.py
+++ b/src/backend/core/projects/user_folders.py
@@ -7,15 +7,14 @@
import os
import uuid
from core.db.models import UserShadow
-from core.services.folder_batch import lock_folder_owner
+from core.db.repository.folder import lock_folder_owner
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-from core.db.models import Artifact, UserFolder
-from core.db.repository import ArtifactRepository, AuditLogRepository
-from core.services.artifact_edition import is_personal_artifact, personal_artifact_create_fields
-from core.storage import get_storage
+from core.db.models import UserFolder
+from core.db.repository import AuditLogRepository
+from core.files.storage import get_storage
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ def _sanitize_name(name: str) -> Tuple[bool, str]:
return True, cleaned
-from core.services.personal_file_operations import PersonalFileOperations
+from core.files.personal_operations import PersonalFileOperations
class UserFolderService(PersonalFileOperations):
@@ -150,7 +149,7 @@ def create_folder(
parent_folder_id=parent_folder_id,
name=cleaned,
)
- from core.spaces.files import SpaceFiles
+ from core.files.spaces.files import SpaceFiles
if actor != user_id:
return FolderResult(False, "无权创建该用户的文件夹")
path = self._folder_rel(folder)
@@ -223,7 +222,7 @@ def rename_folder(self, folder_id: str, name: str, actor: str) -> FolderResult:
old_rel = self._folder_rel(folder)
parent_path = old_rel.rpartition("/")[0]
new_rel = parent_path + "/" + cleaned if parent_path else cleaned
- from core.spaces.files import SpaceFiles
+ from core.files.spaces.files import SpaceFiles
SpaceFiles(self.db, actor, get_storage()).move("personal", old_rel, new_rel)
folder.name = cleaned
folder.updated_at = utc_now()
@@ -294,7 +293,7 @@ def move_folder(
old_rel = self._folder_rel(folder)
parent_path = self._folder_rel(parent) if new_parent_id else ""
new_rel = parent_path + "/" + folder.name if parent_path else folder.name
- from core.spaces.files import SpaceFiles
+ from core.files.spaces.files import SpaceFiles
SpaceFiles(self.db, actor, get_storage()).move("personal", old_rel, new_rel)
folder.parent_folder_id = new_parent_id
folder.updated_at = utc_now()
@@ -341,7 +340,7 @@ def delete_folder(self, folder_id: str, actor: str) -> Tuple[FolderResult, int]:
ids_to_delete = self._collect_descendants(folder_id)
now = utc_now()
- from core.spaces.files import SpaceFiles
+ from core.files.spaces.files import SpaceFiles
affected = SpaceFiles(self.db, actor, get_storage()).delete_tree("personal", self._folder_rel(folder))
self.db.query(UserFolder).filter(
@@ -367,8 +366,8 @@ def delete_folder(self, folder_id: str, actor: str) -> Tuple[FolderResult, int]:
return FolderResult(True, "文件夹已删除", folder_id=folder_id), int(affected or 0)
def count_affected_artifacts(self, folder_id: str, user_id: str) -> int:
- from core.spaces.access import namespace
- from core.spaces.tree import tree_entries
+ from core.files.spaces.access import namespace
+ from core.files.spaces.tree import tree_entries
folder = self.get_owned(folder_id, user_id)
if folder is None:
return 0
diff --git a/src/backend/core/runs/__init__.py b/src/backend/core/runs/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/backend/orchestration/agent_api_recovery.py b/src/backend/core/runs/api_recovery.py
similarity index 94%
rename from src/backend/orchestration/agent_api_recovery.py
rename to src/backend/core/runs/api_recovery.py
index 7427dde6..68737228 100644
--- a/src/backend/orchestration/agent_api_recovery.py
+++ b/src/backend/core/runs/api_recovery.py
@@ -3,9 +3,9 @@
from __future__ import annotations
from core.db.models import ChatSession
-from core.llm.agent_api_runtime import apply_api_scope, parse_api_scope
-from core.services.agent_api_service import session_matches_scope
-from core.services.project_api_service import project_session_matches
+from core.llm.api_runtime.agent import apply_api_scope, parse_api_scope
+from core.auth.agent_api import session_matches_scope
+from core.projects.api import project_session_matches
def validate_recovery_context(db, run, context, *, user_id: str, chat_id: str) -> dict:
diff --git a/src/backend/core/services/run_binding.py b/src/backend/core/runs/binding.py
similarity index 73%
rename from src/backend/core/services/run_binding.py
rename to src/backend/core/runs/binding.py
index 88374833..53455534 100644
--- a/src/backend/core/services/run_binding.py
+++ b/src/backend/core/runs/binding.py
@@ -9,40 +9,33 @@
from core.db.engine import SessionLocal
from core.db.models import ChatRun, ChatSession
-from core.services.run_journal_domain import (
- DurableRunBinding,
- RunAlreadyExists,
- RunJournalError,
- RunLeaseLost,
-)
+from core.db.threads import run_db
+from core.runs.journal_domain import DurableRunBinding, RunAlreadyExists, RunJournalError, RunLeaseLost
from sqlalchemy.orm import sessionmaker
-@asynccontextmanager
-async def durable_run_binding(
+def _admit(
+ session_factory: sessionmaker,
+ journal,
*,
user_id: str,
- chat_id: Optional[str],
+ chat_id: str,
kind: str,
external_id: str,
- request_payload: Optional[Mapping[str, Any]] = None,
- recovery_snapshot: Optional[Mapping[str, Any]] = None,
- session_factory: sessionmaker = SessionLocal,
- lease_seconds: int = 300,
- heartbeat_interval: Optional[float] = None,
- binding_run_id: Optional[str] = None,
-):
- """Give non-chat Agent entry points the same durable run/heartbeat contract."""
-
- from core.services.run_journal import RunJournal
-
- actual_chat_id = str(chat_id or f"internal_{uuid.uuid4().hex[:24]}")
+ run_id: str,
+ owner: str,
+ request_payload: Optional[Mapping[str, Any]],
+ recovery_snapshot: Optional[Mapping[str, Any]],
+ lease_seconds: int,
+ deterministic: bool,
+) -> None:
+ """Create the run's chat if needed, accept the run and claim its lease."""
with session_factory() as db:
- session = db.get(ChatSession, actual_chat_id)
+ session = db.get(ChatSession, chat_id)
if session is None:
db.add(
ChatSession(
- chat_id=actual_chat_id,
+ chat_id=chat_id,
user_id=user_id,
title=f"{kind} run",
)
@@ -51,16 +44,13 @@ async def durable_run_binding(
elif session.user_id != user_id:
raise RunJournalError("durable run chat belongs to another user")
- run_id = str(binding_run_id or f"run_{uuid.uuid4().hex}")
- owner = f"{kind}:{uuid.uuid4().hex}"
- journal = RunJournal(session_factory)
snapshot = {"kind": kind, "external_id": external_id}
snapshot.update(dict(recovery_snapshot or {}))
try:
journal.accept(
run_id=run_id,
message_id=f"msg_{uuid.uuid4().hex}",
- chat_id=actual_chat_id,
+ chat_id=chat_id,
user_id=user_id,
request_payload={
"kind": kind,
@@ -74,7 +64,7 @@ async def durable_run_binding(
# The primary key resolves the check-then-create race across processes;
# only translate an error when that exact row now exists so unrelated
# database failures retain their original signal.
- if binding_run_id:
+ if deterministic:
with session_factory() as db:
if db.get(ChatRun, run_id) is not None:
raise RunAlreadyExists(run_id) from exc
@@ -82,6 +72,45 @@ async def durable_run_binding(
if not journal.claim(run_id, owner=owner, lease_seconds=lease_seconds):
raise RunLeaseLost(run_id)
+
+@asynccontextmanager
+async def durable_run_binding(
+ *,
+ user_id: str,
+ chat_id: Optional[str],
+ kind: str,
+ external_id: str,
+ request_payload: Optional[Mapping[str, Any]] = None,
+ recovery_snapshot: Optional[Mapping[str, Any]] = None,
+ session_factory: sessionmaker = SessionLocal,
+ lease_seconds: int = 300,
+ heartbeat_interval: Optional[float] = None,
+ binding_run_id: Optional[str] = None,
+):
+ """Give non-chat Agent entry points the same durable run/heartbeat contract."""
+
+ from core.runs.journal import RunJournal
+
+ actual_chat_id = str(chat_id or f"internal_{uuid.uuid4().hex[:24]}")
+ run_id = str(binding_run_id or f"run_{uuid.uuid4().hex}")
+ owner = f"{kind}:{uuid.uuid4().hex}"
+ journal = RunJournal(session_factory)
+ await run_db(
+ _admit,
+ session_factory,
+ journal,
+ user_id=user_id,
+ chat_id=actual_chat_id,
+ kind=kind,
+ external_id=external_id,
+ run_id=run_id,
+ owner=owner,
+ request_payload=request_payload,
+ recovery_snapshot=recovery_snapshot,
+ lease_seconds=lease_seconds,
+ deterministic=bool(binding_run_id),
+ )
+
lost = asyncio.Event()
worker_task = asyncio.current_task()
if worker_task is None: # pragma: no cover - async context always has a task
@@ -108,7 +137,7 @@ async def _heartbeat() -> None:
return
heartbeat = asyncio.create_task(_heartbeat())
- from core.llm.context_safety import request_admission_scope
+ from core.llm.context.safety import request_admission_scope
with request_admission_scope(
context={"run_id": run_id, "journal_owner": owner},
@@ -120,7 +149,8 @@ async def _heartbeat() -> None:
admission.check()
except asyncio.CancelledError:
if not lost.is_set():
- journal.complete(
+ await run_db(
+ journal.complete,
run_id,
owner=owner,
status="cancelled",
@@ -128,25 +158,28 @@ async def _heartbeat() -> None:
)
raise
except Exception as exc:
- from core.services.tool_effect_ledger import find_tool_outcome_unknown
+ from core.runs.tool_effects.errors import find_tool_outcome_unknown
unknown = find_tool_outcome_unknown(exc)
if admission.stopped is not None:
- from core.services.context_budget_checkpoint import ContextBudgetJournal
+ from core.chat.compaction.budget_checkpoint import ContextBudgetJournal
- ContextBudgetJournal(session_factory).pause_context(
+ await run_db(
+ ContextBudgetJournal(session_factory).pause_context,
{"run_id": run_id, "journal_owner": owner},
admission.stopped,
admission.checkpoint,
)
elif unknown is not None:
- journal.needs_attention(
+ await run_db(
+ journal.needs_attention,
run_id,
owner=owner,
reason=f"tool outcome pending policy recovery: {unknown}",
)
else:
- journal.complete(
+ await run_db(
+ journal.complete,
run_id,
owner=owner,
status="failed",
@@ -154,7 +187,7 @@ async def _heartbeat() -> None:
)
raise
else:
- journal.complete(run_id, owner=owner, status="completed")
+ await run_db(journal.complete, run_id, owner=owner, status="completed")
finally:
heartbeat.cancel()
with suppress(asyncio.CancelledError):
diff --git a/src/backend/core/runs/clock.py b/src/backend/core/runs/clock.py
new file mode 100644
index 00000000..8db190c5
--- /dev/null
+++ b/src/backend/core/runs/clock.py
@@ -0,0 +1,55 @@
+"""UTC timestamps shared by run records and emitted events."""
+
+from __future__ import annotations
+
+import datetime as _datetime
+from datetime import datetime
+from typing import Optional
+
+import core.db.engine as engine
+import core.db.models as models
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def _utcnow() -> datetime:
+ return _datetime.datetime.now(_datetime.timezone.utc)
+
+
+def _epoch_ms(value: Optional[datetime]) -> Optional[int]:
+ """Wall-clock milliseconds for a stored timestamp.
+
+ SQLite hands back naive datetimes; everything written here is UTC, so an
+ absent tzinfo is read as UTC rather than as the host's local zone.
+ """
+ if value is None:
+ return None
+ if value.tzinfo is None:
+ value = value.replace(tzinfo=_datetime.timezone.utc)
+ return int(value.timestamp() * 1000)
+
+
+def _run_started_at_ms(run_id: str) -> Optional[int]:
+ """When this run actually began, in wall-clock milliseconds.
+
+ Read from ``chat_runs.started_at``, which is written once when the run is
+ claimed and is never rewritten by a recovering worker. Sending it with the
+ first frame is what lets a second device — or a reloaded tab — join a run
+ already in flight and show the true elapsed time instead of starting its
+ own stopwatch from zero.
+
+ Only a worker resuming someone else's run needs to look this up: a worker
+ that claimed the run already holds the instant it wrote. Keeping the read
+ off the normal send path matters — it sits directly in front of the first
+ frame, where every millisecond is one the user spends staring at nothing.
+ """
+ with engine.SessionLocal() as db:
+ row = (
+ db.query(models.ChatRun.started_at, models.ChatRun.created_at)
+ .filter(models.ChatRun.run_id == run_id)
+ .first()
+ )
+ if row is None:
+ return None
+ return _epoch_ms(row.started_at or row.created_at)
diff --git a/src/backend/core/runs/events/__init__.py b/src/backend/core/runs/events/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/backend/core/runs/events/projection.py b/src/backend/core/runs/events/projection.py
new file mode 100644
index 00000000..3dd8caf3
--- /dev/null
+++ b/src/backend/core/runs/events/projection.py
@@ -0,0 +1,13 @@
+"""Projection functions supplied by the event log's owning domain."""
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ProjectionCodec:
+ new_projection: Callable[[str], dict[str, Any]]
+ fold_projection: Callable[[dict[str, Any], dict[str, Any]], dict[str, Any]]
+ affected_tools: Callable[[dict[str, Any], dict[str, Any]], set[int]]
+ encode_checkpoint: Callable[..., dict[str, str]]
+ decode_checkpoint: Callable[..., dict[str, Any] | None]
diff --git a/src/backend/orchestration/run_event_stream.py b/src/backend/core/runs/events/store.py
similarity index 84%
rename from src/backend/orchestration/run_event_stream.py
rename to src/backend/core/runs/events/store.py
index 33e68099..d081c709 100644
--- a/src/backend/orchestration/run_event_stream.py
+++ b/src/backend/core/runs/events/store.py
@@ -19,16 +19,15 @@
import asyncio
import json
-from copy import deepcopy
-import time
import threading
+import time
from collections import deque
+from copy import deepcopy
from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable
+from core.runs.events.projection import ProjectionCodec
from core.infra.redis import get_redis, redis_configured
from redis.exceptions import WatchError
-from orchestration.run_projection import fold_projection, new_projection
-from orchestration.run_projection_storage import affected_tools, decode_checkpoint, encode_checkpoint
# Cursor meaning "from the very beginning of the log".
START = ""
@@ -59,7 +58,7 @@ def cursor_millis(cursor: str) -> Optional[int]:
return None
-def _sort_key(cursor: str) -> Tuple[int, int]:
+def sort_key(cursor: str) -> Tuple[int, int]:
if not cursor:
return (-1, -1)
stamp, _, seq = cursor.partition("-")
@@ -134,7 +133,8 @@ def _decode(entries: Any) -> List[Entry]:
def _snapshot_key(self, run_id: str) -> str:
return f"jx:chat:run:{run_id}:projection"
- def __init__(self) -> None:
+ def __init__(self, projection: ProjectionCodec) -> None:
+ self.projection = projection
self._projections: Dict[str, Dict[str, Any]] = {}
self._pending: Dict[str, int] = {}
self._versions: Dict[str, str] = {}
@@ -145,7 +145,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None:
event = {**event, "event_offset": int(event.get("_offset") or 0)}
for attempt in range(8):
if attempt:
- await asyncio.sleep(min(0.005 * 2 ** attempt, 0.1))
+ await asyncio.sleep(min(0.005 * 2**attempt, 0.1))
async with get_redis().pipeline(transaction=True) as pipe:
try:
await pipe.watch(self._key(run_id))
@@ -156,7 +156,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None:
previous, captured_version = await self.capture(run_id)
if captured_version != version:
continue
- previous = previous or new_projection(run_id)
+ previous = previous or self.projection.new_projection(run_id)
count = 1
dirty = set(range(len(previous["tools"])))
history_dirty = True
@@ -167,24 +167,36 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None:
offset = int(event.get("_offset") or 0)
if offset and offset <= previous["event_offset"]:
raise ValueError("run event offset must advance")
- state = fold_projection(previous, event)
+ state = self.projection.fold_projection(previous, event)
if event.get("type") in {"run_started", "steer_applied"}:
- dirty = affected_tools(state, event)
+ dirty = self.projection.affected_tools(state, event)
else:
- dirty.update(affected_tools(state, event))
+ dirty.update(self.projection.affected_tools(state, event))
if len(state["tools"]) > len(previous["tools"]):
dirty.update(range(len(previous["tools"]), len(state["tools"])))
- history_dirty = history_dirty or event.get("type") in {"run_started", "steer_applied"}
+ history_dirty = history_dirty or event.get("type") in {
+ "run_started",
+ "steer_applied",
+ }
interval = min(64, max(1, MAXLEN // 2))
distance = offset - previous.get("_checkpoint_offset", 0) if offset else count
checkpoint = distance >= interval or event.get("type") in {
- "run_started", "steer_applied", "__terminal__"
+ "run_started",
+ "steer_applied",
+ "__terminal__",
}
pipe.multi()
- pipe.xadd(self._key(run_id), {"data": json.dumps(event, ensure_ascii=False)}, maxlen=MAXLEN)
+ pipe.xadd(
+ self._key(run_id),
+ {"data": json.dumps(event, ensure_ascii=False)},
+ maxlen=MAXLEN,
+ )
if checkpoint:
state["_checkpoint_offset"] = offset
- pipe.hset(self._snapshot_key(run_id), mapping=encode_checkpoint(state, dirty, history_dirty=history_dirty))
+ pipe.hset(
+ self._snapshot_key(run_id),
+ mapping=self.projection.encode_checkpoint(state, dirty, history_dirty=history_dirty),
+ )
result = await pipe.execute()
self._projections[run_id] = state
self._versions[run_id] = result[0]
@@ -209,14 +221,14 @@ async def capture(self, run_id: str) -> Tuple[Optional[Dict[str, Any]], str]:
pipe.hgetall(self._snapshot_key(run_id))
pipe.xrange(self._key(run_id), min="-", max="+")
raw, entries = await pipe.execute()
- state = decode_checkpoint(raw)
+ state = self.projection.decode_checkpoint(raw)
decoded = self._decode(entries)
if state is not None:
floor = state["event_offset"]
state["_checkpoint_offset"] = floor
for _, event in decoded:
if int(event.get("_offset") or 0) > floor:
- state = fold_projection(state, event)
+ state = self.projection.fold_projection(state, event)
return state, decoded[-1][0] if decoded else START
async def read(
@@ -246,8 +258,9 @@ async def wait(self, run_id: str, *, after: str, limit: int, timeout_ms: int) ->
async def _drop_connections(redis: Any) -> None:
try:
await redis.connection_pool.disconnect(inuse_connections=False)
- except Exception: # noqa: BLE001 - recovery must not mask the real error
- pass
+ except Exception as fallback_error: # noqa: BLE001 - recovery must not mask the real error
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.runs.events.store:_drop_connections', fallback_error)
async def last_write_ms(self, run_id: str) -> Optional[int]:
entries = await get_redis().xrevrange(self._key(run_id), max="+", min="-", count=1)
@@ -281,7 +294,15 @@ async def clear(self, run_id: str) -> None:
class _RunLog:
"""One run's in-process buffer plus the followers parked on it."""
- __slots__ = ("entries", "expires_at", "last_millis", "sequence", "waiters", "projection", "lock")
+ __slots__ = (
+ "entries",
+ "expires_at",
+ "last_millis",
+ "sequence",
+ "waiters",
+ "projection",
+ "lock",
+ )
def __init__(self) -> None:
self.entries: deque[Entry] = deque(maxlen=MAXLEN)
@@ -300,9 +321,9 @@ def mint(self) -> str:
return f"{millis}-{self.sequence}"
def since(self, after: str, limit: Optional[int]) -> List[Entry]:
- floor = _sort_key(after)
+ floor = sort_key(after)
with self.lock:
- found = [entry for entry in self.entries if _sort_key(entry[0]) > floor]
+ found = [entry for entry in self.entries if sort_key(entry[0]) > floor]
return found[:limit] if limit else found
@@ -314,7 +335,8 @@ class LocalRunEventStream:
with an asyncio primitive shared across loops.
"""
- def __init__(self) -> None:
+ def __init__(self, projection: ProjectionCodec) -> None:
+ self.projection = projection
self._logs: Dict[str, _RunLog] = {}
def _sweep(self) -> None:
@@ -337,7 +359,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None:
log = self._log(run_id, create=True)
event = {**event, "event_offset": int(event.get("_offset") or 0)}
with log.lock:
- log.projection = fold_projection(log.projection or new_projection(run_id), event)
+ log.projection = self.projection.fold_projection(log.projection or self.projection.new_projection(run_id), event)
log.entries.append((log.mint(), event))
waiters, log.waiters = log.waiters, []
for loop, ready in waiters:
@@ -393,13 +415,25 @@ async def clear(self, run_id: str) -> None:
self._logs.pop(run_id, None)
-_REDIS_STREAM = RedisRunEventStream()
-_LOCAL_STREAM = LocalRunEventStream()
+_REDIS_STREAM: RedisRunEventStream | None = None
+_LOCAL_STREAM: LocalRunEventStream | None = None
+
+
+def configure_projection(projection: ProjectionCodec) -> None:
+ """Initialize once; repeated host setup must preserve active streams."""
+ global _REDIS_STREAM, _LOCAL_STREAM
+ if _REDIS_STREAM is None:
+ _REDIS_STREAM = RedisRunEventStream(projection)
+ if _LOCAL_STREAM is None:
+ _LOCAL_STREAM = LocalRunEventStream(projection)
def get_run_event_stream() -> RunEventStream:
"""The run event log this deployment should use."""
- return _REDIS_STREAM if redis_configured() else _LOCAL_STREAM
+ stream = _REDIS_STREAM if redis_configured() else _LOCAL_STREAM
+ if stream is None:
+ raise RuntimeError("Run event projection has not been configured")
+ return stream
__all__ = [
diff --git a/src/backend/core/runs/events/subscription.py b/src/backend/core/runs/events/subscription.py
new file mode 100644
index 00000000..9ce2181e
--- /dev/null
+++ b/src/backend/core/runs/events/subscription.py
@@ -0,0 +1,81 @@
+"""Replay and follow durable run events from the requested offset."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, AsyncIterator, Dict
+
+import core.chat.runtime.followups as followups
+import core.runs.events.store as store
+import core.runs.events.writer as writer
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+import redis.exceptions as _redis_exceptions
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def follow_run(run_id: str, *, from_offset: int = 0) -> AsyncIterator[Dict[str, Any]]:
+ "Read the run's event log, delivering events after ``from_offset``.\n\n Flow:\n 1. Replay everything recorded so far; deliver those with ``_offset > from_offset``\n 2. Block on the log waiting for new events\n 3. Stop when a ``__terminal__``-typed event arrives\n 4. Wait timed out + run in a terminal state → drain once more, then exit\n gracefully (backstop so the follower never hangs forever)\n\n Which log backs this — Redis Streams or the in-process one — is the\n deployment's choice; see :mod:`core.runs.events.store`.\n"
+ events = writer._events()
+
+ run = runtime.get_run(run_id)
+ if run is None:
+ # The chats.py route layer should have validated this already; this is a second line of defense
+ raise runtime.ChatRunNotFound(f"chat run {run_id} not found")
+
+ cursor = store.START
+
+ # ── Phase 1: replay historical events ──
+ try:
+ history = await events.read(run_id)
+ except Exception as exc:
+ logger.warning("chat_run_xrange_failed", run_id=run_id, error=str(exc), exc_info=exc)
+ history = []
+
+ delivered, cursor, terminal_seen = followups._drain(history, from_offset, cursor)
+ for event in delivered:
+ yield event
+ if terminal_seen:
+ return
+
+ # ── Phase 2: blocking tail ──
+ while True:
+ try:
+ batch = await events.wait(
+ run_id, after=cursor, limit=100, timeout_ms=settings._XREAD_BLOCK_MS
+ )
+ except _redis_exceptions.TimeoutError:
+ # Benign: the block window elapsed with no new events — semantically
+ # identical to an empty result. (redis-py 8.0 defaults socket_timeout
+ # to 5s; if it ever equals _XREAD_BLOCK_MS the read raises here on
+ # every idle window instead of returning nil. We size socket_timeout
+ # well above the block in core/infra/redis.py, but treat the timeout
+ # as "no events" regardless so a long, quiet run never spams logs.)
+ batch = []
+ except Exception as exc:
+ logger.warning("chat_run_xread_failed", run_id=run_id, error=str(exc), exc_info=exc)
+ await asyncio.sleep(0.5)
+ batch = []
+
+ if not batch:
+ # Nothing arrived: check whether the run reached a terminal state
+ current = runtime.get_run(run_id)
+ if current is not None and current.status in settings._TERMINAL_STATUSES:
+ # After the terminal state, read the latest events once more (covers the race)
+ try:
+ tail = await events.read(run_id, after=cursor, limit=200)
+ except Exception:
+ tail = []
+ delivered, cursor, _terminal = followups._drain(tail, from_offset, cursor)
+ for event in delivered:
+ yield event
+ return
+ continue
+
+ delivered, cursor, terminal_seen = followups._drain(batch, from_offset, cursor)
+ for event in delivered:
+ yield event
+ if terminal_seen:
+ return
diff --git a/src/backend/core/runs/events/writer.py b/src/backend/core/runs/events/writer.py
new file mode 100644
index 00000000..2a6ebec0
--- /dev/null
+++ b/src/backend/core/runs/events/writer.py
@@ -0,0 +1,197 @@
+"""Append ordered run events and persist terminal stream markers."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+import typing as _typing
+from typing import Any, Dict
+
+import core.runs.events.store as store
+import core.runs.ownership as ownership
+import core.runs.settings as settings
+import core.runs.journal as run_journal
+from core.infra.logging import get_logger
+from core.runs.events.store import RunEventStream
+from core.runs.journal import RunJournal
+
+logger = get_logger(__name__)
+
+
+def _events() -> RunEventStream:
+ """This deployment's run event log (Redis Streams, or in-process)."""
+ return store.get_run_event_stream()
+
+
+class EventOffsets:
+ """Durable SSE offsets for one worker, without a database write per event.
+
+ Allocating an offset and proving the worker still owns the run used to be
+ the same write, so every token cost a serialized transaction — invisible
+ against a server database, and a disk flush per token on the desktop, where
+ the journal is SQLite and the person is watching the answer appear.
+
+ The two are separable. Offsets come from reservations the journal hands out
+ in runs, and this keeps the unused remainder. Ownership is proven by reading
+ the lease rather than writing to it, and on a bounded schedule rather than
+ once per event: measured in the deployment that read costs ~0.5 ms, which at
+ forty concurrent streams is a whole core spent asking a question whose
+ answer almost never changes.
+
+ Ownership can stop being ours two ways, and the bound is sized against the
+ slower one:
+
+ * This process notices itself — its own heartbeat fails to renew, or the
+ user cancelled here. ``lease_lost`` is set, and the very next event is
+ fenced without leaving the loop at all, so this path got *faster*, not
+ looser.
+ * Another worker decided it — a cancel that landed on a different process,
+ or a recovery that re-claimed this run. That can only arrive through the
+ shared journal, so the read cannot be dropped, only bounded: at most one
+ per ``_CANCEL_VISIBILITY_SEC``, the same cadence the chunk loop already
+ polls cancellation on. Re-claiming requires the lease to have *expired*,
+ which means this worker went a full lease without renewing — it was not
+ emitting during that stretch either — so the window where a fenced worker
+ could still append is bounded by a quarter second and reached only by a
+ process that had stopped producing anyway.
+
+ A terminal event is allocated on its own: it has to sit after every live
+ offset, including the ones a reservation left unused. That allocation is an
+ owned write, so a run always fences hard on the way out.
+ """
+
+ def __init__(
+ self,
+ journal: RunJournal,
+ run_id: str,
+ owner: str,
+ *,
+ lease_lost: asyncio.Event,
+ ):
+ self._journal = journal
+ self._run_id = run_id
+ self._owner = owner
+ self._next = 0
+ self._remaining = 0
+ self._lease_lost = lease_lost
+ self._proven_at = 0.0
+ # The offset most recently handed out. Callers persist it with their
+ # checkpoints so a reconnecting follower knows where the run had got to.
+ self.last = 0
+
+ async def take(self, *, terminal: bool = False) -> int:
+ if terminal:
+ self._remaining = 0
+ self.last = await asyncio.to_thread(
+ self._journal.allocate_event_offset, self._run_id, terminal=True
+ )
+ return self.last
+ if self._remaining == 0:
+ # A reservation is an owned write: it fences on its own, and more
+ # strongly than a read does.
+ self._next = await asyncio.to_thread(self._reserve)
+ self._remaining = settings._OFFSET_RESERVATION
+ self._proven_at = time.monotonic()
+ else:
+ await self._prove_ownership()
+ offset = self._next
+ self._next += 1
+ self._remaining -= 1
+ self.last = offset
+ return offset
+
+ async def _prove_ownership(self) -> None:
+ """Raise ``RunLeaseLost`` unless this worker still owns the run."""
+ if self._lease_lost.is_set():
+ # Our own heartbeat already decided this; the journal cannot
+ # disagree, so there is nothing to go and ask.
+ raise run_journal.RunLeaseLost(self._run_id)
+ now = time.monotonic()
+ if now - self._proven_at < settings._CANCEL_VISIBILITY_SEC:
+ return
+ await asyncio.to_thread(self._journal.require_lease, self._run_id, self._owner)
+ self._proven_at = now
+
+ async def emit(self, event: Dict[str, Any], *, terminal: bool = False) -> None:
+ """Allocate this event's offset and append it to the run's log."""
+ await self.take(terminal=terminal)
+ await xadd_event(self._run_id, self.last, event)
+
+ def _reserve(self) -> int:
+ return self._journal.allocate_event_offset(
+ self._run_id, owner=self._owner, count=settings._OFFSET_RESERVATION
+ )
+
+
+def _json_safe(value: Any) -> Any:
+ return json.loads(json.dumps(value, ensure_ascii=False, default=str))
+
+
+async def _write_terminal_to_stream(
+ run_id: str,
+ *,
+ chat_id: str,
+ error_text: str,
+ cancelled: bool = False,
+) -> None:
+ """Write an error event + terminal marker + EXPIRE.
+
+ Used when the worker isn't around to emit them itself (cross-process cancel,
+ orphan recovery). Failure is logged but not raised.
+ """
+ err_event: _typing.Dict[str, _typing.Any] = {
+ "type": "error",
+ "error": error_text,
+ "delta": error_text,
+ "chat_id": chat_id,
+ }
+ term_event: _typing.Dict[str, _typing.Any] = {
+ "type": settings.TERMINAL_TYPE,
+ "chat_id": chat_id,
+ }
+ if cancelled:
+ err_event["_cancelled"] = True
+ term_event["_cancelled"] = True
+ try:
+ journal = ownership.journal()
+ error_offset = journal.allocate_event_offset(run_id, terminal=True)
+ await xadd_event(run_id, error_offset, err_event)
+ terminal_offset = journal.allocate_event_offset(run_id, terminal=True)
+ await xadd_event(run_id, terminal_offset, term_event)
+ await expire_stream(run_id)
+ except Exception as exc:
+ logger.warning("chat_run_terminal_write_failed", run_id=run_id, error=str(exc))
+
+
+async def xadd_event(run_id: str, offset: int, event: Dict[str, Any]) -> None:
+ """Stamp an SSE event with its offset and wall clock, append it to the run's log.
+
+ ``server_ts`` is when the server produced the event. A client that joins
+ late — after a reconnect, a session switch, or from a second device —
+ replays the log and reads these stamps, so its "waiting since" clock
+ reflects the run's real timeline instead of the moment this browser
+ happened to receive them. It deliberately has no underscore prefix: that
+ prefix marks transport-internal keys, which ``follow_run_as_sse`` strips
+ before the frame goes out, and this one is meant for the client.
+ """
+ payload = {**event, "_offset": offset, "server_ts": int(time.time() * 1000)}
+ try:
+ await _events().append(run_id, payload)
+ except Exception as exc:
+ logger.warning("chat_run_xadd_failed", run_id=run_id, error=str(exc))
+
+
+async def expire_stream(run_id: str) -> None:
+ try:
+ await _events().expire(run_id, settings._STREAM_TTL_SECONDS)
+ except Exception as exc:
+ logger.warning("chat_run_expire_failed", run_id=run_id, error=str(exc))
+
+
+async def _reset_stream_projection(run_id: str) -> None:
+ """Drop a crashed worker's partial projection before replaying its run."""
+ try:
+ await _events().clear(run_id)
+ except Exception as exc:
+ logger.warning("chat_run_projection_reset_failed", run_id=run_id, error=str(exc))
diff --git a/src/backend/core/runs/host.py b/src/backend/core/runs/host.py
new file mode 100644
index 00000000..046fe99e
--- /dev/null
+++ b/src/backend/core/runs/host.py
@@ -0,0 +1,41 @@
+"""Run services installed by the application composition root."""
+from core.harness.registry.run_cancellation import register_run_cancellation
+from core.runs import runtime
+
+
+def register() -> None:
+ from core.jobs.tool_operations import job_operations
+ from core.harness.registry.job_operations import register_job_operations
+ register_job_operations(job_operations)
+ from core.chat.history_operations import history_operations
+ from core.harness.registry.chat_history import register_chat_history
+ register_chat_history(history_operations)
+ from core.harness.registry.compaction import register_compaction_services
+ from core.chat.compaction import service as compaction
+ from core.chat.compaction.budget_checkpoint import ContextStopCheckpoint
+ register_compaction_services(
+ run=compaction.run_compaction, mid_turn=compaction.run_mid_turn_compaction,
+ token_limit=compaction.resolve_token_limit, trigger_ratio=compaction.resolve_trigger_ratio,
+ checkpoint=ContextStopCheckpoint,
+ )
+ from core.harness.registry.hook_factories import register_run_hook_factories
+ from core.chat.steer_hook import SteeringHook
+ from core.sandbox.commands.completion_hook import command_completion_hooks
+ register_run_hook_factories(steering=SteeringHook, command_completion=command_completion_hooks)
+ from core.harness.registry.hook_factories import register_hook_factory
+ from core.jobs.reminder_hook import job_ledger_hooks
+ register_hook_factory(job_ledger_hooks)
+ from core.runs.tool_effects.hooks import tool_effect_hooks
+ register_hook_factory(tool_effect_hooks)
+ from core.llm.ports import register_execution_services
+ from core.sandbox.commands.runtime import runtime as commands
+ from core.sandbox.commands import store as records
+ from core.jobs.agent import snapshot_parent_runtime
+ register_execution_services(commands=commands, records=records, parent_snapshot=snapshot_parent_runtime)
+ from core.chat.projection.codec import chat_projection_codec
+ from core.runs.events.store import configure_projection
+ configure_projection(chat_projection_codec())
+ async def cancel(run_id: str, *, user_id: str, stop_terminal_worker: bool = False) -> bool:
+ return await runtime.cancel_run(run_id, user_id=user_id, stop_terminal_worker=stop_terminal_worker)
+ from core.runs.journal import RunJournal
+ register_run_cancellation(cancel, cancel_in_session=RunJournal.cancel_in_session)
diff --git a/src/backend/core/services/run_journal.py b/src/backend/core/runs/journal.py
similarity index 94%
rename from src/backend/core/services/run_journal.py
rename to src/backend/core/runs/journal.py
index d3b24093..c11ec987 100644
--- a/src/backend/core/services/run_journal.py
+++ b/src/backend/core/runs/journal.py
@@ -11,21 +11,10 @@
from core.db.engine import SessionLocal
from core.db.models import ChatRun, ChatRunOperation
-from core.services.run_binding import durable_run_binding
-from core.services.run_journal_domain import (
- LIVE_STATUSES,
- DurableRunBinding,
- JournalReceipt,
- RecoveryDecision,
- RunAlreadyExists,
- RunJournalError,
- RunLeaseLost,
- RunNotFound,
- _aware,
- _utcnow,
-)
-from core.services.run_journal_lifecycle import RunLifecycle
-from core.services.run_journal_recovery import RunRecovery
+from core.runs.binding import durable_run_binding
+from core.runs.journal_domain import LIVE_STATUSES, DurableRunBinding, JournalReceipt, RecoveryDecision, RunAlreadyExists, RunJournalError, RunLeaseLost, RunNotFound, _aware, _utcnow
+from core.runs.journal_lifecycle import RunLifecycle
+from core.runs.journal_recovery import RunRecovery
from sqlalchemy import func, or_
from sqlalchemy.orm import Session, sessionmaker, undefer
@@ -51,6 +40,7 @@ def accept(
user_id: str,
request_payload: Mapping[str, Any],
recovery_snapshot: Optional[Mapping[str, Any]] = None,
+ on_accept=None,
) -> ChatRun:
"""Commit acceptance before any in-process worker is created."""
with self._sessions() as db:
@@ -64,6 +54,8 @@ def accept(
recovery_snapshot=recovery_snapshot,
now=self._clock(),
)
+ if on_accept is not None:
+ on_accept(db, row)
db.commit()
db.refresh(row)
db.expunge(row)
@@ -87,7 +79,7 @@ def accept_in_session(
) -> ChatRun:
"""Stage a pending run inside the caller's existing transaction."""
accepted_at = now or _utcnow()
- from core.services.api_key_runs import bind_request_key, require_admission_key
+ from core.auth.api_key_runs import bind_request_key, require_admission_key
request_payload = bind_request_key(request_payload, user_id)
require_admission_key(db, request_payload, user_id, accepted_at)
@@ -116,14 +108,14 @@ def accept_in_session(
db.flush()
return row
- def claim(self, run_id: str, *, owner: str, lease_seconds: int) -> bool:
+ def claim(self, run_id: str, *, owner: str, lease_seconds: int, pending_only: bool = False) -> bool:
"""Atomically acquire a free/expired lease; exactly one claimant wins."""
if not owner:
raise ValueError("owner is required")
now = self._clock()
expires = now + timedelta(seconds=max(1, int(lease_seconds)))
with self._sessions() as db:
- from core.services.api_key_runs import authorize_run_execution
+ from core.auth.api_key_runs import authorize_run_execution
if not authorize_run_execution(db, run_id, now):
return False
@@ -131,7 +123,7 @@ def claim(self, run_id: str, *, owner: str, lease_seconds: int) -> bool:
db.query(ChatRun)
.filter(
ChatRun.run_id == run_id,
- ChatRun.status.in_(LIVE_STATUSES),
+ ChatRun.status.in_(("pending",) if pending_only else LIVE_STATUSES),
or_(
ChatRun.lease_owner.is_(None),
ChatRun.lease_expires_at.is_(None),
@@ -157,7 +149,7 @@ def renew(self, run_id: str, *, owner: str, lease_seconds: int) -> bool:
now = self._clock()
expires = now + timedelta(seconds=max(1, int(lease_seconds)))
with self._sessions() as db:
- from core.services.api_key_runs import authorize_run_execution
+ from core.auth.api_key_runs import authorize_run_execution
if not authorize_run_execution(db, run_id, now):
return False
diff --git a/src/backend/core/services/run_journal_domain.py b/src/backend/core/runs/journal_domain.py
similarity index 100%
rename from src/backend/core/services/run_journal_domain.py
rename to src/backend/core/runs/journal_domain.py
diff --git a/src/backend/core/services/run_journal_lifecycle.py b/src/backend/core/runs/journal_lifecycle.py
similarity index 93%
rename from src/backend/core/services/run_journal_lifecycle.py
rename to src/backend/core/runs/journal_lifecycle.py
index 33456d8d..5ec43b88 100644
--- a/src/backend/core/services/run_journal_lifecycle.py
+++ b/src/backend/core/runs/journal_lifecycle.py
@@ -6,13 +6,7 @@
from typing import Any, Callable, Mapping, Optional
from core.db.models import ChatRun
-from core.services.run_journal_domain import (
- LIVE_STATUSES,
- TERMINAL_STATUSES,
- RunLeaseLost,
- RunNotFound,
- _aware,
-)
+from core.runs.journal_domain import LIVE_STATUSES, TERMINAL_STATUSES, RunLeaseLost, RunNotFound, _aware
from sqlalchemy.orm import Session
@@ -85,9 +79,19 @@ def complete(
db.commit()
return True
- def cancel(self, run_id: str, *, reason: str) -> bool:
+ def cancel(
+ self,
+ run_id: str,
+ *,
+ reason: str,
+ cancel_guard: Optional[Callable[[Session], bool]] = None,
+ ) -> bool:
now = self._clock()
with self._sessions() as db:
+ # The caller may lock its ownership row here. Keep that lock and
+ # the child transition in one short transaction, without external I/O.
+ if cancel_guard is not None and not cancel_guard(db):
+ return False
row = db.query(ChatRun).filter(ChatRun.run_id == run_id).with_for_update().one_or_none()
if row is None or row.status not in LIVE_STATUSES:
return False
diff --git a/src/backend/core/services/run_journal_recovery.py b/src/backend/core/runs/journal_recovery.py
similarity index 97%
rename from src/backend/core/services/run_journal_recovery.py
rename to src/backend/core/runs/journal_recovery.py
index 74aa3ea6..fccd6a27 100644
--- a/src/backend/core/services/run_journal_recovery.py
+++ b/src/backend/core/runs/journal_recovery.py
@@ -3,14 +3,7 @@
from __future__ import annotations
from core.db.models import ChatRun
-from core.services.run_journal_domain import (
- LIVE_STATUSES,
- NON_CHAT_RUN_KINDS,
- RECOVERABLE_PRE_MODEL_PHASES,
- RECOVERABLE_TOOL_PHASES,
- SAFE_STOP_ORPHAN_KINDS,
- RecoveryDecision,
-)
+from core.runs.journal_domain import LIVE_STATUSES, NON_CHAT_RUN_KINDS, RECOVERABLE_PRE_MODEL_PHASES, RECOVERABLE_TOOL_PHASES, SAFE_STOP_ORPHAN_KINDS, RecoveryDecision
from sqlalchemy import or_
from sqlalchemy.orm import undefer
diff --git a/src/backend/core/runs/ownership.py b/src/backend/core/runs/ownership.py
new file mode 100644
index 00000000..42908987
--- /dev/null
+++ b/src/backend/core/runs/ownership.py
@@ -0,0 +1,276 @@
+"""Own run leases and journal-bound creation, fencing and terminal transitions."""
+
+from __future__ import annotations
+
+import asyncio
+import typing as _typing
+import uuid
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+import core.db.engine as engine
+import core.db.models as models
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.runtime as runtime
+import core.runs.settings as settings
+import core.runs.journal as run_journal
+from core.db.models import ChatRun
+from core.infra.logging import get_logger
+from core.runs.journal import RunJournal
+
+logger = get_logger(__name__)
+
+
+def journal() -> RunJournal:
+ # Resolve SessionLocal at call time so tests and alternate runtime profiles
+ # can replace the factory without leaving the journal bound to another DB.
+ return run_journal.RunJournal(engine.SessionLocal)
+
+
+def new_worker_owner(run_id: str) -> str:
+ return f"{settings._WORKER_INSTANCE_ID}:{run_id}:{uuid.uuid4().hex[:12]}"
+
+
+async def lease_heartbeat(
+ run_id: str,
+ owner: str,
+ worker: asyncio.Task,
+ lease_lost: asyncio.Event,
+) -> None:
+ """Renew ownership and fence the local coroutine immediately on lease loss."""
+ interval = settings._RUN_LEASE_HEARTBEAT_SEC
+ try:
+ run = await asyncio.to_thread(runtime.get_run, run_id)
+ payload = (run.request_payload or {}) if run else {}
+ if payload.get("api_key_id") or payload.get("agent_api_scope"):
+ # A revoke handled by another worker must also stop a silent model call.
+ interval = min(interval, 1.0)
+ while True:
+ await asyncio.sleep(interval)
+ # ``renew`` is a synchronous UPDATE on chat_runs. Run it off the
+ # event loop: whenever that row is locked by another transaction
+ # this call waits inside the driver, and waiting *on the loop*
+ # freezes every coroutine in the process — including the one
+ # holding the lock, which then can never commit and release it.
+ # A blocked heartbeat must only cost this run its lease, never the
+ # whole backend.
+ renewed = await asyncio.to_thread(
+ journal().renew,
+ run_id,
+ owner=owner,
+ lease_seconds=settings.RUN_LEASE_SECONDS,
+ )
+ if not renewed:
+ logger.warning("chat_run_lease_lost", run_id=run_id, owner=owner)
+ lease_lost.set()
+ worker.cancel()
+ return
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # DB/driver failures must fail closed
+ logger.error(
+ "chat_run_lease_renew_failed",
+ run_id=run_id,
+ owner=owner,
+ error=str(exc),
+ exc_info=True,
+ )
+ lease_lost.set()
+ worker.cancel()
+
+
+def _update_run_status(run_id: str, **fields: Any) -> None:
+ """Single-shot UPDATE on chat_runs; isolated session so it doesn't pollute caller's txn."""
+ if not fields:
+ return
+ if fields.get("status") in settings._TERMINAL_STATUSES:
+ fields.setdefault("writer_slot", None)
+ with engine.SessionLocal() as db:
+ affected = (
+ db.query(models.ChatRun)
+ .filter(models.ChatRun.run_id == run_id)
+ .update(fields, synchronize_session=False)
+ )
+ if not affected:
+ logger.warning("chat_run_update_missing", run_id=run_id, fields=list(fields.keys()))
+ db.commit()
+
+
+def _claim_run_execution(run_id: str) -> Optional[datetime]:
+ """Fence duplicate initial workers before acquiring the durable lease.
+
+ Returns the instant the run was marked running, or None when another worker
+ got there first. The caller needs that instant for the first frame, and
+ handing it back keeps the send path free of a second read of the same row.
+ """
+
+ started_at = clock._utcnow()
+ with engine.SessionLocal() as db:
+ affected = (
+ db.query(models.ChatRun)
+ .filter(models.ChatRun.run_id == run_id, models.ChatRun.status == "pending")
+ .update(
+ {"status": "running", "started_at": started_at},
+ synchronize_session=False,
+ )
+ )
+ db.commit()
+ return started_at if affected else None
+
+
+def _finalize_run(
+ run_id: str,
+ *,
+ require_expired_lease: bool = False,
+ **fields: Any,
+) -> bool:
+ """CAS variant of writing the terminal state: only updates while the run is still live.
+
+ Prevents the race where "the reaper / cancel has already moved the run to a
+ terminal state, and the worker's late completed/failed overwrites it back" —
+ the side that loses the race abandons its write and returns False.
+ """
+ fields.setdefault("lease_owner", None)
+ fields.setdefault("lease_expires_at", None)
+ if fields.get("error_message") and not fields.get("failure_reason"):
+ fields["failure_reason"] = fields["error_message"]
+ if fields.get("status") in settings._TERMINAL_STATUSES:
+ fields.setdefault("writer_slot", None)
+ with engine.SessionLocal() as db:
+ query = db.query(models.ChatRun).filter(
+ models.ChatRun.run_id == run_id,
+ models.ChatRun.status.in_(settings.LIVE_STATUSES),
+ )
+ if require_expired_lease:
+ from sqlalchemy import or_
+
+ now = clock._utcnow()
+ query = query.filter(
+ or_(
+ models.ChatRun.lease_owner.is_(None),
+ models.ChatRun.lease_expires_at.is_(None),
+ models.ChatRun.lease_expires_at <= now,
+ )
+ )
+ affected = query.update(fields, synchronize_session=False)
+ db.commit()
+ if not affected:
+ logger.info("chat_run_finalize_skipped", run_id=run_id, fields=list(fields.keys()))
+ return bool(affected)
+
+
+def _request_run_stop(
+ run_id: str,
+ *,
+ status: settings.RunStatus,
+ error_message: Optional[str] = None,
+ require_expired_lease: bool = False,
+) -> bool:
+ """Write an external terminal verdict while retaining the writer fence."""
+
+ if status not in settings._TERMINAL_STATUSES:
+ raise ValueError(f"not a terminal status: {status}")
+ fields: _typing.Dict[str, _typing.Any] = {"status": status, "completed_at": clock._utcnow()}
+ if error_message:
+ fields["error_message"] = error_message[:1000]
+ fields["failure_reason"] = error_message[:1000]
+ with engine.SessionLocal() as db:
+ query = db.query(models.ChatRun).filter(
+ models.ChatRun.run_id == run_id,
+ models.ChatRun.status.in_(settings.LIVE_STATUSES),
+ )
+ if require_expired_lease:
+ from sqlalchemy import or_
+
+ now = clock._utcnow()
+ query = query.filter(
+ or_(
+ models.ChatRun.lease_owner.is_(None),
+ models.ChatRun.lease_expires_at.is_(None),
+ models.ChatRun.lease_expires_at <= now,
+ )
+ )
+ affected = query.update(fields, synchronize_session=False)
+ db.commit()
+ return bool(affected)
+
+
+def _request_run_cancel(run_id: str) -> bool:
+ return _request_run_stop(run_id, status="cancelled")
+
+
+def _acknowledge_terminal_writer(run_id: str) -> bool:
+ """Release the writer fence only after the owning worker has stopped writing."""
+
+ with engine.SessionLocal() as db:
+ affected = (
+ db.query(models.ChatRun)
+ .filter(
+ models.ChatRun.run_id == run_id,
+ models.ChatRun.status.in_(settings._TERMINAL_STATUSES),
+ models.ChatRun.writer_slot.isnot(None),
+ )
+ .update({"writer_slot": None}, synchronize_session=False)
+ )
+ db.commit()
+ return bool(affected)
+
+
+def _acknowledge_never_started_terminal_writer(run_id: str) -> bool:
+ """Release a terminal fence only when no worker ever claimed the run.
+
+ A duplicate worker can lose the pending-to-running claim while the real
+ worker is active, then observe a concurrent cancellation. It must not
+ release the real worker's fence in that case. ``started_at IS NULL`` is
+ the durable proof that there is no owning worker to wait for.
+ """
+
+ with engine.SessionLocal() as db:
+ affected = (
+ db.query(models.ChatRun)
+ .filter(
+ models.ChatRun.run_id == run_id,
+ models.ChatRun.status.in_(settings._TERMINAL_STATUSES),
+ models.ChatRun.writer_slot.isnot(None),
+ models.ChatRun.started_at.is_(None),
+ )
+ .update({"writer_slot": None}, synchronize_session=False)
+ )
+ db.commit()
+ return bool(affected)
+
+
+def create_run_record(
+ *,
+ chat_id: str,
+ user_id: str,
+ request_payload: Dict[str, Any],
+ recovery_snapshot: Optional[Dict[str, Any]] = None,
+ on_accept=None,
+) -> ChatRun:
+ """Durably accept a run before any worker task can be spawned."""
+ run_id = f"run_{uuid.uuid4().hex[:16]}"
+ message_id = f"msg_{uuid.uuid4().hex[:16]}"
+ return journal().accept(
+ run_id=run_id,
+ message_id=message_id,
+ chat_id=chat_id,
+ user_id=user_id,
+ request_payload=writer._json_safe(request_payload),
+ recovery_snapshot=writer._json_safe(recovery_snapshot or {}),
+ on_accept=on_accept,
+ )
+
+
+def cancelled_by_user(run_id: str) -> bool:
+ """租约被清掉有两种可能:别的 worker 接管了恢复,或者用户按了停止。
+
+ cancel_run 会先把行写成终态 cancelled 再清租约,之后不会再有人接手这一轮 ——
+ 被围栏的 worker 若也闭嘴,用户已经看到的半截回答就没人落库了。
+ """
+ try:
+ current = runtime.get_run(run_id)
+ except Exception: # noqa: BLE001 - 读不到就按"别人接管"处理,宁可不写
+ return False
+ return current is not None and current.status == "cancelled"
diff --git a/src/backend/core/runs/runtime.py b/src/backend/core/runs/runtime.py
new file mode 100644
index 00000000..76790c07
--- /dev/null
+++ b/src/backend/core/runs/runtime.py
@@ -0,0 +1,296 @@
+"""Query and cancel durable runs and track their local asyncio workers."""
+
+from __future__ import annotations
+
+import asyncio
+import datetime as _datetime
+import typing as _typing
+from typing import Any, Awaitable, Callable, List, Optional, Tuple
+
+import core.db.engine as engine
+import core.db.models as models
+import core.db.reserved_chats as reserved_chats
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.settings as settings
+import core.runs.tasks as tasks
+from core.db.models import ChatRun
+from core.db.threads import run_db
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ChatRunNotFound(Exception):
+ pass
+
+
+class ChatRunPermissionDenied(Exception):
+ pass
+
+
+def register_run_task(run_id: str, coro: Awaitable[None], *, name: str) -> None:
+ """Spawn the worker coroutine, register it in _active_runs, auto-cleanup on done."""
+ active = tasks.process_state().active_runs
+ if run_id in active and not active[run_id].done():
+ coro.close()
+ return
+ task = asyncio.create_task(coro, name=name)
+ tasks.process_state().active_runs[run_id] = task
+
+ def finished(worker):
+ if active.get(run_id) is worker:
+ active.pop(run_id, None)
+ if not worker.cancelled() and worker.exception() is not None:
+ logger.error(
+ "run_worker_unhandled_error",
+ run_id=run_id,
+ error=str(worker.exception()),
+ exc_info=worker.exception(),
+ )
+
+ task.add_done_callback(finished)
+
+
+async def wait_run(run_id: str) -> ChatRun:
+ """Wait for a locally launched run and return its durable terminal row.
+
+ Shielding prevents a disconnected non-stream HTTP client from cancelling
+ the detached worker. Explicit ``cancel_run`` still targets the registered
+ task and keeps the database writer fence until that task has stopped.
+ """
+
+ task = tasks.process_state().active_runs.get(run_id)
+ if task is not None:
+ try:
+ await asyncio.shield(task)
+ except asyncio.CancelledError:
+ if not task.done():
+ # The waiter (for example a disconnected HTTP request) was
+ # cancelled; shielding keeps the worker alive, and the caller
+ # must still observe its own cancellation.
+ raise
+ # Explicit run cancellation stopped the worker; the durable row
+ # below is the authoritative outcome.
+ run = await run_db(get_run, run_id)
+ if run is None:
+ raise ChatRunNotFound(f"chat run {run_id} not found")
+ if run.status in settings.LIVE_STATUSES:
+ raise RuntimeError(f"chat run {run_id} lost its local worker before completion")
+ return run
+
+
+def get_run(run_id: str) -> Optional[ChatRun]:
+ """Read a single run (used by the route layer for authorization)."""
+ with engine.SessionLocal() as db:
+ return db.query(models.ChatRun).filter(models.ChatRun.run_id == run_id).first()
+
+
+def is_run_cancelled(run_id: str) -> bool:
+ """Polled by plan_mode every ~15s heartbeat; see ``cancel_run`` for why.
+
+ Any terminal state stops the worker (not just ``cancelled``): a run reaped
+ to ``failed`` by the stale reaper must also make the cooperative worker stop
+ itself as soon as possible, or we get the split-brain of "DB says dead, task
+ keeps running". The worker itself only writes the terminal state during
+ wrap-up, so any terminal state seen while polling must come from an external
+ verdict.
+ """
+ with engine.SessionLocal() as db:
+ row = db.query(models.ChatRun.status).filter(models.ChatRun.run_id == run_id).first()
+ return bool(row and row[0] in settings._TERMINAL_STATUSES)
+
+
+def _claim_cancellation(
+ run_id: str,
+ *,
+ user_id: str,
+ stop_terminal_worker: bool,
+ cancel_guard: Optional[Callable[[Any], bool]],
+) -> Optional[ChatRun]:
+ """Write the cancelled state; returns the run, or None when there is nothing to stop."""
+ run = get_run(run_id)
+ if run is None:
+ raise ChatRunNotFound(f"chat run {run_id} not found")
+ if run.user_id != user_id:
+ raise ChatRunPermissionDenied(f"run {run_id} owned by another user")
+ already_cancelled = stop_terminal_worker and run.status == "cancelled"
+ if run.status not in settings.LIVE_STATUSES and not already_cancelled:
+ return None
+
+ if already_cancelled and cancel_guard is not None:
+ # Even terminal-worker cleanup needs current caller ownership. Release
+ # the short guard transaction before stream or worker operations.
+ with engine.SessionLocal() as db:
+ if not cancel_guard(db):
+ return None
+
+ if not already_cancelled and not ownership.journal().cancel(
+ run_id,
+ reason="user cancelled",
+ **({"cancel_guard": cancel_guard} if cancel_guard is not None else {}),
+ ):
+ # Race: the worker/reaper just beat us to writing the terminal state — treat as "no longer running".
+ return None
+ return run
+
+
+async def cancel_run(
+ run_id: str,
+ *,
+ user_id: str,
+ stop_terminal_worker: bool = False,
+ cancel_guard: Optional[Callable[[Any], bool]] = None,
+) -> bool:
+ """Mark status=cancelled. Idempotent on terminal runs.
+
+ For chat-run kinds (regular chat / plan-generate) we forcefully cancel the
+ underlying asyncio task — these don't hold long-lived per-step HTTP MCP
+ clients so anyio cancel propagation is safe.
+
+ For plan-execute kinds we DO NOT call task.cancel(). plan_mode polls
+ is_run_cancelled() between LLM streaming heartbeats and self-cancels
+ inside its own task. Cross-task task.cancel() on a plan-execute worker
+ risks an anyio cancel-scope deadlock: the worker holds per-step
+ streamable_http MCP clients whose internal SSE tasks may be blocked on
+ socket recv at cancel time, causing anyio's _deliver_cancellation to
+ self-reschedule every 10ms and starve the whole event loop.
+ """
+ run = await run_db(
+ _claim_cancellation,
+ run_id,
+ user_id=user_id,
+ stop_terminal_worker=stop_terminal_worker,
+ cancel_guard=cancel_guard,
+ )
+ if run is None:
+ return False
+
+ payload = run.request_payload if isinstance(run.request_payload, dict) else {}
+ cooperative_only = payload.get("kind") in settings._COOPERATIVE_KINDS
+
+ # Cooperative workers are fenced by the cancelled DB row and therefore
+ # deliberately stay silent when their heartbeat cancels them. Project the
+ # authoritative cancellation here so followers never wait on the old task.
+ if cooperative_only:
+ await writer._write_terminal_to_stream(
+ run_id,
+ chat_id=run.chat_id,
+ error_text="任务已被用户取消",
+ cancelled=True,
+ )
+
+ task = tasks.process_state().active_runs.get(run_id)
+ if task is not None and not task.done():
+ if cooperative_only:
+ target = asyncio.shield(task)
+ timeout = 20.0
+ else:
+ task.cancel()
+ target = task
+ timeout = 2.0
+ try:
+ await asyncio.wait_for(target, timeout=timeout)
+ except (asyncio.TimeoutError, asyncio.CancelledError):
+ pass
+ except Exception as exc:
+ logger.warning("cancel_run_await_failed", run_id=run_id, error=str(exc))
+ if task.done():
+ await run_db(ownership._acknowledge_terminal_writer, run_id)
+ elif not cooperative_only:
+ # Cross-process / post-restart: worker not in this process. Inject the
+ # terminal markers ourselves so any active follower exits cleanly.
+ await writer._write_terminal_to_stream(
+ run_id,
+ chat_id=run.chat_id,
+ error_text="任务已被用户取消",
+ cancelled=True,
+ )
+
+ return True
+
+
+def _run_kind(run: ChatRun) -> Any:
+ payload = run.request_payload if isinstance(run.request_payload, dict) else {}
+ return payload.get("kind")
+
+
+def get_active_run_for_chat(chat_id: str, user_id: str) -> Optional[ChatRun]:
+ """Backing query for GET /v1/chats/{chat_id}/active-run."""
+ with engine.SessionLocal() as db:
+ rows = (
+ db.query(models.ChatRun)
+ .filter(
+ models.ChatRun.chat_id == chat_id,
+ models.ChatRun.user_id == user_id,
+ (
+ models.ChatRun.status.in_(settings.LIVE_STATUSES)
+ | (models.ChatRun.writer_slot.isnot(None))
+ ),
+ )
+ .order_by(models.ChatRun.created_at.desc())
+ .all()
+ )
+ for row in rows:
+ if _run_kind(row) not in settings._BACKGROUND_RUN_KINDS:
+ return row
+ return None
+
+
+def has_local_runs() -> bool:
+ """这个进程手上有没有正在跑的 run。
+
+ 用来在查库之前便宜地排掉"本进程什么都没在跑"的情况 —— 谁在跑哪个 run 只有进程自己
+ 知道(``_active_runs`` 是进程内的),而有些判断必须落到具体进程上:确认条的 pending
+ 与前端信号队列都活在进程里,只有执行这个 run 的进程弹得出来。
+ """
+ return bool(tasks.process_state().active_runs)
+
+
+def is_local_run(run_id: str) -> bool:
+ """这个 run 是不是由本进程在执行。"""
+ return run_id in tasks.process_state().active_runs
+
+
+def list_active_runs_for_user(
+ user_id: str, recent_seconds: int = 0
+) -> Tuple[List[ChatRun], List[str]]:
+ """Backing query for GET /v1/chats/active-runs, in one query.
+
+ Returns one live run per chat: only genuinely live rows (``pending`` /
+ ``running``) qualify. The per-chat probe additionally accepts a lingering
+ ``writer_slot`` so a reconnecting client can clean its own zombie UI up;
+ this listing feeds the sidebar's "still running" dot, where a terminal row
+ must not light anything.
+
+ With ``recent_seconds`` it also returns the chats whose runs started in that
+ window, newest first, so a page finds a chat started elsewhere (Personal
+ Agent, API, another device) even when its run ended between two checks.
+ """
+ live = models.ChatRun.status.in_(settings.LIVE_STATUSES)
+ since = clock._utcnow() - _datetime.timedelta(
+ seconds=min(recent_seconds, settings.MAX_RECENT_RUN_SECONDS)
+ )
+ with engine.SessionLocal() as db:
+ rows = (
+ db.query(models.ChatRun)
+ .filter(
+ models.ChatRun.user_id == user_id,
+ (live | (models.ChatRun.created_at >= since)) if recent_seconds else live,
+ # Module-owned chats (Personal Agent) never belong in the ordinary sidebar.
+ reserved_chats.ordinary_chat_filter(models.ChatRun.chat_id),
+ )
+ .order_by(models.ChatRun.created_at.desc())
+ .all()
+ )
+ latest_per_chat: _typing.Dict[str, models.ChatRun] = {}
+ recent: _typing.Dict[str, None] = {}
+ for row in rows:
+ if _run_kind(row) in settings._BACKGROUND_RUN_KINDS:
+ continue
+ if row.status in settings.LIVE_STATUSES:
+ latest_per_chat.setdefault(row.chat_id, row)
+ if recent_seconds and row.created_at >= since:
+ recent.setdefault(row.chat_id)
+ return list(latest_per_chat.values()), list(recent)
diff --git a/src/backend/core/runs/settings.py b/src/backend/core/runs/settings.py
new file mode 100644
index 00000000..0e406f73
--- /dev/null
+++ b/src/backend/core/runs/settings.py
@@ -0,0 +1,98 @@
+"""Existing run kinds, statuses and execution deadlines shared by run owners."""
+
+from __future__ import annotations
+
+import os
+import socket
+import typing as _typing
+import uuid
+
+import core.llm.runtime.human_interaction as human_interaction
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+RunKind = _typing.Literal["chat", "plan_execute", "plan_generate", "autonomous_loop"]
+
+
+_COOPERATIVE_KINDS: tuple[str, ...] = ("plan_execute", "autonomous_loop")
+
+
+RunStatus = _typing.Literal[
+ "pending", "running", "needs_attention", "completed", "failed", "cancelled"
+]
+
+
+_TERMINAL_STATUSES: tuple[RunStatus, ...] = (
+ "needs_attention",
+ "completed",
+ "failed",
+ "cancelled",
+)
+
+
+LIVE_STATUSES: tuple[RunStatus, ...] = ("pending", "running")
+
+
+_STREAM_TTL_SECONDS = human_interaction.STREAM_TTL_SECONDS
+
+
+TERMINAL_TYPE = "__terminal__"
+
+
+_XREAD_BLOCK_MS = 5000
+
+
+HEARTBEAT_INTERVAL_SEC = 15.0
+
+
+_FIRST_RESPONSE_TIMEOUT_SEC = 600.0
+
+
+_INACTIVITY_TIMEOUT_SEC = float(os.getenv("CHAT_RUN_INACTIVITY_TIMEOUT_SEC", "600"))
+
+
+_STALE_RUN_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_MAX_AGE_SEC", "1800"))
+
+
+STALE_REAPER_INTERVAL_SEC = float(os.getenv("CHAT_RUN_REAPER_INTERVAL_SEC", "300"))
+
+
+_STALE_QUIET_SEC = float(os.getenv("CHAT_RUN_STALE_QUIET_SEC", str(_INACTIVITY_TIMEOUT_SEC)))
+
+
+_HARD_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_HARD_MAX_AGE_SEC", "21600"))
+
+
+RUN_LEASE_SECONDS = max(15, int(os.getenv("CHAT_RUN_LEASE_SECONDS", "90")))
+
+
+_RUN_LEASE_HEARTBEAT_SEC = max(1.0, RUN_LEASE_SECONDS / 3)
+
+
+RUN_RECOVERY_INTERVAL_SEC = max(
+ 1.0,
+ float(os.getenv("CHAT_RUN_RECOVERY_INTERVAL_SEC", "15")),
+)
+
+
+_WORKER_INSTANCE_ID = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:12]}"
+
+
+_OFFSET_RESERVATION = 64
+
+
+_CANCEL_VISIBILITY_SEC = 0.25
+
+
+_BACKGROUND_RUN_KINDS = frozenset(
+ {
+ "automation_plan",
+ "automation_prompt",
+ "internal_job_agent",
+ "legacy_chat_stream",
+ }
+)
+
+
+MAX_RECENT_RUN_SECONDS = 86400
diff --git a/src/backend/core/runs/tasks.py b/src/backend/core/runs/tasks.py
new file mode 100644
index 00000000..746f7350
--- /dev/null
+++ b/src/backend/core/runs/tasks.py
@@ -0,0 +1,23 @@
+"""Local worker handles, keyed by durable run identity."""
+
+from __future__ import annotations
+
+import asyncio
+import typing as _typing
+from dataclasses import dataclass, field
+
+from core.infra.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class TasksState:
+ active_runs: _typing.Dict[str, asyncio.Task] = field(default_factory=lambda: {})
+
+
+_STATE = TasksState()
+
+
+def process_state() -> TasksState:
+ return _STATE
diff --git a/src/backend/orchestration/tool_effect_recovery.py b/src/backend/core/runs/tool_effect_recovery.py
similarity index 68%
rename from src/backend/orchestration/tool_effect_recovery.py
rename to src/backend/core/runs/tool_effect_recovery.py
index ae38417d..7a137b11 100644
--- a/src/backend/orchestration/tool_effect_recovery.py
+++ b/src/backend/core/runs/tool_effect_recovery.py
@@ -6,16 +6,17 @@
"""
from __future__ import annotations
+from core.harness.registry import execution_identity as bridge_identity
import json
from typing import Any
from agentscope.message import ToolCallBlock
from agentscope.tool._response import ToolResponse
-
-from core.db.engine import SessionLocal
-from core.db.models import ChatRun, ToolEffectReceipt, RemoteToolEffect
-from core.services.tool_effect_ledger import ReconciliationResult, ToolEffectError, ToolIntent
+from core.db.models import ChatRun, RemoteToolEffect, ToolEffectReceipt
+from core.runs.tool_effects.types import ReconciliationResult, ToolIntent
+from core.runs.tool_effects.errors import ToolEffectError
+from core.db.threads import run_in_session
def _contains_withheld(value: Any) -> bool:
@@ -41,9 +42,9 @@ async def replay_tool_intent(intent: ToolIntent) -> dict[str, Any]:
"""Rebuild the frozen chat tool surface and invoke one adapter, without an LLM call."""
from core.llm.factory import create_agent_executor
- from core.llm.mcp_manager import close_clients
+ from core.llm.mcp.manager import close_clients
- with SessionLocal() as db:
+ def _replay_context(db):
run = db.get(ChatRun, intent.run_id)
if run is None:
raise ToolEffectError(f"missing run for tool recovery: {intent.run_id}")
@@ -54,12 +55,14 @@ async def replay_tool_intent(intent: ToolIntent) -> dict[str, Any]:
context = dict(worker_args.get("context") or {})
user_id = run.user_id
chat_id = run.chat_id
+ return context, user_id, chat_id
+ context, user_id, chat_id = await run_in_session(_replay_context)
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled():
- from core.capabilities.errors import IntegrityFailed
- from core.capabilities.runtime import require_root_tool_scope
+ if has_run_capabilities():
+ from core.harness.capability_errors import IntegrityFailed
+ from core.harness.registry.capability_runtime import require_root_tool_scope
try:
require_root_tool_scope(intent.run_id, user_id, intent.tool_call_id, intent.tool_name)
@@ -125,59 +128,63 @@ async def replay_tool_intent(intent: ToolIntent) -> dict[str, Any]:
await close_clients(clients)
+def _scheduled_task_receipt(db, intent: ToolIntent):
+ """Settle from the local receipt, or return the cloud call that can settle it."""
+ run = db.get(ChatRun, intent.run_id)
+ if run is None:
+ return ReconciliationResult.unknown()
+ receipt = db.get(ToolEffectReceipt, intent.effect_id)
+ if receipt is not None:
+ if receipt.user_id != run.user_id or receipt.tool_name != intent.tool_name:
+ return ReconciliationResult.unknown()
+ return ReconciliationResult.applied(dict(receipt.result_payload or {}))
+ remote = db.get(RemoteToolEffect, intent.effect_id)
+ if remote is not None and remote.gateway_url:
+ return {"url": remote.gateway_url, "schema_hash": remote.schema_hash, "user_id": run.user_id}
+ # A device operation may predate persisted cross-end bindings.
+ # Its missing local receipt says nothing about cloud state.
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
+
+ if bridge_enabled() and intent.redacted_args.get("execution_location") != "local":
+ return ReconciliationResult.unknown()
+ return ReconciliationResult.not_applied()
+
+
async def reconcile_scheduled_task_intent(intent: ToolIntent) -> ReconciliationResult:
"""Use the adapter receipt committed in the same transaction as the mutation."""
_plain_replay_args(intent)
- with SessionLocal() as db:
- run = db.get(ChatRun, intent.run_id)
- if run is None:
- return ReconciliationResult.unknown()
- receipt = db.get(ToolEffectReceipt, intent.effect_id)
- if receipt is None:
- remote = db.get(RemoteToolEffect, intent.effect_id)
- if remote is not None and remote.gateway_url:
- remote_call = {
- "url": remote.gateway_url, "schema_hash": remote.schema_hash,
- "user_id": run.user_id,
- }
- else:
- remote_call = None
- if remote_call is None:
- # A device operation may predate persisted cross-end bindings.
- # Its missing local receipt says nothing about cloud state.
- from core.auth.desktop_bridge import bridge_enabled
- if bridge_enabled() and intent.redacted_args.get("execution_location") != "local":
- return ReconciliationResult.unknown()
- return ReconciliationResult.not_applied()
- else:
- if receipt.user_id != run.user_id or receipt.tool_name != intent.tool_name:
- return ReconciliationResult.unknown()
- return ReconciliationResult.applied(dict(receipt.result_payload or {}))
- return await _reconcile_cloud_receipt(intent, remote_call)
+ settled = await run_in_session(_scheduled_task_receipt, intent)
+ if isinstance(settled, ReconciliationResult):
+ return settled
+ return await _reconcile_cloud_receipt(intent, settled)
async def _reconcile_cloud_receipt(intent, remote_call):
import httpx
- from core.services import desktop_cloud_bridge as bridge
- from core.services.desktop_capability_protocol import token_subject
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
+ from core.capabilities.manifest_protocol import token_subject
- state = bridge.get_state()
+ state = bridge_identity.get_state()
if not state or token_subject(state.get("token", "")) != remote_call["user_id"]:
return ReconciliationResult.unknown()
- if not remote_call["url"].startswith(state.get("cloud_base", "").rstrip("/") + "/api/v1/desktop/capability/gateway/"):
+ if not remote_call["url"].startswith(
+ state.get("cloud_base", "").rstrip("/") + "/api/v1/desktop/capability/gateway/"
+ ):
return ReconciliationResult.unknown()
try:
- bridge.ensure_current_authorization(state)
+ bridge_identity.ensure_current_authorization(state)
async with httpx.AsyncClient(timeout=15) as client:
response = await client.post(
remote_call["url"].rsplit("/", 1)[0] + "/receipt",
- headers=bridge.cloud_headers(state),
- json={"tool_name": intent.tool_name, "operation_id": intent.effect_id,
- "schema_hash": remote_call["schema_hash"]},
+ headers=bridge_identity.cloud_headers(state),
+ json={
+ "tool_name": intent.tool_name,
+ "operation_id": intent.effect_id,
+ "schema_hash": remote_call["schema_hash"],
+ },
)
- bridge.require_current_account(state)
+ bridge_identity.require_current_account(state)
response.raise_for_status()
data = response.json()["data"]
if not isinstance(data, dict):
diff --git a/src/backend/core/runs/tool_effects/__init__.py b/src/backend/core/runs/tool_effects/__init__.py
new file mode 100644
index 00000000..877f5963
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/__init__.py
@@ -0,0 +1 @@
+"""Durable tool invocation intents, results and recovery."""
diff --git a/src/backend/core/runs/tool_effects/arguments.py b/src/backend/core/runs/tool_effects/arguments.py
new file mode 100644
index 00000000..84c89911
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/arguments.py
@@ -0,0 +1,65 @@
+"""Canonicalise and redact tool arguments and derive idempotency keys."""
+from __future__ import annotations
+import hashlib
+import json
+from typing import Any, Mapping, Optional
+from core.infra.data_masking import mask_sensitive_data
+from core.runs.tool_effects import errors
+from core.runs.tool_effects import journal_constants
+
+def embedded_redact(value: str) -> str:
+ cleaned = value
+ for pattern in journal_constants.EMBEDDED_SECRET_PATTERNS:
+ cleaned = pattern.sub(r"\1[REDACTED]", cleaned)
+ return cleaned
+
+
+def sanitize_value(value: Any, *, key: str = "") -> Any:
+ lower = key.lower()
+ if isinstance(value, Mapping):
+ return {str(k): sanitize_value(v, key=str(k)) for k, v in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [sanitize_value(item, key=key) for item in value]
+ if isinstance(value, str):
+ if lower in journal_constants.WITHHOLD_FIELDS:
+ digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
+ return {"_withheld": True, "sha256": digest, "length": len(value)}
+ return embedded_redact(value)
+ return value
+
+
+def canonical_tool_args(args: Mapping[str, Any]) -> tuple[str, str, Mapping[str, Any]]:
+ canonical = json.dumps(
+ args or {}, sort_keys=True, separators=(",", ":"), default=str
+ )
+ args_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+ by_field = mask_sensitive_data(dict(args or {}), field_patterns=journal_constants.REDACT_FIELDS)
+ return canonical, args_hash, sanitize_value(by_field)
+
+
+def tool_args_hash(args: Mapping[str, Any]) -> str:
+ return canonical_tool_args(args)[1]
+
+
+def stable_idempotency_key(run_id: str, tool_name: str, invocation_id: str) -> str:
+ raw = f"{run_id}\x00{tool_name}\x00{invocation_id}".encode("utf-8")
+ return "tool:" + hashlib.sha256(raw).hexdigest()
+
+
+def resolved_idempotency_key(
+ *,
+ run_id: str,
+ principal_scope: str,
+ tool_name: str,
+ tool_call_id: str,
+ explicit_key: Optional[str],
+) -> tuple[str, bool]:
+ caller_key = str(explicit_key or "").strip()
+ if caller_key:
+ scoped = f"{principal_scope}\x00{caller_key}".encode("utf-8")
+ return "caller:" + hashlib.sha256(scoped).hexdigest(), True
+ if not tool_call_id:
+ raise errors.ToolIntentCommitError(
+ "tool_call_id or explicit idempotency_key is required"
+ )
+ return stable_idempotency_key(run_id, tool_name, tool_call_id), False
diff --git a/src/backend/core/runs/tool_effects/errors.py b/src/backend/core/runs/tool_effects/errors.py
new file mode 100644
index 00000000..dcaf1373
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/errors.py
@@ -0,0 +1,43 @@
+"""Errors raised by the tool effect ledger."""
+from __future__ import annotations
+from typing import Optional
+
+class ToolEffectError(RuntimeError):
+ pass
+
+
+class ToolIntentCommitError(ToolEffectError):
+ """The intent could not be durably committed; invocation is forbidden."""
+
+
+class ToolOutcomeUnknown(ToolEffectError):
+ """The ledger proves intent but cannot prove the external outcome."""
+
+
+def find_tool_outcome_unknown(exc: BaseException) -> Optional[ToolOutcomeUnknown]:
+ """Unwrap AgentScope/anyio ExceptionGroup layers without losing safety meaning."""
+ pending = [exc]
+ seen: set[int] = set()
+ while pending:
+ current = pending.pop()
+ if id(current) in seen:
+ continue
+ seen.add(id(current))
+ if isinstance(current, ToolOutcomeUnknown):
+ return current
+ nested = getattr(current, "exceptions", None)
+ if isinstance(nested, tuple):
+ pending.extend(item for item in nested if isinstance(item, BaseException))
+ if isinstance(current.__cause__, BaseException):
+ pending.append(current.__cause__)
+ if isinstance(current.__context__, BaseException):
+ pending.append(current.__context__)
+ return None
+
+
+class ToolEffectLeaseLost(ToolEffectError):
+ pass
+
+
+class ToolIdempotencyConflict(ToolEffectError):
+ """A caller reused one explicit key for different logical input."""
diff --git a/src/backend/core/runs/tool_effects/gateway.py b/src/backend/core/runs/tool_effects/gateway.py
new file mode 100644
index 00000000..293e64c5
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/gateway.py
@@ -0,0 +1,256 @@
+"""The state machine every side-effecting tool call goes through."""
+from __future__ import annotations
+from core.harness import tool_effect_context
+from core.runs.tool_effects.journal import ToolEffectJournal
+import asyncio
+import inspect
+import uuid
+from contextlib import suppress
+from typing import Any, Awaitable, Callable, Mapping, Optional
+from core.runs.tool_effects import errors
+from core.runs.tool_effects.registry import ToolRecoveryRegistry
+from core.runs.tool_effects import types
+
+class ToolEffectGateway:
+ """The single execution state machine used by production middleware and tests."""
+
+ def __init__(
+ self,
+ journal: ToolEffectJournal,
+ registry: ToolRecoveryRegistry,
+ *,
+ poll_interval: float = 0.05,
+ lease_seconds: int = 300,
+ ) -> None:
+ self.journal = journal
+ self.registry = registry
+ self.poll_interval = poll_interval
+ self.lease_seconds = lease_seconds
+
+ @staticmethod
+ async def _call(handler, intent: types.ToolIntent): # noqa: ANN001, ANN205
+ value = handler(intent) if handler is not None else None
+ return await value if inspect.isawaitable(value) else value
+
+ @staticmethod
+ def _reconciliation(value: Any) -> types.ReconciliationResult:
+ if isinstance(value, types.ReconciliationResult):
+ return value
+ return (
+ types.ReconciliationResult.applied(value)
+ if value is not None
+ else types.ReconciliationResult.unknown()
+ )
+
+ def _start_heartbeat(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ ) -> tuple[asyncio.Task, asyncio.Event]:
+ lost = asyncio.Event()
+ interval = max(0.05, min(30.0, self.lease_seconds / 3))
+
+ async def _beat() -> None:
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ ok = await asyncio.to_thread(
+ self.journal.renew_invocation_claim,
+ effect_id,
+ run_owner=run_owner,
+ claim_owner=claim_owner,
+ lease_seconds=self.lease_seconds,
+ )
+ except Exception: # noqa: BLE001 - lost heartbeat means lost ownership
+ ok = False
+ if not ok:
+ lost.set()
+ return
+
+ return asyncio.create_task(_beat()), lost
+
+ @staticmethod
+ async def _stop_heartbeat(task: asyncio.Task) -> None:
+ task.cancel()
+ with suppress(asyncio.CancelledError):
+ await task
+
+ async def execute_outcome(
+ self,
+ *,
+ run_id: str,
+ owner: str,
+ tool_call_id: str,
+ tool_name: str,
+ args: Mapping[str, Any],
+ invoke: Callable[[], Awaitable[Any] | Any],
+ idempotency_key: Optional[str] = None,
+ classify_failure: Optional[Callable[[Any], bool]] = None,
+ ) -> types.ToolExecutionOutcome:
+ claim_owner = f"tool:{uuid.uuid4().hex}"
+ policy = self.registry.resolve(tool_name)
+ while True:
+ decision = await asyncio.to_thread(
+ self.journal.begin_intent,
+ run_id=run_id,
+ owner=owner,
+ claim_owner=claim_owner,
+ tool_call_id=tool_call_id,
+ tool_name=tool_name,
+ args=args,
+ recovery_policy=policy.policy,
+ idempotency_key=idempotency_key,
+ lease_seconds=self.lease_seconds,
+ )
+ if decision.action == "known":
+ return types.ToolExecutionOutcome(
+ decision.result,
+ decision.intent.effect_id,
+ decision.intent.result_id,
+ decision.result_state,
+ False,
+ )
+ if decision.action == "unknown":
+ raise errors.ToolOutcomeUnknown(decision.effect_id)
+ if decision.action == "wait":
+ await asyncio.sleep(self.poll_interval)
+ terminal = await asyncio.to_thread(
+ self.journal.terminal_decision, decision.effect_id
+ )
+ if terminal is None:
+ continue
+ if terminal.action == "unknown":
+ raise errors.ToolOutcomeUnknown(decision.effect_id)
+ return types.ToolExecutionOutcome(
+ terminal.result,
+ terminal.intent.effect_id,
+ terminal.intent.result_id,
+ terminal.result_state,
+ False,
+ )
+ if decision.action not in ("execute", "reconcile"):
+ raise errors.ToolEffectError(
+ f"unsupported tool intent action: {decision.action}"
+ )
+
+ await asyncio.to_thread(
+ self.journal.acquire_invocation_claim,
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ lease_seconds=self.lease_seconds,
+ )
+ heartbeat, claim_lost = self._start_heartbeat(
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ )
+ token = tool_effect_context.CURRENT_TOOL_EFFECT.set(
+ tool_effect_context.ToolInvocationContext(
+ decision.intent.effect_id,
+ decision.intent.result_id,
+ decision.intent.idempotency_key,
+ decision.intent.run_id,
+ claim_owner,
+ )
+ )
+ try:
+ if decision.action == "reconcile":
+ if policy.reconciler is None:
+ self.journal.mark_unknown(
+ decision.effect_id,
+ owner=owner,
+ claim_owner=claim_owner,
+ reason="reconcile policy has no reconciliation adapter",
+ )
+ raise errors.ToolOutcomeUnknown(decision.effect_id)
+ reconciliation = self._reconciliation(
+ await self._call(policy.reconciler, decision.intent)
+ )
+ if claim_lost.is_set():
+ raise errors.ToolEffectLeaseLost(decision.effect_id)
+ if reconciliation.outcome == "applied":
+ result = await asyncio.to_thread(
+ self.journal.commit_result,
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ result=reconciliation.result,
+ reconciled=True,
+ )
+ return types.ToolExecutionOutcome(
+ result,
+ decision.intent.effect_id,
+ decision.intent.result_id,
+ "success",
+ False,
+ )
+ if reconciliation.outcome != "not_applied":
+ self.journal.mark_unknown(
+ decision.effect_id,
+ owner=owner,
+ claim_owner=claim_owner,
+ reason="reconciliation could not determine external outcome",
+ )
+ raise errors.ToolOutcomeUnknown(decision.effect_id)
+ value = invoke()
+ result = await value if inspect.isawaitable(value) else value
+ if claim_lost.is_set():
+ raise errors.ToolEffectLeaseLost(decision.effect_id)
+ failed = (
+ bool(classify_failure(result))
+ if classify_failure is not None
+ else False
+ )
+ # A returned error payload is still a definitive tool outcome.
+ # Persist it and give the original error back to the model so
+ # it can correct the request or explain an authorization issue.
+ # ToolOutcomeUnknown is reserved for invocations that never
+ # produced a result at all (exceptions, disconnects, crashes).
+ result = await asyncio.to_thread(
+ self.journal.commit_result,
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ result=result,
+ failed=failed,
+ )
+ return types.ToolExecutionOutcome(
+ result,
+ decision.intent.effect_id,
+ decision.intent.result_id,
+ "error" if failed else "success",
+ True,
+ )
+ except asyncio.CancelledError:
+ with suppress(Exception):
+ self.journal.abandon_invocation_claim(
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ )
+ raise
+ except errors.ToolOutcomeUnknown:
+ with suppress(Exception):
+ self.journal.abandon_invocation_claim(
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ )
+ raise
+ except Exception as exc:
+ with suppress(Exception):
+ self.journal.abandon_invocation_claim(
+ decision.effect_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ )
+ raise errors.ToolOutcomeUnknown(decision.effect_id) from exc
+ finally:
+ await self._stop_heartbeat(heartbeat)
+ tool_effect_context.CURRENT_TOOL_EFFECT.reset(token)
+
+ async def execute(self, **kwargs) -> Any: # noqa: ANN003
+ return (await self.execute_outcome(**kwargs)).result
diff --git a/src/backend/core/runs/tool_effects/hooks.py b/src/backend/core/runs/tool_effects/hooks.py
new file mode 100644
index 00000000..769a40e9
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/hooks.py
@@ -0,0 +1,14 @@
+"""Durable tool admission is an execution-layer policy."""
+from core.harness.hooks import Decision, HookSpec, HookStage
+
+
+def require_durable_tool_binding(invocation):
+ identity = invocation.data.get("identity", {})
+ if not invocation.run_id or not identity.get("journal_owner") or not invocation.operation_name:
+ return Decision.reject("tool execution requires a durable run_id and journal_owner binding")
+ return Decision()
+
+
+def tool_effect_hooks(request):
+ yield HookSpec("durable_tool_binding", HookStage.BEFORE_TOOL,
+ require_durable_tool_binding, priority=800)
diff --git a/src/backend/core/runs/tool_effects/host.py b/src/backend/core/runs/tool_effects/host.py
new file mode 100644
index 00000000..a71c9eab
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/host.py
@@ -0,0 +1,28 @@
+"""Compose durable tool-effect execution for model runtime ports."""
+from core.db.engine import SessionLocal
+from core.llm.ports import register_tool_effect_runtime
+from core.runs.tool_effects.errors import ToolEffectError, ToolIntentCommitError
+from core.runs.tool_effects.gateway import ToolEffectGateway
+from core.runs.tool_effects.journal import ToolEffectJournal
+from core.runs.tool_effects.registry import DEFAULT_TOOL_RECOVERY_REGISTRY
+
+
+class DurableToolEffects:
+ def __init__(self, session_factory=None, registry=None):
+ self.gateway = ToolEffectGateway(
+ ToolEffectJournal(session_factory or SessionLocal),
+ registry or DEFAULT_TOOL_RECOVERY_REGISTRY,
+ )
+
+ async def execute_outcome(self, **kwargs):
+ return await self.gateway.execute_outcome(**kwargs)
+
+ def binding_error(self, message):
+ return ToolIntentCommitError(message)
+
+ def response_error(self, message):
+ return ToolEffectError(message)
+
+
+def register():
+ register_tool_effect_runtime(DurableToolEffects)
diff --git a/src/backend/core/runs/tool_effects/intent_transactions.py b/src/backend/core/runs/tool_effects/intent_transactions.py
new file mode 100644
index 00000000..753386cf
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/intent_transactions.py
@@ -0,0 +1,342 @@
+"""Record tool intents and decide whether a call may run."""
+from __future__ import annotations
+import uuid
+from datetime import timedelta
+from typing import Any, Mapping, Optional
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from core.runs.journal import LIVE_STATUSES, RunLeaseLost, RunNotFound
+from sqlalchemy import or_
+from sqlalchemy.exc import IntegrityError, OperationalError
+from sqlalchemy.orm import Session
+from core.runs.tool_effects import arguments
+from core.runs.tool_effects import errors
+from core.runs.tool_effects import registry
+from core.runs.tool_effects import types
+from core.infra.time import as_utc
+
+def intent_from_row(row: ToolEffectLedger) -> types.ToolIntent:
+ return types.ToolIntent(
+ effect_id=row.effect_id,
+ result_id=row.result_id,
+ run_id=row.run_id,
+ operation_seq=int(row.operation_seq),
+ tool_name=row.tool_name,
+ tool_call_id=row.tool_call_id or "",
+ args_hash=row.args_hash,
+ redacted_args=dict(row.redacted_args or {}),
+ idempotency_key=row.idempotency_key or "",
+ recovery_policy=row.recovery_policy,
+ created_at=as_utc(row.created_at),
+ )
+
+
+def terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]:
+ return (
+ db.query(ToolEffectLedger)
+ .filter(ToolEffectLedger.terminal_effect_id == effect_id)
+ .one_or_none()
+ )
+
+
+def decision_from_terminal(
+ intent: types.ToolIntent, terminal: ToolEffectLedger
+) -> types.IntentDecision:
+ if terminal.event_type == "unknown_outcome":
+ return types.IntentDecision("unknown", intent, result_state="unknown")
+ return types.IntentDecision(
+ "known",
+ intent,
+ terminal.result_payload,
+ "success" if terminal.event_type == "result" else "error",
+ )
+
+
+def validate_identity(
+ row: ToolEffectLedger, *, tool_name: str, args_hash: str
+) -> None:
+ if row.tool_name != tool_name or row.args_hash != args_hash:
+ raise errors.ToolIdempotencyConflict(
+ "idempotency_key already belongs to different tool arguments"
+ )
+
+
+def existing_decision(
+ self,
+ *,
+ key: str,
+ caller_run_id: str,
+ run_owner: str,
+ claim_owner: str,
+ tool_name: str,
+ args_hash: str,
+ lease_seconds: int,
+) -> types.IntentDecision:
+ with self._sessions() as db:
+ intent_row = (
+ db.query(ToolEffectLedger)
+ .filter(ToolEffectLedger.idempotency_key == key)
+ .one()
+ )
+ self._validate_identity(
+ intent_row, tool_name=tool_name, args_hash=args_hash
+ )
+ intent = self._intent_from_row(intent_row)
+ caller_run = db.get(ChatRun, caller_run_id)
+ original_run = db.get(ChatRun, intent.run_id)
+ if caller_run is None or original_run is None:
+ raise errors.ToolIntentCommitError(f"run does not exist: {caller_run_id}")
+ if original_run.user_id != caller_run.user_id:
+ raise errors.ToolIdempotencyConflict(
+ "idempotency_key belongs to a different principal"
+ )
+ terminal = self._terminal_row(db, intent.effect_id)
+ if terminal is not None and caller_run_id == intent.run_id:
+ return self._decision_from_terminal(intent, terminal)
+ now = as_utc(self._clock())
+ caller_expiry = as_utc(caller_run.lease_expires_at)
+ if (
+ caller_run.status not in LIVE_STATUSES
+ or caller_run.lease_owner != run_owner
+ or caller_expiry is None
+ or now is None
+ or caller_expiry <= now
+ ):
+ raise errors.ToolIntentCommitError(f"run lease is not owned: {caller_run_id}")
+ if terminal is not None:
+ return self._decision_from_terminal(intent, terminal)
+ if intent.run_id != caller_run_id:
+ return types.IntentDecision("wait", intent)
+
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == intent.effect_id)
+ .with_for_update()
+ .one()
+ )
+ run = self._runs._locked_owned_run(db, caller_run_id, run_owner)
+ terminal = self._terminal_row(db, intent.effect_id)
+ if terminal is not None:
+ return self._decision_from_terminal(intent, terminal)
+ lease_expiry = as_utc(lease.lease_expires_at)
+ if (
+ lease.run_owner == run_owner
+ and lease_expiry is not None
+ and lease_expiry > now
+ ):
+ if lease.claim_owner == claim_owner:
+ return types.IntentDecision(
+ "reconcile"
+ if intent.recovery_policy == "reconcile"
+ else "execute",
+ intent,
+ )
+ return types.IntentDecision("wait", intent)
+
+ previous_owner = lease.run_owner
+ previous_claim = lease.claim_owner
+ previous_expiry = lease.lease_expires_at
+ affected = (
+ db.query(ToolEffectLease)
+ .filter(
+ ToolEffectLease.effect_id == intent.effect_id,
+ ToolEffectLease.run_owner == previous_owner,
+ ToolEffectLease.claim_owner == previous_claim,
+ ToolEffectLease.lease_expires_at == previous_expiry,
+ or_(
+ ToolEffectLease.lease_expires_at.is_(None),
+ ToolEffectLease.lease_expires_at <= self._clock(),
+ ),
+ )
+ .update(
+ {
+ ToolEffectLease.run_owner: run_owner,
+ ToolEffectLease.claim_owner: claim_owner,
+ ToolEffectLease.lease_expires_at: self._clock()
+ + timedelta(seconds=max(1, lease_seconds)),
+ ToolEffectLease.updated_at: self._clock(),
+ },
+ synchronize_session=False,
+ )
+ )
+ if not affected:
+ db.rollback()
+ return types.IntentDecision("wait", intent)
+ db.expire_all()
+ lease = db.get(ToolEffectLease, intent.effect_id)
+ self._append_event(
+ db,
+ run,
+ intent_row,
+ owner=run_owner,
+ event_type="recovery_claim",
+ operation_type="tool_effect_recovery_claimed",
+ phase=f"tool_intent_{intent.recovery_policy}",
+ safety=intent.recovery_policy,
+ )
+ if intent.recovery_policy == "never_replay":
+ self._mark_unknown_locked(
+ db,
+ run,
+ intent_row,
+ lease,
+ owner=run_owner,
+ claim_owner=claim_owner,
+ reason="non-replayable tool intent has no committed result",
+ )
+ db.commit()
+ return types.IntentDecision("unknown", intent)
+ db.commit()
+ return types.IntentDecision(
+ "reconcile" if intent.recovery_policy == "reconcile" else "execute",
+ intent,
+ )
+
+
+def begin_intent(
+ self,
+ *,
+ run_id: str,
+ owner: str,
+ claim_owner: str,
+ tool_call_id: str,
+ tool_name: str,
+ args: Mapping[str, Any],
+ recovery_policy: str,
+ idempotency_key: Optional[str] = None,
+ lease_seconds: int = 300,
+ _conflict_retries: int = 4,
+) -> types.IntentDecision:
+ if recovery_policy not in registry.RECOVERY_POLICIES:
+ raise ValueError(f"invalid recovery policy: {recovery_policy}")
+ _canonical, args_hash, redacted = arguments.canonical_tool_args(args)
+ with self._sessions() as db:
+ caller_run = db.get(ChatRun, run_id)
+ if caller_run is None:
+ raise errors.ToolIntentCommitError(f"run does not exist: {run_id}")
+ principal_scope = caller_run.user_id
+ key, _explicit = arguments.resolved_idempotency_key(
+ run_id=run_id,
+ principal_scope=str(principal_scope),
+ tool_name=tool_name,
+ tool_call_id=tool_call_id,
+ explicit_key=idempotency_key
+ or str(args.get("idempotency_key") or args.get("request_id") or "").strip(),
+ )
+ try:
+ with self._sessions() as db:
+ found = (
+ db.query(ToolEffectLedger.event_id)
+ .filter(ToolEffectLedger.idempotency_key == key)
+ .first()
+ )
+ if found is not None:
+ return self._existing_decision(
+ key=key,
+ caller_run_id=run_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ tool_name=tool_name,
+ args_hash=args_hash,
+ lease_seconds=lease_seconds,
+ )
+
+ with self._sessions() as db:
+ run = self._runs._locked_owned_run(db, run_id, owner)
+ found = (
+ db.query(ToolEffectLedger.event_id)
+ .filter(ToolEffectLedger.idempotency_key == key)
+ .first()
+ )
+ if found is not None:
+ db.rollback()
+ return self._existing_decision(
+ key=key,
+ caller_run_id=run_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ tool_name=tool_name,
+ args_hash=args_hash,
+ lease_seconds=lease_seconds,
+ )
+ effect_id = f"eff_{uuid.uuid4().hex}"
+ result_id = f"toolres_{uuid.uuid4().hex}"
+ receipt = self._runs._append_locked(
+ db,
+ run,
+ owner=owner,
+ operation_type="tool_intent_committed",
+ phase=f"tool_intent_{recovery_policy}",
+ safety=recovery_policy,
+ payload={
+ "effect_id": effect_id,
+ "result_id": result_id,
+ "tool_name": tool_name,
+ "tool_call_id": tool_call_id,
+ "args_hash": args_hash,
+ },
+ )
+ row = ToolEffectLedger(
+ effect_id=effect_id,
+ result_id=result_id,
+ run_id=run_id,
+ operation_seq=receipt.operation_seq,
+ event_type="intent",
+ tool_name=tool_name,
+ tool_call_id=tool_call_id,
+ args_hash=args_hash,
+ redacted_args=dict(redacted),
+ idempotency_key=key,
+ recovery_policy=recovery_policy,
+ )
+ db.add(row)
+ db.add(
+ ToolEffectLease(
+ effect_id=effect_id,
+ run_id=run_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ lease_expires_at=self._clock()
+ + timedelta(seconds=max(1, lease_seconds)),
+ updated_at=self._clock(),
+ )
+ )
+ run.updated_at = self._clock()
+ db.commit()
+ return types.IntentDecision("execute", self._intent_from_row(row))
+ except errors.ToolEffectError:
+ raise
+ except (IntegrityError, OperationalError) as exc:
+ with self._sessions() as db:
+ found = (
+ db.query(ToolEffectLedger.event_id)
+ .filter(ToolEffectLedger.idempotency_key == key)
+ .first()
+ )
+ if found is not None:
+ return self._existing_decision(
+ key=key,
+ caller_run_id=run_id,
+ run_owner=owner,
+ claim_owner=claim_owner,
+ tool_name=tool_name,
+ args_hash=args_hash,
+ lease_seconds=lease_seconds,
+ )
+ if _conflict_retries > 0:
+ return self.begin_intent(
+ run_id=run_id,
+ owner=owner,
+ claim_owner=claim_owner,
+ tool_call_id=tool_call_id,
+ tool_name=tool_name,
+ args=args,
+ recovery_policy=recovery_policy,
+ idempotency_key=idempotency_key,
+ lease_seconds=lease_seconds,
+ _conflict_retries=_conflict_retries - 1,
+ )
+ raise errors.ToolIntentCommitError(str(exc)) from exc
+ except (RunNotFound, RunLeaseLost) as exc:
+ raise errors.ToolIntentCommitError(str(exc)) from exc
+ except Exception as exc: # noqa: BLE001
+ raise errors.ToolIntentCommitError(str(exc)) from exc
diff --git a/src/backend/core/runs/tool_effects/invocation_claims.py b/src/backend/core/runs/tool_effects/invocation_claims.py
new file mode 100644
index 00000000..c67ae972
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/invocation_claims.py
@@ -0,0 +1,123 @@
+"""Acquire, renew and abandon invocation claims."""
+from __future__ import annotations
+from datetime import timedelta
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from core.runs.journal import LIVE_STATUSES
+from core.runs.tool_effects import errors
+
+def acquire_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ lease_seconds: int = 300,
+) -> None:
+ """Validate and extend one invocation claim in a short atomic transaction."""
+ now = self._clock()
+ expires = now + timedelta(seconds=max(1, lease_seconds))
+ with self._sessions() as db:
+ live_run = db.query(ChatRun.run_id).filter(
+ ChatRun.status.in_(LIVE_STATUSES),
+ ChatRun.lease_owner == run_owner,
+ ChatRun.lease_expires_at > now,
+ )
+ affected = (
+ db.query(ToolEffectLease)
+ .filter(
+ ToolEffectLease.effect_id == effect_id,
+ ToolEffectLease.run_owner == run_owner,
+ ToolEffectLease.claim_owner == claim_owner,
+ ToolEffectLease.lease_expires_at > now,
+ ToolEffectLease.run_id.in_(live_run),
+ )
+ .update(
+ {
+ ToolEffectLease.lease_expires_at: expires,
+ ToolEffectLease.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ )
+ if not affected or self._terminal_row(db, effect_id) is not None:
+ db.rollback()
+ raise errors.ToolEffectLeaseLost(effect_id)
+ db.commit()
+
+
+def renew_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ lease_seconds: int,
+) -> bool:
+ if not self._runs.renew(
+ self._intent_run_id(effect_id),
+ owner=run_owner,
+ lease_seconds=lease_seconds,
+ ):
+ return False
+ try:
+ self.acquire_invocation_claim(
+ effect_id,
+ run_owner=run_owner,
+ claim_owner=claim_owner,
+ lease_seconds=lease_seconds,
+ )
+ return True
+ except errors.ToolEffectLeaseLost:
+ return False
+
+
+def renew_recovery_claim(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ lease_seconds: int,
+) -> bool:
+ return self.renew_invocation_claim(
+ effect_id,
+ run_owner=recovery_owner,
+ claim_owner=recovery_owner,
+ lease_seconds=lease_seconds,
+ )
+
+
+def intent_run_id(self, effect_id: str) -> str:
+ with self._sessions() as db:
+ row = (
+ db.query(ToolEffectLedger.run_id)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one()
+ )
+ return str(row[0])
+
+
+def abandon_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+) -> None:
+ """Expire a cancelled/ambiguous call so policy recovery can take over."""
+ now = self._clock()
+ with self._sessions() as db:
+ db.query(ToolEffectLease).filter(
+ ToolEffectLease.effect_id == effect_id,
+ ToolEffectLease.run_owner == run_owner,
+ ToolEffectLease.claim_owner == claim_owner,
+ ).update(
+ {
+ ToolEffectLease.lease_expires_at: now,
+ ToolEffectLease.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ db.commit()
diff --git a/src/backend/core/runs/tool_effects/journal.py b/src/backend/core/runs/tool_effects/journal.py
new file mode 100644
index 00000000..7b593351
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/journal.py
@@ -0,0 +1,223 @@
+"""Synchronous transaction boundary behind the tool effect gateway."""
+from __future__ import annotations
+from core.runs.tool_effects import intent_transactions
+from core.runs.tool_effects import result_transactions
+from core.runs.tool_effects import invocation_claims
+from core.runs.tool_effects import retention
+from core.runs.tool_effects import recovery_claims
+from datetime import datetime
+from typing import Any, Callable, Mapping, Optional
+from core.db.engine import SessionLocal
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from core.runs.journal import RunJournal
+from sqlalchemy.orm import Session, sessionmaker
+from core.runs.tool_effects import types
+from core.infra.time import utc_now
+
+class ToolEffectJournal:
+ """Small synchronous transaction boundary behind the async gateway."""
+
+ def __init__(
+ self,
+ session_factory: sessionmaker = SessionLocal,
+ *,
+ clock: Callable[[], datetime] = utc_now,
+ ) -> None:
+ self._sessions = session_factory
+ self._clock = clock
+ self._runs = RunJournal(session_factory, clock=clock)
+
+ @staticmethod
+ def _intent_from_row(row: ToolEffectLedger) -> types.ToolIntent:
+ return intent_transactions.intent_from_row(row)
+
+ @staticmethod
+ def _terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]:
+ return intent_transactions.terminal_row(db, effect_id)
+
+ @staticmethod
+ def _decision_from_terminal(
+ intent: types.ToolIntent, terminal: ToolEffectLedger
+ ) -> types.IntentDecision:
+ return intent_transactions.decision_from_terminal(intent, terminal)
+
+ @staticmethod
+ def _validate_identity(
+ row: ToolEffectLedger, *, tool_name: str, args_hash: str
+ ) -> None:
+ return intent_transactions.validate_identity(row, tool_name=tool_name, args_hash=args_hash)
+
+ def _append_event(
+ self,
+ db: Session,
+ run: ChatRun,
+ intent: ToolEffectLedger,
+ *,
+ owner: str,
+ event_type: str,
+ operation_type: str,
+ phase: str,
+ safety: str,
+ result: Any = None,
+ error: Optional[str] = None,
+ terminal: bool = False,
+ ) -> None:
+ return result_transactions.append_event(self, db, run, intent, owner=owner, event_type=event_type, operation_type=operation_type, phase=phase, safety=safety, result=result, error=error, terminal=terminal)
+
+ def _mark_unknown_locked(
+ self,
+ db: Session,
+ run: ChatRun,
+ intent: ToolEffectLedger,
+ lease: ToolEffectLease,
+ *,
+ owner: str,
+ claim_owner: str,
+ reason: str,
+ ) -> None:
+ return result_transactions.mark_unknown_locked(self, db, run, intent, lease, owner=owner, claim_owner=claim_owner, reason=reason)
+
+ def _existing_decision(
+ self,
+ *,
+ key: str,
+ caller_run_id: str,
+ run_owner: str,
+ claim_owner: str,
+ tool_name: str,
+ args_hash: str,
+ lease_seconds: int,
+ ) -> types.IntentDecision:
+ return intent_transactions.existing_decision(self, key=key, caller_run_id=caller_run_id, run_owner=run_owner, claim_owner=claim_owner, tool_name=tool_name, args_hash=args_hash, lease_seconds=lease_seconds)
+
+ def begin_intent(
+ self,
+ *,
+ run_id: str,
+ owner: str,
+ claim_owner: str,
+ tool_call_id: str,
+ tool_name: str,
+ args: Mapping[str, Any],
+ recovery_policy: str,
+ idempotency_key: Optional[str] = None,
+ lease_seconds: int = 300,
+ _conflict_retries: int = 4,
+ ) -> types.IntentDecision:
+ return intent_transactions.begin_intent(self, run_id=run_id, owner=owner, claim_owner=claim_owner, tool_call_id=tool_call_id, tool_name=tool_name, args=args, recovery_policy=recovery_policy, idempotency_key=idempotency_key, lease_seconds=lease_seconds, _conflict_retries=_conflict_retries)
+
+ def acquire_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ lease_seconds: int = 300,
+ ) -> None:
+ return invocation_claims.acquire_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, lease_seconds=lease_seconds)
+
+ def renew_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ lease_seconds: int,
+ ) -> bool:
+ return invocation_claims.renew_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, lease_seconds=lease_seconds)
+
+ def renew_recovery_claim(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ lease_seconds: int,
+ ) -> bool:
+ return invocation_claims.renew_recovery_claim(self, effect_id, recovery_owner=recovery_owner, lease_seconds=lease_seconds)
+
+ def _intent_run_id(self, effect_id: str) -> str:
+ return invocation_claims.intent_run_id(self, effect_id)
+
+ def abandon_invocation_claim(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ ) -> None:
+ return invocation_claims.abandon_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner)
+
+ def _commit_result_locked(
+ self,
+ db: Session,
+ intent: ToolEffectLedger,
+ lease: ToolEffectLease,
+ *,
+ owner: str,
+ result: Any,
+ failed: bool,
+ reconciled: bool,
+ ) -> Any:
+ return result_transactions.commit_result_locked(self, db, intent, lease, owner=owner, result=result, failed=failed, reconciled=reconciled)
+
+ def commit_result(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ result: Any,
+ failed: bool = False,
+ reconciled: bool = False,
+ _conflict_retries: int = 4,
+ ) -> Any:
+ return result_transactions.commit_result(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, result=result, failed=failed, reconciled=reconciled, _conflict_retries=_conflict_retries)
+
+ def mark_unknown(
+ self,
+ effect_id: str,
+ *,
+ owner: str,
+ claim_owner: str,
+ reason: str,
+ ) -> None:
+ return result_transactions.mark_unknown(self, effect_id, owner=owner, claim_owner=claim_owner, reason=reason)
+
+ def terminal_decision(self, effect_id: str) -> Optional[types.IntentDecision]:
+ return result_transactions.terminal_decision(self, effect_id)
+
+ def pending_effect_ids(self) -> list[str]:
+ return result_transactions.pending_effect_ids(self)
+
+ def prune_settled(self, *, retention_days: int = 7) -> int:
+ return retention.prune_settled(self, retention_days=retention_days)
+
+ def settle_terminal_run_intent(self, effect_id: str) -> Optional[types.ToolIntent]:
+ return retention.settle_terminal_run_intent(self, effect_id)
+
+ def claim_recovery_intent(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ lease_seconds: int = 120,
+ ) -> Optional[types.ToolIntent]:
+ return recovery_claims.claim_recovery_intent(self, effect_id, recovery_owner=recovery_owner, lease_seconds=lease_seconds)
+
+ def commit_recovered_result(
+ self,
+ effect_id: str,
+ result: Any,
+ *,
+ recovery_owner: str,
+ ) -> Any:
+ return recovery_claims.commit_recovered_result(self, effect_id, result, recovery_owner=recovery_owner)
+
+ def release_recovery_claim(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ retry_seconds: int = 30,
+ ) -> None:
+ return recovery_claims.release_recovery_claim(self, effect_id, recovery_owner=recovery_owner, retry_seconds=retry_seconds)
diff --git a/src/backend/core/runs/tool_effects/journal_constants.py b/src/backend/core/runs/tool_effects/journal_constants.py
new file mode 100644
index 00000000..6963b27b
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/journal_constants.py
@@ -0,0 +1,32 @@
+"""Retention and redaction settings for the tool effect journal."""
+from __future__ import annotations
+import re
+
+ATTENTION_RETENTION_FACTOR = 4
+
+
+REDACT_FIELDS = [
+ "password",
+ "token",
+ "api_key",
+ "apikey",
+ "authorization",
+ "secret",
+ "access_key",
+ "cookie",
+]
+
+
+WITHHOLD_FIELDS = {"command", "content", "stdin", "script", "code"}
+
+
+EMBEDDED_SECRET_PATTERNS = (
+ re.compile(r"(?i)(authorization\s*:\s*bearer\s+)[^\s'\";]+"),
+ re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}"),
+ re.compile(
+ r"(?i)([?&](?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)=)[^\s]+"
+ ),
+ re.compile(
+ r"(?i)((?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)\s*[=:]\s*)[^\s'\";]+"
+ ),
+)
diff --git a/src/backend/core/runs/tool_effects/recovery.py b/src/backend/core/runs/tool_effects/recovery.py
new file mode 100644
index 00000000..5fd565b3
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/recovery.py
@@ -0,0 +1,176 @@
+"""Recover tool effects left incomplete by a lost process."""
+from __future__ import annotations
+from core.runs.tool_effects.journal import ToolEffectJournal
+import asyncio
+import time
+import uuid
+from contextlib import suppress
+from typing import Optional
+import logging
+from core.runs.tool_effects import errors
+from core.runs.tool_effects import gateway
+from core.runs.tool_effects.registry import DEFAULT_TOOL_RECOVERY_REGISTRY, ToolRecoveryRegistry
+from core.runs.tool_effects import types
+
+logger = logging.getLogger(__name__)
+
+async def recover_incomplete_tool_effects(
+ *,
+ journal: Optional[ToolEffectJournal] = None,
+ registry: Optional[ToolRecoveryRegistry] = None,
+ lease_seconds: int = 120,
+ heartbeat_interval: Optional[float] = None,
+) -> list[types.RecoverySweepDecision]:
+ """Recover each orphan independently; one broken adapter cannot starve later work."""
+
+ journal = journal or ToolEffectJournal()
+ registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY
+ from core.harness.usage import UsageAttempt, attempt_status_for_exception, record_usage_safely
+ from core.harness.ledger import HarnessUsageLedger
+
+ usage_ledger = HarnessUsageLedger(journal._sessions)
+
+ async def _recorded_recovery_call(handler, intent, phase): # noqa: ANN001
+ started = time.monotonic()
+ status = "failed"
+ try:
+ result = await gateway.ToolEffectGateway._call(handler, intent)
+ status = "success"
+ return result
+ except asyncio.CancelledError:
+ status = "cancelled"
+ raise
+ except Exception as exc:
+ status = attempt_status_for_exception(exc)
+ raise
+ finally:
+ try:
+ await record_usage_safely(
+ usage_ledger,
+ UsageAttempt(
+ run_id=intent.run_id,
+ kind="tool",
+ operation_name=intent.tool_name,
+ effect_id=intent.effect_id,
+ status=status,
+ latency_ms=int((time.monotonic() - started) * 1_000),
+ metadata={"recovery_phase": phase},
+ ),
+ )
+ except Exception: # noqa: BLE001
+ logger.debug("tool recovery usage persistence failed", exc_info=True)
+
+ decisions: list[types.RecoverySweepDecision] = []
+ for effect_id in await asyncio.to_thread(journal.pending_effect_ids):
+ terminal_intent = journal.settle_terminal_run_intent(effect_id)
+ if terminal_intent is not None:
+ decisions.append(
+ types.RecoverySweepDecision(
+ effect_id, terminal_intent.run_id, "terminal_unknown"
+ )
+ )
+ continue
+ recovery_owner = f"tool-recovery:{uuid.uuid4().hex}"
+ intent = journal.claim_recovery_intent(
+ effect_id,
+ recovery_owner=recovery_owner,
+ lease_seconds=lease_seconds,
+ )
+ if intent is None:
+ continue
+ policy = registry.resolve(intent.tool_name)
+ claim_lost = asyncio.Event()
+
+ async def _heartbeat() -> None:
+ interval = heartbeat_interval
+ if interval is None:
+ interval = max(0.05, min(30.0, lease_seconds / 3))
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ ok = await asyncio.to_thread(
+ journal.renew_recovery_claim,
+ effect_id,
+ recovery_owner=recovery_owner,
+ lease_seconds=lease_seconds,
+ )
+ except Exception: # noqa: BLE001 - lost heartbeat means lost ownership
+ ok = False
+ if not ok:
+ claim_lost.set()
+ return
+
+ heartbeat = asyncio.create_task(_heartbeat())
+ try:
+ if intent.recovery_policy == "never_replay":
+ journal.mark_unknown(
+ effect_id,
+ owner=recovery_owner,
+ claim_owner=recovery_owner,
+ reason="non-replayable tool intent has no committed result",
+ )
+ decisions.append(
+ types.RecoverySweepDecision(effect_id, intent.run_id, "needs_attention")
+ )
+ continue
+ if intent.recovery_policy == "replay_safe":
+ if policy.replayer is None:
+ raise errors.ToolEffectError("replay_safe policy has no replay adapter")
+ result = await _recorded_recovery_call(
+ policy.replayer, intent, "replay"
+ )
+ if claim_lost.is_set():
+ raise errors.ToolEffectLeaseLost(effect_id)
+ journal.commit_recovered_result(
+ effect_id, result, recovery_owner=recovery_owner
+ )
+ decisions.append(
+ types.RecoverySweepDecision(effect_id, intent.run_id, "replayed")
+ )
+ continue
+
+ if policy.reconciler is None:
+ raise errors.ToolEffectError("reconcile policy has no reconciliation adapter")
+ reconciliation = gateway.ToolEffectGateway._reconciliation(
+ await _recorded_recovery_call(policy.reconciler, intent, "reconcile")
+ )
+ if claim_lost.is_set():
+ raise errors.ToolEffectLeaseLost(effect_id)
+ if reconciliation.outcome == "applied":
+ result = reconciliation.result
+ elif (
+ reconciliation.outcome == "not_applied" and policy.replayer is not None
+ ):
+ result = await _recorded_recovery_call(
+ policy.replayer, intent, "replay_after_reconcile"
+ )
+ else:
+ journal.mark_unknown(
+ effect_id,
+ owner=recovery_owner,
+ claim_owner=recovery_owner,
+ reason="reconciliation could not determine external outcome",
+ )
+ decisions.append(
+ types.RecoverySweepDecision(effect_id, intent.run_id, "needs_attention")
+ )
+ continue
+ if claim_lost.is_set():
+ raise errors.ToolEffectLeaseLost(effect_id)
+ journal.commit_recovered_result(
+ effect_id, result, recovery_owner=recovery_owner
+ )
+ decisions.append(
+ types.RecoverySweepDecision(effect_id, intent.run_id, "reconciled")
+ )
+ except Exception: # noqa: BLE001 - release this claim and continue the sweep
+ try:
+ journal.release_recovery_claim(effect_id, recovery_owner=recovery_owner)
+ except errors.ToolEffectError:
+ pass
+ decisions.append(types.RecoverySweepDecision(effect_id, intent.run_id, "retry"))
+ finally:
+ heartbeat.cancel()
+ with suppress(asyncio.CancelledError):
+ await heartbeat
+ return decisions
diff --git a/src/backend/core/runs/tool_effects/recovery_claims.py b/src/backend/core/runs/tool_effects/recovery_claims.py
new file mode 100644
index 00000000..c2f72da2
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/recovery_claims.py
@@ -0,0 +1,166 @@
+"""Claim and settle intents during recovery."""
+from __future__ import annotations
+from datetime import timedelta
+from typing import Any, Optional
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from core.runs.journal import LIVE_STATUSES
+from sqlalchemy import or_
+from core.runs.tool_effects import errors
+from core.runs.tool_effects import types
+
+def claim_recovery_intent(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ lease_seconds: int = 120,
+) -> Optional[types.ToolIntent]:
+ with self._sessions() as db:
+ intent = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one_or_none()
+ )
+ if intent is None or self._terminal_row(db, effect_id) is not None:
+ return None
+ now = self._clock()
+ from core.auth.api_key_runs import authorize_run_execution
+
+ if not authorize_run_execution(db, intent.run_id, now):
+ return None
+ expires = now + timedelta(seconds=max(1, lease_seconds))
+ run_affected = (
+ db.query(ChatRun)
+ .filter(
+ ChatRun.run_id == intent.run_id,
+ ChatRun.status.in_((*LIVE_STATUSES, "needs_attention")),
+ or_(
+ ChatRun.lease_owner.is_(None),
+ ChatRun.lease_expires_at.is_(None),
+ ChatRun.lease_expires_at <= now,
+ ChatRun.lease_owner == recovery_owner,
+ ),
+ )
+ .update(
+ {
+ ChatRun.status: "running",
+ ChatRun.lease_owner: recovery_owner,
+ ChatRun.lease_expires_at: expires,
+ ChatRun.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ )
+ if not run_affected:
+ db.rollback()
+ return None
+ lease_affected = (
+ db.query(ToolEffectLease)
+ .filter(
+ ToolEffectLease.effect_id == effect_id,
+ or_(
+ ToolEffectLease.lease_expires_at.is_(None),
+ ToolEffectLease.lease_expires_at <= now,
+ ToolEffectLease.claim_owner == recovery_owner,
+ ),
+ )
+ .update(
+ {
+ ToolEffectLease.run_owner: recovery_owner,
+ ToolEffectLease.claim_owner: recovery_owner,
+ ToolEffectLease.lease_expires_at: expires,
+ ToolEffectLease.updated_at: now,
+ },
+ synchronize_session=False,
+ )
+ )
+ if not lease_affected or self._terminal_row(db, effect_id) is not None:
+ db.rollback()
+ return None
+ db.expire_all()
+ run = db.get(ChatRun, intent.run_id)
+ self._append_event(
+ db,
+ run,
+ intent,
+ owner=recovery_owner,
+ event_type="recovery_claim",
+ operation_type="tool_effect_recovery_claimed",
+ phase=f"tool_intent_{intent.recovery_policy}",
+ safety=intent.recovery_policy,
+ )
+ db.commit()
+ return self._intent_from_row(intent)
+
+
+def commit_recovered_result(
+ self,
+ effect_id: str,
+ result: Any,
+ *,
+ recovery_owner: str,
+) -> Any:
+ value = self.commit_result(
+ effect_id,
+ run_owner=recovery_owner,
+ claim_owner=recovery_owner,
+ result=result,
+ reconciled=True,
+ )
+ with self._sessions() as db:
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == effect_id)
+ .with_for_update()
+ .one()
+ )
+ run = (
+ db.query(ChatRun)
+ .filter(ChatRun.run_id == lease.run_id)
+ .with_for_update()
+ .one()
+ )
+ if lease.run_owner != recovery_owner or run.lease_owner != recovery_owner:
+ raise errors.ToolEffectLeaseLost(effect_id)
+ run.lease_owner = None
+ run.lease_expires_at = None
+ run.updated_at = self._clock()
+ db.commit()
+ return value
+
+
+def release_recovery_claim(
+ self,
+ effect_id: str,
+ *,
+ recovery_owner: str,
+ retry_seconds: int = 30,
+) -> None:
+ """Defer a failed adapter without letting RunJournal resume past the pending effect."""
+ with self._sessions() as db:
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == effect_id)
+ .with_for_update()
+ .one()
+ )
+ if lease.run_owner != recovery_owner or lease.claim_owner != recovery_owner:
+ raise errors.ToolEffectLeaseLost(effect_id)
+ run = (
+ db.query(ChatRun)
+ .filter(ChatRun.run_id == lease.run_id)
+ .with_for_update()
+ .one()
+ )
+ if run.lease_owner != recovery_owner:
+ raise errors.ToolEffectLeaseLost(effect_id)
+ retry_at = self._clock() + timedelta(seconds=max(1, retry_seconds))
+ lease.claim_owner = None
+ lease.lease_expires_at = retry_at
+ lease.updated_at = self._clock()
+ run.lease_expires_at = retry_at
+ run.updated_at = self._clock()
+ db.commit()
diff --git a/src/backend/core/runs/tool_effects/registry.py b/src/backend/core/runs/tool_effects/registry.py
new file mode 100644
index 00000000..36d1608e
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/registry.py
@@ -0,0 +1,95 @@
+"""Recovery policy for each tool; unknown tools are never replayed."""
+from __future__ import annotations
+from typing import Any, Awaitable, Callable, Optional, cast
+from core.runs.tool_effects import types
+
+RECOVERY_POLICIES = ("replay_safe", "reconcile", "never_replay")
+
+
+class ToolRecoveryRegistry:
+ """Explicit adapter contracts. Unknown tools fail closed as never-replay."""
+
+ def __init__(self) -> None:
+ self._policies: dict[str, types.ToolPolicy] = {}
+
+ def register(
+ self,
+ tool_name: str,
+ policy: str,
+ *,
+ replayer: Optional[Callable[[types.ToolIntent], Awaitable[Any] | Any]] = None,
+ reconciler: Optional[
+ Callable[[types.ToolIntent], Awaitable[types.ReconciliationResult | Any] | Any]
+ ] = None,
+ ) -> None:
+ if policy not in RECOVERY_POLICIES:
+ raise ValueError(f"invalid recovery policy: {policy}")
+ if policy == "replay_safe" and reconciler is not None:
+ raise ValueError("replay_safe does not accept a reconciler")
+ if policy == "never_replay" and (
+ replayer is not None or reconciler is not None
+ ):
+ raise ValueError("never_replay does not accept recovery adapters")
+ self._policies[str(tool_name)] = types.ToolPolicy(policy, replayer, reconciler)
+
+ def resolve(self, tool_name: str) -> types.ToolPolicy:
+ return self._policies.get(str(tool_name), types.ToolPolicy("never_replay"))
+
+
+async def default_replayer(intent: types.ToolIntent) -> Any:
+ from core.runs.tool_effect_recovery import replay_tool_intent
+
+ return await replay_tool_intent(intent)
+
+
+async def scheduled_task_reconciler(intent: types.ToolIntent) -> types.ReconciliationResult:
+ from core.runs.tool_effect_recovery import reconcile_scheduled_task_intent
+
+ return cast(types.ReconciliationResult, await reconcile_scheduled_task_intent(intent))
+
+
+def build_default_tool_recovery_registry() -> ToolRecoveryRegistry:
+ registry = ToolRecoveryRegistry()
+ for name in (
+ "Read",
+ "read_project_instructions",
+ "list_sites",
+ "Glob",
+ "Grep",
+ "view_text_file",
+ "get_data_context",
+ "read_image",
+ "read_artifact",
+ "channel_read_attachment",
+ "sandbox_get_artifact",
+ ):
+ registry.register(name, "replay_safe", replayer=default_replayer)
+ for name in (
+ "Write",
+ "save_project_instructions",
+ "Edit",
+ "Bash",
+ "sandbox_put_artifact",
+ "pin_to_workspace",
+ "run_job",
+ "choose_design",
+ "call_subagent",
+ "update_plan",
+ "load_plugin",
+ ):
+ registry.register(name, "never_replay")
+ for name in (
+ "create_scheduled_task",
+ "update_scheduled_task",
+ "delete_scheduled_task",
+ ):
+ registry.register(
+ name,
+ "reconcile",
+ replayer=default_replayer,
+ reconciler=scheduled_task_reconciler,
+ )
+ return registry
+
+
+DEFAULT_TOOL_RECOVERY_REGISTRY = build_default_tool_recovery_registry()
diff --git a/src/backend/core/runs/tool_effects/result_transactions.py b/src/backend/core/runs/tool_effects/result_transactions.py
new file mode 100644
index 00000000..b66b8e3b
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/result_transactions.py
@@ -0,0 +1,285 @@
+"""Commit tool results or mark their outcome unknown."""
+from __future__ import annotations
+from typing import Any, Optional
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from sqlalchemy.exc import IntegrityError, OperationalError
+from sqlalchemy.orm import Session, aliased
+from core.runs.tool_effects import errors
+from core.runs.tool_effects import types
+from core.infra.time import as_utc
+
+def append_event(
+ self,
+ db: Session,
+ run: ChatRun,
+ intent: ToolEffectLedger,
+ *,
+ owner: str,
+ event_type: str,
+ operation_type: str,
+ phase: str,
+ safety: str,
+ result: Any = None,
+ error: Optional[str] = None,
+ terminal: bool = False,
+) -> None:
+ receipt = self._runs._append_locked(
+ db,
+ run,
+ owner=owner,
+ operation_type=operation_type,
+ phase=phase,
+ safety=safety,
+ payload={"effect_id": intent.effect_id, "result_id": intent.result_id},
+ )
+ db.add(
+ ToolEffectLedger(
+ effect_id=intent.effect_id,
+ result_id=intent.result_id,
+ run_id=intent.run_id,
+ operation_seq=receipt.operation_seq,
+ event_type=event_type,
+ tool_name=intent.tool_name,
+ tool_call_id=intent.tool_call_id,
+ args_hash=intent.args_hash,
+ redacted_args=intent.redacted_args,
+ recovery_policy=intent.recovery_policy,
+ result_payload=result,
+ error_message=error,
+ terminal_effect_id=intent.effect_id if terminal else None,
+ )
+ )
+
+
+def mark_unknown_locked(
+ self,
+ db: Session,
+ run: ChatRun,
+ intent: ToolEffectLedger,
+ lease: ToolEffectLease,
+ *,
+ owner: str,
+ claim_owner: str,
+ reason: str,
+) -> None:
+ if self._terminal_row(db, intent.effect_id) is not None:
+ return
+ if lease.run_owner != owner or lease.claim_owner != claim_owner:
+ raise errors.ToolEffectLeaseLost(intent.effect_id)
+ self._append_event(
+ db,
+ run,
+ intent,
+ owner=owner,
+ event_type="unknown_outcome",
+ operation_type="tool_outcome_unknown",
+ phase="needs_attention",
+ safety="manual_review",
+ error=reason,
+ terminal=True,
+ )
+ run.status = "needs_attention"
+ run.failure_reason = reason
+ run.lease_owner = None
+ run.lease_expires_at = None
+ run.updated_at = self._clock()
+ lease.claim_owner = None
+ lease.lease_expires_at = None
+ lease.updated_at = self._clock()
+
+
+def commit_result_locked(
+ self,
+ db: Session,
+ intent: ToolEffectLedger,
+ lease: ToolEffectLease,
+ *,
+ owner: str,
+ result: Any,
+ failed: bool,
+ reconciled: bool,
+) -> Any:
+ terminal = self._terminal_row(db, intent.effect_id)
+ if terminal is not None:
+ if terminal.event_type == "unknown_outcome":
+ raise errors.ToolOutcomeUnknown(intent.effect_id)
+ return terminal.result_payload
+ if lease.run_owner != owner:
+ raise errors.ToolEffectLeaseLost(intent.effect_id)
+ run = self._runs._locked_owned_run(db, intent.run_id, owner)
+ self._append_event(
+ db,
+ run,
+ intent,
+ owner=owner,
+ event_type="failure" if failed else "result",
+ operation_type="tool_failure_committed"
+ if failed
+ else "tool_result_committed",
+ phase="tool_result_committed",
+ safety="reconciled" if reconciled else intent.recovery_policy,
+ result=result,
+ error=str(result)[:4000] if failed else None,
+ terminal=True,
+ )
+ lease.claim_owner = None
+ lease.lease_expires_at = None
+ lease.updated_at = self._clock()
+ run.updated_at = self._clock()
+ return result
+
+
+def commit_result(
+ self,
+ effect_id: str,
+ *,
+ run_owner: str,
+ claim_owner: str,
+ result: Any,
+ failed: bool = False,
+ reconciled: bool = False,
+ _conflict_retries: int = 4,
+) -> Any:
+ with self._sessions() as db:
+ intent = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one()
+ )
+ terminal = self._terminal_row(db, effect_id)
+ if terminal is not None:
+ if terminal.event_type == "unknown_outcome":
+ raise errors.ToolOutcomeUnknown(effect_id)
+ return terminal.result_payload
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == effect_id)
+ .with_for_update()
+ .one()
+ )
+ now = as_utc(self._clock())
+ lease_expiry = as_utc(lease.lease_expires_at)
+ if (
+ lease.run_owner != run_owner
+ or lease.claim_owner != claim_owner
+ or lease_expiry is None
+ or now is None
+ or lease_expiry <= now
+ ):
+ # A concurrent retry can observe the lease only after the
+ # winning commit cleared its claim. Re-check the durable
+ # terminal decision before reporting lease loss: identical
+ # result commits are idempotent even when the first pre-lock
+ # terminal read raced with the winner.
+ terminal = self._terminal_row(db, effect_id)
+ if terminal is not None:
+ if terminal.event_type == "unknown_outcome":
+ raise errors.ToolOutcomeUnknown(effect_id)
+ return terminal.result_payload
+ raise errors.ToolEffectLeaseLost(effect_id)
+ value = self._commit_result_locked(
+ db,
+ intent,
+ lease,
+ owner=run_owner,
+ result=result,
+ failed=failed,
+ reconciled=reconciled,
+ )
+ try:
+ db.commit()
+ except (IntegrityError, OperationalError) as exc:
+ db.rollback()
+ terminal = self.terminal_decision(effect_id)
+ if terminal is not None:
+ if terminal.action == "unknown":
+ raise errors.ToolOutcomeUnknown(effect_id)
+ return terminal.result
+ if _conflict_retries > 0:
+ return self.commit_result(
+ effect_id,
+ run_owner=run_owner,
+ claim_owner=claim_owner,
+ result=result,
+ failed=failed,
+ reconciled=reconciled,
+ _conflict_retries=_conflict_retries - 1,
+ )
+ raise errors.ToolEffectError(str(exc)) from exc
+ return value
+
+
+def mark_unknown(
+ self,
+ effect_id: str,
+ *,
+ owner: str,
+ claim_owner: str,
+ reason: str,
+) -> None:
+ with self._sessions() as db:
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == effect_id)
+ .with_for_update()
+ .one()
+ )
+ intent = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one()
+ )
+ run = self._runs._locked_owned_run(db, intent.run_id, owner)
+ self._mark_unknown_locked(
+ db,
+ run,
+ intent,
+ lease,
+ owner=owner,
+ claim_owner=claim_owner,
+ reason=reason,
+ )
+ db.commit()
+
+
+def terminal_decision(self, effect_id: str) -> Optional[types.IntentDecision]:
+ with self._sessions() as db:
+ intent = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .one_or_none()
+ )
+ if intent is None:
+ return None
+ terminal = self._terminal_row(db, effect_id)
+ return (
+ self._decision_from_terminal(self._intent_from_row(intent), terminal)
+ if terminal is not None
+ else None
+ )
+
+
+def pending_effect_ids(self) -> list[str]:
+ with self._sessions() as db:
+ terminal = aliased(ToolEffectLedger)
+ rows = (
+ db.query(ToolEffectLedger.effect_id)
+ .filter(
+ ToolEffectLedger.event_type == "intent",
+ ~db.query(terminal.event_id)
+ .filter(terminal.terminal_effect_id == ToolEffectLedger.effect_id)
+ .exists(),
+ )
+ .order_by(ToolEffectLedger.created_at, ToolEffectLedger.event_id)
+ .all()
+ )
+ return [row.effect_id for row in rows]
diff --git a/src/backend/core/runs/tool_effects/retention.py b/src/backend/core/runs/tool_effects/retention.py
new file mode 100644
index 00000000..0d089fe1
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/retention.py
@@ -0,0 +1,154 @@
+"""Prune settled tool effect records and close orphaned intents."""
+from __future__ import annotations
+from datetime import timedelta
+from typing import Optional
+from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
+from core.runs.journal import LIVE_STATUSES
+from sqlalchemy import and_, case
+from core.runs.tool_effects import journal_constants
+from core.runs.tool_effects import types
+from core.infra.time import as_utc
+
+def prune_settled(self, *, retention_days: int = 7) -> int:
+ """Drop per-run recovery/observability facts of long-terminal runs.
+
+ Covers the tool effect ledger and leases plus the other append-only
+ per-run journals (harness usage attempts, harness events, run
+ operations); without this they grow without bound.
+
+ The run's own recovery columns go the same way. ``recovery_snapshot``
+ carries the whole conversation as of that turn, so a long chat stores
+ roughly its own history once per turn — the single largest thing in a
+ desktop database. Only crash recovery reads it, and only for runs that
+ are still live and still replayable, which is exactly what the cutoff
+ and the settled filter below exclude.
+
+ ``needs_attention`` is kept ``_ATTENTION_RETENTION_FACTOR`` times longer
+ rather than exempted, and that applies to every tier above. Exempting it
+ means never: nothing in the backend transitions a run out of that status,
+ so an exemption pins its journals — and its snapshot — for good. In
+ production every record that had outlived the retention window belonged
+ to such a run.
+ """
+ from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt
+
+ now = as_utc(self._clock())
+ cutoff = now - timedelta(days=retention_days)
+ attention_cutoff = now - timedelta(days=retention_days * journal_constants.ATTENTION_RETENTION_FACTOR)
+ with self._sessions() as db:
+
+ earliest_cutoff = max(cutoff, attention_cutoff)
+
+ def _prunable(run_col, stamp):
+ """Row whose run is terminal and past that run's retention tier.
+
+ ``stamp < earliest_cutoff`` is implied by the EXISTS (it is the
+ loosest of the tiers) and stated anyway: it is the only
+ predicate on these tables an index can serve, and without it the
+ planner has nothing to range-scan on.
+ """
+ tier = case(
+ (ChatRun.status == "needs_attention", attention_cutoff),
+ else_=cutoff,
+ )
+ return and_(
+ stamp < earliest_cutoff,
+ db.query(ChatRun.run_id)
+ .filter(
+ ChatRun.run_id == run_col,
+ ChatRun.status.notin_(LIVE_STATUSES),
+ stamp < tier,
+ )
+ .exists(),
+ )
+
+ deleted = (
+ db.query(ToolEffectLedger)
+ .filter(_prunable(ToolEffectLedger.run_id, ToolEffectLedger.created_at))
+ .delete(synchronize_session=False)
+ )
+ db.query(ToolEffectLease).filter(
+ _prunable(ToolEffectLease.run_id, ToolEffectLease.updated_at)
+ ).delete(synchronize_session=False)
+ for model in (HarnessUsageAttempt, HarnessEventLog, ChatRunOperation):
+ deleted += (
+ db.query(model)
+ .filter(_prunable(model.run_id, model.created_at))
+ .delete(synchronize_session=False)
+ )
+ deleted += (
+ db.query(ChatRun)
+ .filter(
+ ChatRun.status.notin_(LIVE_STATUSES),
+ ChatRun.completed_at < earliest_cutoff,
+ ChatRun.completed_at
+ < case(
+ (ChatRun.status == "needs_attention", attention_cutoff),
+ else_=cutoff,
+ ),
+ ChatRun.recovery_snapshot.isnot(None),
+ )
+ .update(
+ {ChatRun.recovery_snapshot: None, ChatRun.request_payload: None},
+ synchronize_session=False,
+ )
+ )
+ db.commit()
+ return deleted
+
+
+def settle_terminal_run_intent(self, effect_id: str) -> Optional[types.ToolIntent]:
+ """Close an orphan intent without replay after its owning run is terminal.
+
+ Cancellation/failure is a hard replay boundary: the user's stopped run
+ must never perform a delayed external action. We still append an
+ ``unknown_outcome`` fact so the intent cannot remain pending forever.
+ """
+ with self._sessions() as db:
+ intent = (
+ db.query(ToolEffectLedger)
+ .filter(
+ ToolEffectLedger.effect_id == effect_id,
+ ToolEffectLedger.event_type == "intent",
+ )
+ .with_for_update()
+ .one_or_none()
+ )
+ if intent is None or self._terminal_row(db, effect_id) is not None:
+ return None
+ run = (
+ db.query(ChatRun)
+ .filter(ChatRun.run_id == intent.run_id)
+ .with_for_update()
+ .one_or_none()
+ )
+ if run is None or run.status in (*LIVE_STATUSES, "needs_attention"):
+ return None
+ lease = (
+ db.query(ToolEffectLease)
+ .filter(ToolEffectLease.effect_id == effect_id)
+ .with_for_update()
+ .one()
+ )
+ terminal_status = str(run.status)
+ self._append_event(
+ db,
+ run,
+ intent,
+ owner="system:terminal-settlement",
+ event_type="unknown_outcome",
+ operation_type="tool_outcome_unknown_after_terminal_run",
+ phase=str(run.run_phase or terminal_status),
+ safety="manual_review",
+ error=f"run became {terminal_status} before the tool outcome committed",
+ terminal=True,
+ )
+ # _append_event advances only the operation ledger. Preserve the
+ # authoritative terminal verdict and merely close the effect claim.
+ run.status = terminal_status
+ lease.claim_owner = None
+ lease.lease_expires_at = None
+ lease.updated_at = self._clock()
+ run.updated_at = self._clock()
+ db.commit()
+ return self._intent_from_row(intent)
diff --git a/src/backend/core/runs/tool_effects/types.py b/src/backend/core/runs/tool_effects/types.py
new file mode 100644
index 00000000..2052f25d
--- /dev/null
+++ b/src/backend/core/runs/tool_effects/types.py
@@ -0,0 +1,74 @@
+"""Value types for tool intents, decisions and outcomes."""
+from __future__ import annotations
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Awaitable, Callable, Mapping, Optional
+
+@dataclass(frozen=True)
+class ReconciliationResult:
+ outcome: str
+ result: Any = None
+
+ @classmethod
+ def applied(cls, result: Any) -> "ReconciliationResult":
+ return cls("applied", result)
+
+ @classmethod
+ def not_applied(cls) -> "ReconciliationResult":
+ return cls("not_applied")
+
+ @classmethod
+ def unknown(cls) -> "ReconciliationResult":
+ return cls("unknown")
+
+
+@dataclass(frozen=True)
+class ToolPolicy:
+ policy: str
+ replayer: Optional[Callable[["ToolIntent"], Awaitable[Any] | Any]] = None
+ reconciler: Optional[
+ Callable[["ToolIntent"], Awaitable[ReconciliationResult | Any] | Any]
+ ] = None
+
+
+@dataclass(frozen=True)
+class ToolIntent:
+ effect_id: str
+ result_id: str
+ run_id: str
+ operation_seq: int
+ tool_name: str
+ tool_call_id: str
+ args_hash: str
+ redacted_args: Mapping[str, Any]
+ idempotency_key: str
+ recovery_policy: str
+ created_at: Optional[datetime] = None
+
+
+@dataclass(frozen=True)
+class IntentDecision:
+ action: str
+ intent: ToolIntent
+ result: Any = None
+ result_state: str = "success"
+
+ @property
+ def effect_id(self) -> str:
+ return self.intent.effect_id
+
+
+@dataclass(frozen=True)
+class ToolExecutionOutcome:
+ result: Any
+ effect_id: str
+ result_id: str
+ result_state: str
+ invoked: bool
+
+
+@dataclass(frozen=True)
+class RecoverySweepDecision:
+ effect_id: str
+ run_id: str
+ action: str
diff --git a/src/backend/core/runs/watchdog.py b/src/backend/core/runs/watchdog.py
new file mode 100644
index 00000000..6521a8e5
--- /dev/null
+++ b/src/backend/core/runs/watchdog.py
@@ -0,0 +1,322 @@
+"""Detect stalled streams and reap expired runs without replacing a live writer."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import datetime as _datetime
+from datetime import datetime
+from typing import Any, AsyncIterator, Callable, Optional
+
+import core.db.engine as engine
+import core.db.models as models
+import core.llm.runtime.human_interaction as human_interaction
+import core.runs.clock as clock
+import core.runs.events.writer as writer
+import core.runs.ownership as ownership
+import core.runs.settings as settings
+import core.runs.tasks as tasks
+from core.infra.logging import get_logger
+from core.db.threads import run_in_session
+
+logger = get_logger(__name__)
+
+
+async def _aiter_with_inactivity_timeout(
+ aiter: AsyncIterator[Any],
+ timeout: float,
+ *,
+ is_activity: Callable[[Any], bool] | None = None,
+ first_response_timeout: float | None = None,
+):
+ """Yield from an async iterator and enforce a meaningful-activity deadline.
+
+ Guards against a hung workflow (model loop / blocked MCP / asyncio
+ deadlock) leaving the chat run stuck in 'running' forever — on timeout
+ the underlying generator is closed and the error propagates into the
+ existing ``except Exception`` path that writes the ``failed`` terminal.
+
+ Transport keep-alives may still be yielded to the caller, but they do not
+ reset the deadline when ``is_activity`` marks them as non-meaningful.
+ The optional first-response deadline is absolute until visible output;
+ it is then disabled for the remainder of this workflow.
+ """
+ activity_check = is_activity or (lambda _: True)
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + timeout
+ first_deadline = (
+ loop.time() + first_response_timeout if first_response_timeout is not None else None
+ )
+
+ def timeout_error() -> TimeoutError:
+ from core.infra.exceptions import FirstResponseTimeoutError
+
+ if first_deadline is not None and loop.time() >= first_deadline:
+ return FirstResponseTimeoutError()
+ return TimeoutError(f"工作流 {timeout:.0f}s 内无有效输出,已判定为卡死并中止")
+
+ while True:
+ effective_deadline = (
+ min(deadline, first_deadline) if first_deadline is not None else deadline
+ )
+ remaining = effective_deadline - loop.time()
+ if remaining <= 0:
+ with contextlib.suppress(Exception):
+ await aiter.aclose() # type: ignore[attr-defined]
+ raise timeout_error()
+ try:
+ item = await asyncio.wait_for(aiter.__anext__(), timeout=remaining)
+ except StopAsyncIteration:
+ return
+ except asyncio.TimeoutError as exc:
+ with contextlib.suppress(Exception):
+ await aiter.aclose() # type: ignore[attr-defined]
+ raise timeout_error() from exc
+ if first_deadline is not None and _is_first_response(item):
+ first_deadline = None
+ if activity_check(item):
+ deadline = loop.time() + timeout
+ yield item
+
+
+def _is_first_response(item: Any) -> bool:
+ """Only visible model output or a tool interaction ends the initial wait."""
+ kind = item.get("type")
+ if kind in {"ai_message", "content", "thinking"}:
+ return bool(item.get("delta"))
+ if kind == "content_replace":
+ return bool(item.get("content"))
+ return kind in {
+ "tool_call_start",
+ "tool_call_delta",
+ "tool_call",
+ "tool_result",
+ "tool_pending",
+ "file_confirm",
+ "plan_update",
+ "compaction_progress",
+ }
+
+
+def _is_chat_run_activity(item: Any, chat_id: str) -> bool:
+ """Classify workflow output for the inactivity watchdog.
+
+ A transport heartbeat is normally not meaningful progress. It becomes
+ progress only while a registered tool is legitimately suspended on a
+ human answer; this keeps true HITL waits alive without weakening zombie
+ detection for every other silent run.
+ """
+
+ return item.get("type") != "heartbeat" or human_interaction.has_pending(chat_id)
+
+
+async def _stream_last_write_ms(run_id: str) -> Optional[int]:
+ """Write time of the last event on the run's log (epoch ms).
+
+ Cursors are minted as ``-`` by every backend, so the millisecond
+ half is the write time — no extra bookkeeping. Returns None when there is
+ no log / no events / the read fails (callers treat that as "no activity").
+ """
+ try:
+ return await writer._events().last_write_ms(run_id)
+ except Exception as exc:
+ logger.warning("chat_run_activity_check_failed", run_id=run_id, error=str(exc))
+ return None
+
+
+def _chat_has_live_job(chat_id: Optional[str], now: datetime) -> bool:
+ """这个会话是否挂着**还在推进**的批量作业(跨进程可见的活性证据)。
+
+ 判据是 ``jobs.updated_at`` 在静默窗口内前进过:作业每完成一次子调用都会写 usage,
+ onupdate 随之推进 updated_at。只要作业还在动,这个 run 就不是僵尸——哪怕它的
+ asyncio task 属于另一个进程(多 worker / 多副本),也哪怕主链路一个流事件都没有。
+
+ 作业本身有 max_seconds 墙钟预算与熔断,所以这条豁免不会让 run 永生。
+ """
+ if not chat_id:
+ return False
+ try:
+ from core.db.models import Job
+
+ cutoff = now - _datetime.timedelta(seconds=settings._STALE_QUIET_SEC)
+ with engine.SessionLocal() as db:
+ return (
+ db.query(Job.job_id)
+ .filter(
+ Job.chat_id == chat_id,
+ Job.status.in_(("pending", "running")),
+ Job.updated_at >= cutoff,
+ )
+ .first()
+ is not None
+ )
+ except Exception: # noqa: BLE001 —— 证据查不到就按原判据走,绝不因此放过真僵尸
+ return False
+
+
+async def reap_stale_runs() -> int:
+ """Periodic safety net: fail 'running' runs that show no sign of life.
+
+ The verdict is activity-aware, in two tiers (both tunable via env vars):
+
+ - Reap only when **over-age + quiet**: lifetime exceeds
+ ``CHAT_RUN_MAX_AGE_SEC`` AND the Redis Stream's last write is older than
+ ``CHAT_RUN_STALE_QUIET_SEC``. Long tasks still steadily producing
+ tool_call/content are no longer falsely killed (historical bug: purely
+ age-based reaping choked even active runs mid-tool-call once the 30-minute
+ mark hit).
+ - **Absolute cap**: runs older than ``CHAT_RUN_HARD_MAX_AGE_SEC`` are reaped
+ even when actively emitting, except for automation conversations and
+ registered bounded human interactions (which keep their own deadline).
+
+ Reaping also cancels the in-process worker task, keeping the DB terminal
+ state aligned with the actual task (historical bug: only flipping the DB
+ without killing the task let the worker silently run another 1.5h and
+ overwrite the terminal state back to completed). plan_execute is the
+ exception — cross-task cancel risks the anyio cancel-scope deadlock (see the
+ ``cancel_run`` docstring); it stops itself after ``is_run_cancelled``
+ polling observes the terminal state.
+
+ Complements periodic ``recover_orphan_runs`` and the per-run
+ inactivity watchdog — also sweeps up historical zombie runs left by
+ older code paths.
+ """
+ from core.db.models import ChatSession
+ from sqlalchemy import func
+
+ now = clock._utcnow()
+ cutoff = now - _datetime.timedelta(seconds=settings._STALE_RUN_MAX_AGE_SEC)
+ hard_cutoff = now - _datetime.timedelta(seconds=settings._HARD_MAX_AGE_SEC)
+ def _stale_candidates(db):
+ candidates = (
+ db.query(
+ models.ChatRun.run_id,
+ models.ChatRun.chat_id,
+ models.ChatRun.request_payload,
+ models.ChatRun.lease_expires_at,
+ ChatSession.extra_data.label("session_metadata"),
+ func.coalesce(models.ChatRun.started_at, models.ChatRun.created_at).label(
+ "began_at"
+ ),
+ )
+ .outerjoin(ChatSession, ChatSession.chat_id == models.ChatRun.chat_id)
+ .filter(models.ChatRun.status == "running")
+ .filter(func.coalesce(models.ChatRun.started_at, models.ChatRun.created_at) < cutoff)
+ .all()
+ )
+ return candidates
+ candidates = await run_in_session(_stale_candidates)
+ if not candidates:
+ return 0
+
+ now_ms = int(now.timestamp() * 1000)
+ reaped = 0
+ for row in candidates:
+ rid, cid = row.run_id, row.chat_id
+ lease_expires_at = row.lease_expires_at
+ if lease_expires_at is not None and lease_expires_at.tzinfo is None:
+ lease_expires_at = lease_expires_at.replace(tzinfo=_datetime.timezone.utc)
+ began_at = row.began_at
+ if began_at is not None and began_at.tzinfo is None: # SQLite stores naive UTC
+ began_at = began_at.replace(tzinfo=_datetime.timezone.utc)
+ session_metadata = row.session_metadata or {}
+ automation_run = bool(session_metadata.get("automation_run"))
+ hard_expired = not automation_run and began_at is not None and began_at < hard_cutoff
+ if not hard_expired and lease_expires_at is not None and lease_expires_at > now:
+ logger.info(
+ "chat_run_stale_skip_live_lease",
+ run_id=rid,
+ lease_expires_at=lease_expires_at.isoformat(),
+ )
+ continue
+ payload = row.request_payload if isinstance(row.request_payload, dict) else {}
+
+ # 自主循环豁免统一年龄硬顶:loop 的使命就是「跑到任务完成为止」(去预算化),
+ # 且它有自己的停滞护栏/熔断/硬后备轮数。进程内 task 还活着的 loop 绝不按
+ # 年龄杀(历史竞态:6h 硬顶 == loop 旧默认预算,跑满预算的健康 loop 在优雅
+ # 收尾前被硬顶抢杀,报错还误导为「无响应」)。孤儿 loop(进程重启遗留)
+ # 走下面的静默判据正常清理。
+ if hard_expired and payload.get("kind") == "autonomous_loop":
+ task = tasks.process_state().active_runs.get(rid)
+ if task is not None and not task.done():
+ logger.info("chat_run_stale_skip_loop_inprocess", run_id=rid)
+ continue
+ hard_expired = False # 孤儿 loop:不按年龄硬杀,交给静默判据
+
+ # A legitimate user decision may begin near the run's absolute-age
+ # ceiling. Its own bounded wait deadline is the authority; do not cut a
+ # newly opened question/confirmation down to the remainder of the run
+ # hard cap. Once the registry resolves or times out, this exemption
+ # disappears automatically on the next sweep.
+ if hard_expired and human_interaction.has_pending(cid):
+ logger.info("chat_run_stale_skip_human_wait", run_id=rid, chat_id=cid)
+ continue
+
+ # 工作流模式的批量作业:run 卡在 run_job(wait=True) 里等作业,期间主链路一个
+ # 流事件都不产生(逐项结果按设计不进对话),"流静默"判据必然误判。进程内 task
+ # 豁免只在本进程有效——多 worker / 多副本部署里,另一个进程看不到这个 task,
+ # 照样会把健康的长作业杀掉。所以这里用**跨进程可见的证据**:DB 里这个会话是否
+ # 还有正在推进的作业(每次子调用都会 add_usage → jobs.updated_at 前进)。
+ if _chat_has_live_job(cid, now):
+ logger.info("chat_run_stale_skip_live_job", run_id=rid, chat_id=cid)
+ continue
+
+ if not hard_expired:
+ # A run whose asyncio task is alive in THIS process is never a
+ # zombie — the in-process inactivity watchdog owns hang detection
+ # for it. The stream-quiet check below is only for orphans whose
+ # worker vanished (process restart/crash). Without this guard, a
+ # healthy long run streaming huge tool-call args (which map to no
+ # stream events) past 30min would be reaped as "quiet".
+ task = tasks.process_state().active_runs.get(rid)
+ if task is not None and not task.done():
+ logger.info("chat_run_stale_skip_inprocess", run_id=rid)
+ continue
+ last_ms = await _stream_last_write_ms(rid)
+ if last_ms is not None and (now_ms - last_ms) < settings._STALE_QUIET_SEC * 1000:
+ # The stream is still producing: this is a long task, not a zombie — skip this round
+ logger.info(
+ "chat_run_stale_skip_active",
+ run_id=rid,
+ quiet_sec=round((now_ms - last_ms) / 1000, 1),
+ )
+ continue
+ reason = "run stalled: no stream activity (stale watchdog)"
+ else:
+ reason = "run exceeded hard max age (stale watchdog)"
+
+ if not ownership._request_run_stop(
+ rid,
+ status="failed",
+ error_message=reason,
+ require_expired_lease=not hard_expired,
+ ):
+ continue # race: the worker just beat us to writing the terminal state
+
+ await writer._write_terminal_to_stream(
+ rid,
+ chat_id=cid or "",
+ error_text="任务长时间无响应,已被系统中止,请重新发起",
+ )
+
+ payload = row.request_payload if isinstance(row.request_payload, dict) else {}
+ task = tasks.process_state().active_runs.get(rid)
+ if (
+ task is not None
+ and not task.done()
+ and payload.get("kind") not in settings._COOPERATIVE_KINDS
+ ):
+ task.cancel()
+ try:
+ await asyncio.wait_for(task, timeout=2.0)
+ except (asyncio.TimeoutError, asyncio.CancelledError):
+ pass
+ except Exception as exc:
+ logger.warning("stale_reaper_await_failed", run_id=rid, error=str(exc))
+ if task.done():
+ ownership._acknowledge_terminal_writer(rid)
+ reaped += 1
+
+ if reaped:
+ logger.info("chat_run_stale_reaped", count=reaped)
+ return reaped
diff --git a/src/backend/core/sandbox/commands/__init__.py b/src/backend/core/sandbox/commands/__init__.py
new file mode 100644
index 00000000..ebd4bf0b
--- /dev/null
+++ b/src/backend/core/sandbox/commands/__init__.py
@@ -0,0 +1 @@
+"""Sandbox commands, owned lifecycle and bounded output storage."""
diff --git a/src/backend/core/sandbox/commands/completion_hook.py b/src/backend/core/sandbox/commands/completion_hook.py
new file mode 100644
index 00000000..86fe23a0
--- /dev/null
+++ b/src/backend/core/sandbox/commands/completion_hook.py
@@ -0,0 +1,44 @@
+"""Expose terminal command facts at safe reasoning and finish boundaries."""
+import asyncio
+from core.harness.hooks import Decision, HookSpec, HookStage
+from core.sandbox.commands import delivery, store
+
+
+async def inject_completion(invocation):
+ if invocation.operation_name != "agent_reasoning":
+ return Decision()
+ identity = invocation.data.get("identity", {})
+ chat_id, user_id = identity.get("chat_id"), identity.get("user_id")
+ if not (invocation.run_id and chat_id and user_id):
+ return Decision()
+ if not await asyncio.to_thread(store.has_pending, chat_id, user_id):
+ return Decision()
+ rows = await asyncio.to_thread(delivery.reserve_boundary, chat_id, user_id, invocation.run_id)
+ if not rows:
+ return Decision()
+ item = dict(content=delivery.notice_text(rows), kind="reminder",
+ origin="harness:command_completion", trust="system", priority=950,
+ token_budget=4000, truncation_policy="never")
+ return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), item]})
+
+
+async def pending_completion(invocation):
+ identity = invocation.data.get("identity", {})
+ chat_id, user_id = identity.get("chat_id"), identity.get("user_id")
+ if not (invocation.run_id and chat_id and user_id):
+ return Decision()
+ rows = await asyncio.to_thread(store.pending_notices, chat_id)
+ if any(row["notice_status"] == "pending" and row["user_id"] == user_id for row in rows):
+ return Decision.modify({"continue_run": True})
+ return Decision()
+
+
+def command_completion_hooks():
+ return (
+ HookSpec("command_completion_context", HookStage.TRANSFORM_CONTEXT,
+ inject_completion, priority=90, timeout_ms=30000,
+ mutable_fields=frozenset({"context_items"})),
+ HookSpec("command_completion_finish", HookStage.BEFORE_FINISH,
+ pending_completion, priority=90, timeout_ms=30000,
+ mutable_fields=frozenset({"continue_run"})),
+ )
diff --git a/src/backend/core/services/commands/context.py b/src/backend/core/sandbox/commands/context.py
similarity index 85%
rename from src/backend/core/services/commands/context.py
rename to src/backend/core/sandbox/commands/context.py
index c323401e..3a412598 100644
--- a/src/backend/core/services/commands/context.py
+++ b/src/backend/core/sandbox/commands/context.py
@@ -1,15 +1,13 @@
"""Reconstruct active command events at the persisted-history boundary."""
from core.db.models import ChatRun
from core.db.models.command import CommandTask
-from core.services.commands.delivery import notice_text
+from core.sandbox.commands.delivery import notice_text
def completion_context(row):
"""One stable sideband record per completed command, never a chat row."""
- from core.llm.context_ir import (
- KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY,
- make_text_context_item, session_context_metadata,
- )
+ from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY
+ from core.llm.context.items import make_text_context_item, session_context_metadata
event_id = "command_event_" + row["task_id"]
prompt = notice_text([row])
item = make_text_context_item(
@@ -34,7 +32,7 @@ def append_active_completions(db, chat_id, history):
ChatRun.writer_slot == "main")
.order_by(CommandTask.task_id).all())
existing = {message.get("message_id") for message in history}
- from core.services.commands.store import view
+ from core.sandbox.commands.store import view
events = [completion_context(view(row)) for row in rows
if "command_event_" + row.task_id not in existing]
return [*history, *events]
diff --git a/src/backend/core/services/commands/delivery.py b/src/backend/core/sandbox/commands/delivery.py
similarity index 92%
rename from src/backend/core/services/commands/delivery.py
rename to src/backend/core/sandbox/commands/delivery.py
index 1acdf1ca..bb969009 100644
--- a/src/backend/core/services/commands/delivery.py
+++ b/src/backend/core/sandbox/commands/delivery.py
@@ -4,8 +4,8 @@
from sqlalchemy import update
from core.db.models import ChatRun, ChatSession
from core.db.models.command import CommandTask
-from core.services.commands import store
-from core.services.chat_sequencer import ChatSequencer
+from core.sandbox.commands import store
+from core.chat.sequencer import ChatSequencer
def notice_text(rows):
facts = ["[Background command completion]",
@@ -23,7 +23,7 @@ def acknowledge(task_ids, run_id):
def reserve_boundary(chat_id, user_id, run_id):
"""Persist facts and their receipt together before injecting live context."""
- from core.services.commands.context import completion_context
+ from core.sandbox.commands.context import completion_context
with store.SessionLocal() as db:
db.execute(update(ChatSession).where(ChatSession.chat_id == chat_id)
.values(updated_at=ChatSession.updated_at))
@@ -63,7 +63,8 @@ def attach(db, run):
return attach
def prepare(rows, owner):
- from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY, make_text_context_item, session_context_metadata
+ from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY
+ from core.llm.context.items import make_text_context_item, session_context_metadata
first = rows[0]
with store.SessionLocal() as db:
chat = db.get(ChatSession, first["chat_id"])
@@ -100,8 +101,8 @@ def claim_retry(task_id, owner):
return row.notice_run_id
def load_launch(run_id):
- from core.services.chat_service import ChatService
- from core.services.session_operations.session_context import _load_session_messages
+ from core.chat.service import ChatService
+ from core.chat.operations.session_context import load_session_messages as _load_session_messages
with store.SessionLocal() as db:
rows = db.query(CommandTask).filter_by(notice_run_id=run_id, notice_status="reserved").all()
if not rows:
diff --git a/src/backend/orchestration/commands/events.py b/src/backend/core/sandbox/commands/events.py
similarity index 93%
rename from src/backend/orchestration/commands/events.py
rename to src/backend/core/sandbox/commands/events.py
index cd9e23b2..6f4089a2 100644
--- a/src/backend/orchestration/commands/events.py
+++ b/src/backend/core/sandbox/commands/events.py
@@ -1,9 +1,11 @@
"""Bounded command wake events, separate from per-run output projections."""
+
import asyncio
import hashlib
import threading
import time
from collections import deque
+
from core.infra.redis import get_redis, redis_configured
TTL_SECONDS = 3600
@@ -82,8 +84,11 @@ async def wait(self, user_id, chat_id, cursor, timeout=15):
with self.lock:
log = self._log(key)
batch = [item for item in log if item[0] > cursor]
- return (batch[-1][0] if batch else cursor,
- [run for _, run in batch], len(log) == MAX_EVENTS)
+ return (
+ batch[-1][0] if batch else cursor,
+ [run for _, run in batch],
+ len(log) == MAX_EVENTS,
+ )
def _wake(future):
@@ -105,8 +110,9 @@ async def publish(self, user_id, chat_id, run_id):
async def wait(self, user_id, chat_id, cursor, timeout=15):
key = topic(user_id, chat_id)
- rows = await get_redis(blocking=True).xread({key: cursor},
- count=MAX_EVENTS, block=max(1, int(timeout * 1000)))
+ rows = await get_redis(blocking=True).xread(
+ {key: cursor}, count=MAX_EVENTS, block=max(1, int(timeout * 1000))
+ )
if not rows:
return cursor, [], False
batch = rows[0][1]
@@ -123,6 +129,7 @@ def get_command_events():
if redis_configured():
return _redis
from core.infra.worker_count import resolve
+
if resolve() != 1:
raise RuntimeError("In-memory command events require a single worker")
return _memory
diff --git a/src/backend/orchestration/commands/notifications.py b/src/backend/core/sandbox/commands/notifications.py
similarity index 77%
rename from src/backend/orchestration/commands/notifications.py
rename to src/backend/core/sandbox/commands/notifications.py
index b15d7c22..1f70c7b9 100644
--- a/src/backend/orchestration/commands/notifications.py
+++ b/src/backend/core/sandbox/commands/notifications.py
@@ -1,27 +1,39 @@
"""Deliver command completion without blocking the event loop on database I/O."""
+
import asyncio
import logging
import uuid
-from core.services.commands import delivery, store
-from core.services.chat_sequencer import ChatBusyError
+
+import core.chat.runtime.admission as admission
+from core.chat.sequencer import ChatBusyError
+from core.sandbox.commands import delivery, store
logger = logging.getLogger(__name__)
+
async def launch(run_id):
- from orchestration import chat_run_executor
prepared = await asyncio.to_thread(delivery.load_launch, run_id)
if prepared is None:
return
rows, run, history, args = prepared
if args is not None:
prompt = delivery.notice_text(rows)
- await chat_run_executor.start_run(accepted_run=run, chat_id=run.chat_id,
- user_id=run.user_id, session_messages=history, effective_user_message=prompt,
- raw_user_message=prompt, context=args["context"], model_name=args.get("model_name"))
- from orchestration.commands.events import get_command_events
+ await admission.start_run(
+ accepted_run=run,
+ chat_id=run.chat_id,
+ user_id=run.user_id,
+ session_messages=history,
+ effective_user_message=prompt,
+ raw_user_message=prompt,
+ context=args["context"],
+ model_name=args.get("model_name"),
+ )
+ from core.sandbox.commands.events import get_command_events
+
await get_command_events().publish(run.user_id, run.chat_id, run_id)
await asyncio.to_thread(delivery.acknowledge, [r["task_id"] for r in rows], run_id)
+
async def dispatch():
owner = uuid.uuid4().hex
after = None
@@ -32,6 +44,7 @@ async def dispatch():
await dispatch_batch(rows, owner)
after = rows[-1]["task_id"]
+
async def dispatch_batch(batch, owner):
groups = {}
for row in batch:
diff --git a/src/backend/core/sandbox/commands/runtime.py b/src/backend/core/sandbox/commands/runtime.py
new file mode 100644
index 00000000..c9e1c2a6
--- /dev/null
+++ b/src/backend/core/sandbox/commands/runtime.py
@@ -0,0 +1,449 @@
+"""One output collector per command; no model-driven polling."""
+
+import asyncio
+import logging
+import time
+import uuid
+from functools import wraps
+
+from core.sandbox.protocol import ProcessRequest
+from core.sandbox.providers.opensandbox.recovery import (
+ execution_reference,
+ recover_execution,
+ release_execution,
+)
+from core.sandbox.commands import store
+
+logger = logging.getLogger(__name__)
+LAUNCH_RENEW_SECONDS = 10
+
+
+def service_loop(method):
+ """Keep command collectors on their service loop across agent worker threads."""
+
+ @wraps(method)
+ async def invoke(self, *args, **kwargs):
+ current = asyncio.get_running_loop()
+ if self.loop is None:
+ self.loop = current
+ if self.loop is current:
+ return await method(self, *args, **kwargs)
+ if not self.loop.is_running():
+ raise RuntimeError("Command service loop is not running")
+ future = asyncio.run_coroutine_threadsafe(method(self, *args, **kwargs), self.loop)
+ return await asyncio.wrap_future(future)
+
+ return invoke
+
+
+class CommandRuntime:
+ def __init__(self):
+ self.loop = None
+ self.owner = uuid.uuid4().hex
+ self.tasks = {}
+ self.providers = {}
+ self._locks = {}
+ self._heartbeat = None
+
+ def _ensure_heartbeat(self):
+ if self._heartbeat is None or self._heartbeat.done():
+ self._heartbeat = asyncio.create_task(self._renew_handles())
+
+ async def _renew_handles(self):
+ while self.providers:
+ try:
+ await asyncio.wait_for(
+ asyncio.to_thread(store.renew_owned, list(self.providers), self.owner),
+ timeout=store.LEASE_SECONDS / 3,
+ )
+ except Exception:
+ logger.exception("Command handle renewal failed")
+ await asyncio.sleep(store.LEASE_SECONDS / 3)
+
+ def bind_loop(self):
+ """Initialize every worker before a temporary agent thread can call it."""
+ current = asyncio.get_running_loop()
+ if self.loop is not None and self.loop is not current:
+ raise RuntimeError("Command runtime already belongs to another event loop")
+ self.loop = current
+
+ @service_loop
+ async def start(
+ self, provider, req, *, chat_id, source_run_id, background=False, yield_time_ms=60000
+ ):
+ row = await asyncio.to_thread(store.create, req, chat_id, source_run_id, self.owner)
+ tid = row["task_id"]
+ result = None
+ try:
+ result = await self._launch(
+ tid, provider, req, yield_time_ms=250 if background else yield_time_ms
+ )
+ return await self._attach(tid, provider, req, result)
+ except BaseException as exc:
+ if result and result.get("status") == "running" and tid not in self.providers:
+ # Preserve control even if persistence or the first stop RPC fails.
+ self.providers[tid] = provider, req, result["session_id"]
+ self._ensure_heartbeat()
+ await self._interrupt(tid)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ try:
+ await asyncio.to_thread(
+ store.finish, tid, "unknown", "Launch state unknown; do not rerun: " + str(exc)
+ )
+ except LookupError:
+ pass # A hard-deleted chat already removed the task.
+ raise
+
+ async def _launch(self, tid, provider, req, *, yield_time_ms):
+ async def renew():
+ while True:
+ await asyncio.sleep(LAUNCH_RENEW_SECONDS)
+ await asyncio.to_thread(store.renew_launch, tid, self.owner)
+
+ launch = asyncio.create_task(provider.start_process(req, yield_time_ms=yield_time_ms))
+ lease = asyncio.create_task(renew())
+ try:
+ done, _ = await asyncio.wait((launch, lease), return_when=asyncio.FIRST_COMPLETED)
+ if launch in done:
+ return await launch
+ await lease # Lease loss cancels the pending launch; never restart it.
+ raise RuntimeError("Command launch lease ended unexpectedly")
+ finally:
+ for task in (launch, lease):
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(launch, lease, return_exceptions=True)
+
+ async def _attach(self, tid, provider, req, result):
+ reference = None
+ if result.get("status") == "running":
+ reference = await execution_reference(provider, result["session_id"], req)
+ if reference is not None:
+ reference["cursor"] = result["execution_cursor"]
+ row = await asyncio.to_thread(store.attach, tid, self.owner, result, reference)
+ payload = await asyncio.to_thread(
+ store.read, tid, req.user_id, req.session_id, consume_notice=False
+ )
+ if row["status"] == "running":
+ self.providers[tid] = provider, req, result["session_id"]
+ self._ensure_heartbeat()
+ self.tasks[tid] = asyncio.create_task(self._monitor(tid, row))
+ payload["next"] = (
+ "Command is managed in the background. Completion will notify this chat. Do independent work; do not poll or restart it. If no independent work remains, end this response once; completion will resume this chat."
+ )
+ return payload
+
+ @service_loop
+ async def adopt(self, provider, process_id, *, user_id, session_id, chat_id, source_run_id):
+ if process_id.startswith("cmd_"):
+ await asyncio.to_thread(store.owned, process_id, user_id, session_id)
+ return await asyncio.to_thread(
+ store.read, process_id, user_id, session_id, consume_notice=False
+ )
+ req = ProcessRequest(
+ script_content="[adopted command]",
+ script_name="_managed.sh",
+ language="bash",
+ session_id=session_id,
+ user_id=user_id,
+ )
+ row = await asyncio.to_thread(
+ store.create, req, chat_id, source_run_id, self.owner, process_id
+ )
+ if not row["created"]:
+ return await asyncio.to_thread(
+ store.read, row["task_id"], user_id, session_id, consume_notice=False
+ )
+ try:
+ result = await provider.write_stdin(
+ process_id,
+ sandbox_session_id=session_id,
+ user_id=user_id,
+ chars="",
+ yield_time_ms=0,
+ )
+ return await self._attach(row["task_id"], provider, req, result)
+ except BaseException:
+ await asyncio.to_thread(
+ store.finish, row["task_id"], "unknown", "Process adoption failed"
+ )
+ raise
+
+ async def _monitor(self, tid, row):
+ provider, req, pid = self.providers[tid]
+ last_touch = 0
+ failures = 0
+ try:
+ while True:
+ try:
+ if not await asyncio.to_thread(store.chat_available, tid):
+ result = await self._interrupt(tid)
+ try:
+ await asyncio.to_thread(
+ store.control_result,
+ tid,
+ self.owner,
+ "cancelled",
+ "Conversation removed",
+ True,
+ result,
+ )
+ except LookupError:
+ pass # Physical deletion removed the row; the owned process was stopped.
+ break
+ row = await asyncio.to_thread(store.snapshot, tid)
+ if row["status"] not in store.LIVE or row["owner"] != self.owner:
+ if row["execution_ref"]:
+ await release_execution(provider, pid, req)
+ else:
+ await self._interrupt(tid)
+ break
+ if row["cancel_requested"]:
+ result = await self._interrupt(tid)
+ await asyncio.to_thread(
+ store.control_result, tid, self.owner, "cancelled", "", True, result
+ )
+ break
+ if row["deadline_at"] and time.time() >= row["deadline_at"]:
+ result = await self._interrupt(tid)
+ await asyncio.to_thread(
+ store.control_result,
+ tid,
+ self.owner,
+ "failed",
+ "Command execution deadline exceeded",
+ False,
+ result,
+ )
+ break
+ async with self._locks.setdefault(tid, asyncio.Lock()):
+ result = await provider.write_stdin(
+ pid,
+ sandbox_session_id=req.session_id,
+ user_id=req.user_id,
+ chars="",
+ yield_time_ms=250,
+ )
+ row = await asyncio.to_thread(store.collect, tid, self.owner, result)
+ if row["status"] not in store.LIVE:
+ break
+ if time.monotonic() - last_touch >= 10:
+ await provider.touch_session(req.session_id)
+ last_touch = time.monotonic()
+ failures = 0
+ await asyncio.sleep(0.5)
+ except Exception:
+ failures += 1
+ logger.warning("Command observation failed task=%s attempt=%s",
+ tid, failures, exc_info=True)
+ if failures >= 3:
+ if row["execution_ref"]:
+ # Durable recovery can retry without rerunning the command.
+ await release_execution(provider, pid, req)
+ await asyncio.to_thread(store.release_lease, tid, self.owner)
+ break
+ # A nonrecoverable local process must be stopped, never
+ # reported terminal merely because observation failed.
+ result = await self._interrupt(tid)
+ await asyncio.to_thread(
+ store.control_result, tid, self.owner, "failed",
+ "Command observation failed", False, result,
+ )
+ break
+ await asyncio.sleep(2 ** (failures - 1))
+ except asyncio.CancelledError:
+ if row["execution_ref"]:
+ await release_execution(provider, pid, req)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ await asyncio.to_thread(store.release_lease, tid, self.owner)
+ else:
+ result = await self._interrupt(tid)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ await asyncio.to_thread(
+ store.control_result,
+ tid,
+ self.owner,
+ "cancelled",
+ "Command owner stopped",
+ True,
+ result,
+ )
+ raise
+ else:
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ finally:
+ self.tasks.pop(tid, None)
+
+ async def _interrupt(self, tid):
+ provider, req, pid = self.providers[tid]
+ async with self._locks.setdefault(tid, asyncio.Lock()):
+ result = await provider.write_stdin(
+ pid,
+ sandbox_session_id=req.session_id,
+ user_id=req.user_id,
+ chars="\x03",
+ yield_time_ms=1000,
+ )
+ while result.get("status") == "running":
+ try:
+ await asyncio.to_thread(store.collect, tid, self.owner, result)
+ except Exception:
+ logger.warning("Command stop bookkeeping deferred task=%s", tid)
+ next_result = await provider.write_stdin(
+ pid,
+ sandbox_session_id=req.session_id,
+ user_id=req.user_id,
+ chars="",
+ yield_time_ms=1000,
+ )
+ result = next_result
+ if result.get("status") != "exited":
+ raise RuntimeError("Process stop not confirmed")
+ return result
+
+ async def cancel(self, tid, user_id, session_id):
+ await asyncio.to_thread(store.owned, tid, user_id, session_id)
+ row = await asyncio.to_thread(store.snapshot, tid)
+ if row["status"] in store.TERMINAL:
+ return
+ await asyncio.to_thread(store.request_cancel, tid)
+ await self.wait(tid, user_id, session_id, 3000)
+
+ async def wait(self, tid, user_id, session_id, wait_ms=0):
+ await asyncio.to_thread(store.owned, tid, user_id, session_id)
+ deadline = time.monotonic() + max(0, min(wait_ms, 300000)) / 1000
+ while time.monotonic() < deadline:
+ if (await asyncio.to_thread(store.snapshot, tid))["status"] in store.TERMINAL:
+ break
+ await asyncio.sleep(0.05)
+
+ @service_loop
+ async def recover(self, provider):
+ # Failed local stops remain owned and are retried by this worker's
+ # existing recovery supervisor before any no-reference classification.
+ for tid in list(self.providers):
+ if tid not in self.tasks:
+ self.tasks[tid] = asyncio.create_task(self._stop_retained(tid))
+ for row in await asyncio.to_thread(store.pending_recovery):
+ tid = row["task_id"]
+ if tid in self.tasks and not self.tasks[tid].done():
+ continue
+ if not await asyncio.to_thread(store.claim, tid, self.owner):
+ continue
+ if not row["execution_ref"]:
+ await asyncio.to_thread(
+ store.finish,
+ tid,
+ "unknown",
+ "No recoverable execution reference; original command will not be rerun",
+ owner=self.owner,
+ )
+ continue
+ req = ProcessRequest(
+ script_content="",
+ script_name="_managed.sh",
+ language="bash",
+ session_id=row["sandbox_session_id"],
+ user_id=row["user_id"],
+ )
+ result = None
+ try:
+ result = await recover_execution(provider, row["execution_ref"], req)
+ ref = {**row["execution_ref"], "cursor": result["execution_cursor"]}
+ recovered = await asyncio.to_thread(
+ store.save_recovery, tid, self.owner, result, ref
+ )
+ if recovered["status"] == "running":
+ self.providers[tid] = provider, req, result["session_id"]
+ self._ensure_heartbeat()
+ self.tasks[tid] = asyncio.create_task(self._monitor(tid, recovered))
+ except BaseException as exc:
+ if result and result.get("status") == "running":
+ await release_execution(provider, result["session_id"], req)
+ await asyncio.to_thread(store.release_lease, tid, self.owner)
+ if isinstance(exc, asyncio.CancelledError):
+ raise
+ logger.warning("Command recovery deferred task=%s", tid, exc_info=True)
+
+ async def _stop_retained(self, tid):
+ try:
+ try:
+ row = await asyncio.to_thread(store.snapshot, tid)
+ except LookupError:
+ row = None
+ if row and row["execution_ref"]:
+ provider, req, pid = self.providers[tid]
+ await release_execution(provider, pid, req)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ await asyncio.to_thread(store.release_lease, tid, self.owner)
+ return
+ result = await self._interrupt(tid)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ try:
+ await asyncio.to_thread(
+ store.control_result, tid, self.owner, "failed",
+ "Command observation failed", False, result,
+ )
+ except LookupError:
+ pass
+ except Exception:
+ logger.exception("Command stop requires another recovery pass task=%s", tid)
+ finally:
+ self.tasks.pop(tid, None)
+
+ @service_loop
+ async def shutdown(self):
+ tasks = list(self.tasks.values())
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ # A task cancelled before its first instruction cannot run its finally.
+ for tid, (provider, req, pid) in list(self.providers.items()):
+ try:
+ await self._shutdown_handle(tid, provider, req, pid)
+ except Exception as exc:
+ results.append(exc)
+ for result in results:
+ if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError):
+ raise result
+ self.tasks.clear()
+ if self._heartbeat is not None:
+ self._heartbeat.cancel()
+ await asyncio.gather(self._heartbeat, return_exceptions=True)
+ self._heartbeat = None
+ self.loop = None
+
+ async def _shutdown_handle(self, tid, provider, req, pid):
+ try:
+ row = await asyncio.to_thread(store.snapshot, tid)
+ except LookupError:
+ await self._interrupt(tid)
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+ return
+ if row["execution_ref"]:
+ await release_execution(provider, pid, req)
+ await asyncio.to_thread(store.release_lease, tid, self.owner)
+ else:
+ result = await self._interrupt(tid)
+ await asyncio.to_thread(
+ store.control_result,
+ tid,
+ self.owner,
+ "cancelled",
+ "Command owner stopped",
+ True,
+ result,
+ )
+ self.providers.pop(tid, None)
+ self._locks.pop(tid, None)
+
+
+runtime = CommandRuntime()
diff --git a/src/backend/core/services/commands/store.py b/src/backend/core/sandbox/commands/store.py
similarity index 95%
rename from src/backend/core/services/commands/store.py
rename to src/backend/core/sandbox/commands/store.py
index 869b890e..2ddf87a5 100644
--- a/src/backend/core/services/commands/store.py
+++ b/src/backend/core/sandbox/commands/store.py
@@ -18,7 +18,7 @@ def view(row):
return {c.name: getattr(row, c.name) for c in row.__table__.columns}
def can_edit(db, chat_id, user_id):
- from core.services.chat_service import ChatService
+ from core.chat.service import ChatService
pair = ChatService(db).get_session_with_access(chat_id, user_id)
return pair is not None and pair[1] in ("admin", "edit")
@@ -248,3 +248,14 @@ def renew_launch(task_id, owner):
if row.owner != owner or row.status != "starting":
raise RuntimeError("Command launch ownership lost")
row.lease_until = time.time() + LAUNCH_LEASE_SECONDS
+
+
+def renew_owned(task_ids, owner):
+ """Keep live local handles fenced even while output/stop RPCs are blocked."""
+ now = time.time()
+ with SessionLocal() as db:
+ db.execute(update(CommandTask).where(
+ CommandTask.task_id.in_(task_ids), CommandTask.owner == owner,
+ CommandTask.status.in_(LIVE), CommandTask.lease_until > now,
+ ).values(lease_until=now + LEASE_SECONDS))
+ db.commit()
diff --git a/src/backend/orchestration/commands/subscription.py b/src/backend/core/sandbox/commands/subscription.py
similarity index 66%
rename from src/backend/orchestration/commands/subscription.py
rename to src/backend/core/sandbox/commands/subscription.py
index 04ba3f6d..e7fd3725 100644
--- a/src/backend/orchestration/commands/subscription.py
+++ b/src/backend/core/sandbox/commands/subscription.py
@@ -1,12 +1,14 @@
"""Authenticated snapshot-and-tail subscription for ordinary-chat wake events."""
+
import asyncio
import json
-from fastapi import HTTPException
+
from core.auth.backend import get_current_user, security
-from core.services.commands import store
from core.db.models import ChatRun
from core.db.models.command import CommandTask
-from orchestration.commands.events import MAX_EVENTS, get_command_events
+from core.sandbox.commands.events import MAX_EVENTS, get_command_events
+from core.sandbox.commands import store
+from fastapi import HTTPException
HEARTBEAT_SECONDS = 15
@@ -18,9 +20,7 @@ def require_edit(chat_id, user_id):
async def authorize(request, chat_id, expected_user=None):
- with store.SessionLocal() as db:
- credentials = await security(request)
- user = await get_current_user(request, credentials=credentials, db=db)
+ user = await get_current_user(request, await security(request))
if user.api_key_id or user.public_chat_visitor:
raise HTTPException(403, "Command events require an account session")
if expected_user is not None and user.user_id != expected_user:
@@ -31,18 +31,31 @@ async def authorize(request, chat_id, expected_user=None):
def snapshot(chat_id, user_id):
with store.SessionLocal() as db:
- rows = (db.query(ChatRun).join(CommandTask, CommandTask.notice_run_id == ChatRun.run_id)
- .filter(CommandTask.chat_id == chat_id, CommandTask.user_id == user_id,
- CommandTask.delivery_mode == "followup",
- CommandTask.notice_status.in_(("reserved", "delivered")),
- ChatRun.chat_id == chat_id, ChatRun.user_id == user_id)
- .distinct().order_by(ChatRun.created_at.desc(), ChatRun.run_id.desc())
- .limit(MAX_EVENTS).all())
+ rows = (
+ db.query(ChatRun)
+ .join(CommandTask, CommandTask.notice_run_id == ChatRun.run_id)
+ .filter(
+ CommandTask.chat_id == chat_id,
+ CommandTask.user_id == user_id,
+ CommandTask.delivery_mode == "followup",
+ CommandTask.notice_status.in_(("reserved", "delivered")),
+ ChatRun.chat_id == chat_id,
+ ChatRun.user_id == user_id,
+ )
+ .distinct()
+ .order_by(ChatRun.created_at.desc(), ChatRun.run_id.desc())
+ .limit(MAX_EVENTS)
+ .all()
+ )
return [run.run_id for run in rows if run.recovery_snapshot]
def frame(run_ids):
- return "data: " + json.dumps({"type": "command_runs", "run_ids": list(dict.fromkeys(run_ids))}) + "\n\n"
+ return (
+ "data: "
+ + json.dumps({"type": "command_runs", "run_ids": list(dict.fromkeys(run_ids))})
+ + "\n\n"
+ )
async def subscribe(request, chat_id, user_id):
@@ -55,7 +68,8 @@ async def subscribe(request, chat_id, user_id):
yield frame(runs)
while True:
cursor, runs, resync = await events.wait(
- user_id, chat_id, cursor, timeout=HEARTBEAT_SECONDS)
+ user_id, chat_id, cursor, timeout=HEARTBEAT_SECONDS
+ )
await authorize(request, chat_id, user_id)
if resync:
runs = await asyncio.to_thread(snapshot, chat_id, user_id)
diff --git a/src/backend/core/sandbox/credential_dispatch.py b/src/backend/core/sandbox/credential_dispatch.py
new file mode 100644
index 00000000..fcb13137
--- /dev/null
+++ b/src/backend/core/sandbox/credential_dispatch.py
@@ -0,0 +1,30 @@
+"""Provider-neutral credential dispatch for account-owned cloud commands."""
+from core.harness.registry import sandbox_credentials
+
+def credential_environment():
+ environment = {}
+ for provider in sandbox_credentials():
+ if provider.environment:
+ environment.update(provider.environment())
+ return environment
+
+
+def credential_mounts(user_id):
+ return [mount for provider in sandbox_credentials() if provider.mounts
+ for mount in provider.mounts(user_id)]
+
+
+async def prepare_credentials(host, sandbox, user_id, script):
+ if not user_id:
+ return
+ for provider in sorted(sandbox_credentials(), key=lambda item: item.prepare_order):
+ if provider.before_command:
+ await provider.before_command(host, sandbox, user_id, script)
+
+
+async def persist_credentials(host, sandbox, user_id, script):
+ if not user_id:
+ return
+ for provider in sandbox_credentials():
+ if provider.after_command:
+ await provider.after_command(host, sandbox, user_id, script)
diff --git a/src/backend/core/sandbox/factory.py b/src/backend/core/sandbox/factory.py
index 406d0b93..7a57ccfb 100644
--- a/src/backend/core/sandbox/factory.py
+++ b/src/backend/core/sandbox/factory.py
@@ -8,6 +8,7 @@
from __future__ import annotations
import logging
+from dataclasses import dataclass
from core.config.settings import settings
@@ -15,34 +16,31 @@
logger = logging.getLogger(__name__)
-_provider: SandboxProvider | None = None
-
def _ordinary_provider() -> SandboxProvider:
- global _provider
- if _provider is not None:
- return _provider
+ if process_state().provider is not None:
+ return process_state().provider
kind = settings.sandbox.provider
if kind == "script_runner":
from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider
- _provider = ScriptRunnerProvider()
+ process_state().provider = ScriptRunnerProvider()
elif kind == "opensandbox":
from core.sandbox.providers.edition import PersistentSandboxProvider
- _provider = PersistentSandboxProvider()
+ process_state().provider = PersistentSandboxProvider()
elif kind == "cube":
from core.sandbox.providers.cube.provider import CubeSandboxProvider
- _provider = CubeSandboxProvider()
+ process_state().provider = CubeSandboxProvider()
else:
raise ValueError(
f"未知 SANDBOX_PROVIDER: {kind!r},可选: script_runner / opensandbox / cube"
)
- logger.info("[sandbox] activated provider=%s", _provider.name)
- return _provider
+ logger.info("[sandbox] activated provider=%s", process_state().provider.name)
+ return process_state().provider
def get_sandbox_provider() -> SandboxProvider:
@@ -53,5 +51,16 @@ def get_sandbox_provider() -> SandboxProvider:
def reset_provider_cache() -> None:
"""For testing use only."""
- global _provider
- _provider = None
+ process_state().provider = None
+
+
+@dataclass
+class FactoryState:
+ provider: SandboxProvider | None = None
+
+
+_STATE = FactoryState()
+
+
+def process_state() -> FactoryState:
+ return _STATE
diff --git a/src/backend/core/sandbox/oslayer/windows_runtime.py b/src/backend/core/sandbox/oslayer/windows_runtime.py
index ef2062b2..2a0f8397 100644
--- a/src/backend/core/sandbox/oslayer/windows_runtime.py
+++ b/src/backend/core/sandbox/oslayer/windows_runtime.py
@@ -27,15 +27,12 @@
import os
import threading
from collections import Counter
+from dataclasses import dataclass, field
# Environment variables Windows programs read to find their temp directory.
TEMP_ENV_KEYS = ("TEMP", "TMP")
ROOTS_FILENAME = "sandbox_labeled_roots.json"
-_LOCK = threading.Lock()
-_IN_USE: Counter[str] = Counter()
-_TOKEN = None
-
def root_key(path: str) -> str:
return os.path.normcase(os.path.normpath(os.path.abspath(path)))
@@ -73,7 +70,7 @@ def apply_labels(plan: dict) -> None:
roots = {root_key(root["path"]): root for root in plan.get("writable_roots") or []}
store = LabeledRoots(plan["state_dir"])
labeled = store.load()
- in_use = {key for key, count in _IN_USE.items() if count > 0}
+ in_use = {key for key, count in process_state().IN_USE.items() if count > 0}
stale = labeled - set(roots) - in_use
for key in stale:
remove_label(key)
@@ -99,17 +96,16 @@ def child_environment(plan: dict, env: dict) -> dict:
def _sandbox_token():
"""The low-integrity token, built once: it takes nothing from the plan."""
- global _TOKEN
- if _TOKEN is None:
+ if process_state().TOKEN is None:
from .win32 import ffi
from .win32.token import create_low_integrity_token, open_current_process_token
base_token = open_current_process_token()
try:
- _TOKEN = create_low_integrity_token(base_token)
+ process_state().TOKEN = create_low_integrity_token(base_token)
finally:
ffi.CloseHandle(base_token)
- return _TOKEN
+ return process_state().TOKEN
def spawn_confined(
@@ -136,13 +132,13 @@ def spawn_confined(
keys = [root_key(root["path"]) for root in plan.get("writable_roots") or []]
def release() -> None:
- with _LOCK:
- _IN_USE.subtract(keys)
+ with process_state().LOCK:
+ process_state().IN_USE.subtract(keys)
- with _LOCK:
+ with process_state().LOCK:
apply_labels(plan)
token = _sandbox_token()
- _IN_USE.update(keys)
+ process_state().IN_USE.update(keys)
try:
return spawn_with_token(
token,
@@ -168,3 +164,17 @@ def release() -> None:
"root_key",
"spawn_confined",
]
+
+
+@dataclass
+class WindowsRuntimeState:
+ LOCK: object = field(default_factory=lambda: threading.Lock())
+ IN_USE: Counter[str] = field(default_factory=lambda: Counter())
+ TOKEN: object = None
+
+
+_STATE = WindowsRuntimeState()
+
+
+def process_state() -> WindowsRuntimeState:
+ return _STATE
diff --git a/src/backend/core/llm/tool_permissions/account.py b/src/backend/core/sandbox/permissions/account.py
similarity index 90%
rename from src/backend/core/llm/tool_permissions/account.py
rename to src/backend/core/sandbox/permissions/account.py
index e653eb32..941fd786 100644
--- a/src/backend/core/llm/tool_permissions/account.py
+++ b/src/backend/core/sandbox/permissions/account.py
@@ -2,8 +2,8 @@
from core.db.models import ChatSession, UserShadow
from core.db.reserved_chats import reserved_chat_owner
-from core.services.user_service import UserService
-from .preferences import normalize_approval_mode
+from core.auth.users import UserService
+from core.sandbox.permissions.approval import normalize_approval_mode
def read_account_mode(user_id: str, chat_id: str | None = None) -> str | None:
diff --git a/src/backend/core/llm/tool_permissions/preferences.py b/src/backend/core/sandbox/permissions/approval.py
similarity index 94%
rename from src/backend/core/llm/tool_permissions/preferences.py
rename to src/backend/core/sandbox/permissions/approval.py
index 1cfa8ce9..2737d0e8 100644
--- a/src/backend/core/llm/tool_permissions/preferences.py
+++ b/src/backend/core/sandbox/permissions/approval.py
@@ -38,7 +38,7 @@ def resolve_approval_mode(explicit: Any, *, user_id: Optional[str]) -> str:
return APPROVAL_ASK
try:
from core.db.engine import SessionLocal
- from core.services.user_service import UserService
+ from core.auth.users import UserService
with SessionLocal() as db:
stored = UserService(db).get_user_settings(str(user_id)).get("tool_approval_mode")
@@ -72,8 +72,8 @@ async def refresh_runtime(runtime):
"""Resolve one immutable dispatch policy; a failed read is an error, never a prompt."""
if not runtime.account_approval or not runtime.user_id or runtime.default_allow:
return runtime
- from .account import read_account_mode
- from .models import PermissionEnforcementError
+ from core.sandbox.permissions.account import read_account_mode
+ from core.sandbox.permissions.dispatch_models import PermissionEnforcementError
try:
mode = await asyncio.to_thread(read_account_mode, runtime.user_id, runtime.chat_id)
diff --git a/src/backend/core/llm/tool_permissions/models.py b/src/backend/core/sandbox/permissions/dispatch_models.py
similarity index 98%
rename from src/backend/core/llm/tool_permissions/models.py
rename to src/backend/core/sandbox/permissions/dispatch_models.py
index 1d8d8c9a..70bc82ee 100644
--- a/src/backend/core/llm/tool_permissions/models.py
+++ b/src/backend/core/sandbox/permissions/dispatch_models.py
@@ -7,12 +7,12 @@
from contextvars import ContextVar
from dataclasses import dataclass, field, replace
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Sequence
-from .preferences import APPROVAL_ASK, APPROVAL_FULL
+from core.sandbox.permissions.approval import APPROVAL_ASK, APPROVAL_FULL
if TYPE_CHECKING:
from core.sandbox.permissions.os_gate import LocalAccessDecision
from core.sandbox.oslayer import SandboxLaunch
- from core.services.project_scope import ProjectScope
+ from core.projects.edition import ProjectScope
DOMAIN_LOCAL_PATH = "local_path"
DOMAIN_LOCAL_COMMAND = "local_command"
diff --git a/src/backend/core/sandbox/permissions/os_gate.py b/src/backend/core/sandbox/permissions/os_gate.py
index d68e7e51..34065398 100644
--- a/src/backend/core/sandbox/permissions/os_gate.py
+++ b/src/backend/core/sandbox/permissions/os_gate.py
@@ -109,7 +109,7 @@ def protected_read_paths() -> tuple[str, ...]:
resolves the *narrowest* matching entry, and the writable workspace root is
deeper than the data root denied here.
"""
- from core.capabilities.paths import CAPS_ROOT_ENV
+ from core.config.runtime_env import CAPS_ROOT_ENV
candidates = [local_state_dir()]
for name in (CAPS_ROOT_ENV, _INSTALL_ROOT_ENV):
diff --git a/src/backend/core/llm/tool_permissions/specs.py b/src/backend/core/sandbox/permissions/tool_specs.py
similarity index 95%
rename from src/backend/core/llm/tool_permissions/specs.py
rename to src/backend/core/sandbox/permissions/tool_specs.py
index 0ed3f93e..1796292a 100644
--- a/src/backend/core/llm/tool_permissions/specs.py
+++ b/src/backend/core/sandbox/permissions/tool_specs.py
@@ -4,19 +4,7 @@
import os
import logging
from typing import Sequence, Any, Mapping, Optional
-from .models import (
- PermissionRuntime,
- PermissionIntent,
- ToolPermissionSpec,
- DOMAIN_DENY,
- DOMAIN_MYSPACE,
- DOMAIN_LOCAL_PATH,
- DOMAIN_LOCAL_COMMAND,
- READ,
- WRITE,
- EXECUTE,
- _canonical_path,
-)
+from core.sandbox.permissions.dispatch_models import PermissionRuntime, PermissionIntent, ToolPermissionSpec, DOMAIN_DENY, DOMAIN_MYSPACE, DOMAIN_LOCAL_PATH, DOMAIN_LOCAL_COMMAND, READ, WRITE, EXECUTE, _canonical_path
logger = logging.getLogger(__name__)
@@ -88,7 +76,7 @@ def _is_myspace_target(logical: str, physical: str, user_id: Optional[str]) -> b
"""
if logical == "/myspace" or logical.startswith("/myspace/"):
return True
- from core.llm.tools._paths import is_myspace_physical
+ from core.sandbox.workspace.resolution import is_myspace_physical
return bool(is_myspace_physical(physical, user_id))
@@ -137,7 +125,7 @@ def resolve(args: Mapping[str, Any], runtime: PermissionRuntime):
return ()
intents: list[PermissionIntent] = []
first: Optional[tuple[str, str]] = None
- from core.llm.tools._paths import to_physical_path
+ from core.sandbox.workspace.resolution import to_physical_path
for index, (arg, mode) in enumerate(path_specs):
logical = str(args.get(arg) or "").strip()
@@ -196,7 +184,7 @@ def resolve(args: Mapping[str, Any], _runtime: PermissionRuntime):
def builtin_tool_permission(tool_name: str) -> Optional[ToolPermissionSpec]:
"""Return the rule for a governed native tool, otherwise pass through."""
- from core.llm.tools._tool_confirmation import OP_EDIT, OP_WRITE
+ from core.sandbox.permissions.vocabulary import OP_EDIT, OP_WRITE
governed: dict[str, ToolPermissionSpec] = {
"Read": local_path_tool("file_path", READ, tool_name="Read"),
diff --git a/src/backend/core/sandbox/permissions/vocabulary.py b/src/backend/core/sandbox/permissions/vocabulary.py
new file mode 100644
index 00000000..fd4c534f
--- /dev/null
+++ b/src/backend/core/sandbox/permissions/vocabulary.py
@@ -0,0 +1,26 @@
+"""Stable operation and decision vocabulary for execution approvals."""
+
+KIND_MYSPACE = "myspace"
+KIND_AUTOMATION = "automation"
+KIND_DESIGN_PICK = "design_pick"
+KIND_LOCAL_CMD = "local_cmd"
+KIND_LOCAL_PATH_PREFIX = "local_path:"
+KIND_TOOL_PERMISSION = "tool_permission"
+KIND_TOOL_PREFIX = "tool:"
+OP_WRITE = "write"
+OP_EDIT = "edit"
+OP_DELETE = "delete"
+OP_LOCAL_EXEC = "local_exec"
+OP_LOCAL_READ = "local_read"
+OP_LOCAL_WRITE = "local_write"
+OP_MOVE = "move"
+OP_MKDIR = "mkdir"
+OP_CRON_CREATE = "cron_create"
+OP_CRON_UPDATE = "cron_update"
+OP_CRON_DELETE = "cron_delete"
+DECISION_ALLOW = "allow"
+DECISION_ALLOW_SESSION = "allow_session"
+DECISION_DENY = "deny"
+DECISION_CHOICE = "choice"
+DECISION_SKIP = "skip"
+STATUS_BLOCKED = "blocked_non_interactive"
diff --git a/src/backend/core/sandbox/processes/cloud.py b/src/backend/core/sandbox/processes/cloud.py
index 10df65a9..24a3692a 100644
--- a/src/backend/core/sandbox/processes/cloud.py
+++ b/src/backend/core/sandbox/processes/cloud.py
@@ -17,9 +17,13 @@
from core.sandbox.processes.sessions import ProcessSessionError, ProcessSessions
+# No process outlives its command, officecli's resident included: write each edit to disk at once.
+_COMMAND_ENVS = {"OFFICECLI_RESIDENT_FLUSH": "each"}
+
+
def _module_owned(session_id) -> bool:
"""A module's dedicated sandbox never receives the platform's extra credentials."""
- from core.extensions import sandbox_route
+ from core.harness.registry import sandbox_route
return sandbox_route(str(session_id or "")) is not None
@@ -85,7 +89,8 @@ async def close_all(self):
async def _spawn(self, req):
from core.sandbox.workspace.identity import SANDBOX_RUN_GID, SANDBOX_RUN_UID
from core.sandbox.workspace.paths import WORKSPACE
- from core.sandbox.providers.account_env import dws_extra_envs, _firecrawl_extra_envs
+ from core.sandbox.providers.account_env import _firecrawl_extra_envs
+ from core.sandbox.credential_dispatch import credential_environment
provider = self.provider
interpreters = {"bash": "bash", "python": "python3 -u", "javascript": "node"}
@@ -138,9 +143,13 @@ async def _spawn(self, req):
working_directory=WORKSPACE,
uid=SANDBOX_RUN_UID,
gid=SANDBOX_RUN_GID,
- envs=({**_EXECD_BASH_ENVS, "EXECD_ACCESS_TOKEN": ""} if _module_owned(req.session_id) else {**_EXECD_BASH_ENVS, **dws_extra_envs(), **_firecrawl_extra_envs()}),
+ envs=({**_EXECD_BASH_ENVS, **_COMMAND_ENVS, "EXECD_ACCESS_TOKEN": ""} if _module_owned(req.session_id) else {**_EXECD_BASH_ENVS, **_COMMAND_ENVS, **credential_environment(), **_firecrawl_extra_envs()}),
+ )
+ # As a subreaper the command keeps everything it starts, however detached,
+ # until it ends; whatever is left then reaches PID 1 and is killed there.
+ execution = await sess.sandbox.commands.run(
+ "sandbox-init --subreaper " + command, opts=opts
)
- execution = await sess.sandbox.commands.run(command, opts=opts)
if not execution.id:
raise SandboxError("Background command returned no execution ID")
return OpenSandboxHandle(provider, req, sess.sandbox.commands, execution.id)
@@ -156,9 +165,9 @@ async def _spawn(self, req):
cwd=WORKSPACE,
user="root",
envs=(
- {"EXECD_ACCESS_TOKEN": ""}
+ {**_COMMAND_ENVS, "EXECD_ACCESS_TOKEN": ""}
if _module_owned(req.session_id)
- else {**dws_extra_envs(), **_firecrawl_extra_envs()} or None
+ else {**_COMMAND_ENVS, **credential_environment(), **_firecrawl_extra_envs()}
),
stdin=False,
timeout=0,
@@ -276,8 +285,11 @@ async def poll(self):
raise
code = exc.exit_code
self.finished = True
- if self.req.user_id and "openyida" in self.req.script_content:
- await self.provider._persist_yida_state(self.sandbox, self.req.user_id)
+ from core.sandbox.credential_dispatch import persist_credentials
+
+ await persist_credentials(
+ self.provider, self.sandbox, self.req.user_id, self.req.script_content
+ )
return out, err, code
async def interrupt(self):
diff --git a/src/backend/core/sandbox/processes/completion.py b/src/backend/core/sandbox/processes/completion.py
index 71c1f8b1..b7e926e3 100644
--- a/src/backend/core/sandbox/processes/completion.py
+++ b/src/backend/core/sandbox/processes/completion.py
@@ -36,8 +36,9 @@ async def stop_after_start():
chars="\x03",
yield_time_ms=1000,
)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.processes.completion:stop_after_start', fallback_error)
cleanup = asyncio.create_task(stop_after_start())
await asyncio.shield(cleanup)
@@ -88,6 +89,7 @@ async def stop_after_start():
yield_time_ms=1000,
)
)
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.processes.completion:run_to_completion', fallback_error)
raise
diff --git a/src/backend/core/sandbox/processes/dispatch.py b/src/backend/core/sandbox/processes/dispatch.py
index d8862f1e..06a780e4 100644
--- a/src/backend/core/sandbox/processes/dispatch.py
+++ b/src/backend/core/sandbox/processes/dispatch.py
@@ -12,7 +12,7 @@ def _on_service_loop(method: Callable[..., Any]) -> Callable[..., Any]:
"""把碰 SDK / 池 / httpx 的入口钉在 provider 的服务事件循环上执行。
provider 是进程级单例,但调用方不止一个事件循环:子智能体跑在
- ``_subagent_pool`` 线程里的独立循环(``core/llm/subagent_tool.py`` 的
+ ``_subagent_pool`` 线程里的独立循环(``core/llm/subagents/tool.py`` 的
``asyncio.new_event_loop()``),且**平台内置子智能体复用主对话的
session_id**(explorer/worker/reviewer 要看同一份 workspace),于是拿到的是
主循环创建的那个 ``_Session``。
diff --git a/src/backend/core/sandbox/processes/interactive.py b/src/backend/core/sandbox/processes/interactive.py
index 1f171f21..ccbd5848 100644
--- a/src/backend/core/sandbox/processes/interactive.py
+++ b/src/backend/core/sandbox/processes/interactive.py
@@ -7,7 +7,6 @@
import logging
import secrets
import zipfile
-from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
import httpx
@@ -108,7 +107,7 @@ async def launch(provider, installation, module, user_id, chat_id, config):
raise ValueError("runtime_package_too_large")
archive_name = ".__interactive_" + secrets.token_hex(12) + ".zip"
body = {**config, "archive": archive_name, "callable": runtime["callable"]}
- from core.plugins.resources.confinement import launch_policy
+ from core.capabilities.plugins.resources.confinement import launch_policy
confinement = launch_policy(provider, user_id, chat_id)
start = asyncio.create_task(
diff --git a/src/backend/core/sandbox/providers/account_env.py b/src/backend/core/sandbox/providers/account_env.py
index 050a110e..d4400d6f 100644
--- a/src/backend/core/sandbox/providers/account_env.py
+++ b/src/backend/core/sandbox/providers/account_env.py
@@ -7,31 +7,6 @@
logger = logging.getLogger(__name__)
-def dws_extra_envs() -> dict[str, str]:
- """Return deployment-level dws Custom App environment variables.
-
- This helper lives in the provider-neutral sandbox module because both the
- backend OAuth subprocess and optional sandbox providers need it. Keeping
- it here also lets DingTalk configuration load without importing any concrete
- sandbox provider.
- """
- out: dict[str, str] = {}
- try:
- from core.services.system_config import SystemConfigService
-
- svc = SystemConfigService.get_instance()
- client_id = (svc.get("dingtalk.client_id") or "").strip()
- client_secret = (svc.get("dingtalk.client_secret") or "").strip()
- trusted_domains = (svc.get("dingtalk.trusted_domains") or "").strip()
- except Exception as exc: # noqa: BLE001
- logger.warning("[dws] read dingtalk config failed: %s", exc)
- return out
- if client_id and client_secret:
- out["DWS_CLIENT_ID"] = client_id
- out["DWS_CLIENT_SECRET"] = client_secret
- if trusted_domains:
- out["DWS_TRUSTED_DOMAINS"] = trusted_domains
- return out
def _firecrawl_extra_envs() -> dict[str, str]:
@@ -51,7 +26,7 @@ def _firecrawl_extra_envs() -> dict[str, str]:
"""
out: dict[str, str] = {}
try:
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
svc = SystemConfigService.get_instance()
api_key = (svc.get("firecrawl.api_key") or "").strip()
diff --git a/src/backend/core/sandbox/providers/edition.py b/src/backend/core/sandbox/providers/edition.py
index e507d98b..5d1f4527 100644
--- a/src/backend/core/sandbox/providers/edition.py
+++ b/src/backend/core/sandbox/providers/edition.py
@@ -1,6 +1,6 @@
"""Persistent sandbox provider for personal accounts."""
from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider
-from core.spaces.sandbox import SpaceSandboxMixin
+from core.files.spaces.sandbox import SpaceSandboxMixin
class PersistentSandboxProvider(SpaceSandboxMixin, OpenSandboxProvider):
diff --git a/src/backend/core/sandbox/providers/opensandbox/account_volumes.py b/src/backend/core/sandbox/providers/opensandbox/account_volumes.py
deleted file mode 100644
index d00832e1..00000000
--- a/src/backend/core/sandbox/providers/opensandbox/account_volumes.py
+++ /dev/null
@@ -1,292 +0,0 @@
-"""Account-specific OpenSandbox volumes; shared account paths live in workspace."""
-
-from __future__ import annotations
-
-import logging
-import re
-from typing import Any, Optional
-
-from core.config.settings import settings
-from core.sandbox.workspace.credentials import (
- _YIDA_WORKSPACE_MOUNT,
- dws_cache_dir,
- email_cache_dir,
- lark_cache_dir,
- yida_cache_dir,
-)
-from core.sandbox.workspace.identity import safe_user_id
-
-logger = logging.getLogger(__name__)
-
-# dws (DingTalk CLI) credential persistence volumes: bind per-user host directories to the
-# two paths under the sandbox ubuntu user's home that dws reads/writes by default. Measured
-# behavior: the dws keychain/cache only honors $HOME, env redirection has no effect, so these
-# two absolute paths must be bind-mounted (see the workspace.credentials.dws_cache_dir comment).
-_DWS_HOME = "/home/ubuntu"
-
-_DWS_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = (
- # (host subpath under dws_cache/{uid}, sandbox mount point)
- # Matches the backend login layout HOME=dws_cache/{uid}/home: login writes home/.dws and
- # home/.local/share/dws-cli; here we bind those two directories into the sandbox ubuntu
- # user's home.
- ("home/.dws", f"{_DWS_HOME}/.dws"),
- ("home/.local/share/dws-cli", f"{_DWS_HOME}/.local/share/dws-cli"),
-)
-
-
-def _make_dws_creds_volumes(user_id: Optional[str]) -> list[Any]:
- """Build the two read-write bind-mount Volumes for dws credential persistence (one set per user).
-
- Empty list is returned when: flag off / invalid user_id / host_storage not configured /
- SDK has no Volume model — any unmet condition quietly degrades to "no credential
- volumes mounted"; dws then has no credentials in the sandbox, business commands fail
- with exit code 2 (not logged in), and the dingtalk skill guides the user to
- "Settings → Connect DingTalk account" first. Same approach as _make_myspace_volume:
- pre-create the host directories as backend (UID 1000) to avoid docker's root
- auto-mkdir at bind time leaving the backend unable to sync/clean credentials.
- """
- if not settings.sandbox.dws_creds_bind_mount_enabled:
- return []
- uid = safe_user_id(user_id or "")
- if not uid:
- return []
- host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/")
- if not host_storage:
- logger.warning(
- "[opensandbox] dws creds bind mount enabled but HOST_STORAGE_PATH "
- "not configured; dws will have no persisted credentials"
- )
- return []
- cache_dir = dws_cache_dir(uid)
- try:
- for sub, _mount in _DWS_CREDS_SUBPATHS:
- (cache_dir / sub).mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[opensandbox] failed to pre-create dws_cache dir %s: %s; "
- "skipping dws creds bind mount",
- cache_dir,
- exc,
- )
- return []
- try:
- from opensandbox.models.sandboxes import Host, Volume
- except ImportError:
- return []
- slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u"
- out: list[Any] = []
- for idx, (sub, mount_path) in enumerate(_DWS_CREDS_SUBPATHS):
- # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains
- # / and ., so use an index instead.
- vol_name = f"dws{idx}-{slug}"[:63].rstrip("-") or f"dws{idx}"
- out.append(
- Volume(
- name=vol_name,
- host=Host(path=f"{host_storage}/dws_cache/{uid}/{sub}"),
- mount_path=mount_path,
- read_only=False,
- )
- )
- return out
-
-
-# lark-cli (Feishu/Lark CLI) credential persistence volumes: same structure as dws. On Linux
-# lark-cli uses a file-based encrypted store (master.key + *.enc) and only honors these two
-# directories under $HOME, so both absolute paths must be bind-mounted (see the
-# _common.lark_cache_dir comment). Note: we do **not** inject LARK_APP_* via env — when
-# LARK_APP_ID/SECRET are present in the env, lark-cli defaults to the tenant/bot token flow,
-# breaking --as user; app config is instead seeded by the backend's config init into the
-# per-user HOME (same directory as the user token, entering the sandbox with the volume).
-_LARK_HOME = "/home/ubuntu"
-
-_LARK_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = (
- # (host subpath under lark_cache/{uid}, sandbox mount point); matches the backend login
- # layout HOME=lark_cache/{uid}/home: login writes home/.lark-cli and
- # home/.local/share/lark-cli; here we bind them into the sandbox ubuntu home.
- ("home/.lark-cli", f"{_LARK_HOME}/.lark-cli"),
- ("home/.local/share/lark-cli", f"{_LARK_HOME}/.local/share/lark-cli"),
-)
-
-
-def _make_lark_creds_volumes(user_id: Optional[str]) -> list[Any]:
- """Build the two read-write bind-mount Volumes for lark-cli credential persistence (one set per user).
-
- Same approach as [[_make_dws_creds_volumes]]: flag off / invalid user_id / host_storage
- not configured / SDK has no Volume model — any unmet condition quietly degrades to "no
- credential volumes mounted"; lark-cli then has no credentials in the sandbox, business
- commands fail as not-logged-in, and the lark-shared skill guides the user to complete
- auth login first. Pre-create the host directories as backend (UID 1000) to avoid
- docker's root auto-mkdir at bind time leaving the backend unable to sync/clean.
- """
- if not settings.sandbox.lark_creds_bind_mount_enabled:
- return []
- uid = safe_user_id(user_id or "")
- if not uid:
- return []
- host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/")
- if not host_storage:
- logger.warning(
- "[opensandbox] lark creds bind mount enabled but HOST_STORAGE_PATH "
- "not configured; lark-cli will have no persisted credentials"
- )
- return []
- cache_dir = lark_cache_dir(uid)
- try:
- for sub, _mount in _LARK_CREDS_SUBPATHS:
- (cache_dir / sub).mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[opensandbox] failed to pre-create lark_cache dir %s: %s; "
- "skipping lark creds bind mount",
- cache_dir,
- exc,
- )
- return []
- try:
- from opensandbox.models.sandboxes import Host, Volume
- except ImportError:
- return []
- slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u"
- out: list[Any] = []
- for idx, (sub, mount_path) in enumerate(_LARK_CREDS_SUBPATHS):
- # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains
- # / and ., so use an index instead.
- vol_name = f"lark{idx}-{slug}"[:63].rstrip("-") or f"lark{idx}"
- out.append(
- Volume(
- name=vol_name,
- host=Host(path=f"{host_storage}/lark_cache/{uid}/{sub}"),
- mount_path=mount_path,
- read_only=False,
- )
- )
- return out
-
-
-# himalaya (email CLI) credential persistence volume: same structure as lark/dws. himalaya's
-# config is a plain-file config.toml (containing account blocks + raw app-password lines) and
-# only honors the config path, so binding this one directory suffices (no keychain, no
-# secondary store). The app password is a user-supplied IMAP/SMTP app password written into
-# config.toml by the backend; no env is injected in the sandbox (unlike the dingtalk Custom
-# App / firecrawl, which need deployment-level envs).
-_EMAIL_HOME = "/home/ubuntu"
-
-_EMAIL_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = (
- # (host subpath under email_cache/{uid}, sandbox mount point); matches the backend
- # layout HOME=email_cache/{uid}/home: config lands in home/.config/himalaya; here we
- # bind it to the same path under the sandbox ubuntu home.
- ("home/.config/himalaya", f"{_EMAIL_HOME}/.config/himalaya"),
-)
-
-
-def _make_email_creds_volumes(user_id: Optional[str]) -> list[Any]:
- """Build the read-write bind-mount Volume for himalaya credential persistence (one per user, single directory).
-
- Same approach as [[_make_lark_creds_volumes]]: flag off / invalid user_id / host_storage
- not configured / SDK has no Volume model — any unmet condition quietly degrades to "no
- credential volume mounted"; himalaya then has no config in the sandbox, business
- commands fail as unconfigured, and the email skill guides the user to bind a mailbox on
- the plugin detail page first. Pre-create the host directories as backend (UID 1000) to
- avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean.
- """
- if not settings.sandbox.email_creds_bind_mount_enabled:
- return []
- uid = safe_user_id(user_id or "")
- if not uid:
- return []
- host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/")
- if not host_storage:
- logger.warning(
- "[opensandbox] email creds bind mount enabled but HOST_STORAGE_PATH "
- "not configured; himalaya will have no persisted credentials"
- )
- return []
- cache_dir = email_cache_dir(uid)
- try:
- for sub, _mount in _EMAIL_CREDS_SUBPATHS:
- (cache_dir / sub).mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[opensandbox] failed to pre-create email_cache dir %s: %s; "
- "skipping email creds bind mount",
- cache_dir,
- exc,
- )
- return []
- try:
- from opensandbox.models.sandboxes import Host, Volume
- except ImportError:
- return []
- slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u"
- out: list[Any] = []
- for idx, (sub, mount_path) in enumerate(_EMAIL_CREDS_SUBPATHS):
- vol_name = f"email{idx}-{slug}"[:63].rstrip("-") or f"email{idx}"
- out.append(
- Volume(
- name=vol_name,
- host=Host(path=f"{host_storage}/email_cache/{uid}/{sub}"),
- mount_path=mount_path,
- read_only=False,
- )
- )
- return out
-
-
-_YIDA_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = (
- # (host subpath under yida_cache/{uid}, sandbox mount point)
- ("workspace", _YIDA_WORKSPACE_MOUNT),
-)
-
-
-def _make_yida_creds_volumes(user_id: Optional[str]) -> list[Any]:
- """Build the read-write bind-mount Volume for openyida login-state persistence (one per user, single directory).
-
- Same approach as [[_make_email_creds_volumes]]: flag off / invalid user_id /
- host_storage not configured / SDK has no Volume model — any unmet condition quietly
- degrades to "no credential volume mounted"; openyida then has no login state in the
- sandbox, business commands fail as not-logged-in, and the yida skill guides the user
- through QR-code login in the conversation (still usable within the session, just not
- surviving across sessions). Pre-create the host directories as backend (UID 1000) to
- avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean.
- """
- if not settings.sandbox.yida_creds_bind_mount_enabled:
- return []
- uid = safe_user_id(user_id or "")
- if not uid:
- return []
- host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/")
- if not host_storage:
- logger.warning(
- "[opensandbox] yida creds bind mount enabled but HOST_STORAGE_PATH "
- "not configured; openyida will have no persisted login state"
- )
- return []
- cache_dir = yida_cache_dir(uid)
- try:
- for sub, _mount in _YIDA_CREDS_SUBPATHS:
- (cache_dir / sub).mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[opensandbox] failed to pre-create yida_cache dir %s: %s; "
- "skipping yida creds bind mount",
- cache_dir,
- exc,
- )
- return []
- try:
- from opensandbox.models.sandboxes import Host, Volume
- except ImportError:
- return []
- slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u"
- out: list[Any] = []
- for idx, (sub, mount_path) in enumerate(_YIDA_CREDS_SUBPATHS):
- vol_name = f"yida{idx}-{slug}"[:63].rstrip("-") or f"yida{idx}"
- out.append(
- Volume(
- name=vol_name,
- host=Host(path=f"{host_storage}/yida_cache/{uid}/{sub}"),
- mount_path=mount_path,
- read_only=False,
- )
- )
- return out
diff --git a/src/backend/core/sandbox/providers/opensandbox/admin.py b/src/backend/core/sandbox/providers/opensandbox/admin.py
index 81ac1781..d22adc74 100644
--- a/src/backend/core/sandbox/providers/opensandbox/admin.py
+++ b/src/backend/core/sandbox/providers/opensandbox/admin.py
@@ -94,8 +94,9 @@ async def _iter_server_sandbox_infos(self) -> Any:
finally:
try:
await mgr.close()
- except Exception: # noqa: BLE001
- pass
+ except Exception as fallback_error: # noqa: BLE001
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.providers.opensandbox.admin:_iter_server_sandbox_infos', fallback_error)
@_on_service_loop
async def admin_get_sandbox(self, sandbox_id: str) -> Optional[SandboxInfo]:
diff --git a/src/backend/core/sandbox/providers/opensandbox/adoption.py b/src/backend/core/sandbox/providers/opensandbox/adoption.py
index a21591a6..c247e48d 100644
--- a/src/backend/core/sandbox/providers/opensandbox/adoption.py
+++ b/src/backend/core/sandbox/providers/opensandbox/adoption.py
@@ -6,10 +6,6 @@
import core.sandbox.sessions.registry as session_registry
from core.sandbox.providers.opensandbox.runtime import (
- _LEGACY_ENTRYPOINTS,
- _LIGHT_ENTRYPOINT,
- _SETUP_ENTRYPOINT,
- _user_bound_sandbox_required,
logger,
)
from core.sandbox.providers.opensandbox.volumes import (
@@ -24,28 +20,12 @@
class _OpenSandboxAdoptionMixin:
async def _adopt_existing_sandboxes(self) -> None:
- """Find sandboxes this app previously left on the OpenSandbox server and route
- them to adopt / kill.
-
- Classification rules (strictest to loosest):
- 1. ``state != "Running"`` → kill (container already dead, metadata leftover)
- 2. ``metadata[hugagent-owner] == "backend"`` → 先看 session_registry:
- 仍绑在会话上的原样留着(会话下一轮自己 attach 回来),无主的才 adopt 进对应的池
- 3. ``entrypoint`` matches ``_LIGHT_ENTRYPOINT`` / a legacy entrypoint,
- but the ``hugagent-owner`` tag is missing → legacy orphan, kill
- 4. anything else (someone else's sandbox, or an unrecognized entrypoint) → leave alone
-
- Adopted sandboxes go into ``_pool.idle``; surplus adopt candidates beyond
- ``max_idle`` are also killed to avoid unbounded pile-up. This method is called
- once before ``shutdown``; calling it again is idempotent (sandboxes scanned a
- second time are already in the pool, and an SDK ``connect`` failure
- auto-kills).
- """
+ """Adopt explicitly owned, unbound sandboxes; leave other clients alone."""
stats = {
"adopted_jupyter": 0,
"adopted_light": 0,
"skipped_bound": 0,
- "killed_legacy": 0,
+ "killed_invalid": 0,
"killed_dead": 0,
"killed_overflow": 0,
}
@@ -56,11 +36,11 @@ async def _adopt_existing_sandboxes(self) -> None:
logger.info(
"[opensandbox] sandbox adoption complete: "
"adopted jupyter=%d light=%d; left to sessions=%d; "
- "killed legacy=%d dead=%d overflow=%d",
+ "killed invalid=%d dead=%d overflow=%d",
stats["adopted_jupyter"],
stats["adopted_light"],
stats["skipped_bound"],
- stats["killed_legacy"],
+ stats["killed_invalid"],
stats["killed_dead"],
stats["killed_overflow"],
)
@@ -77,9 +57,8 @@ async def _classify_and_adopt(
try:
state = (getattr(info.status, "state", None) or "").lower()
metadata = info.metadata or {}
- entrypoint = list(info.entrypoint or [])
owner_tag = metadata.get(_METADATA_OWNER_KEY)
- if owner_tag and owner_tag != _METADATA_OWNER_VALUE:
+ if owner_tag != _METADATA_OWNER_VALUE:
# Another owner's sandbox (another deployment, or a module's own
# controller-managed computers) is never adopted or reaped here.
return
@@ -88,10 +67,6 @@ async def _classify_and_adopt(
return # The creator has not yet delivered this instance.
kind_tag = metadata.get(_METADATA_POOL_KIND_KEY)
- known_legacy = entrypoint in (_LIGHT_ENTRYPOINT, _SETUP_ENTRYPOINT) or entrypoint in _LEGACY_ENTRYPOINTS
- # Untagged instances with unknown entrypoints belong to other clients.
- if owner_tag != _METADATA_OWNER_VALUE and not known_legacy:
- return
# 1. Dead owned sandbox → guarded cleanup
if state != "running":
if await self._kill_by_id(mgr, info.id, reason=f"non-running state={state}"):
@@ -103,145 +78,114 @@ async def _classify_and_adopt(
return
# 2. Our own sandbox → adopt
- if owner_tag == _METADATA_OWNER_VALUE:
- if kind_tag not in ("jupyter", "light"):
- logger.warning(
- "[opensandbox] adopt: own sandbox %s missing pool-kind tag, killing",
- info.id,
- )
- if await self._kill_by_id(mgr, info.id, reason="missing pool-kind"):
- stats["killed_legacy"] += 1
- return
+ if kind_tag not in ("jupyter", "light"):
+ logger.warning(
+ "[opensandbox] adopt: own sandbox %s missing pool-kind tag, killing",
+ info.id,
+ )
+ if await self._kill_by_id(mgr, info.id, reason="missing pool-kind"):
+ stats["killed_invalid"] += 1
+ return
- # User-bound jupyter sandboxes are re-homed by user_id. When any
- # private-data mount is enabled, a sandbox without a user tag cannot
- # carry the current user's volumes; kill it rather than silently hand
- # out a credential-less or cross-user container.
- user_tag = metadata.get(_METADATA_USER_ID_KEY)
- user_bound_enabled = _user_bound_sandbox_required()
- if kind_tag == "jupyter" and user_bound_enabled and not user_tag:
- if await self._kill_by_id(
- mgr,
- info.id,
- reason="jupyter sandbox missing user-id tag (legacy / cross-user unsafe)",
- ):
- stats["killed_legacy"] += 1
- return
- # Created before the per-user skill view: its /workspace/skills is the
- # whole shared dir, which no longer holds this user's private skills.
- # Mounts can't be changed after creation, so drop it.
- if (
- kind_tag == "jupyter"
- and user_tag
- and metadata.get(_METADATA_SKILLS_LAYOUT_KEY) != _METADATA_SKILLS_LAYOUT_VALUE
+ # Persistent sandboxes require an explicit user owner so adoption
+ # cannot assign private mounts to another account.
+ user_tag = metadata.get(_METADATA_USER_ID_KEY)
+ if kind_tag == "jupyter" and not user_tag:
+ if await self._kill_by_id(
+ mgr,
+ info.id,
+ reason="jupyter sandbox missing user-id tag",
):
- if await self._kill_by_id(
- mgr, info.id, reason="jupyter sandbox predates the per-user skill view"
- ):
- stats["killed_legacy"] += 1
- return
- # Reverse case: the sandbox still carries a user tag after all
- # user-bound mounts were disabled. Its original volumes remain mounted;
- # generic-pool reuse could expose them to another user, so kill it.
- if kind_tag == "jupyter" and user_tag and not user_bound_enabled:
- if await self._kill_by_id(
- mgr,
- info.id,
- reason="jupyter sandbox carries user-id tag but user-bound mounts "
- "are disabled (stale private mounts would cross-user leak)",
- ):
- stats["killed_legacy"] += 1
- return
-
- # 仍绑在会话上的容器不是无主的,收进空闲池就会被按空闲回收掉——而会话
- # 那边还在用它。绑定活在 session_registry 里(跨进程、比空闲窗口活得
- # 久),重启并不会让它消失,所以这里原样留着:会话下一轮照登记 attach
- # 回来。真没人来接,服务端自己的 TTL 会收走。
- owner_session = await session_registry.bound_session(info.id)
- if owner_session:
- stats["skipped_bound"] += 1
- logger.info(
- "[opensandbox] adopt: sandbox %s still bound to session %s; "
- "leaving it to that session",
- info.id,
- owner_session,
- )
- return
-
- if not await self._track_sandbox(str(info.id)):
- return # Another live worker already owns this idle handle.
- try:
- sbx = await Sandbox.connect(
- info.id,
- connection_config=self._make_config(),
- skip_health_check=(kind_tag == "light"),
- )
- except Exception as exc: # noqa: BLE001
- logger.warning(
- "[opensandbox] adopt: connect %s failed (%s), retaining container",
- info.id, exc,
- )
- await session_registry.untrack(str(info.id), self._tracking_owner_token())
- self._tracking_ids.discard(str(info.id))
- return
+ stats["killed_invalid"] += 1
+ return
+ # Created before the per-user skill view: its /workspace/skills is the
+ # whole shared dir, which no longer holds this user's private skills.
+ # Mounts can't be changed after creation, so drop it.
+ if (
+ kind_tag == "jupyter"
+ and user_tag
+ and metadata.get(_METADATA_SKILLS_LAYOUT_KEY) != _METADATA_SKILLS_LAYOUT_VALUE
+ ):
+ if await self._kill_by_id(
+ mgr, info.id, reason="jupyter sandbox predates the per-user skill view"
+ ):
+ stats["killed_invalid"] += 1
+ return
+ # 仍绑在会话上的容器不是无主的,收进空闲池就会被按空闲回收掉——而会话
+ # 那边还在用它。绑定活在 session_registry 里(跨进程、比空闲窗口活得
+ # 久),重启并不会让它消失,所以这里原样留着:会话下一轮照登记 attach
+ # 回来。真没人来接,服务端自己的 TTL 会收走。
+ owner_session = await session_registry.bound_session(info.id)
+ if owner_session:
+ stats["skipped_bound"] += 1
+ logger.info(
+ "[opensandbox] adopt: sandbox %s still bound to session %s; "
+ "leaving it to that session",
+ info.id,
+ owner_session,
+ )
+ return
- # Direct-connect to execd (bypasses the server proxy's ~3s/request overhead); falls back automatically when unreachable.
- await self._repoint_execd_direct(sbx)
+ if not await self._track_sandbox(str(info.id)):
+ return # Another live worker already owns this idle handle.
+ try:
+ sbx = await Sandbox.connect(
+ info.id,
+ connection_config=self._make_config(),
+ skip_health_check=(kind_tag == "light"),
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ "[opensandbox] adopt: connect %s failed (%s), retaining container",
+ info.id, exc,
+ )
+ await session_registry.untrack(str(info.id), self._tracking_owner_token())
+ self._tracking_ids.discard(str(info.id))
+ return
- # jupyter + user tag → user-bound pool
- if kind_tag == "jupyter" and user_tag and user_bound_enabled:
- ok = await self._jupyter_user_pool.adopt(user_tag, sbx)
- if not ok:
- await self._destroy_sandbox(sbx)
- stats["killed_overflow"] += 1
- return
- stats["adopted_jupyter"] += 1
- logger.info(
- "[opensandbox] adopt: claimed jupyter sandbox %s for user=%s",
- info.id,
- user_tag,
- )
- return
+ # Direct-connect to execd (bypasses the server proxy's ~3s/request overhead); falls back automatically when unreachable.
+ await self._repoint_execd_direct(sbx)
- # light (or fallback jupyter with all user-bound mounts disabled) → generic pool
- # pool full → kill rather than leak
- bucket = self._pool._buckets[kind_tag] # noqa: SLF001
- if len(bucket.idle) >= bucket.max_idle:
- if await self._kill_by_id(
- mgr,
- info.id,
- reason=f"{kind_tag} bucket already at max_idle={bucket.max_idle}",
- ):
- stats["killed_overflow"] += 1
- return
- ok = await self._pool.adopt_idle(kind_tag, sbx)
+ # jupyter + user tag → user-bound pool
+ if kind_tag == "jupyter" and user_tag:
+ ok = await self._jupyter_user_pool.adopt(user_tag, sbx)
if not ok:
- # adopt_idle refused (race: another entry enqueued concurrently filled the capacity) — destroy
await self._destroy_sandbox(sbx)
stats["killed_overflow"] += 1
return
- stats[f"adopted_{kind_tag}"] += 1
+ stats["adopted_jupyter"] += 1
logger.info(
- "[opensandbox] adopt: claimed %s sandbox %s",
- kind_tag,
+ "[opensandbox] adopt: claimed jupyter sandbox %s for user=%s",
info.id,
+ user_tag,
)
return
- # 3. No tag, but the entrypoint looks like one we previously created (orphan left by an older version) → kill
- if (
- entrypoint in (_LIGHT_ENTRYPOINT, _SETUP_ENTRYPOINT)
- or entrypoint in _LEGACY_ENTRYPOINTS
- ):
+ # Light sandboxes use the shared pool.
+ # pool full → kill rather than leak
+ bucket = self._pool._buckets[kind_tag] # noqa: SLF001
+ if len(bucket.idle) >= bucket.max_idle:
if await self._kill_by_id(
mgr,
info.id,
- reason=f"legacy untagged sandbox with our entrypoint {entrypoint}",
+ reason=f"{kind_tag} bucket already at max_idle={bucket.max_idle}",
):
- stats["killed_legacy"] += 1
+ stats["killed_overflow"] += 1
return
+ ok = await self._pool.adopt_idle(kind_tag, sbx)
+ if not ok:
+ # adopt_idle refused (race: another entry enqueued concurrently filled the capacity) — destroy
+ await self._destroy_sandbox(sbx)
+ stats["killed_overflow"] += 1
+ return
+ stats[f"adopted_{kind_tag}"] += 1
+ logger.info(
+ "[opensandbox] adopt: claimed %s sandbox %s",
+ kind_tag,
+ info.id,
+ )
+ return
- # 4. Someone else's sandbox / unrecognized entrypoint → leave alone
except Exception as exc: # noqa: BLE001
logger.warning(
"[opensandbox] adopt: classifying %s failed: %s",
diff --git a/src/backend/core/sandbox/providers/opensandbox/binding.py b/src/backend/core/sandbox/providers/opensandbox/binding.py
index 426f3ca8..acc7983c 100644
--- a/src/backend/core/sandbox/providers/opensandbox/binding.py
+++ b/src/backend/core/sandbox/providers/opensandbox/binding.py
@@ -49,7 +49,7 @@ async def _resolve_bound_session(self, session_id, user_id):
await session_registry.remember(session_id, str(sess.sandbox.id))
self._maybe_schedule_renew(session_id, sess)
if newly_bound:
- from core.plugins.resources.prewarm import schedule
+ from core.capabilities.plugins.resources.prewarm import schedule
schedule(self, session_id, sess.user_id, str(sess.sandbox.id))
return sess
diff --git a/src/backend/core/sandbox/providers/opensandbox/creation.py b/src/backend/core/sandbox/providers/opensandbox/creation.py
index a78197c1..f4854490 100644
--- a/src/backend/core/sandbox/providers/opensandbox/creation.py
+++ b/src/backend/core/sandbox/providers/opensandbox/creation.py
@@ -10,13 +10,8 @@
import core.sandbox.sessions.registry as session_registry
from core.config.settings import settings
from core.sandbox.errors import SandboxConnectError
-from core.sandbox.providers.opensandbox.account_volumes import (
- _make_dws_creds_volumes,
- _make_email_creds_volumes,
- _make_lark_creds_volumes,
- _make_yida_creds_volumes,
-)
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.credential_dispatch import credential_mounts
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
# Snapshot persistence (see internal design docs). Imports are
# deferred to runtime to avoid cross-module circular dependencies at startup
@@ -24,7 +19,6 @@
from core.sandbox.providers.opensandbox.runtime import (
_LIGHT_ENTRYPOINT,
_SETUP_ENTRYPOINT,
- _user_bound_sandbox_required,
logger,
)
from core.sandbox.providers.opensandbox.volumes import _make_skills_volumes, _pool_metadata
@@ -38,11 +32,11 @@ def _session_entrypoint(self, user_id):
return _SETUP_ENTRYPOINT if user_id else _LIGHT_ENTRYPOINT
async def _additional_workspace_mounts(self, user_id):
- from core.spaces.mounts import prepare_volumes
+ from core.files.spaces.mounts import provision_volumes
if not user_id:
return [], {}
- return await asyncio.to_thread(prepare_volumes, user_id)
+ return await asyncio.to_thread(provision_volumes, user_id)
async def _create_jupyter_sandbox(
self,
@@ -59,23 +53,7 @@ async def _create_jupyter_sandbox(
volumes = []
else:
skills_vols = _make_skills_volumes(user_id)
- # dws (DingTalk CLI) credential persistence volumes: two bind-mounts per user,
- # mounted at the sandbox home's ~/.dws and ~/.local/share/dws-cli (where dws
- # reads/writes by default), so OAuth credentials survive across sessions.
- dws_vols = _make_dws_creds_volumes(user_id)
- # lark-cli (Feishu CLI) credential persistence volumes: likewise mounts
- # ~/.lark-cli and ~/.local/share/lark-cli (including master.key), so the
- # user's OAuth user_access_token survives across sessions.
- lark_vols = _make_lark_creds_volumes(user_id)
- # himalaya (email CLI) credential persistence volume: mounts ~/.config/himalaya
- # (including config.toml + IMAP/SMTP authorization codes), so the user's
- # mailbox configuration survives across sessions.
- email_vols = _make_email_creds_volumes(user_id)
- # openyida (Yida CLI) login-state persistence volume: mounts the fixed working
- # directory ~/yida-workspace (including .cache/cookies-*.json QR-login cookies
- # + project artifacts), so the login state survives across sessions.
- yida_vols = _make_yida_creds_volumes(user_id)
- volumes = skills_vols + extra_volumes + dws_vols + lark_vols + email_vols + yida_vols
+ volumes = skills_vols + extra_volumes + credential_mounts(user_id)
# entrypoint / volumes / metadata / timeout are exactly the same on the image
# and snapshot paths; volumes must also be passed explicitly on the snapshot
# path — docker commit does not preserve mount configuration, otherwise the
@@ -103,19 +81,18 @@ async def _create_jupyter_sandbox(
except Exception as e:
raise SandboxConnectError(f"无法创建 OpenSandbox 持久会话沙盒: {e}") from e
- # 目录约定已由 entrypoint 在启动时建好,直接打标,让 _persistent_setup 整段跳过
- # —— 这才是首条命令省下那一秒的地方。老沙箱(旧 entrypoint 建的、adopt 回来的)
- # 没有这个标记,仍按原路在首次执行前补建,不受影响。
+ # The entrypoint prepared the workspace; avoid repeating setup on first use.
if setup_uid:
try:
sbx._jx_setup_done = True # type: ignore[attr-defined]
sbx._jx_myspace_uid = setup_uid # type: ignore[attr-defined]
- except Exception: # noqa: BLE001 — defensive: opaque SDK object
- pass
+ except Exception as fallback_error: # noqa: BLE001 — defensive: opaque SDK object
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.providers.opensandbox.creation:_create_jupyter_sandbox', fallback_error)
return sbx
async def _create_light_sandbox(self) -> Any:
- """Create a light sandbox: just the sleep entrypoint + execd, no Jupyter.
+ """Create a light sandbox: just the init entrypoint + execd, no Jupyter.
Used for ephemeral one-shot execution (the frontend "re-run" button, skill calls without a chat_id).
Key point: ``skip_health_check=True`` ── the SDK's default readiness probe
@@ -132,9 +109,7 @@ async def _create_light_sandbox(self) -> Any:
try:
sbx = await self._create_owned_sandbox(
self._image,
- timeout=timedelta(
- seconds=300
- ), # ephemeral is short-lived; 5min cap as a safety net
+ timeout=timedelta(seconds=300), # ephemeral is short-lived; 5min cap as a safety net
entrypoint=_LIGHT_ENTRYPOINT,
connection_config=config,
skip_health_check=True,
@@ -265,8 +240,9 @@ async def _destroy_sandbox(
finally:
try:
await sbx.close()
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.providers.opensandbox.creation:_destroy_sandbox', fallback_error)
async def _pooled_sandbox_owner(self, sbx: Any) -> Optional[str]:
"""池子用的归属谓词:这台容器当前属于哪个会话,无主返回 ``None``。
@@ -355,18 +331,8 @@ async def _probe_pooled_sandbox_alive(self, sbx: Any) -> bool:
return True
async def _create_session(self, user_id: Optional[str] = None) -> _Session:
- """Construct a new persistent session (user-bound sandbox, running directly via execd, no Jupyter kernel).
-
- Plan F: when ``user_id`` is given and any private-data bind mount is on,
- **bypass the general warm pool** — the pool's sandboxes have no user-bound
- MySpace/account-credential volumes. Instead, take the user-bound path via
- ``_create_jupyter_sandbox(user_id=...)``. Adopted old sandboxes have already
- been slotted per user into ``_jupyter_user_pool`` (see B.7).
-
- When ``user_id`` is absent (legacy / fallback), take the old path: grab one
- from the general pool without account volumes.
- """
- use_user_pool = bool(user_id and (_user_bound_sandbox_required() or str(user_id).startswith("api_ak_")))
+ """Route account sessions to their user pool; anonymous sessions use the general pool."""
+ use_user_pool = bool(user_id)
if use_user_pool:
sandbox = await self._jupyter_user_pool.acquire(user_id)
@@ -382,9 +348,7 @@ async def _create_session(self, user_id: Optional[str] = None) -> _Session:
)
async def _create_session_from_snapshot(
- self,
- snapshot_id: str,
- user_id: Optional[str] = None,
+ self, snapshot_id: str, user_id: Optional[str] = None,
) -> _Session:
"""Start a persistent session sandbox directly from an opensandbox snapshot + wrap it into a _Session.
@@ -399,8 +363,7 @@ async def _create_session_from_snapshot(
usable as soon as it is up.
"""
sandbox = await self._create_jupyter_sandbox(
- from_snapshot_id=snapshot_id,
- user_id=user_id,
+ from_snapshot_id=snapshot_id, user_id=user_id,
)
return _Session(
sandbox=sandbox,
@@ -422,6 +385,5 @@ async def _create_owned_sandbox(self, *args, **kwargs):
sandbox = await Sandbox.create(*args, **kwargs)
await self._repoint_execd_direct(sandbox)
if not await self._track_sandbox(str(sandbox.id)):
- # Another owner may already be using it. Never kill or deliver it.
raise SandboxConnectError("sandbox ownership was acquired elsewhere")
return sandbox
diff --git a/src/backend/core/sandbox/providers/opensandbox/execution.py b/src/backend/core/sandbox/providers/opensandbox/execution.py
index 9eccacc8..10e8ab6f 100644
--- a/src/backend/core/sandbox/providers/opensandbox/execution.py
+++ b/src/backend/core/sandbox/providers/opensandbox/execution.py
@@ -10,7 +10,7 @@
from core.config.settings import settings
from core.sandbox.protocol import ProcessRequest
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
# Snapshot persistence (see internal design docs). Deferred to a
# runtime import to avoid a cross-module circular dependency at startup
@@ -143,8 +143,9 @@ async def _persistent_setup(self, sess: _Session, req: ProcessRequest) -> None:
sess.sandbox._jx_setup_done = True # type: ignore[attr-defined]
if uid:
sess.sandbox._jx_myspace_uid = uid # type: ignore[attr-defined]
- except Exception: # noqa: BLE001 — defensive: opaque SDK object
- pass
+ except Exception as fallback_error: # noqa: BLE001 — defensive: opaque SDK object
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.providers.opensandbox.execution:_persistent_setup', fallback_error)
else:
stderr = _join_logs(getattr(getattr(result, "logs", None), "stderr", []) or [])
logger.warning(
diff --git a/src/backend/core/sandbox/providers/opensandbox/lifecycle.py b/src/backend/core/sandbox/providers/opensandbox/lifecycle.py
index 00a74034..ee4db477 100644
--- a/src/backend/core/sandbox/providers/opensandbox/lifecycle.py
+++ b/src/backend/core/sandbox/providers/opensandbox/lifecycle.py
@@ -9,7 +9,7 @@
from core.config.settings import settings
from core.sandbox.errors import SandboxNotFoundError, SandboxError
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
# Snapshot persistence (see internal design docs). Imports are
# deferred to runtime to avoid cross-module circular dependencies at startup
@@ -40,7 +40,7 @@ async def _create_session_for(
workspace it built up. The ~7s warm path is only correct for a chat with
nothing to restore, so the idle pool is checked after the lookup misses.
"""
- from core.agent_skills.publication import prepare_skill_view
+ from core.capabilities.skills.publication import prepare_skill_view
# This runs even on a pre-warmed pool hit, before any snapshot/fresh
# fallback can hide a publication failure.
diff --git a/src/backend/core/sandbox/providers/opensandbox/models.py b/src/backend/core/sandbox/providers/opensandbox/models.py
index de15e34f..4f256929 100644
--- a/src/backend/core/sandbox/providers/opensandbox/models.py
+++ b/src/backend/core/sandbox/providers/opensandbox/models.py
@@ -7,14 +7,11 @@
@dataclass
-class _Session:
+class SandboxSessionRecord:
sandbox: Any # opensandbox.Sandbox
- # The user_id owning this sandbox (binds MySpace/account-credential volumes +
- # user re-homing on adopt across restarts). None when legacy paths call
- # _get_or_create_session without a user_id — execution then falls back to a
- # credential-less general sandbox.
+ # None identifies an anonymous session without account volumes.
user_id: Optional[str] = None
- # Plan F: which pool this sandbox was actually taken from. **release must follow this,
+ # Which pool this sandbox was actually taken from. **release must follow this,
# never re-decide from the current user-bound-volume settings** — a mid-flight
# flag change (ops toggle, hot reload) would route release to a different pool,
# corrupting counts / cross-user contamination. The snapshot restore path builds
diff --git a/src/backend/core/sandbox/providers/opensandbox/pool.py b/src/backend/core/sandbox/providers/opensandbox/pool.py
index b85cb96e..597c604a 100644
--- a/src/backend/core/sandbox/providers/opensandbox/pool.py
+++ b/src/backend/core/sandbox/providers/opensandbox/pool.py
@@ -224,8 +224,7 @@ async def probe(sbx):
logger.info(
"[sandbox-pool] sweep: evicting dead idle sandbox kind=%s "
"id=%s (server-side reaped); refilling",
- kind,
- getattr(sbx, "id", "?"),
+ kind, getattr(sbx, "id", "?"),
)
if (id(sbx) not in foreign
and (self._owner is None or not await self._owner(sbx))):
diff --git a/src/backend/core/sandbox/providers/opensandbox/provider.py b/src/backend/core/sandbox/providers/opensandbox/provider.py
index c724577e..813832c7 100644
--- a/src/backend/core/sandbox/providers/opensandbox/provider.py
+++ b/src/backend/core/sandbox/providers/opensandbox/provider.py
@@ -18,10 +18,9 @@
from core.sandbox.providers.opensandbox.creation import _OpenSandboxCreationMixin
from core.sandbox.providers.opensandbox.execution import _OpenSandboxExecMixin
from core.sandbox.providers.opensandbox.lifecycle import _OpenSandboxLifecycleMixin
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
from core.sandbox.providers.opensandbox.pool import SandboxPool
from core.sandbox.providers.opensandbox.runtime import (
- _user_bound_sandbox_required,
logger,
write_file,
)
@@ -51,9 +50,7 @@ class OpenSandboxProvider(
_service_loop: Optional[asyncio.AbstractEventLoop] = None
def __init__(self) -> None:
- # Only the opensandbox SDK is needed (execd commands.run); no longer depends on
- # code_interpreter (the Jupyter kernel was removed, all languages go through
- # execd, see cloud_processes).
+ # All languages execute through execd.
try:
from opensandbox import Sandbox
except ImportError as e:
@@ -63,7 +60,10 @@ def __init__(self) -> None:
) from e
s = settings.sandbox
- self._domain = s.opensandbox_domain
+ # SDK and direct health/renew requests must use the same absolute endpoint.
+ self._domain = s.opensandbox_domain.rstrip("/")
+ if not self._domain.startswith(("http://", "https://")):
+ self._domain = "http://" + self._domain
self._api_key = s.opensandbox_api_key
self._image = s.opensandbox_image
self._ttl_s = s.idle_ttl_s
@@ -93,7 +93,7 @@ def __init__(self) -> None:
except RuntimeError:
self._service_loop = None
# ⚠️ 锁必须按**事件循环**分桶。这个 provider 是进程级单例,但调用方不止一个 loop:
- # 子智能体跑在独立线程的独立事件循环里(core/llm/subagent_tool.py 的
+ # 子智能体跑在独立线程的独立事件循环里(core/llm/subagents/tool.py 的
# asyncio.new_event_loop())。asyncio.Lock 在**首次 await 时**绑定当时的 running loop,
# 之后被另一个 loop 用到就抛 "is bound to a different event loop"。
# 事故形态极隐蔽:谁先碰 provider 谁把锁绑走,另一个 loop 的沙箱保活 touch_session
@@ -118,9 +118,7 @@ def __init__(self) -> None:
# serves the fallback path where user_id is missing; normal user-bound traffic
# goes through _jupyter_user_pool.
self._pool = SandboxPool(
- jupyter_min_idle=(
- s.opensandbox_pool_jupyter_min_idle if not _user_bound_sandbox_required() else 0
- ),
+ jupyter_min_idle=0,
jupyter_max_idle=s.opensandbox_pool_jupyter_max_idle,
light_min_idle=s.opensandbox_pool_light_min_idle,
light_max_idle=s.opensandbox_pool_light_max_idle,
@@ -131,9 +129,7 @@ def __init__(self) -> None:
liveness_fn=self._probe_pooled_sandbox_alive,
owner_fn=self._pooled_sandbox_owner,
)
- # Plan F: jupyter sandboxes are bucketed by user_id. When bind mount is enabled,
- # all jupyter acquires go through here; adopted old sandboxes are placed into the
- # matching bucket by their hugagent-user-id metadata.
+ # Account sessions and adopted instances share the same per-user pool.
self._jupyter_user_pool = _JupyterUserPool(
factory=lambda uid: self._create_jupyter_sandbox(user_id=uid),
destroy_fn=self._destroy_sandbox,
@@ -338,23 +334,15 @@ async def health(self) -> bool:
# instances (still alive but not tracked in memory, backend_managed=False).
# Pool figures come from the stats of the two pools.
+ def bind_loop(self) -> None:
+ loop = asyncio.get_running_loop()
+ if self._service_loop is not None and self._service_loop is not loop:
+ raise RuntimeError("Sandbox provider already belongs to another event loop")
+ self._service_loop = loop
+
async def warmup(self) -> None:
- """Startup-time pool warmup entry point (called by the FastAPI startup hook,
- repeatable, idempotent).
-
- Steps:
- 1. ``_adopt_existing_sandboxes``: scan all sandboxes on the OpenSandbox server;
- live sandboxes carrying this app's metadata tag are reclaimed into
- ``_pool.idle``; those with "our previous entrypoint but missing the tag" or
- in an unusable state are killed outright to clean up orphans.
- 2. ``_pool.warmup``: if adoption did not fill min_idle, top up the remaining
- deficit via ``factory()`` (same behavior as before).
- Effect: after a backend restart, idle fills almost instantly with no need to
- recreate 4 sandboxes (saves ~60s).
- """
- # 锚定服务循环:warmup 由 FastAPI 启动钩子在主循环上调用,之后所有跨循环的
- # provider 调用都转发到这里(见 ``_on_service_loop``)。
- self._service_loop = asyncio.get_running_loop()
+ """Adopt owned instances and start pool/snapshot maintenance."""
+ self.bind_loop()
try:
await self._adopt_existing_sandboxes()
except Exception as exc: # noqa: BLE001 — an adoption failure must not block warmup
@@ -371,11 +359,7 @@ async def warmup(self) -> None:
if self._snapshot_gc_task is None or self._snapshot_gc_task.done():
self._snapshot_gc_task = asyncio.create_task(self._snapshot_gc_loop())
- async def shutdown(self) -> None:
- """Called on process exit: gracefully shut down all persistent sessions + idle sandboxes in the pools + snapshot workers."""
- processes = getattr(self, "_command_processes", None)
- if processes:
- await processes.close_all()
+ async def stop_maintenance(self) -> None:
# Stop the snapshot workers first: no more new park / GC triggers
self._snapshot_workers_shutdown.set()
for task in (self._snapshot_idle_task, self._snapshot_gc_task):
@@ -389,6 +373,13 @@ async def shutdown(self) -> None:
self._snapshot_idle_task = None
self._snapshot_gc_task = None
+ async def shutdown(self) -> None:
+ """Called on process exit: gracefully shut down all persistent sessions + idle sandboxes in the pools + snapshot workers."""
+ processes = getattr(self, "_command_processes", None)
+ if processes:
+ await processes.close_all()
+ await self.stop_maintenance()
+
for sid, sess in list(self._sessions.items()):
try:
await self._release_session(sid, sess, detach_only=True)
diff --git a/src/backend/core/sandbox/providers/opensandbox/reclamation.py b/src/backend/core/sandbox/providers/opensandbox/reclamation.py
index 0ed0f7f6..e0a767f4 100644
--- a/src/backend/core/sandbox/providers/opensandbox/reclamation.py
+++ b/src/backend/core/sandbox/providers/opensandbox/reclamation.py
@@ -6,7 +6,7 @@
import core.sandbox.sessions.registry as session_registry
from core.sandbox.providers.opensandbox.accounting import finish_accounting
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
from core.sandbox.providers.opensandbox.runtime import logger
@@ -50,7 +50,7 @@ async def protect(chat_id, sess):
await asyncio.gather(*(protect(chat_id, sess) for chat_id, sess in entries))
async def _destroy_session(self, session_id, sess, *, reuse_sandbox=False):
- from core.plugins.resources.prewarm import closing, stop
+ from core.capabilities.plugins.resources.prewarm import closing, stop
import asyncio
async with closing(session_id):
clean = False
diff --git a/src/backend/core/sandbox/providers/opensandbox/runtime.py b/src/backend/core/sandbox/providers/opensandbox/runtime.py
index 63e27793..6871edfc 100644
--- a/src/backend/core/sandbox/providers/opensandbox/runtime.py
+++ b/src/backend/core/sandbox/providers/opensandbox/runtime.py
@@ -45,20 +45,7 @@ async def write_file(sbx: Any, path: str, content: Any) -> None:
logger.warning("[opensandbox] chown %s 失败(继续): %r", path, exc)
-def _user_bound_sandbox_required() -> bool:
- """Account sessions always require account-specific volumes."""
- return True
-
-
-# Entrypoints earlier versions created sandboxes with. Nothing is created with them
-# any more (every sandbox uses _LIGHT_ENTRYPOINT), but adopt still recognizes them so
-# orphans left by an older backend get killed instead of leaking.
-_LEGACY_ENTRYPOINTS = (
- ["/opt/opensandbox/code-interpreter.sh"],
- ["/opt/hugagent/entrypoint-as-ubuntu.sh"],
-)
-
-# Plan F Q1: environment variables used when running bash via sandbox.commands.run (execd).
+# Environment variables used when running bash via sandbox.commands.run (execd).
# execd does not source /opt/opensandbox/code-interpreter-env.sh; PATH then only contains the
# system bin paths, so python resolves to a "system python" like /usr/bin/python3 that has no
# pandas/docx etc. installed. We explicitly provide a PATH matching where packages are
@@ -82,14 +69,16 @@ def _user_bound_sandbox_required() -> bool:
# The ephemeral path does not start Jupyter; the injected execd alone provides commands.run.
# Startup drops from ~10s to ~3s.
-_LIGHT_ENTRYPOINT = ["sleep", "infinity"]
+# PID 1 is sandbox-init (docker/sandbox/sandbox-init.c): it reaps every orphan and kills
+# what user commands, which run as SANDBOX_RUN_UID, leave behind.
+_LIGHT_ENTRYPOINT = ["sandbox-init", str(SANDBOX_RUN_UID)]
# 用户绑定沙箱的 entrypoint:容器启动时就建好目录约定(scratch + /myspace 软链),
# 首条命令不必再为此花一次 execd 往返(实测固定 ~1s,而这段 shell 只要 2ms)。uid 走
# 环境变量,entrypoint 保持常量,adopt 仍能按它认出自己创建的沙箱。
#
# 两个约束:``/myspace`` 已是真目录时不能直接 ln(软链会建到目录里面),所以先判断;
-# 整段不许中途退出,任何一步失败都要走到 exec sleep infinity,否则容器直接死掉。
+# 整段不许中途退出,任何一步失败都要走到 exec sandbox-init,否则容器直接死掉。
_SETUP_ENTRYPOINT = [
"/bin/sh",
"-c",
@@ -102,7 +91,7 @@ def _user_bound_sandbox_required() -> bool:
'rm -f /myspace; ln -s "/workspace/myspace/$JX_MYSPACE_UID" /myspace 2>/dev/null; '
"fi; "
"fi; "
- "exec sleep infinity",
+ f"exec sandbox-init {SANDBOX_RUN_UID}",
]
@@ -123,7 +112,7 @@ def _join_logs(parts: list) -> str:
"自带换行的不再补" 这一条正是为它留的:两者合起来逐字节还原
(``A``、``\n``、``\n``、``B`` → ``A\n\n\nB\n``),补重了空行会翻倍。
execd 早于 v1.0.17 时根本不为空行发事件,那种情况下空行在到达后端之前就没了,
- 这里补不回来——版本钉在 ``docker/opensandbox-config.toml.tpl``。
+ 这里补不回来——版本钉在 ``docker/opensandbox-server/opensandbox-config.toml.tpl``。
"""
if not parts:
return ""
diff --git a/src/backend/core/sandbox/providers/opensandbox/snapshots.py b/src/backend/core/sandbox/providers/opensandbox/snapshots.py
index 6627255b..c871cd5d 100644
--- a/src/backend/core/sandbox/providers/opensandbox/snapshots.py
+++ b/src/backend/core/sandbox/providers/opensandbox/snapshots.py
@@ -9,7 +9,7 @@
import core.sandbox.sessions.registry as session_registry
from core.config.settings import settings
from core.sandbox.errors import SandboxNotFoundError, SandboxError, SandboxTimeoutError
-from core.sandbox.providers.opensandbox.models import _Session
+from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session
# Snapshot persistence (see internal design docs). Imports are
# deferred to runtime to avoid cross-module circular dependencies at startup
@@ -94,7 +94,7 @@ def _upsert() -> Optional[str]:
return await asyncio.to_thread(_upsert)
- async def _delete_snapshot_row(self, chat_id: str) -> None:
+ async def _delete_snapshot_row(self, chat_id: str, snapshot_id: str) -> None:
"""Delete the chat_sandbox_snapshots row (leaves the remote snapshot untouched)."""
def _delete() -> None:
@@ -102,7 +102,8 @@ def _delete() -> None:
ChatSandboxSnapshot = _get_snapshot_model()
with SessionLocal() as db:
db.query(ChatSandboxSnapshot).filter(
- ChatSandboxSnapshot.chat_id == chat_id
+ ChatSandboxSnapshot.chat_id == chat_id,
+ ChatSandboxSnapshot.snapshot_id == snapshot_id,
).delete()
db.commit()
@@ -213,6 +214,7 @@ async def _snapshot_idle_sessions_once(self) -> None:
The idle threshold starts snapshotting; server expiry includes a grace
period so that the scan and snapshot can finish before container removal.
"""
+ await self._discover_idle_sessions()
threshold = settings.sandbox.idle_ttl_s
now = time.monotonic()
candidates: list[tuple[str, _Session]] = []
@@ -233,9 +235,13 @@ async def _snapshot_idle_sessions_once(self) -> None:
candidates.append((chat_id, sess))
if len(candidates) >= self._PARK_BATCH_LIMIT:
break
- for chat_id, sess in candidates:
+ for chat_id, _ in candidates:
self._parking_in_flight.add(chat_id)
- asyncio.create_task(self._park_session_via_snapshot(chat_id, sess))
+ # Children belong to the maintenance task: losing leadership cancels
+ # and joins every in-flight snapshot before releasing ownership.
+ async with asyncio.TaskGroup() as group:
+ for chat_id, sess in candidates:
+ group.create_task(self._park_session_via_snapshot(chat_id, sess))
async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None:
"""take snapshot → wait Ready → upsert DB → release the session.
@@ -271,8 +277,7 @@ async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None
chat_id, settings.sandbox.idle_ttl_s
):
logger.info(
- "[opensandbox] park chat=%s skipped: still in use elsewhere",
- chat_id,
+ "[opensandbox] park chat=%s skipped: still in use elsewhere", chat_id,
)
return
try:
@@ -295,29 +300,23 @@ async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None
return
await self._wait_snapshot_ready(snap_id)
old_snap = await self._upsert_snapshot(
- chat_id,
- snapshot_id=snap_id,
- sandbox_id=sandbox_id,
+ chat_id, snapshot_id=snap_id, sandbox_id=sandbox_id,
)
# The actual release; the container goes back to the idle pool
await self._destroy_session(chat_id, sess, reuse_sandbox=True)
logger.info(
"[opensandbox] parked chat=%s snapshot=%s (replaced_old=%s)",
- chat_id,
- snap_id,
- old_snap or "none",
+ chat_id, snap_id, old_snap or "none",
)
# Clean up the old remote snapshot outside the lock (async)
if old_snap:
- asyncio.create_task(self._delete_snapshot(old_snap))
+ await self._delete_snapshot(old_snap)
except asyncio.CancelledError:
raise
except Exception as e: # noqa: BLE001
logger.warning(
"[opensandbox] park chat=%s sandbox=%s failed: %s",
- chat_id,
- sandbox_id,
- e,
+ chat_id, sandbox_id, e,
)
finally:
self._parking_in_flight.discard(chat_id)
@@ -383,7 +382,7 @@ async def _delete_one(snapshot_id: str) -> bool:
if not ok:
continue
try:
- await self._delete_snapshot_row(chat_id)
+ await self._delete_snapshot_row(chat_id, snapshot_id)
except Exception as e: # noqa: BLE001
logger.warning(
"[opensandbox] gc: delete db row chat=%s failed: %s",
@@ -417,14 +416,13 @@ async def _take_snapshot(self, sandbox_id: str) -> str:
data = resp.json()
snap_id = data.get("id") or data.get("snapshotId")
if not snap_id:
- raise SandboxError(f"create snapshot returned no id (sandbox={sandbox_id}): {data!r}")
+ raise SandboxError(
+ f"create snapshot returned no id (sandbox={sandbox_id}): {data!r}"
+ )
return str(snap_id)
async def _wait_snapshot_ready(
- self,
- snapshot_id: str,
- *,
- timeout_s: Optional[int] = None,
+ self, snapshot_id: str, *, timeout_s: Optional[int] = None,
) -> None:
"""Poll GET /v1/snapshots/{id} until state == 'Ready'.
diff --git a/src/backend/core/sandbox/providers/opensandbox/tracking.py b/src/backend/core/sandbox/providers/opensandbox/tracking.py
index 04e2005e..6fa04478 100644
--- a/src/backend/core/sandbox/providers/opensandbox/tracking.py
+++ b/src/backend/core/sandbox/providers/opensandbox/tracking.py
@@ -85,3 +85,34 @@ async def _confirm_sandbox_exists(self, sandbox_id):
raise SandboxNotFoundError(f"sandbox {sandbox_id} no longer exists")
if response.status_code >= 400:
raise SandboxError(f"sandbox metadata query failed: HTTP {response.status_code}")
+
+
+ async def _discover_idle_sessions(self):
+ """Maintenance owns shared bindings, regardless of which worker served them."""
+ import time
+ from core.sandbox.providers.opensandbox.volumes import _METADATA_USER_ID_KEY
+ for chat, sandbox_id, active_at in await session_registry.bindings():
+ if chat in self._sessions or time.time() - active_at < settings.sandbox.idle_ttl_s:
+ continue
+ lock = await self._get_session_lock(chat)
+ if lock.locked():
+ continue
+ try:
+ async with lock, session_registry.claim(chat):
+ if await session_registry.lookup(chat) != sandbox_id:
+ continue
+ response = await self._get_http_client().get(
+ f"{self._domain.rstrip('/')}/v1/sandboxes/{sandbox_id}"
+ )
+ if response.status_code == 404:
+ if response.json().get("code") == "DOCKER::SANDBOX_NOT_FOUND":
+ await session_registry.forget(chat, sandbox_id)
+ continue
+ response.raise_for_status()
+ user_id = (response.json().get("metadata") or {}).get(_METADATA_USER_ID_KEY)
+ session = await self._attach_session(sandbox_id, user_id)
+ if session is not None:
+ session.last_active_ts = time.monotonic() - max(0, time.time() - active_at)
+ self._sessions[chat] = session
+ except Exception as exc:
+ logger.warning("sandbox discovery deferred chat=%s: %s", chat, exc)
diff --git a/src/backend/core/sandbox/providers/opensandbox/user_pool.py b/src/backend/core/sandbox/providers/opensandbox/user_pool.py
index 4a2c469c..0c87798d 100644
--- a/src/backend/core/sandbox/providers/opensandbox/user_pool.py
+++ b/src/backend/core/sandbox/providers/opensandbox/user_pool.py
@@ -8,7 +8,7 @@
import logging
import time
from collections import deque
-from typing import Any, Optional
+from typing import Any, Callable, Optional
from core.sandbox.errors import SandboxError
@@ -149,9 +149,7 @@ async def release(self, user_id: str, sbx: Any, *, reuse: bool, destroy: bool =
logger.info(
"[jupyter-user-pool] release user=%s sandbox_id=%s -> idle "
"(idle_count=%d)",
- user_id,
- getattr(sbx, "id", "?"),
- len(q),
+ user_id, getattr(sbx, "id", "?"), len(q),
)
else:
should_destroy = sbx
diff --git a/src/backend/core/sandbox/providers/opensandbox/volumes.py b/src/backend/core/sandbox/providers/opensandbox/volumes.py
index a4cc3bf1..dab22752 100644
--- a/src/backend/core/sandbox/providers/opensandbox/volumes.py
+++ b/src/backend/core/sandbox/providers/opensandbox/volumes.py
@@ -2,55 +2,11 @@
from __future__ import annotations
-import logging
-import os
-import re
-from pathlib import Path
-from typing import Any, Optional
+from typing import Optional
from core.config.settings import settings
from core.sandbox.sessions.metadata import USER_ID_METADATA_KEY
from core.sandbox.workspace.identity import safe_user_id
-from extensions.paths import SKILLS_DIR
-
-logger = logging.getLogger(__name__)
-
-# Built-in skills enter the sandbox at /workspace/skills/ via a host bind mount, read-only.
-#
-# Note: an absolute host path is required — the OpenSandbox server creates sandbox
-# containers through the host docker daemon, and docker.sock sees the host's view.
-# HOST_REPO_PATH is injected by docker-compose (the .env variable of the same name);
-# when missing we fall back to the in-container path derived from Path(__file__) —
-# in dev the backend already bind-mounts `${HOST_REPO_PATH}:${HOST_REPO_PATH}`, so
-# paths inside and outside the container are identical and the derived result is
-# also a valid host path.
-_BUILTIN_SKILLS_HOST_PATH: Optional[str] = None
-
-
-def _resolve_builtin_skills_host_path() -> Optional[str]:
- """Compute the absolute host path of the built-in skills dir, for sandbox volumes."""
- global _BUILTIN_SKILLS_HOST_PATH
- if _BUILTIN_SKILLS_HOST_PATH is not None:
- return _BUILTIN_SKILLS_HOST_PATH or None # "" → None
- host_repo = os.environ.get("HOST_REPO_PATH", "").strip()
- if host_repo:
- candidate = f"{host_repo.rstrip('/')}/src/backend/extensions/skills"
- else:
- # In-container derived path: this file lives in src/backend/core/sandbox/providers/opensandbox/,
- # walk back up to src/backend then down into extensions/skills. In dev the backend
- # bind-mounts the whole repo via HOST_REPO_PATH, so the absolute path inside
- # the container equals the host path.
- candidate = str(SKILLS_DIR)
- if not Path(candidate).is_dir():
- logger.warning(
- "[opensandbox] built-in skills dir not found at %s; skill bind-mount disabled",
- candidate,
- )
- _BUILTIN_SKILLS_HOST_PATH = ""
- return None
- _BUILTIN_SKILLS_HOST_PATH = candidate
- return candidate
-
# Metadata tags used for adoption / cross-process pool reuse. Every sandbox created by
# this app carries them; after a backend restart _adopt_existing_sandboxes uses this tag
@@ -68,7 +24,7 @@ def _resolve_builtin_skills_host_path() -> Optional[str]:
_METADATA_OWNER_KEY = "hugagent-owner"
_METADATA_OWNER_VALUE = "backend"
_METADATA_POOL_KIND_KEY = "hugagent-pool-kind"
-# Plan F: jupyter sandboxes are bucketed by user_id. Every jupyter sandbox created by this
+# Persistent sandboxes are bucketed by user_id. Every jupyter sandbox created by this
# service carries this tag; after a backend restart _adopt_existing_sandboxes uses it to
# reclaim orphan sandboxes into the correct user's pool — cross-user reuse would let one
# user see another's myspace bind contents, a security hole that must be prevented.
@@ -110,46 +66,23 @@ def _make_skills_volumes(user_id: Optional[str] = None) -> list:
whoever asks next and must never carry someone's private files.
Read-only, and the bind is live, so newly imported skills appear at once.
- Without ``HOST_STORAGE_PATH`` fall back to the built-in source tree alone and
- warn; on failure return an empty list — the sandbox is created as usual, just
- without /workspace/skills/, and skill references hit a plain not-found.
+ HOST_STORAGE_PATH must identify the same storage root on the Docker host.
"""
- try:
- from opensandbox.models.sandboxes import Host, Volume
- except ImportError: # older SDKs lack these two — can only continue without mounting
- return []
+ from opensandbox.models.sandboxes import Host, Volume
+ from core.sandbox.errors import SandboxError
- host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/")
+ host_storage = settings.sandbox.opensandbox_host_storage_path.strip().rstrip("/")
if not host_storage:
- # No host storage → no unified dir to bind; mount the built-in source tree alone.
- legacy = _resolve_builtin_skills_host_path()
- logger.warning(
- "[opensandbox] HOST_STORAGE_PATH 未配置;/workspace/skills 仅挂内置源码树,"
- "DB 导入技能在沙盒内不可见"
- )
- if not legacy:
- return []
- return [
- Volume(
- name="skills",
- host=Host(path=legacy),
- mount_path="/workspace/skills",
- read_only=True,
- )
- ]
+ raise SandboxError("OpenSandbox skill mounts require HOST_STORAGE_PATH")
# get_sandbox_skills_dir() / get_user_skills_root() return the container-side
# dirs (the same storage volume as the host bind source) and guarantee they
# exist — OpenSandbox fails to bind a nonexistent source. Only their basenames
# are used to compose host paths, so the names are not written twice.
- from core.agent_skills.config import (
- SHARED_LINK_NAME,
- get_sandbox_skills_dir,
- get_user_skills_root,
- )
+ from core.capabilities.skills.config import SHARED_LINK_NAME, get_sandbox_skills_dir, get_user_skills_root
shared_host = f"{host_storage}/{get_sandbox_skills_dir().name}"
- from core.agent_skills.publication import prepare_skill_view
+ from core.capabilities.skills.publication import prepare_skill_view
view = prepare_skill_view(safe_user_id(user_id) or None)
if view is None:
diff --git a/src/backend/core/sandbox/providers/script_runner/provider.py b/src/backend/core/sandbox/providers/script_runner/provider.py
index db759f1a..0b317a17 100644
--- a/src/backend/core/sandbox/providers/script_runner/provider.py
+++ b/src/backend/core/sandbox/providers/script_runner/provider.py
@@ -7,6 +7,7 @@
import logging
import re
import time
+from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
@@ -25,7 +26,7 @@
SandboxAdminNotSupported,
SandboxInfo,
)
-from core.sandbox.providers.script_runner.auth import auth_headers
+from core.infra.local_runtime.runner_auth import auth_headers
from core.sandbox.workspace.streams import stream_to_file
logger = logging.getLogger(__name__)
@@ -61,11 +62,11 @@ def _base_url(self) -> str:
async def _execution_body(self, req: ProcessRequest) -> dict:
capability_view_key = None
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if capabilities_enabled() and req.capability_run_id:
- from core.capabilities import runtime
- from core.capabilities.errors import IntegrityFailed
+ if has_run_capabilities() and req.capability_run_id:
+ from core.harness.registry import capability_runtime as runtime
+ from core.harness.capability_errors import IntegrityFailed
pinned_view = await asyncio.to_thread(
runtime.view_for_execution,
@@ -163,8 +164,8 @@ async def _process_request(self, route: str, body: dict, wait_ms: int) -> dict:
payload = response.json()
capability = payload.pop("_capability", None)
if capability and capability.get("run_id"):
- from core.capabilities import runtime
- from core.capabilities.errors import IntegrityFailed
+ from core.harness.registry import capability_runtime as runtime
+ from core.harness.capability_errors import IntegrityFailed
try:
prepared = await asyncio.to_thread(
@@ -372,7 +373,6 @@ def admin_pool_stats(self) -> dict:
raise SandboxAdminNotSupported("script_runner 无连接池")
-_SKILL_VIEW_SYNCED: dict[str, tuple[float, int]] = {}
_SKILL_VIEW_TTL_S = 60.0
@@ -386,27 +386,39 @@ def _refresh_skill_view(user_id: Optional[str]) -> None:
an immediate refresh regardless of the TTL. Failure is never worth failing an
execution over.
"""
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
- from core.agent_skills.publication import prepare_skill_view
+ if not has_run_capabilities():
+ from core.capabilities.skills.publication import prepare_skill_view
prepare_skill_view(user_id)
return
uid = (user_id or "").strip()
if not uid:
return
- from core.capabilities.skills import view_generation
+ from core.harness.registry.local_publication import local_publication
now = time.monotonic()
- gen = view_generation()
- last_ts, last_gen = _SKILL_VIEW_SYNCED.get(uid, (0.0, -1))
+ gen = local_publication().view_generation()
+ last_ts, last_gen = process_state().SKILL_VIEW_SYNCED.get(uid, (0.0, -1))
if gen == last_gen and now - last_ts < _SKILL_VIEW_TTL_S:
return
- _SKILL_VIEW_SYNCED[uid] = (now, gen)
+ process_state().SKILL_VIEW_SYNCED[uid] = (now, gen)
try:
- from core.agent_skills.config import sync_user_skill_view
+ from core.capabilities.skills.config import sync_user_skill_view
sync_user_skill_view(uid)
except Exception as exc: # noqa: BLE001
logger.warning("[script_runner] 技能视图刷新失败 user=%s: %s", uid, exc)
+
+
+@dataclass
+class ProviderState:
+ SKILL_VIEW_SYNCED: dict[str, tuple[float, int]] = field(default_factory=lambda: {})
+
+
+_STATE = ProviderState()
+
+
+def process_state() -> ProviderState:
+ return _STATE
diff --git a/src/backend/core/sandbox/runner/native_group.py b/src/backend/core/sandbox/runner/native_group.py
index 7150477d..5f055d04 100644
--- a/src/backend/core/sandbox/runner/native_group.py
+++ b/src/backend/core/sandbox/runner/native_group.py
@@ -1,4 +1,5 @@
"""Linux owner process groups retain their unreaped leader until the whole group stops."""
+from __future__ import annotations
import asyncio
import os
@@ -7,6 +8,7 @@
import subprocess
import sys
import threading
+from dataclasses import dataclass, field
from pathlib import Path
@@ -170,13 +172,8 @@ def close(self):
self._namespace_fd = None
-_namespace_lock = threading.Lock()
-_namespace_prefix = None
-
-
def lifetime_command(cmd, trusted_prefix=()):
"""Preserve trusted existing confinement; otherwise add only PID lifetime isolation."""
- global _namespace_prefix
if (
trusted_prefix
and Path(trusted_prefix[0]).name == "bwrap"
@@ -184,8 +181,8 @@ def lifetime_command(cmd, trusted_prefix=()):
and "--die-with-parent" in trusted_prefix
):
return cmd, True
- with _namespace_lock:
- if _namespace_prefix is None:
+ with process_state().namespace_lock:
+ if process_state().namespace_prefix is None:
executable = shutil.which("bwrap")
prefix = (
[
@@ -216,6 +213,19 @@ def lifetime_command(cmd, trusted_prefix=()):
prefix = []
except (OSError, subprocess.TimeoutExpired):
prefix = []
- _namespace_prefix = prefix
- prefix = list(_namespace_prefix)
+ process_state().namespace_prefix = prefix
+ prefix = list(process_state().namespace_prefix)
return [*prefix, *cmd], bool(prefix)
+
+
+@dataclass
+class NativeGroupState:
+ namespace_lock: object = field(default_factory=lambda: threading.Lock())
+ namespace_prefix: object = None
+
+
+_STATE = NativeGroupState()
+
+
+def process_state() -> NativeGroupState:
+ return _STATE
diff --git a/src/backend/core/sandbox/runner/server.py b/src/backend/core/sandbox/runner/server.py
index 9d591c4f..515103f4 100644
--- a/src/backend/core/sandbox/runner/server.py
+++ b/src/backend/core/sandbox/runner/server.py
@@ -14,7 +14,6 @@
import os
import shutil
import signal
-import subprocess
import sys
import tempfile
from pathlib import Path
@@ -150,6 +149,7 @@ def _session_workspace(
"DOTNET_CLI_TELEMETRY_OPTOUT": "1", # disable dotnet telemetry
"DOTNET_NOLOGO": "1", # suppress dotnet startup banner
"DOTNET_EnableDiagnostics": "0", # stop dotnet from creating diagnostic pipes/core dump files
+ "OFFICECLI_RESIDENT_FLUSH": "each", # the resident dies with the command's process group; flush every edit
}
if os.name != "nt":
SAFE_ENV.update(
diff --git a/src/backend/core/sandbox/sessions/registry.py b/src/backend/core/sandbox/sessions/registry.py
index 0bdb12bf..53eb0e17 100644
--- a/src/backend/core/sandbox/sessions/registry.py
+++ b/src/backend/core/sandbox/sessions/registry.py
@@ -127,7 +127,9 @@ async def renew_lease() -> None:
while True:
await asyncio.sleep(min(_LOCK_RENEW_INTERVAL_S, ttl / 3))
try:
- retained = await state.hold(key, token, ttl=ttl)
+ retained = await asyncio.wait_for(
+ state.compare_exchange(key, token, token, ttl=ttl), timeout=ttl / 3
+ )
except Exception:
retained = False
if not retained:
@@ -312,7 +314,9 @@ async def renew_destruction():
while True:
await asyncio.sleep(min(_LOCK_RENEW_INTERVAL_S, ttl / 3))
try:
- retained = await state.hold(key, token, ttl=ttl)
+ retained = await asyncio.wait_for(
+ state.compare_exchange(key, token, token, ttl=ttl), timeout=ttl / 3
+ )
except Exception:
retained = False
if not retained:
@@ -378,3 +382,16 @@ async def creation_pending(token: str) -> bool:
return bool(await get_ephemeral_state().get(
_LOCK_KEY.format(session_id=f"creating:{token}")
))
+
+
+async def bindings():
+ """Snapshot shared bindings without extending activity or their lifetime."""
+ prefix = _KEY.split("{")[0]
+ state = get_ephemeral_state()
+ entries = []
+ for key in await state.keys(prefix):
+ session_id = key[len(prefix):]
+ entry = await _read(session_id)
+ if entry is not None:
+ entries.append((session_id, entry[0], entry[1]))
+ return entries
diff --git a/src/backend/core/sandbox/sessions/router.py b/src/backend/core/sandbox/sessions/router.py
index 2946b775..8de0dbbb 100644
--- a/src/backend/core/sandbox/sessions/router.py
+++ b/src/backend/core/sandbox/sessions/router.py
@@ -11,22 +11,17 @@ def __getattr__(self, name):
return getattr(self._ordinary, name)
def _for(self, session_id):
- from core.extensions import sandbox_route
+ from core.harness.registry import sandbox_route
module_provider = sandbox_route(str(session_id or ""))
if module_provider is not None:
return module_provider()
if str(session_id or "").startswith("eval_"):
- from core.config.settings import settings
- from core.sandbox.providers.evaluation.provider import get_evaluation_provider
-
- if settings.sandbox.provider != "opensandbox":
- raise SandboxError("Evaluation requires OpenSandbox; fallback is forbidden")
- return get_evaluation_provider()
+ raise SandboxError("Evaluation extension is not registered; fallback is forbidden")
return self._ordinary
async def _authorized_for(self, session_id, user_id):
- from core.extensions import sandbox_route
+ from core.harness.registry import sandbox_route
provider = self._for(session_id)
if (
@@ -45,7 +40,7 @@ async def _ready(self, provider, session_id, user_id):
if str(session_id or "").startswith("eval_"):
return
if provider.name != "opensandbox" and user_id and session_id:
- from core.plugins.resources.prewarm import schedule
+ from core.capabilities.plugins.resources.prewarm import schedule
sandbox_id = await provider.current_sandbox_id(session_id)
schedule(provider, session_id, user_id, sandbox_id)
@@ -110,7 +105,7 @@ async def close_session(self, session_id):
provider = self._for(session_id)
if not session_id or str(session_id).startswith("eval_") or provider.name == "opensandbox":
return await provider.close_session(session_id)
- from core.plugins.resources.prewarm import closing, stop
+ from core.capabilities.plugins.resources.prewarm import closing, stop
async with closing(session_id):
await stop(session_id)
diff --git a/src/backend/core/sandbox/workspace/container.py b/src/backend/core/sandbox/workspace/container.py
index 986bb047..b0a83550 100644
--- a/src/backend/core/sandbox/workspace/container.py
+++ b/src/backend/core/sandbox/workspace/container.py
@@ -5,7 +5,7 @@
import posixpath
from typing import Any, Optional
-from core.agent_skills.loader import get_skill_loader
+from core.capabilities.skills.loader_factory import get_skill_loader
def _is_traversal_or_bad(path: str) -> Optional[str]:
@@ -23,7 +23,7 @@ def validate_path(path: str, root: str) -> str | None:
return error
if ".." in path.replace(chr(92), "/").split("/"):
return f"path 不允许包含 ..: {path}"
- from core.services.edition_workspace import resolve_alias
+ from core.files.edition import resolve_alias
if resolve_alias(path) is not None or path == "/myspace" or path.startswith("/myspace/"):
return None
@@ -44,7 +44,7 @@ def workspace_directory(root: str, session_id: str | None) -> str:
def resolve_path(path: str, root: str, session_id: str | None, user_id=None) -> str:
- from core.services.edition_workspace import resolve_alias
+ from core.files.edition import resolve_alias
resolved = resolve_alias(path)
if resolved is not None:
@@ -98,7 +98,7 @@ def resolve_skill_file(file_path: str, loader: Any = None) -> str | None:
return candidate
return None
- from core.agent_skills.config import get_sandbox_skills_dir
+ from core.capabilities.skills.config import get_sandbox_skills_dir
cache_root = str(get_sandbox_skills_dir())
for skill_id, rel_path in reversed(candidates):
@@ -113,8 +113,9 @@ def resolve_skill_file(file_path: str, loader: Any = None) -> str | None:
candidate = os.path.join(skill_dir, rel_path)
if os.path.exists(candidate):
return candidate
- except Exception:
- pass
+ except Exception as fallback_error:
+ from core.infra.failures import record_best_effort_failure
+ record_best_effort_failure('core.sandbox.workspace.container:resolve_skill_file', fallback_error)
return None
@@ -156,7 +157,7 @@ def is_myspace_physical(physical_path: str, user_id: Optional[str], root: str) -
def bash_workspace_instructions(root: str, session_id: str | None) -> str:
_WS = root
- from core.services.edition_workspace import workspace_instructions
+ from core.files.edition import workspace_instructions
return (
workspace_instructions() + "\n"
diff --git a/src/backend/core/sandbox/workspace/credentials.py b/src/backend/core/sandbox/workspace/credentials.py
index 4cc87d5f..541f8515 100644
--- a/src/backend/core/sandbox/workspace/credentials.py
+++ b/src/backend/core/sandbox/workspace/credentials.py
@@ -44,166 +44,26 @@ def purge_credential_dir(root: Path) -> None:
logger.warning("[sandbox] purge credential file failed %s: %s", fp, exc)
-def dws_cache_dir(user_id: str) -> Path:
- """Backend-local dws (DingTalk CLI) credential persistence directory (one per user).
-
- Contains two subdirectories, bind-mounted into the sandbox DingTalk CLI's default
- ``$HOME`` paths:
- - ``dws/`` → sandbox ``/home/ubuntu/.dws`` (config + market cache + app.json)
- - ``share/`` → sandbox ``/home/ubuntu/.local/share/dws-cli`` (encrypted keychain:
- ``dek`` + ``auth-token.enc``)
-
- Empirically: dws's encrypted keychain and the ``.dws`` cache only respect ``$HOME``;
- ``DWS_CONFIG_DIR`` only relocates identity.json/logs, and ``XDG_DATA_HOME`` is
- ignored — so login must run with a per-user ``$HOME`` and these two home subpaths
- must be bind-mounted, rather than relying on env redirection. The sandbox runs as
- ubuntu (UID 1000) and the backend is also 1000, so ownership matches
- (see [[project_dingtalk_dws_integration]]).
- """
- return settings.storage.root / "dws_cache" / user_id
-
-
-def dws_home_dir(user_id: str) -> Path:
- """Per-user ``$HOME`` for dws login (the unified root for backend subprocesses +
- opensandbox bind sources).
-
- The backend runs ``dws auth login`` with this as ``HOME`` → credentials land in
- ``{home}/.dws`` and ``{home}/.local/share/dws-cli``; opensandbox bind-mounts these
- two subpaths into the same home paths inside the sandbox, so **that user's** session
- sandboxes (bucketed per user) see their own credentials. The root of multi-tenant
- isolation: each user gets an independent home directory, never shared.
- """
- return dws_cache_dir(user_id) / "home"
-
-
-def lark_cache_dir(user_id: str) -> Path:
- """Backend-local lark-cli (Feishu/Lark CLI) credential persistence directory (one per user).
-
- Contains two subdirectories, bind-mounted into the sandbox lark-cli's default
- ``$HOME`` paths:
- - ``home/.lark-cli/`` → sandbox ``/home/ubuntu/.lark-cli`` (config.json
- app config + cache + logs)
- - ``home/.local/share/lark-cli/`` → sandbox ``/home/ubuntu/.local/share/lark-cli``
- (file-based encrypted store: ``master.key`` +
- ``appsecret_*.enc`` + per-user ``cli_*_ou_*.enc``)
-
- Empirically: on Linux, lark-cli does **not** use the OS keychain but a file-based
- encrypted store — it only respects these two directories under ``$HOME``;
- ``master.key`` is a local DEK, so decryption works wherever the files travel.
- Isomorphic to dws's "Linux file-based DEK" scheme, so the per-user bind-mount
- persistence is copied verbatim (see [[dws_cache_dir]] and
- internal design docs).
- The sandbox runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches.
- """
- return settings.storage.root / "lark_cache" / user_id
-
-
-def lark_home_dir(user_id: str) -> Path:
- """Per-user ``$HOME`` for lark-cli login (the unified root for the backend
- config init / auth login subprocesses + opensandbox bind sources).
-
- The backend runs ``lark-cli config init`` (writes app config) and ``auth login``
- (device flow, writes user_access_token) with this as ``HOME`` → credentials land in
- ``{home}/.lark-cli`` and ``{home}/.local/share/lark-cli``; opensandbox bind-mounts
- these two subpaths into the same home paths inside the sandbox, so **that user's**
- session sandboxes see their own credentials. The root of multi-tenant isolation:
- each user gets an independent home directory, never shared.
- """
- return lark_cache_dir(user_id) / "home"
-def lark_app_home_dir() -> Path:
- """Org-wide shared Feishu **app** config HOME (the admin's one-click init
- ``config init --new`` lands here).
- Contains ``~/.lark-cli/config.json`` + ``~/.local/share/lark-cli/{master.key,
- appsecret_*.enc}`` (the app foundation — contains **no user tokens whatsoever**).
- These three items are copied into every user's ``lark_home_dir`` as the app
- foundation for their ``auth login`` — the whole org shares one app, and the admin
- configures it only once.
- """
- return settings.storage.root / "lark_cache" / "__app__" / "home"
-def email_cache_dir(user_id: str) -> Path:
- """Backend-local email (himalaya CLI) credential persistence directory (one per user).
- Contains one subdirectory, bind-mounted into the sandbox himalaya's default
- config path:
- - ``home/.config/himalaya/`` → sandbox ``/home/ubuntu/.config/himalaya``
- (``config.toml``: account blocks + IMAP/SMTP servers + auth-code raw lines, file mode 0600)
- Isomorphic to lark/dws's "Linux file-based credentials": himalaya config is plain
- files that only respect the config path, so the per-user bind-mount persistence is
- copied verbatim. No OS keychain, no OAuth token refresh — auth codes stay valid
- long-term (governed by the mail provider).
- See internal design docs.
- """
- return settings.storage.root / "email_cache" / user_id
-def email_home_dir(user_id: str) -> Path:
- """Per-user ``$HOME`` for himalaya login (the unified root for the backend
- verification subprocess + opensandbox bind sources).
- The backend runs ``himalaya folder list`` with this as ``HOME`` as a connectivity
- check; the config lands at ``{home}/.config/himalaya/config.toml``; opensandbox
- bind-mounts this subdirectory into the same home path inside the sandbox. The root
- of multi-tenant isolation: each user gets an independent home directory, never shared.
- """
- return email_cache_dir(user_id) / "home"
-def email_himalaya_config(user_id: str) -> Path:
- """Backend absolute path of this user's himalaya ``config.toml`` (under email_home_dir)."""
- return email_home_dir(user_id) / ".config" / "himalaya" / "config.toml"
-def yida_cache_dir(user_id: str) -> Path:
- """Backend-local openyida (Yida CLI) login-state persistence directory (one per user).
- Contains one subdirectory, bind-mounted into the sandbox's fixed Yida working directory:
- - ``workspace/`` → sandbox ``/home/ubuntu/yida-workspace`` (openyida's projectRoot:
- ``.cache/cookies-*.json`` login cookies + ``.cache/openyida-envs.json`` env config +
- project artifacts such as ``prd/`` and schema cache)
- Difference from dws/lark: openyida's login state does **not** respect $HOME; it lands
- under the "project root"'s ``.cache/`` (projectRoot = the cwd at command execution
- time, see the cwd fallback of findProjectRoot in openyida lib/core/utils.js). So the
- persistence scheme is not binding home subpaths but pinning each user a fixed
- in-sandbox working directory and binding it wholesale: the skill forces all openyida
- commands to run inside that directory, so cookies survive across sessions with the
- volume. Login is completed by QR scan inside the conversation
- (``openyida login --agent-qr``); the backend runs no openyida subprocess. The sandbox
- runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches.
- """
- return settings.storage.root / "yida_cache" / user_id
-def yida_workspace_dir(user_id: str) -> Path:
- """Backend absolute path of this user's fixed Yida working directory (opensandbox
- bind source / cube inject-and-return source).
-
- The sandbox mount point is fixed at ``/home/ubuntu/yida-workspace``; the yida skill
- requires all openyida commands to ``cd`` into that directory, so the ``.cache/``
- login state and project artifacts all land on the persistent volume.
- """
- return yida_cache_dir(user_id) / "workspace"
-def yida_shared_workspace_dir() -> Path:
- """Yida working directory for the script-runner shared sandbox (**one per deployment**,
- not per-user).
-
- script-runner is a shared container with no user concept (the provider ignores
- user_id; rootfs is read_only) and cannot do per-user credential isolation — compose
- mounts this directory at the container's ``/home/ubuntu/yida-workspace``, so the Yida
- login state is shared deployment-wide. Only suitable for single-user/dev environments;
- multi-user production should use opensandbox (per-user bind-mount) or cube (per-user
- inject + return). The backend pre-creates it with 0777 at startup (runner uid 1001 ≠
- backend uid 1000; cross-uid writes need wide-open permissions).
- """
- return settings.storage.root / "yida_cache" / "__shared__" / "workspace"
# openyida (Yida CLI) login-state persistence volume: unlike the "$HOME subpath" pattern of
@@ -212,4 +72,3 @@ def yida_shared_workspace_dir() -> Path:
# So each user gets a fixed working directory bind-mounted into the sandbox, and the yida
# skill forces all openyida commands to cd into that directory before running (see the
# _common.yida_cache_dir comment).
-_YIDA_WORKSPACE_MOUNT = "/home/ubuntu/yida-workspace"
diff --git a/src/backend/core/sandbox/workspace/desktop.py b/src/backend/core/sandbox/workspace/desktop.py
index 6f599435..5ab3991a 100644
--- a/src/backend/core/sandbox/workspace/desktop.py
+++ b/src/backend/core/sandbox/workspace/desktop.py
@@ -8,7 +8,7 @@
def workspace_directory(root: str, session_id: str | None, *, scope=None, validate=True) -> str:
if scope is not None and scope.is_local:
- from core.services.local_project_workspace import project_directory
+ from core.projects.local_workspace import project_directory
return project_directory(scope, validate=validate)
return session_root(root, session_id) if session_id else root
diff --git a/src/backend/core/sandbox/workspace/identity.py b/src/backend/core/sandbox/workspace/identity.py
index 9e83289c..975a604c 100644
--- a/src/backend/core/sandbox/workspace/identity.py
+++ b/src/backend/core/sandbox/workspace/identity.py
@@ -6,7 +6,7 @@
# 沙盒内运行用户代码的身份。与 backend 容器的 appuser 同为 UID 1000,这样 bind mount
# 进来的 myspace 目录两边同属主,沙盒写的文件 backend 能覆盖和删除。
-# 镜像里 /workspace、/home/ubuntu 也归这个 UID(见 docker/Dockerfile.opensandbox)。
+# 镜像里 /workspace、/home/ubuntu 也归这个 UID(见 docker/sandbox/Dockerfile.opensandbox)。
SANDBOX_RUN_UID = 1000
SANDBOX_RUN_GID = 1000
diff --git a/src/backend/core/llm/tools/_paths.py b/src/backend/core/sandbox/workspace/resolution.py
similarity index 88%
rename from src/backend/core/llm/tools/_paths.py
rename to src/backend/core/sandbox/workspace/resolution.py
index 9d261501..58ff4c3d 100644
--- a/src/backend/core/llm/tools/_paths.py
+++ b/src/backend/core/sandbox/workspace/resolution.py
@@ -8,12 +8,10 @@
def path_rules():
- from core.llm.evaluation_runtime import CURRENT_EVALUATION_SCOPE
-
- if CURRENT_EVALUATION_SCOPE.get() is not None:
- from core.llm import evaluation_paths
-
- return evaluation_paths
+ from core.harness.registry.evaluation import current_path_rules
+ restricted_paths = current_path_rules()
+ if restricted_paths is not None:
+ return restricted_paths
from core.config.local_mode import local_mode_enabled
if local_mode_enabled():
@@ -46,7 +44,7 @@ def to_physical_path(
if rules is desktop_paths:
return rules.resolve_path(path, WORKSPACE_ROOT, session_id, user_id, scope=scope)
- from core.services.edition_workspace import resolve_project_path
+ from core.files.edition import resolve_project_path
resolved = resolve_project_path(path, user_id, WORKSPACE_ROOT, scope)
if resolved is not None:
@@ -82,7 +80,7 @@ def space_metadata(physical_path: str, user_id: Optional[str]) -> dict:
if is_myspace_physical(physical_path, user_id):
kind = "personal"
else:
- from core.services.edition_workspace import is_organization_path
+ from core.files.edition import is_organization_path
kind = "team" if is_organization_path(None, user_id, physical_path) else "temporary"
return {"persistent": kind != "temporary", "space_type": kind}
diff --git a/src/backend/core/services/__init__.py b/src/backend/core/services/__init__.py
deleted file mode 100644
index bd408101..00000000
--- a/src/backend/core/services/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-"""Community-edition service exports."""
-
-from core.services.api_key_service import ApiKeyService
-from core.services.artifact_service import ArtifactService
-from core.services.catalog_service import CatalogService
-from core.services.chat_service import ChatService
-from core.services.kb_service import KBService
-from core.services.user_agent_service import UserAgentService
-from core.services.user_service import UserService
-
-__all__ = [
- "ApiKeyService",
- "ArtifactService",
- "CatalogService",
- "ChatService",
- "KBService",
- "UserAgentService",
- "UserService",
-]
diff --git a/src/backend/core/services/agent_market_categories.py b/src/backend/core/services/agent_market_categories.py
deleted file mode 100644
index 301bb298..00000000
--- a/src/backend/core/services/agent_market_categories.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""Sub-agent marketplace categories (single source of truth) + Cherry Studio group → category mapping.
-
-The sub-agent marketplace categories are **independent of the skill marketplace's 8
-categories** (role-positioning semantics fit agents better). The preset list, community
-listing applications, and admin review category changes may only take these 9 values;
-the frontend mirror is ``AGENT_MARKETPLACE_CATEGORIES`` in ``utils/constants.ts``.
-"""
-
-from __future__ import annotations
-
-from typing import List
-
-from core.infra.exceptions import BadRequestError
-
-# Sub-agent marketplace fixed categories (stable order, frontend mirror must match)
-AGENT_MARKETPLACE_CATEGORIES: List[str] = [
- "通用助手",
- "职场办公",
- "商业分析",
- "数据分析",
- "研发编程",
- "翻译写作",
- "创意设计",
- "政策法务",
- "教育科研",
-]
-
-DEFAULT_AGENT_CATEGORY = "通用助手"
-
-# Cherry Studio's group tags (dozens of them) → normalized mapping to this project's 9 categories.
-# Only used by scripts/import_cherry_agents.py during import; an unmatched group falls to 「通用助手」.
-CHERRY_GROUP_TO_CATEGORY = {
- # General
- "通用": "通用助手",
- "工具": "通用助手",
- "百科": "通用助手",
- # Workplace / office
- "职业": "职场办公",
- "办公": "职场办公",
- "管理": "职场办公",
- "咨询": "职场办公",
- # Business analysis
- "商业": "商业分析",
- "营销": "商业分析",
- "金融": "商业分析",
- "分析": "商业分析",
- "点评": "商业分析",
- # Data analysis (Cherry has no dedicated group; decided secondarily by keywords in the import script)
- # R&D / programming
- "编程": "研发编程",
- "科学": "研发编程",
- # Translation / writing
- "翻译": "翻译写作",
- "写作": "翻译写作",
- "文案": "翻译写作",
- "语言": "翻译写作",
- "学术": "翻译写作",
- # Creative design
- "创意": "创意设计",
- "设计": "创意设计",
- "艺术": "创意设计",
- "音乐": "创意设计",
- # Policy / legal
- "法律": "政策法务",
- # Education / research
- "教育": "教育科研",
-}
-
-
-def validate_category(category: str) -> str:
- """Validate and return a legal category; anything not in the fixed set 400s directly."""
- category = (category or "").strip()
- if category not in AGENT_MARKETPLACE_CATEGORIES:
- raise BadRequestError(
- message=f"请从固定分类中选择:{'、'.join(AGENT_MARKETPLACE_CATEGORIES)}"
- )
- return category
diff --git a/src/backend/core/services/agent_market_service.py b/src/backend/core/services/agent_market_service.py
deleted file mode 100644
index b7ae7260..00000000
--- a/src/backend/core/services/agent_market_service.py
+++ /dev/null
@@ -1,940 +0,0 @@
-"""Sub-Agent Marketplace service.
-
-The sub-agent marketplace replicates the skill marketplace's "user submits → admin reviews →
-everyone can install" loop, but the marketplace items are **sub-agents** (role prompt +
-capability bindings) rather than skill files:
-
-- Preset sub-agents: shipped with the repo, stored at ``extensions/agents/marketplace//agent.json``
- (pure data, no attachments). The initial content was curated and adapted from Cherry Studio.
-- Community listings (``agent_market_submissions``): users **submit** their self-built sub-agents
- for listing; once an admin approves, they appear in the marketplace with ``source=community``
- and anyone can install. A content snapshot of the source ``UserAgent`` is taken at submit time.
-
-Install semantics = **clone as a private sub-agent**: create a new ``UserAgent`` under the
-installer (``owner_type=user``; admin installs get ``owner_type=admin``, global), copying the
-prompt/welcome message/suggested questions/model config, and applying "install-on-demand" to
-capability bindings — MCP/default skills bind directly, marketplace skills/plugins are installed
-as the installer's private copies as needed and then bound, and unresolvable items are dropped.
-The clone records ``source_market_slug`` to mark it "installed".
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-import uuid
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-
-from core.db.models import AgentMarketSubmission, UserAgent
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.infra.time import utc_now
-from core.services import marketplace_listing as ml
-from core.services.agent_market_categories import (
- AGENT_MARKETPLACE_CATEGORIES,
- DEFAULT_AGENT_CATEGORY,
- validate_category,
-)
-from extensions.paths import AGENTS_DIR
-from fastapi import HTTPException
-from sqlalchemy.orm import Session
-from sqlalchemy.orm.attributes import flag_modified
-
-logger = logging.getLogger(__name__)
-
-# Preset sub-agent root directory: ``src/backend/extensions/agents/marketplace//agent.json``.
-AGENT_MARKET_DIR = AGENTS_DIR / "marketplace"
-AGENT_JSON = "agent.json"
-
-# Sentinel owner for admin direct uploads to the marketplace (distinguishes "admin upload" from real users' community submissions).
-ADMIN_UPLOAD_OWNER = "__admin_upload__"
-
-# The clone's description field matches the user-side creation form's constraint (≤20 chars); the full copy lives in system_prompt.
-_MAX_DESC = 20
-
-
-# ── Preset bundle reading ────────────────────────────────────────────────────
-
-
-def _read_bundle(slug: str) -> Optional[Dict[str, Any]]:
- """Read and validate a single preset sub-agent's ``agent.json`` (fault-tolerant; returns None if broken)."""
- if not slug or "/" in slug or ".." in slug:
- return None
- path = AGENT_MARKET_DIR / slug / AGENT_JSON
- if not path.is_file():
- return None
- try:
- b = json.loads(path.read_text(encoding="utf-8"))
- except Exception as exc: # noqa: BLE001
- logger.warning("agent market bundle broken: %s (%s)", slug, exc)
- return None
- if not isinstance(b, dict) or not b.get("name"):
- return None
- b.setdefault("slug", slug)
- return b
-
-
-def _normalize_binding_lists(raw: Optional[Dict[str, Any]]) -> Dict[str, List[str]]:
- """Normalize a bindings snapshot into the four id lists (missing ones become empty)."""
- b = raw or {}
- return {
- "skill_ids": list(b.get("skill_ids") or []),
- "mcp_server_ids": list(b.get("mcp_server_ids") or []),
- "plugin_ids": list(b.get("plugin_ids") or []),
- "kb_ids": list(b.get("kb_ids") or []),
- }
-
-
-def _bindings_of(d: Dict[str, Any]) -> Dict[str, List[str]]:
- """Get the four binding id lists from a preset bundle's ``bindings``."""
- return _normalize_binding_lists(d.get("bindings"))
-
-
-def _binding_counts(bindings: Dict[str, List[str]]) -> Dict[str, int]:
- return {
- "skill_count": len(bindings.get("skill_ids") or []),
- "mcp_count": len(bindings.get("mcp_server_ids") or []),
- "plugin_count": len(bindings.get("plugin_ids") or []),
- "kb_count": len(bindings.get("kb_ids") or []),
- }
-
-
-def _make_market_meta(
- *,
- slug: str,
- name: str,
- avatar: str,
- summary: str,
- description: str,
- category: str,
- tags: Any,
- version: str,
- author: str,
- source: str,
- featured: bool,
- deletable: bool,
- bindings: Dict[str, List[str]],
-) -> Dict[str, Any]:
- """Preset bundle / community listing → isomorphic public list metadata (without the system_prompt body)."""
- return {
- "slug": slug,
- "name": name or slug,
- "avatar": avatar or "",
- "summary": summary or "",
- "description": description or "",
- "category": category or DEFAULT_AGENT_CATEGORY,
- "tags": list(tags or []),
- "version": version or "1.0.0",
- "author": author,
- "source": source,
- "featured": featured,
- "deletable": deletable,
- **_binding_counts(bindings),
- }
-
-
-def _bundle_public_meta(b: Dict[str, Any]) -> Dict[str, Any]:
- # Presets (filesystem) ship with the repo; admins cannot delete them online (deletable=False).
- return _make_market_meta(
- slug=b.get("slug"),
- name=b.get("name"),
- avatar=b.get("avatar"),
- summary=b.get("summary"),
- description=b.get("description"),
- category=b.get("category"),
- tags=b.get("tags"),
- version=b.get("version"),
- author=b.get("author") or "内置",
- source="builtin",
- featured=bool(b.get("featured")),
- deletable=False,
- bindings=_bindings_of(b),
- )
-
-
-def _submission_public_meta(sub: AgentMarketSubmission) -> Dict[str, Any]:
- # DB listing records (admin upload / user community submission) → admins can delete online (deletable=True).
- return _make_market_meta(
- slug=sub.slug,
- name=sub.name,
- avatar=sub.avatar,
- summary=sub.summary,
- description=sub.description,
- category=sub.category,
- tags=sub.tags,
- version=sub.version,
- author=sub.submitter_name or sub.owner_user_id,
- source="community",
- featured=False,
- deletable=True,
- bindings=_normalize_binding_lists(sub.bindings_snapshot),
- )
-
-
-def _approved_submissions(db: Session) -> List[AgentMarketSubmission]:
- return (
- db.query(AgentMarketSubmission)
- .filter(AgentMarketSubmission.status == "approved")
- .order_by(AgentMarketSubmission.created_at.desc())
- .all()
- )
-
-
-def _get_approved_submission(db: Session, slug: str) -> Optional[AgentMarketSubmission]:
- return (
- db.query(AgentMarketSubmission)
- .filter(
- AgentMarketSubmission.slug == slug,
- AgentMarketSubmission.status == "approved",
- )
- .first()
- )
-
-
-# ── List / categories / detail ───────────────────────────────────────────────
-
-
-def list_marketplace_agents(
- db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None
-) -> List[Dict[str, Any]]:
- """List all marketplace sub-agents: presets (featured first) + community listings, layered with the enable/disable switch and visibility-scope filtering."""
- items: List[Dict[str, Any]] = []
- if AGENT_MARKET_DIR.is_dir():
- for child in sorted(AGENT_MARKET_DIR.iterdir()):
- if not child.is_dir():
- continue
- b = _read_bundle(child.name)
- if b:
- items.append(_bundle_public_meta(b))
- items.sort(key=lambda x: (0 if x["featured"] else 1, x["slug"]))
- items.extend(_submission_public_meta(s) for s in _approved_submissions(db))
- items = ml.annotate_and_filter(
- db,
- ml.KIND_AGENT,
- items,
- id_key="slug",
- include_disabled=include_disabled,
- viewer_user_id=viewer_user_id,
- )
- return items
-
-
-def categories_from_items(items: List[Dict[str, Any]]) -> List[str]:
- """Derive categories from an already-fetched marketplace list: the fixed 9 major categories first, legacy leftover categories appended after."""
- seen: List[str] = list(AGENT_MARKETPLACE_CATEGORIES)
- for it in items:
- if it["category"] not in seen:
- seen.append(it["category"])
- return seen
-
-
-def list_categories(
- db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None
-) -> List[str]:
- """Marketplace category list (standalone entry point; the list endpoint should reuse categories_from_items to avoid a second scan)."""
- return categories_from_items(
- list_marketplace_agents(
- db, include_disabled=include_disabled, viewer_user_id=viewer_user_id
- )
- )
-
-
-def get_agent_detail(db: Session, slug: str) -> Dict[str, Any]:
- """Detail of a single marketplace sub-agent: metadata + prompt body + capability bindings (for the detail preview).
-
- The preset directory takes priority; falls back to the community listing (approved submission) if not found.
- """
- b = _read_bundle(slug)
- if b:
- data = _bundle_public_meta(b)
- data["system_prompt"] = b.get("system_prompt") or ""
- data["welcome_message"] = b.get("welcome_message") or ""
- data["suggested_questions"] = list(b.get("suggested_questions") or [])
- data["bindings"] = _bindings_of(b)
- return data
- sub = _get_approved_submission(db, slug)
- if sub is None:
- raise ResourceNotFoundError("marketplace_agent", slug)
- data = _submission_public_meta(sub)
- data["system_prompt"] = sub.system_prompt or ""
- data["welcome_message"] = sub.welcome_message or ""
- data["suggested_questions"] = list(sub.suggested_questions or [])
- data["bindings"] = _normalize_binding_lists(sub.bindings_snapshot)
- return data
-
-
-def _installed_slugs(db: Session, owner_user_id: Optional[str], slugs: List[str]) -> set:
- """The subset of these marketplace slugs already installed (cloned) in the given scope (user / global admin)."""
- if not slugs:
- return set()
- q = db.query(UserAgent.source_market_slug).filter(UserAgent.source_market_slug.in_(slugs))
- if owner_user_id is None:
- q = q.filter(UserAgent.owner_type == "admin")
- else:
- q = q.filter(UserAgent.owner_type == "user", UserAgent.user_id == owner_user_id)
- return {row[0] for row in q.all()}
-
-
-def annotate_installed(
- items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str]
-) -> List[Dict[str, Any]]:
- """Add the ``installed`` flag to each marketplace list item (for the given scope)."""
- if not items:
- return items
- installed = _installed_slugs(db, owner_user_id, [it["slug"] for it in items])
- for it in items:
- it["installed"] = it["slug"] in installed
- return items
-
-
-def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool:
- return bool(_installed_slugs(db, owner_user_id, [slug]))
-
-
-# ── Install (clone + install-on-demand dependencies) ─────────────────────────
-
-
-def _resolve_market_entry(db: Session, slug: str) -> Dict[str, Any]:
- """Normalize a preset bundle / community listing into a unified clone-source dict."""
- b = _read_bundle(slug)
- if b:
- return {
- "name": b.get("name") or slug,
- "avatar": b.get("avatar") or "",
- "description": b.get("description") or "",
- "summary": b.get("summary") or "",
- "system_prompt": b.get("system_prompt") or "",
- "welcome_message": b.get("welcome_message") or "",
- "suggested_questions": list(b.get("suggested_questions") or []),
- "model_config": dict(b.get("model_config") or {}),
- "bindings": _bindings_of(b),
- "ontology_tags": [
- str(tag)
- for tag in (b.get("ontology_tags") or b.get("tags") or [])
- if str(tag).startswith("ontology:")
- ],
- }
- sub = _get_approved_submission(db, slug)
- if sub is None:
- raise ResourceNotFoundError("marketplace_agent", slug)
- return {
- "name": sub.name or slug,
- "avatar": sub.avatar or "",
- "description": sub.description or "",
- "summary": sub.summary or "",
- "system_prompt": sub.system_prompt or "",
- "welcome_message": sub.welcome_message or "",
- "suggested_questions": list(sub.suggested_questions or []),
- "model_config": dict(sub.model_config_snapshot or {}),
- "bindings": _normalize_binding_lists(sub.bindings_snapshot),
- "ontology_tags": [str(tag) for tag in (sub.tags or []) if str(tag).startswith("ontology:")],
- }
-
-
-def _resolve_bindings(
- db: Session,
- bindings: Dict[str, List[str]],
- owner_user_id: Optional[str],
- operator_name: Optional[str],
-) -> tuple[Dict[str, List[str]], Dict[str, List[str]]]:
- """Resolve each marketplace binding id into the installer's scope; returns (final bindings, report).
-
- - MCP / default skills / accessible KBs: bind directly if within ``available-resources``;
- - marketplace skill slugs: if not in the available list, install as the installer's private skill and bind the returned id;
- - plugin slugs: bind if already installed, otherwise install as private and then bind;
- - unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``.
- """
- from core.config.catalog import get_enabled_ids
- from core.plugins import management as plugin_service
- from core.services import marketplace_service as mk
- from core.services.user_agent_service import UserAgentService
-
- avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id)
- avail_skill = {s["id"] for s in avail.get("skills", [])}
- # available-resources only lists AdminMcpServer rows; but built-in MCP tools (e.g.
- # database_query) are catalog-defined and loaded at runtime by catalog id,
- # and don't necessarily have a same-named AdminMcpServer row. Merge the catalog-enabled
- # built-in MCP ids into the "resolvable" set, so these legitimate built-in tools aren't
- # misjudged as "unresolvable" and dropped (once bound, the runtime connects on demand).
- avail_mcp = {s["id"] for s in avail.get("mcp_servers", [])} | set(get_enabled_ids("mcp"))
- avail_plugin = {p["id"] for p in avail.get("plugins", [])}
- avail_kb = {k["id"] for k in avail.get("kb_spaces", [])}
-
- final: Dict[str, List[str]] = {
- "skill_ids": [],
- "mcp_server_ids": [],
- "plugin_ids": [],
- "kb_ids": [],
- }
- report: Dict[str, List[str]] = {
- "bound": [],
- "installed": [],
- "dropped": [],
- "needs_secret": [],
- }
-
- # MCP tools
- for mid in bindings.get("mcp_server_ids", []):
- if mid in avail_mcp:
- final["mcp_server_ids"].append(mid)
- report["bound"].append(f"mcp:{mid}")
- else:
- report["dropped"].append(f"mcp:{mid}")
-
- # Skills (default skills bind directly; marketplace skills get auto-installed as private)
- for sid in bindings.get("skill_ids", []):
- if sid in avail_skill:
- final["skill_ids"].append(sid)
- report["bound"].append(f"skill:{sid}")
- continue
- try:
- res = mk.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={})
- final["skill_ids"].append(res["id"])
- report["installed"].append(f"skill:{sid}")
- if mk.market_skill_requires_secrets(sid):
- report["needs_secret"].append(f"skill:{sid}")
- except Exception as exc: # noqa: BLE001
- logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc)
- report["dropped"].append(f"skill:{sid}")
-
- # Plugins (bindings store the slug; bind the install_id if installed, otherwise auto-install)
- for pslug in bindings.get("plugin_ids", []):
- gid = f"{pslug}@global"
- match = (
- gid
- if gid in avail_plugin
- else next((p for p in avail_plugin if p.startswith(f"{pslug}@")), None)
- )
- if match:
- final["plugin_ids"].append(match)
- report["bound"].append(f"plugin:{pslug}")
- continue
- try:
- res = plugin_service.install_plugin(
- db, pslug, owner_user_id=owner_user_id, secrets={}, created_by=operator_name
- )
- final["plugin_ids"].append(res["install_id"])
- report["installed"].append(f"plugin:{pslug}")
- if res.get("required_secrets") or res.get("requires_secret"):
- report["needs_secret"].append(f"plugin:{pslug}")
- except Exception as exc: # noqa: BLE001
- logger.info("agent clone: plugin binding unresolved %s (%s)", pslug, exc)
- report["dropped"].append(f"plugin:{pslug}")
-
- # Knowledge bases (keep only those the installer can access)
- for kid in bindings.get("kb_ids", []):
- if kid in avail_kb:
- final["kb_ids"].append(kid)
- report["bound"].append(f"kb:{kid}")
- else:
- report["dropped"].append(f"kb:{kid}")
-
- return final, report
-
-
-def install_marketplace_agent(
- db: Session,
- slug: str,
- *,
- owner_user_id: Optional[str],
- operator_name: Optional[str] = None,
-) -> Dict[str, Any]:
- """Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent)."""
- from core.services.user_agent_service import UserAgentService
-
- entry = _resolve_market_entry(db, slug)
- final_bindings, report = _resolve_bindings(db, entry["bindings"], owner_user_id, operator_name)
-
- mc = entry.get("model_config") or {}
- data: Dict[str, Any] = {
- "name": entry["name"],
- "avatar": entry.get("avatar") or None,
- "description": (entry.get("summary") or entry.get("description") or "")[:_MAX_DESC],
- "system_prompt": entry.get("system_prompt") or "",
- "welcome_message": entry.get("welcome_message") or "",
- "suggested_questions": list(entry.get("suggested_questions") or []),
- "skill_ids": final_bindings["skill_ids"],
- "mcp_server_ids": final_bindings["mcp_server_ids"],
- "plugin_ids": final_bindings["plugin_ids"],
- "kb_ids": final_bindings["kb_ids"],
- "source_market_slug": slug,
- "ontology_tags": list(entry.get("ontology_tags") or []),
- }
- if mc.get("temperature") is not None:
- data["temperature"] = mc["temperature"]
- if mc.get("max_tokens") is not None:
- data["max_tokens"] = mc["max_tokens"]
- if mc.get("max_iters") is not None:
- data["max_iters"] = mc["max_iters"]
- if mc.get("timeout") is not None:
- data["timeout"] = mc["timeout"]
-
- owner_type = "admin" if owner_user_id is None else "user"
- agent = UserAgentService(db).create(
- user_id=owner_user_id,
- operator_name=operator_name,
- owner_type=owner_type,
- data=data,
- )
- logger.info(
- "agent_market_install: slug=%s owner=%s agent=%s bound=%d installed=%d dropped=%d",
- slug,
- owner_user_id or "global",
- agent["agent_id"],
- len(report["bound"]),
- len(report["installed"]),
- len(report["dropped"]),
- )
- return {
- "agent_id": agent["agent_id"],
- "slug": slug,
- "owner": "self" if owner_user_id else "global",
- "install_report": report,
- "message": "子智能体已安装",
- }
-
-
-# ── Community listing: user submission + admin review ───────────────────────
-
-
-def _existing_slugs(db: Session) -> set:
- """Marketplace slugs currently taken: preset directory + all submissions (including rejected; slugs are table-wide unique)."""
- slugs = set()
- if AGENT_MARKET_DIR.is_dir():
- slugs.update(c.name for c in AGENT_MARKET_DIR.iterdir() if c.is_dir())
- slugs.update(r[0] for r in db.query(AgentMarketSubmission.slug).all())
- return slugs
-
-
-def _derive_slug(db: Session, name: str) -> str:
- """Derive the marketplace slug from the sub-agent name (ASCII-ized + unique); Chinese names fall back to agent-."""
- base = re.sub(r"[^a-z0-9_-]+", "-", (name or "").lower()).strip("-")
- if not base:
- base = f"agent-{uuid.uuid4().hex[:8]}"
- taken = _existing_slugs(db)
- slug = base
- n = 2
- while slug in taken:
- slug = f"{base}-{n}"
- n += 1
- return slug
-
-
-def _model_config_snapshot(agent: UserAgent) -> Dict[str, Any]:
- return {
- "temperature": float(agent.temperature) if agent.temperature is not None else None,
- "max_tokens": agent.max_tokens,
- "max_iters": agent.max_iters,
- "timeout": agent.timeout,
- }
-
-
-def _bindings_snapshot(agent: UserAgent) -> Dict[str, List[str]]:
- return {
- "skill_ids": list(agent.skill_ids or []),
- "mcp_server_ids": list(agent.mcp_server_ids or []),
- "plugin_ids": list(agent.plugin_ids or []),
- "kb_ids": list(agent.kb_ids or []),
- }
-
-
-def _submission_to_dict(
- sub: AgentMarketSubmission, *, with_content: bool = False
-) -> Dict[str, Any]:
- data = {
- "submission_id": sub.submission_id,
- "slug": sub.slug,
- "agent_id": sub.agent_id,
- "owner_user_id": sub.owner_user_id,
- "submitter_name": sub.submitter_name or "",
- "name": sub.name,
- "avatar": sub.avatar or "",
- "description": sub.description or "",
- "summary": sub.summary or "",
- "category": sub.category or DEFAULT_AGENT_CATEGORY,
- "tags": list(sub.tags or []),
- "version": sub.version or "1.0.0",
- "note": sub.note or "",
- "status": sub.status,
- "review_note": sub.review_note or "",
- "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None,
- "created_at": sub.created_at.isoformat() if sub.created_at else None,
- }
- if with_content:
- data["system_prompt"] = sub.system_prompt or ""
- data["welcome_message"] = sub.welcome_message or ""
- data["suggested_questions"] = list(sub.suggested_questions or [])
- data["bindings"] = _normalize_binding_lists(sub.bindings_snapshot)
- return data
-
-
-def _owned_agent(db: Session, agent_id: str, owner_user_id: str) -> UserAgent:
- """Fetch the sub-agent owned by this user (owner_type=user), otherwise 404."""
- agent = (
- db.query(UserAgent)
- .filter(
- UserAgent.agent_id == agent_id,
- UserAgent.owner_type == "user",
- UserAgent.user_id == owner_user_id,
- )
- .first()
- )
- if agent is None:
- raise ResourceNotFoundError("user_agent", agent_id)
- return agent
-
-
-def _new_submission(
- *,
- submission_id: str,
- slug: str,
- agent_id: str,
- owner_user_id: str,
- name: str,
- category: str,
- status: str,
- submitter_name: str = "",
- avatar: Optional[str] = None,
- description: str = "",
- summary: str = "",
- note: str = "",
- system_prompt: str = "",
- welcome_message: str = "",
- suggested_questions: Optional[List[str]] = None,
- model_config_snapshot: Optional[Dict[str, Any]] = None,
- bindings_snapshot: Optional[Dict[str, Any]] = None,
- tags: Optional[List[str]] = None,
- reviewed_at: Optional[datetime] = None,
-) -> AgentMarketSubmission:
- """Uniformly construct one ``AgentMarketSubmission`` (fixed fields and default normalization centralized here).
-
- The three entry points (user submission ``submit_to_marketplace`` / the create branch of the
- admin direct-upload ``publish_agent_to_market`` / admin create-and-list
- ``create_market_agent``) share the same field set; callers pass only what differs.
- """
- now = utc_now()
- return AgentMarketSubmission(
- submission_id=submission_id,
- slug=slug,
- agent_id=agent_id,
- owner_user_id=owner_user_id,
- submitter_name=(submitter_name or "").strip(),
- name=name,
- avatar=avatar,
- description=description or "",
- summary=summary or "",
- category=category,
- tags=list(tags or []),
- version="1.0.0",
- note=(note or "").strip(),
- system_prompt=system_prompt or "",
- welcome_message=welcome_message or "",
- suggested_questions=list(suggested_questions or []),
- model_config_snapshot=dict(model_config_snapshot or {}),
- bindings_snapshot=_normalize_binding_lists(bindings_snapshot),
- status=status,
- reviewed_at=reviewed_at,
- created_at=now,
- updated_at=now,
- )
-
-
-def submit_to_marketplace(
- db: Session,
- agent_id: str,
- *,
- owner_user_id: str,
- submitter_name: str = "",
- note: str = "",
- category: str = "",
- summary: str = "",
-) -> Dict[str, Any]:
- """A user submits their self-built sub-agent for marketplace listing (content snapshot, pending admin review)."""
- agent = _owned_agent(db, agent_id, owner_user_id)
-
- active = (
- db.query(AgentMarketSubmission)
- .filter(
- AgentMarketSubmission.agent_id == agent_id,
- AgentMarketSubmission.owner_user_id == owner_user_id,
- AgentMarketSubmission.status.in_(["pending", "approved"]),
- )
- .first()
- )
- if active is not None:
- state = "待审核" if active.status == "pending" else "已上架"
- raise HTTPException(status_code=409, detail=f"该子智能体已有{state}的上架申请")
-
- sub = _new_submission(
- submission_id=f"asub_{uuid.uuid4().hex[:16]}",
- slug=_derive_slug(db, agent.name),
- agent_id=agent_id,
- owner_user_id=owner_user_id,
- submitter_name=submitter_name,
- name=agent.name,
- avatar=agent.avatar,
- description=agent.description or "",
- summary=(summary or "").strip() or (agent.description or ""),
- category=validate_category(category),
- note=note,
- system_prompt=agent.system_prompt or "",
- welcome_message=agent.welcome_message or "",
- suggested_questions=agent.suggested_questions,
- model_config_snapshot=_model_config_snapshot(agent),
- bindings_snapshot=_bindings_snapshot(agent),
- tags=list((agent.extra_config or {}).get("ontology_tags") or []),
- status="pending",
- )
- db.add(sub)
- db.commit()
- logger.info(
- "agent_market_submission_created: id=%s agent=%s owner=%s slug=%s",
- sub.submission_id,
- agent_id,
- owner_user_id,
- sub.slug,
- )
- return _submission_to_dict(sub)
-
-
-def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
- rows = (
- db.query(AgentMarketSubmission)
- .filter(AgentMarketSubmission.owner_user_id == owner_user_id)
- .order_by(AgentMarketSubmission.created_at.desc())
- .all()
- )
- return [_submission_to_dict(r) for r in rows]
-
-
-def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
- """A user withdraws their own submission: pending/rejected can be deleted; approved requires an admin to delist."""
- sub = (
- db.query(AgentMarketSubmission)
- .filter(
- AgentMarketSubmission.submission_id == submission_id,
- AgentMarketSubmission.owner_user_id == owner_user_id,
- )
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("agent_market_submission", submission_id)
- if sub.status == "approved":
- raise BadRequestError(message="子智能体已上架,如需下架请联系管理员")
- db.delete(sub)
- db.commit()
-
-
-def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
- """Admin view: all listing submissions, filterable by status (pending first, newest → oldest)."""
- q = db.query(AgentMarketSubmission)
- if status:
- q = q.filter(AgentMarketSubmission.status == status)
- rows = q.order_by(AgentMarketSubmission.created_at.desc()).all()
- order = {"pending": 0, "approved": 1, "rejected": 2}
- rows.sort(key=lambda r: order.get(r.status, 3))
- return [_submission_to_dict(r) for r in rows]
-
-
-def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
- sub = (
- db.query(AgentMarketSubmission)
- .filter(AgentMarketSubmission.submission_id == submission_id)
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("agent_market_submission", submission_id)
- return _submission_to_dict(sub, with_content=True)
-
-
-def review_submission(
- db: Session,
- submission_id: str,
- *,
- approve: bool,
- review_note: str = "",
- category: str = "",
-) -> Dict[str, Any]:
- """Admin review: approval lists it immediately; rejecting pending = refuse, rejecting approved = delist."""
- sub = (
- db.query(AgentMarketSubmission)
- .filter(AgentMarketSubmission.submission_id == submission_id)
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("agent_market_submission", submission_id)
- if approve and sub.status != "pending":
- state = "已上架" if sub.status == "approved" else "已驳回"
- raise HTTPException(
- status_code=409,
- detail=f"该申请{state},不能重复审核通过(如需重新上架请让用户重新提交)",
- )
- if not approve and sub.status == "rejected":
- raise HTTPException(status_code=409, detail="该申请已是驳回状态")
- sub.status = "approved" if approve else "rejected"
- sub.review_note = (review_note or "").strip()
- if approve and (category or "").strip():
- sub.category = validate_category(category)
- sub.reviewed_at = utc_now()
- sub.updated_at = utc_now()
- db.commit()
- logger.info(
- "agent_market_submission_reviewed: id=%s slug=%s status=%s",
- sub.submission_id,
- sub.slug,
- sub.status,
- )
- return _submission_to_dict(sub)
-
-
-# ── Admin direct upload + delete + list/delist ───────────────────────────────
-
-
-def publish_agent_to_market(
- db: Session,
- agent_id: str,
- *,
- category: str,
- summary: str = "",
- submitter_name: str = "管理员上传",
-) -> Dict[str, Any]:
- """An admin lists an existing sub-agent directly on the marketplace as approved (no review).
-
- If the same agent_id already has an admin listing record, update it (keeping the original slug); otherwise create a new one.
- """
- category = validate_category(category)
- agent = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first()
- if agent is None:
- raise ResourceNotFoundError("user_agent", agent_id)
-
- now = utc_now()
- existing = (
- db.query(AgentMarketSubmission)
- .filter(
- AgentMarketSubmission.agent_id == agent_id,
- AgentMarketSubmission.owner_user_id == ADMIN_UPLOAD_OWNER,
- )
- .first()
- )
- summary = (summary or "").strip() or (agent.description or "")
- if existing is not None:
- existing.name = agent.name
- existing.avatar = agent.avatar
- existing.description = agent.description or ""
- existing.summary = summary
- existing.category = category
- existing.system_prompt = agent.system_prompt or ""
- existing.welcome_message = agent.welcome_message or ""
- existing.suggested_questions = list(agent.suggested_questions or [])
- existing.model_config_snapshot = _model_config_snapshot(agent)
- existing.bindings_snapshot = _bindings_snapshot(agent)
- existing.tags = list((agent.extra_config or {}).get("ontology_tags") or [])
- existing.status = "approved"
- existing.review_note = ""
- existing.reviewed_at = now
- existing.updated_at = now
- flag_modified(existing, "suggested_questions")
- flag_modified(existing, "model_config_snapshot")
- flag_modified(existing, "bindings_snapshot")
- flag_modified(existing, "tags")
- db.commit()
- return {"slug": existing.slug, "action": "updated", "message": "子智能体市场内容已更新"}
-
- sub = _new_submission(
- submission_id=f"aadm_{uuid.uuid4().hex[:16]}",
- slug=_derive_slug(db, agent.name),
- agent_id=agent_id,
- owner_user_id=ADMIN_UPLOAD_OWNER,
- submitter_name=submitter_name,
- name=agent.name,
- avatar=agent.avatar,
- description=agent.description or "",
- summary=summary,
- category=category,
- note="管理员上传,免审核直接上架",
- system_prompt=agent.system_prompt or "",
- welcome_message=agent.welcome_message or "",
- suggested_questions=agent.suggested_questions,
- model_config_snapshot=_model_config_snapshot(agent),
- bindings_snapshot=_bindings_snapshot(agent),
- tags=list((agent.extra_config or {}).get("ontology_tags") or []),
- status="approved",
- reviewed_at=now,
- )
- db.add(sub)
- db.commit()
- logger.info("agent_market_admin_published: agent=%s slug=%s", agent_id, sub.slug)
- return {"slug": sub.slug, "action": "published", "message": "子智能体已上架市场"}
-
-
-def create_market_agent(
- db: Session,
- *,
- name: str,
- category: str,
- avatar: str = "",
- description: str = "",
- summary: str = "",
- system_prompt: str = "",
- welcome_message: str = "",
- suggested_questions: Optional[List[str]] = None,
- bindings: Optional[Dict[str, Any]] = None,
- model_config: Optional[Dict[str, Any]] = None,
- tags: Optional[List[str]] = None,
- submitter_name: str = "管理员上传",
-) -> Dict[str, Any]:
- """An admin directly "creates" a sub-agent and lists it on the marketplace as approved (no global agent created).
-
- Mirrors the skill side's "creating a skill lists it on the skill marketplace": the new
- sub-agent enters the sub-agent marketplace (explicitly installable as a global / private
- clone) and does not take effect globally right away. Content comes from form fields rather
- than a source ``UserAgent``, hence a synthetic placeholder ``agent_id`` (installs always use
- the snapshot and never look back at a source agent).
- """
- now = utc_now()
- sub = _new_submission(
- submission_id=f"aadm_{uuid.uuid4().hex[:16]}",
- slug=_derive_slug(db, name),
- agent_id=f"__market__{uuid.uuid4().hex[:12]}",
- owner_user_id=ADMIN_UPLOAD_OWNER,
- submitter_name=submitter_name,
- name=name,
- avatar=avatar or "",
- description=description or "",
- summary=(summary or "").strip() or (description or ""),
- category=validate_category(category),
- note="管理员创建,直接上架",
- system_prompt=system_prompt or "",
- welcome_message=welcome_message or "",
- suggested_questions=suggested_questions,
- model_config_snapshot=model_config,
- bindings_snapshot=bindings,
- tags=tags,
- status="approved",
- reviewed_at=now,
- )
- db.add(sub)
- db.commit()
- logger.info("agent_market_admin_created: slug=%s name=%s", sub.slug, name)
- return {"slug": sub.slug, "action": "created", "message": "子智能体已上架子智能体市场"}
-
-
-def delete_market_agent(db: Session, slug: str) -> Dict[str, Any]:
- """An admin deletes one DB listing record from the marketplace. Presets ship with the repo and cannot be deleted online (400)."""
- if _read_bundle(slug) is not None:
- raise BadRequestError(message="预置子智能体随仓库发布,不可在线删除")
- sub = db.query(AgentMarketSubmission).filter(AgentMarketSubmission.slug == slug).first()
- if sub is None:
- raise ResourceNotFoundError("marketplace_agent", slug)
- db.delete(sub)
- db.commit()
- logger.info("agent_market_deleted: slug=%s", slug)
- return {"slug": slug, "deleted": True}
-
-
-def set_market_agent_enabled(
- db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
-) -> Dict[str, Any]:
- """List/delist a marketplace sub-agent (controls whether it shows in the marketplace; installed clones are unaffected)."""
- return ml.set_listing_enabled(db, ml.KIND_AGENT, slug, enabled, updated_by=updated_by)
diff --git a/src/backend/core/services/api_protocol_backfill.py b/src/backend/core/services/api_protocol_backfill.py
deleted file mode 100644
index bf171bbe..00000000
--- a/src/backend/core/services/api_protocol_backfill.py
+++ /dev/null
@@ -1,94 +0,0 @@
-"""给存量模型配置补上 ``extra_config.api_protocol``。
-
-新增/编辑模型时由 ``_autofill_api_protocol`` 在保存那刻探测写入;本模块补的是升级前
-就已存在、没有这个字段的行。运行期不探测(那是配置期行为,不能挂在请求路径上),所以
-启动时一次性补齐,补完落库就不再跑。探不出结论的什么都不写,下次启动再试。
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def _needs_probe(extra: dict) -> bool:
- """还没探过,或探过但用的是比现在宽松的老判据。
-
- 老判据只验证 ``/responses`` 路由存在,会把「路由在、却不消费工具结果」的上游判成
- 可用;那种上游一进智能体循环就卡死。判据变严后,旧结论必须重新验一遍,否则谁也不会
- 去纠正它。人工设定的值(没有 source 标记)永远不动。
- """
- from core.llm.providers.protocol_probe import PROBE_VERSION
-
- if extra.get("api_protocol_source") != "probe":
- return not extra.get("api_protocol")
- return int(extra.get("api_protocol_probe_version") or 1) < PROBE_VERSION
-
-
-async def backfill_missing_api_protocol() -> int:
- """探测所有还没记录协议(或结论已过期)的 chat 上游并写库,返回写入的条数。"""
- from core.db.engine import SessionLocal
- from core.db.model_repository import list_providers, update_provider
- from core.llm.providers.protocol_probe import PROBE_VERSION, detect_api_protocol
- from core.llm.providers.registry import get_spec
-
- with SessionLocal() as db:
- pending = []
- for provider in list_providers(db):
- if provider.provider_type != "chat" or not provider.is_active:
- continue
- extra = provider.extra_config or {}
- if not _needs_probe(extra):
- continue
- if not get_spec(getattr(provider, "provider", "") or "").speaks_responses:
- continue
- pending.append(
- (
- provider.provider_id,
- provider.model_name,
- provider.base_url,
- provider.api_key,
- dict(extra),
- )
- )
-
- if not pending:
- return 0
-
- results = await asyncio.gather(
- *(
- # 超时沿用 detect_api_protocol 自己的默认值:保存时探测与这里补齐是
- # 同一件事,给它第二个数只会让两条路径对同一个上游得出不同结论。
- detect_api_protocol(
- base_url=base_url,
- api_key=api_key,
- model_name=model_name,
- )
- for _, model_name, base_url, api_key, _ in pending
- ),
- return_exceptions=True,
- )
-
- filled = 0
- for (provider_id, model_name, _, _, extra), result in zip(pending, results):
- if isinstance(result, BaseException) or not getattr(result, "found", False):
- logger.info(
- "[startup] api_protocol 未能探测 %s:%s",
- model_name,
- result if isinstance(result, BaseException) else "; ".join(result.notes),
- )
- continue
- extra["api_protocol"] = result.protocol
- extra["api_protocol_source"] = "probe"
- extra["api_protocol_probe_version"] = PROBE_VERSION
- update_provider(db, provider_id, extra_config=extra)
- filled += 1
- logger.info(
- "[startup] api_protocol 已探明 %s -> %s(%s)",
- model_name,
- result.protocol,
- "; ".join(result.notes),
- )
- return filled
diff --git a/src/backend/core/services/artifact_edition.py b/src/backend/core/services/artifact_edition.py
deleted file mode 100644
index daef43ca..00000000
--- a/src/backend/core/services/artifact_edition.py
+++ /dev/null
@@ -1,66 +0,0 @@
-"""Community artifact lists are always personal."""
-
-from typing import Any, Optional
-
-
-def artifact_list_scope() -> str:
- return "personal"
-
-
-def extend_artifact_item(artifact, item: dict) -> dict:
- return item
-
-
-def personal_artifact_predicates(artifact_model: Any) -> list[Any]:
- return []
-
-
-def is_personal_artifact(artifact: Any) -> bool:
- return True
-
-
-def personal_artifact_create_fields() -> dict[str, None]:
- return {}
-
-
-def artifact_scope_fields(scope: Any) -> dict[str, Optional[str]]:
- return {
- "user_folder_id": (
- scope.root_folder_id or None if scope is not None and scope.is_personal else None
- )
- }
-
-
-def artifact_scope_folder_id(scope: Any, artifact: Any) -> Optional[str]:
- return artifact.user_folder_id
-
-
-def can_access_artifact(db: Any, user_id: str, artifact: Any) -> bool:
- return str(artifact.user_id) == str(user_id)
-
-
-def artifact_access_metadata(artifact: Any) -> dict[str, Optional[str]]:
- return {"owner_id": artifact.user_id}
-
-
-def can_access_artifact_metadata(db: Any, user_id: str, metadata: dict[str, Any]) -> bool:
- owner_id = metadata.get("owner_id") or metadata.get("user_id")
- return owner_id is None or str(owner_id) == str(user_id)
-
-
-__all__ = [
- "upload_target_matches",
- "artifact_access_metadata",
- "artifact_list_scope",
- "artifact_scope_fields",
- "artifact_scope_folder_id",
- "can_access_artifact",
- "can_access_artifact_metadata",
- "extend_artifact_item",
- "is_personal_artifact",
- "personal_artifact_create_fields",
- "personal_artifact_predicates",
-]
-
-def upload_target_matches(artifact: Any, scope: str, folder_id: Optional[str]) -> bool:
- return scope == "personal" and artifact.user_folder_id == folder_id
diff --git a/src/backend/core/services/automation_tool_routing.py b/src/backend/core/services/automation_tool_routing.py
deleted file mode 100644
index 8c281e70..00000000
--- a/src/backend/core/services/automation_tool_routing.py
+++ /dev/null
@@ -1,106 +0,0 @@
-"""Automation component execution across a desktop's two authorized backends."""
-from __future__ import annotations
-import asyncio
-import json
-
-TOOLS = {
- "create_scheduled_task": "create_task", "list_scheduled_tasks": "list_tasks",
- "get_scheduled_task": "get_task", "update_scheduled_task": "update_task",
- "pause_scheduled_task": "pause_task", "resume_scheduled_task": "resume_task",
- "delete_scheduled_task": "delete_task",
-}
-
-
-def creation_arguments(arguments, user_id, chat_id):
- from core.db.engine import SessionLocal
- from core.db.models import ChatSession, Project
- from core.auth.desktop_bridge import bridge_enabled
- from core.services.automation_execution import instance_location
-
- args = dict(arguments)
- with SessionLocal() as db:
- chat = db.get(ChatSession, chat_id) if chat_id else None
- if not args.get("project_id") and chat and chat.user_id == user_id and chat.deleted_at is None:
- if chat.project_id:
- args["project_id"] = chat.project_id
- project = db.get(Project, args["project_id"]) if args.get("project_id") else None
- if not args.get("execution_location"):
- args["execution_location"] = (
- "local" if project and project.kind == "local"
- else "cloud" if bridge_enabled() else instance_location()
- )
- return args
-
-
-async def local_automation_result(source_plugin, tool_name, arguments, headers, *, authorize=None):
- """Return None for cloud execution; local dispatch requires the automation component."""
- if source_plugin != "automation" or tool_name not in TOOLS:
- return None
- from core.services import desktop_cloud_bridge as bridge
- from core.db.engine import SessionLocal
- from core.db.models import ChatRun, ScheduledTask, ChatSession
- from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT
- from core.services.automation_execution import instance_location
-
- if instance_location() != "local":
- return None
- identity_state = bridge.get_state()
- bridge.ensure_current_authorization(identity_state)
- args = dict(arguments)
- # User identity belongs to the durable local run, not model arguments.
- effect = CURRENT_TOOL_EFFECT.get()
- with SessionLocal() as db:
- run = db.get(ChatRun, effect.run_id) if effect else None
- chat_id = next((v for k, v in headers.items() if k.lower() == "x-chat-id"), "")
- chat = db.get(ChatSession, chat_id) if chat_id else None
- user_id = run.user_id if run else chat.user_id if chat else None
- if not user_id:
- raise ValueError("无法确定本机任务的当前用户")
- from core.capabilities.skills import current_local_user_id
- # The same cloud account has a device-local shadow ID. Resolve its
- # cloud-address-scoped mapping before comparing local resource owners.
- if current_local_user_id() != user_id:
- raise ValueError("本机任务所属账号与当前云端账号不匹配")
- location = args.get("execution_location") or ""
- if location not in {"", "local", "cloud"}:
- raise ValueError("执行位置必须为本机或云端")
- if tool_name == "create_scheduled_task":
- args = creation_arguments(args, user_id, run.chat_id if run else chat_id)
- location = args["execution_location"]
- if location == "cloud" and args.get("project_id"):
- from core.db.models import Project
- project = db.get(Project, args["project_id"])
- if project is not None and project.kind == "local":
- raise ValueError("本机项目不能在云端执行,请选择本机")
- elif not location and args.get("task_ref"):
- local = db.get(ScheduledTask, args["task_ref"])
- if local and local.user_id == user_id:
- location = "local"
- elif db.query(ScheduledTask).filter(
- ScheduledTask.user_id == user_id,
- ScheduledTask.name.ilike(f"%{args['task_ref']}%"),
- ).first() is not None:
- raise ValueError("本机存在同名任务,请明确 execution_location 或使用任务 ID")
- merge_cloud = tool_name == "list_scheduled_tasks" and not location
- if merge_cloud:
- location = "local"
- if location != "local":
- return None
- if authorize is None:
- raise ValueError("本机工具缺少当前授权校验")
- await authorize()
- bridge.ensure_current_authorization(identity_state)
- if tool_name != "create_scheduled_task":
- args.pop("execution_location", None)
- from extensions.mcp_servers.automation_task_mcp import impl
- def dispatch():
- # Account changes must not race the queued local database operation.
- with bridge.account_scope(identity_state):
- return getattr(impl, TOOLS[tool_name])(user_id=user_id, **args)
-
- result = await asyncio.to_thread(dispatch)
- from agentscope.message import TextBlock, ToolResultState
- from agentscope.tool._response import ToolChunk
- return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))],
- state=ToolResultState.SUCCESS if result.get("ok") else ToolResultState.ERROR,
- metadata={"origin": "local", "execution_location": "local", "merge_cloud": merge_cloud})
diff --git a/src/backend/core/services/chat_service.py b/src/backend/core/services/chat_service.py
deleted file mode 100644
index 90775c5c..00000000
--- a/src/backend/core/services/chat_service.py
+++ /dev/null
@@ -1,1301 +0,0 @@
-"""Chat session and message business logic."""
-from core.infra.time import utc_now
-
-import hashlib
-import json
-import logging
-import uuid
-from dataclasses import dataclass, replace
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Tuple
-
-from core.auth.permissions_iface import (
- ChatAccessLevel,
- can_delete_session,
- resolve_chat_access,
-)
-from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession
-from core.db.repository import (
- AuditLogRepository,
- ChatMessageRepository,
- ChatSessionRepository,
-)
-from core.db.repository.chat import _strip_nul
-from core.db.reserved_chats import reserved_chat_owner
-from sqlalchemy import update
-from core.ontology.revision import is_substantive_revision, normalize_revision_candidate
-from sqlalchemy.orm import Session
-
-logger = logging.getLogger(__name__)
-
-_CONTENT_MAX = 100_000
-_TRUNC_NOTE = "\n\n…(本条回复过长,已截断保存;完整过程见工具调用记录)"
-# 思考的独立额度。content 有 CHECK 约束卡死 10 万字,thinking 是 JSON 列没有
-# 约束,但也不能无限长——跑飞的一轮能产出几十万字推理。
-_THINKING_MAX = 200_000
-
-
-def clamp_message_content(content: str) -> str:
- """Apply the database content limit independently of ChatService wiring."""
-
- text = content or ""
- if len(text) <= _CONTENT_MAX:
- return text
- keep = _CONTENT_MAX - len(_TRUNC_NOTE)
- logger.warning("[chat] message content truncated: %d -> %d chars", len(text), _CONTENT_MAX)
- kept = text[:keep]
- # 老格式的思考是以 … 内联在正文里的。截断点可能落在块中间,
- # 闭合标签一丢,历史重建就把整段思考当正文渲染出来(思考泄露到页面上)。
- close = " "
- if kept.count("") > kept.count(close):
- kept = kept[: keep - len(close)] + close
- return kept + _TRUNC_NOTE
-
-
-def clamp_thinking(
- blocks: Optional[List[Dict[str, Any]]],
- segments: Optional[List[Dict[str, Any]]] = None,
-) -> tuple[Optional[List[Dict[str, Any]]], Optional[List[Dict[str, Any]]]]:
- """思考单独存一列,有自己的额度,不再和正文抢 content 的 10 万字上限。
-
- 超额时从最早的块开始丢——最后几轮的推理离最终答案最近,对用户回看最有用。
-
- ``metadata.segments`` 按下标引用思考块,丢块会让下标错位,所以两者必须一起夹取:
- 返回夹取后的块列表,以及下标已重映射、引用了被丢块的段落已剔除的段落表。
- """
- if not blocks:
- return None, segments or None
- if sum(len(str(b.get("content") or "")) for b in blocks) <= _THINKING_MAX:
- return blocks, segments or None
- kept: List[Dict[str, Any]] = []
- kept_indices: List[int] = []
- budget = _THINKING_MAX
- for offset, block in enumerate(reversed(blocks)):
- text = str(block.get("content") or "")
- if not text:
- continue
- if len(text) > budget:
- text = text[len(text) - budget :]
- if not text:
- break
- budget -= len(text)
- kept.append({**block, "content": text})
- kept_indices.append(len(blocks) - 1 - offset)
- if budget <= 0:
- break
- kept.reverse()
- kept_indices.reverse()
- if not kept:
- return None, _drop_thinking_segments(segments, {})
- remap = {old: new for new, old in enumerate(kept_indices)}
- return kept, _drop_thinking_segments(segments, remap)
-
-
-def _drop_thinking_segments(
- segments: Optional[List[Dict[str, Any]]],
- remap: Dict[int, int],
-) -> Optional[List[Dict[str, Any]]]:
- """把段落表里的思考下标重映射到夹取后的位置,引用已丢块的段落直接剔除。"""
- if not segments:
- return segments or None
- rebuilt: List[Dict[str, Any]] = []
- for segment in segments:
- if segment.get("type") != "thinking":
- rebuilt.append(segment)
- continue
- new_index = remap.get(segment.get("index"))
- if new_index is None:
- continue
- rebuilt.append({**segment, "index": new_index})
- return rebuilt or None
-
-
-class CompactionCASConflict(RuntimeError):
- """The checkpoint base/lease is stale and another writer won."""
-
-
-@dataclass(frozen=True)
-class CompactionSourceRow:
- """Detached, immutable copy of one message in the compacted snapshot."""
-
- message_id: str
- chat_seq: int
- role: str
- content: str
- tool_calls: Any
- extra_data: Dict[str, Any]
-
-
-@dataclass(frozen=True)
-class CompactionSnapshot:
- """Immutable source boundary captured before the summary LLM call."""
-
- chat_id: str
- owner: str
- base_checkpoint_id: Optional[str]
- base_checkpoint_version: int
- base_covered_seq: int
- covered_seq: int
- source_hash: str
- replacement_history: tuple[Dict[str, Any], ...]
- source_rows: tuple[CompactionSourceRow, ...]
- source_message_ids: tuple[str, ...]
- source_message_seqs: tuple[int, ...]
-
-
-def _utcnow() -> datetime:
- return datetime.now(timezone.utc)
-
-
-def _as_utc(value: Optional[datetime]) -> Optional[datetime]:
- if value is None:
- return None
- if value.tzinfo is None:
- return value.replace(tzinfo=timezone.utc)
- return value.astimezone(timezone.utc)
-
-
-class ChatService:
- """Service for chat-related operations."""
-
- def __init__(self, db: Session):
- self.db = db
- self.session_repo = ChatSessionRepository(db)
- self.message_repo = ChatMessageRepository(db)
- self.audit_repo = AuditLogRepository(db)
-
- def create_session(
- self,
- user_id: str,
- title: str = "新对话",
- extra_data: Dict = None,
- chat_id: Optional[str] = None,
- project_id: Optional[str] = None,
- ) -> ChatSession:
- """Create a new chat session.
-
- If `chat_id` is provided and belongs to the same user, reuse it.
- If it belongs to another user, generate a new chat_id.
- """
- if chat_id:
- existing = self.session_repo.get_by_id(chat_id)
- if existing:
- if existing.user_id == user_id:
- return existing
- chat_id = None
-
- session_data = {
- "chat_id": chat_id or f"chat_{uuid.uuid4().hex[:16]}",
- "user_id": user_id,
- "title": title,
- "extra_data": extra_data or {},
- "project_id": project_id,
- }
- session = self.session_repo.create(session_data)
-
- # Audit log
- self.audit_repo.create(
- {
- "user_id": user_id,
- "action": "chat.session.created",
- "resource_type": "chat_session",
- "resource_id": session.chat_id,
- "status": "success",
- }
- )
-
- return session
-
- def ensure_session(
- self,
- chat_id: str,
- user_id: str,
- title: str = "新对话",
- extra_data: Optional[Dict] = None,
- project_id: Optional[str] = None,
- ) -> Optional[ChatSession]:
- """Ensure a chat session exists for user and chat_id.
-
- When ``project_id`` is given, attach the session to that project (first write wins; if
- already attached to a different project it stays unchanged — in-project chats never drift
- across projects). All session-creation entry points share this attachment rule.
- """
- existing = self.session_repo.get_by_id(chat_id)
- if existing:
- if existing.user_id != user_id:
- return None
- # Merge any missing metadata flags into existing session
- if extra_data:
- merged = dict(existing.extra_data or {})
- changed = False
- for k, v in extra_data.items():
- if k not in merged:
- merged[k] = v
- changed = True
- if changed:
- existing.extra_data = merged
- self.db.commit()
- session = existing
- else:
- session = self.create_session(
- user_id=user_id,
- title=title,
- extra_data=extra_data or {},
- chat_id=chat_id,
- project_id=project_id,
- )
- if session is not None and project_id and not session.project_id:
- session.project_id = project_id
- self.db.commit()
- return session
-
- def list_sessions(
- self,
- user_id: str,
- page: int = 1,
- page_size: int = 20,
- pinned_only: bool = False,
- favorite_only: bool = False,
- exclude_automation: bool = False,
- ) -> Tuple[List[ChatSession], int, int]:
- """List chat sessions with pagination."""
- sessions, total = self.session_repo.list_by_user(
- user_id,
- page,
- page_size,
- pinned_only,
- favorite_only,
- exclude_automation=exclude_automation,
- )
-
- total_pages = (total + page_size - 1) // page_size
-
- return sessions, total, total_pages
-
- def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]:
- """Get chat session with ownership check.
-
- Historical semantics unchanged: only the owner can get the session. For
- edition-specific sharing, use :py:meth:`get_session_with_access`.
- """
- # Module-owned chats are reachable only through their module, never as ordinary chats.
- if reserved_chat_owner(chat_id):
- return None
- session = self.session_repo.get_by_id(chat_id)
-
- if session and session.user_id != user_id:
- # Access denied - user doesn't own this session
- return None
-
- return session
-
- def get_session_with_access(
- self, chat_id: str, user_id: str
- ) -> Optional[Tuple[ChatSession, ChatAccessLevel]]:
- """Get a session in a sharing context + compute the access level.
-
- - Does not exist / soft-deleted → ``None``
- - No access (``resolve_chat_access`` returns ``'none'``) → ``None``
- - Module-owned (reserved namespace) → ``None``: not an ordinary chat
- - Otherwise returns ``(session, 'admin'|'edit'|'read')``
- """
- if reserved_chat_owner(chat_id):
- return None
- session = self.session_repo.get_by_id(chat_id)
- if session is None:
- return None
- level = resolve_chat_access(self.db, user_id, session)
- if level == "none":
- return None
- return session, level
-
- def update_session_fields(
- self,
- chat_id: str,
- fields: Dict[str, Any],
- *,
- actor_user_id: Optional[str] = None,
- ) -> Optional[ChatSession]:
- """Field update without ownership check (for sharing contexts).
-
- The caller is responsible for permission checks beforehand. ``actor_user_id`` is used
- only for the audit log. The ``extra_data`` field is merged rather than overwritten.
- """
- session = self.session_repo.get_by_id(chat_id)
- if session is None:
- return None
- normalized = dict(fields)
- extra_patch = normalized.get("extra_data")
- if isinstance(extra_patch, dict):
- merged = dict(session.extra_data or {})
- merged.update(extra_patch)
- normalized["extra_data"] = merged
- updated = self.session_repo.update(chat_id, normalized)
- self.audit_repo.create(
- {
- "user_id": actor_user_id or session.user_id,
- "action": "chat.session.updated",
- "resource_type": "chat_session",
- "resource_id": chat_id,
- "details": normalized,
- "status": "success",
- }
- )
- return updated
-
- def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool:
- """Forced delete in a sharing context (no ownership check). Caller handles permissions."""
- session = self.session_repo.get_by_id(chat_id)
- if session is None:
- return False
- result = self.session_repo.soft_delete(chat_id)
- if result:
- self.audit_repo.create(
- {
- "user_id": actor_user_id,
- "action": "chat.session.deleted",
- "resource_type": "chat_session",
- "resource_id": chat_id,
- "details": {
- "owner_user_id": session.user_id,
- "deleted_by_owner": session.user_id == actor_user_id,
- },
- "status": "success",
- }
- )
- return result
-
- def update_session(
- self, chat_id: str, user_id: str, update_data: Dict[str, Any]
- ) -> Optional[ChatSession]:
- """Update chat session."""
- session = self.get_session(chat_id, user_id)
- if not session:
- return None
-
- normalized_update_data = dict(update_data)
- extra_data_patch = normalized_update_data.get("extra_data")
- if isinstance(extra_data_patch, dict):
- merged_extra_data = dict(session.extra_data or {})
- merged_extra_data.update(extra_data_patch)
- normalized_update_data["extra_data"] = merged_extra_data
-
- updated_session = self.session_repo.update(chat_id, normalized_update_data)
-
- # Audit log
- self.audit_repo.create(
- {
- "user_id": user_id,
- "action": "chat.session.updated",
- "resource_type": "chat_session",
- "resource_id": chat_id,
- "details": normalized_update_data,
- "status": "success",
- }
- )
-
- return updated_session
-
- def delete_session(self, chat_id: str, user_id: str) -> bool:
- """Delete chat session (soft delete)."""
- session = self.get_session(chat_id, user_id)
- if not session:
- return False
-
- result = self.session_repo.soft_delete(chat_id)
-
- if result:
- # Audit log
- self.audit_repo.create(
- {
- "user_id": user_id,
- "action": "chat.session.deleted",
- "resource_type": "chat_session",
- "resource_id": chat_id,
- "status": "success",
- }
- )
-
- return result
-
- # chat_messages.content 有 CHECK 约束 char_length <= 100000。长任务的一轮回复
- # (反复重试 + 大量思考正文)真的会撞上,而撞上的后果是**整条 INSERT 抛
- # CheckViolation → 整个 run failed**,一轮的产出全部丢失(实测踩过)。
- # 与其让约束把一轮工作全毁掉,不如夹逼后落库并明确标注被截断。
- def add_message(
- self,
- chat_id: str,
- role: str,
- content: str,
- model: Optional[str] = None,
- thinking: Optional[List[Dict[str, Any]]] = None,
- tool_calls: Optional[List[Dict]] = None,
- usage: Optional[Dict] = None,
- error: Optional[Dict] = None,
- extra_data: Dict = None,
- message_id: Optional[str] = None,
- commit: bool = True,
- chat_seq: Optional[int] = None,
- model_steps: Optional[List[Dict[str, Any]]] = None,
- ) -> ChatMessage:
- """Add a message to a chat session."""
- extra = dict(extra_data or {})
- kept_thinking, kept_segments = clamp_thinking(thinking, extra.get("segments"))
- if "segments" in extra:
- extra["segments"] = kept_segments
- message_data = {
- "message_id": message_id or f"msg_{uuid.uuid4().hex[:16]}",
- "chat_id": chat_id,
- "chat_seq": chat_seq,
- "role": role,
- "content": clamp_message_content(content),
- "model": model,
- "thinking": kept_thinking,
- "tool_calls": tool_calls,
- "model_steps": model_steps,
- "usage": usage,
- "error": error,
- "extra_data": extra,
- }
-
- message = self.message_repo.create(message_data, commit=commit)
-
- # Keep session metadata in sync for list APIs.
- session = self.session_repo.get_by_id(chat_id)
- if session:
- session.message_count = (session.message_count or 0) + 1
- now = utc_now()
- session.updated_at = now
- session.last_message_at = now
- if commit:
- self.db.commit()
-
- return message
-
- def refresh_streaming_message(
- self,
- *,
- run_id: str,
- owner: str,
- message_id: str,
- content: str,
- thinking: Optional[List[Dict[str, Any]]],
- tool_calls: Optional[List[Dict]],
- extra_data: Dict[str, Any],
- model_steps: Optional[List[Dict[str, Any]]] = None,
- ) -> bool:
- """Overwrite the in-flight assistant row while its run streams.
-
- The run row lock serializes this write with its final commit, so a
- superseded or finishing worker cannot clobber the final message. Nothing else is touched — in particular no session
- timestamps, so pollers do not mistake a checkpoint for a new message.
- Returns False when the caller no longer owns the run.
- """
- # Lock the same run row as journal.complete before touching its message.
- # An EXISTS predicate alone can see a live lease, wait for the message
- # lock, and then overwrite a final commit using that stale MVCC view.
- live_run = (self.db.query(ChatRun)
- .filter(ChatRun.run_id == run_id, ChatRun.lease_owner == owner,
- ChatRun.status.in_(("pending", "running")))
- .with_for_update().first())
- if live_run is None:
- return False
- extra = dict(extra_data)
- kept_thinking, kept_segments = clamp_thinking(thinking, extra.get("segments"))
- if "segments" in extra:
- extra["segments"] = kept_segments
- result = self.db.execute(
- update(ChatMessage)
- .where(ChatMessage.message_id == message_id)
- .values(
- content=clamp_message_content(content),
- thinking=_strip_nul(kept_thinking),
- tool_calls=_strip_nul(tool_calls),
- model_steps=_strip_nul(model_steps),
- extra_data=_strip_nul(extra),
- )
- )
- return result.rowcount > 0
-
- def upsert_message(
- self,
- chat_id: str,
- role: str,
- content: str,
- *,
- message_id: str,
- model: Optional[str] = None,
- thinking: Optional[List[Dict[str, Any]]] = None,
- tool_calls: Optional[List[Dict]] = None,
- usage: Optional[Dict] = None,
- error: Optional[Dict] = None,
- extra_data: Dict = None,
- commit: bool = True,
- chat_seq: Optional[int] = None,
- model_steps: Optional[List[Dict[str, Any]]] = None,
- invalidate_mid_turn_checkpoint: bool = False,
- ) -> ChatMessage:
- """Idempotently upsert a message by message_id: overwrite in place if it exists, otherwise create.
-
- The autonomous loop uses this for "incrementally refreshing the same assistant message as
- progress advances" — each requirement processed / each evaluation round flushes the
- current accumulated body + tool cards into this message, so progress is visible in the DB
- even after a mid-run crash/refresh, instead of only being written once at the terminal
- state. The update path does not re-increment message_count (only +1 on creation).
- """
- existing = self.message_repo.get_by_id(message_id)
- if existing is not None:
- update: Dict[str, Any] = {"content": clamp_message_content(content)}
- extra = dict(extra_data) if extra_data is not None else None
- kept_thinking, kept_segments = clamp_thinking(
- thinking, (extra or {}).get("segments")
- )
- if model is not None:
- update["model"] = model
- if thinking is not None:
- update["thinking"] = kept_thinking
- if tool_calls is not None:
- update["tool_calls"] = tool_calls
- if model_steps is not None:
- update["model_steps"] = model_steps
- if usage is not None:
- update["usage"] = usage
- if error is not None:
- update["error"] = error
- if extra is not None:
- if "segments" in extra:
- extra["segments"] = kept_segments
- update["extra_data"] = extra
- if invalidate_mid_turn_checkpoint:
- # The completed turn has already crossed the post-turn trigger.
- # Its new steps cannot be safely sliced from a row covered by a
- # mid-turn checkpoint, so rebuild that successor from originals.
- # Match checkpoint publisher lock order: session, then lineage.
- self.db.query(ChatSession).filter(ChatSession.chat_id == chat_id).with_for_update().first()
- state = (self.db.query(ChatCompactionState)
- .filter(ChatCompactionState.chat_id == chat_id)
- .with_for_update().populate_existing().first())
- if (state is not None and state.active_checkpoint_id
- and int(existing.chat_seq or 0) <= int(state.covered_seq or 0)):
- checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id)
- manifest = ((checkpoint.extra_data or {}).get("replacement_manifest") or {}) if checkpoint else {}
- captured = (extra or {}).get("context_usage", {}).get("captured_at_ms", 0)
- checkpoint_ms = int(_as_utc(checkpoint.created_at).timestamp() * 1000) if checkpoint else 0
- if (manifest.get("phase") == "mid_turn" and type(captured) is int
- and captured > checkpoint_ms):
- self._invalidate_compaction_lineage_for_rewrite(
- chat_id, rewritten_seq=int(existing.chat_seq or 0)
- )
- msg = self.message_repo.update(message_id, update, commit=commit)
- session = self.session_repo.get_by_id(chat_id)
- if session:
- now = utc_now()
- session.updated_at = now
- session.last_message_at = now
- if commit:
- self.db.commit()
- return msg or existing
- return self.add_message(
- chat_id=chat_id,
- role=role,
- content=content,
- model=model,
- thinking=thinking,
- tool_calls=tool_calls,
- usage=usage,
- error=error,
- extra_data=extra_data,
- message_id=message_id,
- commit=commit,
- chat_seq=chat_seq,
- model_steps=model_steps,
- )
-
- def list_all_messages(
- self, chat_id: str, user_id: str
- ) -> Optional[List[ChatMessage]]:
- """List all messages in chronological order with access check.
-
- Edition-specific sharing policies may also grant read access.
-
- **Excludes** compaction checkpoint rows (the only writer of role='system' in
- chat_messages is add_compaction_checkpoint) — internal artifacts, invisible to all
- downstream consumers (replay/export/sharing/title/memory), filtered at the SQL layer
- (a checkpoint's extra_data carries the entire replacement_history; loading it just to
- discard is pure waste). The replay layer (compaction_service) fetches them separately
- via get_latest_compaction_checkpoint.
- """
- pair = self.get_session_with_access(chat_id, user_id)
- if pair is None:
- return None
-
- return (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.role != "system",
- )
- .order_by(ChatMessage.chat_seq)
- .all()
- )
-
- def list_messages(
- self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50
- ) -> Optional[Tuple[List[ChatMessage], int, int]]:
- """List messages in a chat session."""
- # Check ownership
- session = self.get_session(chat_id, user_id)
- if not session:
- return None
-
- messages, total = self.message_repo.list_by_chat(chat_id, page, page_size)
- total_pages = (total + page_size - 1) // page_size
-
- return messages, total, total_pages
-
- def delete_messages_from(self, chat_id: str, message_id: str) -> int:
- """Delete a message and all subsequent messages in the chat.
-
- History rewrites invalidate the entire compaction lineage in the same
- transaction. This fences an in-flight summarizer through the lineage
- version CAS and forces the next replay/compaction to rebuild from raw
- retained messages instead of trusting a stale covered watermark.
-
- Returns the number of messages deleted.
- """
- self._ensure_message_sequences(chat_id)
- target = (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.message_id == message_id,
- )
- .first()
- )
- if not target:
- return 0
-
- self._invalidate_compaction_lineage_for_rewrite(
- chat_id, rewritten_seq=int(target.chat_seq)
- )
-
- deleted = (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.chat_seq >= int(target.chat_seq),
- )
- .delete(synchronize_session="fetch")
- )
-
- # Update session message count
- session = self.session_repo.get_by_id(chat_id)
- if session:
- remaining = (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- )
- .count()
- )
- session.message_count = remaining
- session.updated_at = utc_now()
-
- self.db.commit()
- return deleted
-
- def _invalidate_compaction_lineage_for_rewrite(
- self,
- chat_id: str,
- *,
- rewritten_seq: int,
- ) -> bool:
- """Fence a snapshot that may contain a rewritten/deleted message."""
- state = self._ensure_compaction_state(chat_id)
- # Lock the lineage row where supported. SQLite serializes the following
- # write transaction; PostgreSQL blocks a concurrent publisher here.
- state = (
- self.db.query(ChatCompactionState)
- .filter(ChatCompactionState.chat_id == chat_id)
- .with_for_update()
- .populate_existing()
- .one()
- )
- affects_checkpoint = bool(
- state.active_checkpoint_id
- and int(rewritten_seq) <= int(state.covered_seq or 0)
- )
- affects_snapshot = bool(state.lease_owner)
- if not affects_checkpoint and not affects_snapshot:
- return False
-
- state.active_checkpoint_id = None
- state.checkpoint_version = int(state.checkpoint_version or 0) + 1
- state.covered_seq = 0
- state.lease_owner = None
- state.lease_expires_at = None
- state.updated_at = _utcnow()
-
- from core.llm.compaction import COMPACTION_CHECKPOINT_KIND
-
- checkpoints = (
- self.db.query(ChatMessage)
- .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system")
- .all()
- )
- for checkpoint in checkpoints:
- extra = dict(checkpoint.extra_data or {})
- if extra.get("kind") != COMPACTION_CHECKPOINT_KIND:
- continue
- extra["active"] = False
- extra["notice_pending"] = False
- checkpoint.extra_data = extra
- return True
-
- def add_compaction_checkpoint(
- self,
- chat_id: str,
- *,
- summary_text: str,
- replacement_history: List[Dict],
- covered_seq: Optional[int] = None,
- source_hash: Optional[str] = None,
- base_checkpoint_version: Optional[int] = None,
- replacement_manifest: Optional[Dict[str, Any]] = None,
- ) -> ChatMessage:
- """Compatibility wrapper around the lease + CAS checkpoint protocol."""
- owner = f"direct:{uuid.uuid4().hex}"
- snapshot = self.acquire_compaction_snapshot(
- chat_id, owner=owner, lease_seconds=60
- )
- if snapshot is None:
- raise CompactionCASConflict(
- "no uncompacted source or compaction lease is busy"
- )
- if (
- base_checkpoint_version is not None
- and snapshot.base_checkpoint_version != int(base_checkpoint_version)
- ):
- self.release_compaction_lease(chat_id, owner=owner)
- raise CompactionCASConflict("checkpoint base changed")
- if covered_seq is not None and int(covered_seq) != snapshot.covered_seq:
- self.release_compaction_lease(chat_id, owner=owner)
- raise ValueError("covered_seq must equal the acquired source watermark")
- if source_hash:
- snapshot = replace(snapshot, source_hash=source_hash)
- return self.commit_compaction_checkpoint(
- snapshot,
- owner=owner,
- summary_text=summary_text,
- replacement_history=replacement_history,
- replacement_manifest=replacement_manifest or {},
- )
-
- def _latest_compaction_checkpoint_fallback(
- self, chat_id: str
- ) -> Optional[ChatMessage]:
- """Find the newest checkpoint without timestamps (legacy-state fallback)."""
- from core.llm.compaction import COMPACTION_CHECKPOINT_KIND
-
- from sqlalchemy.orm import defer
-
- rows = (
- self.db.query(ChatMessage).options(defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls))
- .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system")
- .order_by(ChatMessage.chat_seq.desc(), ChatMessage.message_id.desc())
- .limit(20)
- .all()
- )
- return next(
- (
- row
- for row in rows
- if (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND
- ),
- None,
- )
-
- @staticmethod
- def _checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int:
- """Return only an explicitly captured source watermark.
-
- A legacy checkpoint was inserted *after* summarization, so its own
- sequence cannot prove which earlier rows were actually summarized.
- Treating ``checkpoint.chat_seq - 1`` as covered can hide a message
- that arrived while the summary was in flight. Legacy checkpoints are
- therefore conservatively invalidated and rebuilt from raw history.
- """
- if checkpoint is None:
- return 0
- extra = checkpoint.extra_data or {}
- explicit = extra.get("covered_seq")
- if explicit is None:
- explicit = extra.get("covers_up_to_chat_seq")
- if explicit is not None:
- return max(0, int(explicit))
- return 0
-
- def _ensure_message_sequences(self, chat_id: str) -> None:
- """Repair legacy/direct ORM rows before they enter a source snapshot."""
- from sqlalchemy import func
-
- # ``key_share=True`` renders ``FOR NO KEY UPDATE`` on PostgreSQL.
- # A plain ``FOR UPDATE`` here is a full outage risk: inserting any row
- # into ``chat_messages`` takes ``FOR KEY SHARE`` on this parent row for
- # the foreign key, and ``FOR KEY SHARE`` conflicts with ``FOR UPDATE``.
- # Holding that lock therefore blocks every message write for the chat —
- # including the run journal's terminal commit, which runs on the event
- # loop and freezes the whole process when it waits. We only bump the
- # non-key column ``next_message_seq``, so the weaker lock is both
- # sufficient (it still excludes a concurrent sequence repair) and
- # compatible with the foreign-key lock.
- chat = (
- self.db.query(ChatSession)
- .filter(ChatSession.chat_id == chat_id)
- .with_for_update(key_share=True)
- .one_or_none()
- )
- if chat is None:
- return
- max_seq = int(
- self.db.query(func.max(ChatMessage.chat_seq))
- .filter(ChatMessage.chat_id == chat_id)
- .scalar()
- or 0
- )
- next_seq = max(int(chat.next_message_seq or 1), max_seq + 1)
- missing = (
- self.db.query(ChatMessage)
- .filter(ChatMessage.chat_id == chat_id, ChatMessage.chat_seq.is_(None))
- .order_by(ChatMessage.created_at, ChatMessage.message_id)
- .all()
- )
- for row in missing:
- row.chat_seq = next_seq
- next_seq += 1
- chat.next_message_seq = next_seq
- self.db.flush()
-
- def _ensure_compaction_state(self, chat_id: str) -> ChatCompactionState:
- state = self.db.get(ChatCompactionState, chat_id)
- if state is not None:
- return state
- checkpoint = self._latest_compaction_checkpoint_fallback(chat_id)
- extra = (checkpoint.extra_data or {}) if checkpoint is not None else {}
- has_exact_watermark = (
- checkpoint is not None
- and (
- extra.get("covered_seq") is not None
- or extra.get("covers_up_to_chat_seq") is not None
- )
- )
- state = ChatCompactionState(
- chat_id=chat_id,
- active_checkpoint_id=checkpoint.message_id if has_exact_watermark else None,
- checkpoint_version=(
- int(extra.get("checkpoint_version") or 1) if has_exact_watermark else 0
- ),
- covered_seq=self._checkpoint_covered_seq(checkpoint)
- if has_exact_watermark
- else 0,
- updated_at=_utcnow(),
- )
- self.db.add(state)
- self.db.flush()
- return state
-
- def acquire_compaction_snapshot(
- self,
- chat_id: str,
- *,
- owner: str,
- lease_seconds: int,
- ) -> Optional[CompactionSnapshot]:
- """Capture an exact source watermark and acquire a short-lived lease.
-
- Every exit path must end this transaction. The body takes a row lock on
- the chat before it reads, so an exception escaping with the transaction
- still open strands that lock on a pooled connection that no longer has
- a Python owner — an ``idle in transaction`` backend that blocks the
- chat's message writes until the process dies. The rollback below is the
- guarantee that cannot be forgotten by a future early return.
- """
- try:
- return self._acquire_compaction_snapshot(
- chat_id, owner=owner, lease_seconds=lease_seconds
- )
- except BaseException:
- try:
- self.db.rollback()
- except Exception: # noqa: BLE001 - never mask the original failure
- logger.exception(
- "[compaction] rollback failed after a snapshot error (chat=%s)",
- chat_id,
- )
- raise
-
- def _acquire_compaction_snapshot(
- self,
- chat_id: str,
- *,
- owner: str,
- lease_seconds: int,
- ) -> Optional[CompactionSnapshot]:
- from core.llm.compaction import COMPACTION_CHECKPOINT_KIND
-
- self._ensure_message_sequences(chat_id)
- state = self._ensure_compaction_state(chat_id)
- now = _utcnow()
- if state.lease_owner and (_as_utc(state.lease_expires_at) or now) > now:
- self.db.rollback()
- return None
-
- checkpoint = (
- self.db.get(ChatMessage, state.active_checkpoint_id)
- if state.active_checkpoint_id
- else None
- )
- replacement_history = (
- tuple(
- dict(item)
- for item in (
- (checkpoint.extra_data or {}).get("replacement_history") or []
- )
- )
- if checkpoint is not None
- else ()
- )
- candidates = (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.chat_seq > int(state.covered_seq or 0),
- )
- .order_by(ChatMessage.chat_seq)
- .all()
- )
- source_models = tuple(
- row
- for row in candidates
- if not (
- row.role == "system"
- and (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND
- )
- )
- if not source_models:
- self.db.rollback()
- return None
-
- source_rows = tuple(
- CompactionSourceRow(
- message_id=row.message_id,
- chat_seq=int(row.chat_seq or 0),
- role=row.role,
- content=row.content,
- tool_calls=row.tool_calls,
- extra_data=dict(row.extra_data or {}),
- )
- for row in source_models
- )
- covered_seq = max(row.chat_seq for row in source_rows)
- canonical_source = [
- {
- "message_id": row.message_id,
- "chat_seq": row.chat_seq,
- "role": row.role,
- "content": row.content,
- "tool_calls": row.tool_calls,
- "metadata": row.extra_data,
- }
- for row in source_rows
- ]
- source_hash = hashlib.sha256(
- json.dumps(
- {
- "base_checkpoint_version": int(state.checkpoint_version or 0),
- "base_covered_seq": int(state.covered_seq or 0),
- "covered_seq": covered_seq,
- "source": canonical_source,
- },
- ensure_ascii=False,
- sort_keys=True,
- separators=(",", ":"),
- default=str,
- ).encode("utf-8")
- ).hexdigest()
-
- base_version = int(state.checkpoint_version or 0)
- updated = (
- self.db.query(ChatCompactionState)
- .filter(
- ChatCompactionState.chat_id == chat_id,
- ChatCompactionState.checkpoint_version == base_version,
- (
- ChatCompactionState.lease_owner.is_(None)
- | (ChatCompactionState.lease_expires_at <= now)
- | (ChatCompactionState.lease_owner == owner)
- ),
- )
- .update(
- {
- ChatCompactionState.lease_owner: owner,
- ChatCompactionState.lease_expires_at: now
- + timedelta(seconds=max(1, lease_seconds)),
- ChatCompactionState.updated_at: now,
- },
- synchronize_session=False,
- )
- )
- if updated != 1:
- self.db.rollback()
- return None
- base_covered_seq = int(state.covered_seq or 0)
- base_checkpoint_id = checkpoint.message_id if checkpoint is not None else None
- self.db.commit()
- return CompactionSnapshot(
- chat_id=chat_id,
- owner=owner,
- base_checkpoint_id=base_checkpoint_id,
- base_checkpoint_version=base_version,
- base_covered_seq=base_covered_seq,
- covered_seq=covered_seq,
- source_hash=source_hash,
- replacement_history=replacement_history,
- source_rows=source_rows,
- source_message_ids=tuple(row.message_id for row in source_rows),
- source_message_seqs=tuple(row.chat_seq for row in source_rows),
- )
-
- def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool:
- updated = (
- self.db.query(ChatCompactionState)
- .filter(
- ChatCompactionState.chat_id == chat_id,
- ChatCompactionState.lease_owner == owner,
- )
- .update(
- {
- ChatCompactionState.lease_owner: None,
- ChatCompactionState.lease_expires_at: None,
- ChatCompactionState.updated_at: _utcnow(),
- },
- synchronize_session=False,
- )
- )
- self.db.commit()
- return updated == 1
-
- def commit_compaction_checkpoint(
- self,
- snapshot: CompactionSnapshot,
- *,
- owner: str,
- summary_text: str,
- replacement_history: List[Dict[str, Any]],
- replacement_manifest: Dict[str, Any],
- ) -> ChatMessage:
- """Atomically publish one successor from ``snapshot`` using CAS."""
- from core.llm.compaction import COMPACTION_CHECKPOINT_KIND
-
- if owner != snapshot.owner:
- raise CompactionCASConflict("snapshot owner mismatch")
- message_id = f"cmpct_{uuid.uuid4().hex[:16]}"
- next_version = snapshot.base_checkpoint_version + 1
- manifest = dict(replacement_manifest)
- manifest.update(
- {
- "source_message_ids": list(snapshot.source_message_ids),
- "source_message_seqs": list(snapshot.source_message_seqs),
- "source_count": len(snapshot.source_rows),
- "replacement_count": len(replacement_history),
- }
- )
- message = self.message_repo.create(
- {
- "message_id": message_id,
- "chat_id": snapshot.chat_id,
- "role": "system",
- "content": summary_text,
- "extra_data": {
- "kind": COMPACTION_CHECKPOINT_KIND,
- "replacement_history": replacement_history,
- "covered_seq": snapshot.covered_seq,
- "covers_up_to_chat_seq": snapshot.covered_seq,
- "covers_up_to_message_id": (
- snapshot.source_message_ids[-1]
- if snapshot.source_message_ids
- else None
- ),
- "source_hash": snapshot.source_hash,
- "base_checkpoint_version": snapshot.base_checkpoint_version,
- "checkpoint_version": next_version,
- "replacement_manifest": manifest,
- "active": True,
- "notice_pending": True,
- },
- },
- commit=False,
- )
- now = _utcnow()
- updated = (
- self.db.query(ChatCompactionState)
- .filter(
- ChatCompactionState.chat_id == snapshot.chat_id,
- ChatCompactionState.checkpoint_version
- == snapshot.base_checkpoint_version,
- ChatCompactionState.covered_seq == snapshot.base_covered_seq,
- ChatCompactionState.lease_owner == owner,
- )
- .update(
- {
- ChatCompactionState.active_checkpoint_id: message_id,
- ChatCompactionState.checkpoint_version: next_version,
- ChatCompactionState.covered_seq: snapshot.covered_seq,
- ChatCompactionState.lease_owner: None,
- ChatCompactionState.lease_expires_at: None,
- ChatCompactionState.updated_at: now,
- },
- synchronize_session=False,
- )
- )
- if updated != 1:
- self.db.rollback()
- raise CompactionCASConflict("checkpoint base or lease changed")
-
- if snapshot.base_checkpoint_id:
- previous = self.db.get(ChatMessage, snapshot.base_checkpoint_id)
- if previous is not None:
- previous_extra = dict(previous.extra_data or {})
- previous_extra["active"] = False
- previous.extra_data = previous_extra
- self.db.commit()
- return message
-
- def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]:
- """Return the chat's latest compaction checkpoint (None if there is none).
-
- The kind filter is done on the Python side to avoid depending on Postgres-specific
- ``->>`` (SQLite compatible). Each checkpoint's extra_data carries the entire
- replacement_history (can reach ~100KB), and rolling compaction keeps accumulating —
- scan only the latest few rows, never ``.all()`` load everything.
- """
- state = self.db.get(ChatCompactionState, chat_id)
- if state is not None:
- if not state.active_checkpoint_id:
- return None
- from sqlalchemy.orm import defer
- checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id,
- options=[defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)])
- if (
- checkpoint is not None
- and (checkpoint.extra_data or {}).get("covered_seq") is not None
- ):
- return checkpoint
- return None
-
- checkpoint = self._latest_compaction_checkpoint_fallback(chat_id)
- if (
- checkpoint is None
- or (checkpoint.extra_data or {}).get("covered_seq") is None
- ):
- return None
- return checkpoint
-
- def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]:
- """Get a single message by its ID."""
- return (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.message_id == message_id,
- )
- .first()
- )
-
- def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]:
- """Get a message by its durable 0-based position in the chat."""
- return (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- )
- .order_by(ChatMessage.chat_seq)
- .offset(index)
- .limit(1)
- .first()
- )
-
- def get_user_message_before(
- self, chat_id: str, message_id: str
- ) -> Optional[ChatMessage]:
- """Get the user message immediately before the given message."""
- target = self.get_message_by_id(message_id)
- if not target:
- return None
- return (
- self.db.query(ChatMessage)
- .filter(
- ChatMessage.chat_id == chat_id,
- ChatMessage.role == "user",
- ChatMessage.chat_seq < target.chat_seq,
- )
- .order_by(ChatMessage.chat_seq.desc())
- .first()
- )
-
- def update_message_extra_data(
- self,
- message_id: str,
- patch: Dict[str, Any],
- ) -> bool:
- """Merge *patch* into a message's extra_data. Returns True on success."""
- return self.message_repo.update_extra_data(message_id, patch) is not None
-
- def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]:
- """Replace an assistant message with its persisted ontology revision candidate."""
- message = self.message_repo.get_by_id(message_id)
- if not message or message.role != "assistant":
- return None
- extra_data = dict(message.extra_data or {})
- governance = extra_data.get("ontology_governance")
- if not isinstance(governance, dict):
- return None
- review = governance.get("review")
- if not isinstance(review, dict):
- return None
- candidate = normalize_revision_candidate(review.get("candidate_answer"))
- if not is_substantive_revision(candidate):
- return None
- updated_review = {**review, "candidate_answer": candidate, "accepted": True}
- extra_data["ontology_governance"] = {**governance, "review": updated_review}
- # Some pure unit adapters exercise this method without a database.
- if not hasattr(self, "db"):
- return self.message_repo.update(
- message_id,
- {"content": candidate, "extra_data": extra_data},
- )
-
- updated = self.message_repo.update(
- message_id,
- {"content": candidate, "extra_data": extra_data},
- commit=False,
- )
- self._invalidate_compaction_lineage_for_rewrite(
- message.chat_id,
- rewritten_seq=int(message.chat_seq),
- )
- self.db.commit()
- return updated
-
- def search_sessions(
- self,
- user_id: str,
- query: str,
- page: int = 1,
- page_size: int = 20,
- scope: str = "title",
- ) -> Tuple[list, int]:
- """Search chat sessions by title (and optionally message content)."""
- results, total = self.session_repo.search(
- user_id, query, page, page_size, scope=scope
- )
- return results, total
diff --git a/src/backend/core/services/commands/__init__.py b/src/backend/core/services/commands/__init__.py
deleted file mode 100644
index 0280cf1b..00000000
--- a/src/backend/core/services/commands/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Owned command lifecycle and bounded output storage."""
diff --git a/src/backend/core/services/desktop_cloud_bridge.py b/src/backend/core/services/desktop_cloud_bridge.py
deleted file mode 100644
index 2c5860f7..00000000
--- a/src/backend/core/services/desktop_cloud_bridge.py
+++ /dev/null
@@ -1,960 +0,0 @@
-"""桌面双端「云端能力桥」客户端(本机侧)。
-
-双端模式的本机后端通过本模块把**云端授权的 MCP 工具**合并进本机 Agent 装配:
-
- 桌面壳登录云端 → 换取 capability token → 推送 {cloud_base, token} 到本机
- (POST /v1/desktop/capability/cloud-bridge,CONFIG_TOKEN=桥接秘密)
- → 本模块拉取云端 manifest(当前用户最终可用的 MCP 清单)——只在登录、
- 云端能力被改动、以及云端在网关调用里告知能力已变时拉取,没有定时轮询
- → catalog_resolver 解析 enabled_mcp_ids 时把云端 server 追加进清单并按
- 完整 server_id 解析来源(同名候选显式裁决),agent 装配时
- 直接使用 manifest 内完整 schema 注册虚拟 MCP 工具;模型真正调用后
- 才通过普通 JSON 网关在云端网络内执行真实 MCP。
-
-硬边界(与设计文档一致):
-- 本机拿不到云端真实 MCP URL / 密钥——只有网关地址 + capability token;
-- 云端断线不会阻塞 Agent 装配;云端工具真正被调用时会返回明确错误,
- **不**静默回退本机同名旧实现;
-- 工具绑定按当前账号授权清单及显式来源选择解析,不使用工具名称保留名单。
-
-纯本机模式(未配桥)与云端部署(无桥接秘密)零行为变化。
-"""
-
-from __future__ import annotations
-
-from contextlib import contextmanager
-import hashlib
-import logging
-import threading
-import time
-from typing import Any, Dict, List, Optional, Set, Tuple
-
-from core.auth.desktop_bridge import bridge_enabled
-
-logger = logging.getLogger(__name__)
-
-BRIDGE_BLOCK_ID = "desktop_cloud_bridge"
-
-# 拿不到清单时的有界重试间隔(唯一的时间参数;正常情况下不会用到)。
-_MANIFEST_NEG_TTL_S = 30.0
-
-_state_lock = threading.RLock()
-_state: Optional[Dict[str, Any]] = None # {"cloud_base", "token", "expires_at"}
-_credential_rejected = "" # account fingerprint whose current token the cloud refused
-_state_loaded = False
-
-_manifest_lock = threading.Lock()
-_tool_sync_lock = threading.Lock()
-_manifest: Optional[Dict[str, Any]] = None
-_manifest_ts: float = 0.0
-_manifest_error: Optional[str] = None
-_manifest_fetching = False
-_refresh_pending = False # a forced refresh arrived while a pass was running
-
-
-def _state_fingerprint(state: Optional[Dict[str, Any]]) -> str:
- """Bind a manifest snapshot to exactly one cloud/token identity."""
- if not state:
- return ""
- from core.services.desktop_capability_protocol import token_subject, token_claims
-
- token = str(state.get("token") or "")
- # A rotated token for the same account keeps the snapshot; only a real
- # account switch (or an opaque token) invalidates it.
- claims = token_claims(token)
- payload = f"{state.get('cloud_base') or ''}\0{token_subject(token) or token}\0{claims.get('a', '')}\0{claims.get('h', '')}\0{claims.get('d', state.get('device_id') or '')}"
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()
-
-
-def require_current_account(state: Dict[str, Any]) -> None:
- """Reject work captured before logout/account switch, including late HTTP replies."""
- from core.capabilities.errors import CloudUnavailable
-
- if _state_fingerprint(get_state()) != _state_fingerprint(state):
- raise CloudUnavailable("cloud account changed; retry with the current account")
-
-
-@contextmanager
-def account_scope(state: Dict[str, Any]):
- """Serialize a short publication with identity changes; never hold across network IO."""
- with _state_lock:
- require_current_account(state)
- yield
-
-
-def ensure_current_authorization(state: Optional[Dict[str, Any]] = None) -> None:
- """账号仍然有效才允许使用私有字节;这条判定不发网络请求。
-
- 本机不做「事前探测」——云端清单只在登录、云端能力被改动(前端写操作后调
- ``POST /v1/desktop/capabilities/sync``)以及云端在网关调用里明确告知能力已变
- 这三种事件下同步,没有任何定时轮询。真正的撤权由云端在网关调用时裁决:返回
- 401/403 会立刻清掉本机的桥状态,被撤权的能力下一次调用即失败。
- """
- from core.capabilities.errors import CloudUnavailable
-
- st = state or get_state()
- if not st:
- raise CloudUnavailable("cloud authorization expired; sign in again")
- require_current_account(st)
-
-
-def notify_cloud_changed() -> None:
- """云端能力可能已变(前端写操作 / 网关告知):后台同步一次清单与本机文件。"""
- _refresh_manifest_async(force=True)
-
-
-# ── 桥状态(仅内存;壳启动后推送短时运行凭据) ────────────────────
-
-
-def _purge_persisted_state() -> None:
- """Remove credentials left by an earlier release; bridge tokens are memory-only."""
- try:
- from core.services.desktop_model_credentials import scrub_legacy_rows
-
- scrub_legacy_rows()
- except Exception as exc:
- logger.warning(
- "[cloud-bridge] legacy model credential cleanup unavailable: %s", type(exc).__name__
- )
- try:
- from core.db.engine import SessionLocal
- from core.db.models import ContentBlock
-
- with SessionLocal() as db:
- row = db.get(ContentBlock, BRIDGE_BLOCK_ID)
- if row is not None:
- db.delete(row)
- db.commit()
- except Exception as exc:
- logger.warning(
- "[cloud-bridge] legacy credential cleanup unavailable: %s", type(exc).__name__
- )
-
-
-def _load_state_from_db() -> None:
- _purge_persisted_state()
- return None
-
-
-def current_credentials(state: Dict[str, Any]) -> Dict[str, Any]:
- """The credential belongs to the account, not to the snapshot a pass started with.
-
- A sync pass can outlive the token it started with while the shell has already
- renewed it. Same account → newest token; another account → untouched, so the
- surrounding identity checks reject it.
- """
- with _state_lock:
- live = _state
- if live and _state_fingerprint(live) == _state_fingerprint(state):
- return live
- return state
-
-
-def note_rejected_credential(state: Dict[str, Any], exc: BaseException) -> None:
- """Only a refusal of the live token schedules re-sync on its next renewal."""
- global _credential_rejected
- import httpx
-
- if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401:
- authorization = exc.request.headers.get("Authorization")
- with _state_lock:
- if (
- _state
- and authorization == f"Bearer {_state.get('token')}"
- and _state_fingerprint(_state) == _state_fingerprint(state)
- ):
- _credential_rejected = _state_fingerprint(state)
-
-
-def cloud_headers(state: Dict[str, Any]) -> Dict[str, str]:
- from core.services.desktop_capability_protocol import token_claims
-
- state = current_credentials(state)
- device_id = str(
- state.get("device_id") or token_claims(str(state.get("token") or "")).get("d") or ""
- )
- headers = {"Authorization": f"Bearer {state['token']}"}
- if device_id:
- headers["X-Desktop-Device-Id"] = device_id
- return headers
-
-
-def clear_state() -> None:
- """Logout invalidates the account immediately, including in-flight responses."""
- global _state, _state_loaded, _manifest, _manifest_error, _manifest_ts, _credential_rejected
- with _state_lock:
- _state, _state_loaded = None, True
- _credential_rejected = ""
- _clear_partial_selection()
- with _manifest_lock:
- _manifest, _manifest_error, _manifest_ts = None, None, 0.0
- from core.services import desktop_cloud_bundles, desktop_cloud_skills
-
- desktop_cloud_skills.on_account_switch()
- desktop_cloud_bundles.on_account_switch()
- _purge_persisted_state()
- _rebuild_identity_views()
-
-
-def _rebuild_identity_views() -> None:
- from core.capabilities import skills
- from core.capabilities.paths import capabilities_enabled
-
- if not capabilities_enabled():
- return
- from core.agent_skills.cache_refresh import refresh_skill_caches
-
- refresh_skill_caches()
- shared = skills.device_view_dir()
- root = shared.parent / (shared.name + "_u")
- if root.is_dir():
- for child in root.iterdir():
- if child.is_dir():
- skills.rebuild_user_view(child.name)
-
-
-def get_state() -> Optional[Dict[str, Any]]:
- global _state, _state_loaded
- with _state_lock:
- if not _state_loaded:
- _state = _load_state_from_db()
- _state_loaded = True
- st = _state
- if not st:
- return None
- if float(st.get("expires_at") or 0) < time.time():
- return None
- return dict(st)
-
-
-def shell_user_center_id(cloud_base: str, user_center_id: str) -> str:
- """壳送来的桥接用户标识:``cloud:::<云端 user_center_id>``。
-
- 壳按云端地址给用户加命名空间,同一台机器连不同云端时本机身份天然隔离。本机侧
- 用完全相同的规则从凭据里的云端身份推出壳会送来的标识,两者必须逐字相同。
- 端口缺省与壳一致:https 443、其它 80。
- """
- from urllib.parse import urlsplit
-
- parts = urlsplit(cloud_base.strip())
- host = parts.hostname or "cloud"
- port = parts.port or (443 if parts.scheme == "https" else 80)
- return f"cloud:{host}:{port}:{user_center_id}"
-
-
-def get_identity_state() -> Optional[Dict[str, Any]]:
- """Last shell-authenticated identity for offline local resources; never a cloud grant."""
- from core.services.desktop_capability_protocol import token_claims
-
- with _state_lock:
- st = dict(_state) if _state else None
- if not st:
- return None
- claims = token_claims(str(st.get("token") or ""))
- if not claims.get("c"):
- return None
- return {
- "cloud_base": st["cloud_base"],
- "user_center_id": str(claims["c"]),
- "shell_user_center_id": shell_user_center_id(str(st["cloud_base"]), str(claims["c"])),
- "subject": str(claims.get("u") or ""),
- "device_id": str(claims.get("d") or ""),
- "authorization_epoch": claims.get("a"),
- }
-
-
-def set_state(
- cloud_base: str,
- token: str,
- expires_in: int,
- *,
- device_id: Optional[str] = None,
- authorization_epoch: Optional[int] = None,
-) -> None:
- """壳侧推送桥配置(幂等)。立即触发一次后台 manifest 刷新。"""
- global _manifest, _manifest_error, _manifest_ts, _state, _state_loaded, _credential_rejected
- payload = {
- "cloud_base": cloud_base.strip().rstrip("/"),
- "token": token.strip(),
- "expires_at": time.time() + max(1, int(expires_in or 0)),
- "device_id": device_id,
- "authorization_epoch": authorization_epoch,
- }
- from core.services.desktop_capability_protocol import token_claims
-
- claims = token_claims(token)
- if device_id and claims.get("d") and device_id != claims["d"]:
- raise ValueError("desktop device identity does not match token")
- if claims.get("e"):
- payload["expires_at"] = min(payload["expires_at"], float(claims["e"]))
- with _state_lock:
- changed = _state_fingerprint(_state) != _state_fingerprint(payload)
- renewed_after_rejection = not changed and _credential_rejected == _state_fingerprint(payload)
- _credential_rejected = ""
- _state = payload
- _state_loaded = True
- if changed:
- _clear_partial_selection()
- # Never expose a previous cloud account's tools while the replacement
- # dynamic manifest is still in flight. Skill files stay in the previous
- # account's isolated profile directory; only the runtime view changes.
- with _manifest_lock:
- _manifest = None
- _manifest_ts = 0.0
- _manifest_error = None
- from core.services import desktop_cloud_bundles, desktop_cloud_skills
-
- desktop_cloud_skills.on_account_switch()
- desktop_cloud_bundles.on_account_switch()
- _purge_persisted_state()
- if changed:
- _rebuild_identity_views()
- _refresh_manifest_async(force=True)
- elif renewed_after_rejection:
- _refresh_manifest_async(force=True)
-
-
-def bridge_active() -> bool:
- """本进程是否应启用云端能力桥(仅桌面壳孵化的双端本机后端为 True)。"""
- return bridge_enabled() and get_state() is not None
-
-
-# ── manifest 拉取(后台线程,不阻塞事件循环) ──────────────────────────
-
-
-def synced_manifest() -> Optional[Dict[str, Any]]:
- """Read the last accepted connector snapshot without starting a refresh."""
- import copy
-
- with _manifest_lock:
- return copy.deepcopy(_manifest)
-
-
-def sync_tool_manifest_blocking(st: Dict[str, Any]) -> None:
- """Refresh connectors for both login and explicit synchronization."""
- with _tool_sync_lock:
- _sync_tool_manifest(st)
-
-
-def _sync_tool_manifest(st: Dict[str, Any]) -> None:
- global _manifest, _manifest_ts, _manifest_error
- from core.capabilities.errors import CloudUnavailable
-
- require_current_account(st)
- url = f"{st['cloud_base']}/api/v1/desktop/capability/manifest"
- import httpx
- from core.services.desktop_capability_protocol import validate_manifest
-
- headers = {**cloud_headers(st), "Cache-Control": "no-cache"}
- with _manifest_lock:
- current_revision = str((_manifest or {}).get("revision") or "")
- if current_revision:
- headers["If-None-Match"] = f'"{current_revision}"'
-
- resp = httpx.get(
- url,
- headers=headers,
- timeout=httpx.Timeout(10.0, connect=5.0),
- )
- require_current_account(st)
- if resp.status_code in (401, 403):
- with account_scope(st):
- clear_state()
- raise CloudUnavailable("cloud authorization unavailable")
- if resp.status_code == 304:
- with account_scope(st), _manifest_lock:
- _manifest_ts = time.monotonic()
- _manifest_error = None
- else:
- resp.raise_for_status()
- body = resp.json()
- data = body.get("data") if isinstance(body, dict) else None
- manifest = validate_manifest(data)
- with account_scope(st):
- with _manifest_lock:
- _manifest = manifest
- _manifest_ts = time.monotonic()
- _manifest_error = None
- _project_managed_profile(st, manifest)
- logger.info(
- "[cloud-bridge] manifest refreshed revision=%s servers=%d",
- manifest["revision"][:12],
- len(manifest["servers"]),
- )
-
-
-def _fetch_manifest_blocking(st: Dict[str, Any]) -> None:
- global _manifest, _manifest_ts, _manifest_error, _manifest_fetching, _refresh_pending
- refresh_replacement = False
- try:
- sync_tool_manifest_blocking(st)
- # Skills, agents and plugins ride the same poll and token: their intent
- # is reconciled right after the tool manifest is known to be current.
- sync_capabilities_blocking(st)
- except Exception as exc: # noqa: BLE001
- with _state_lock:
- if _state_fingerprint(get_state()) != _state_fingerprint(st):
- refresh_replacement = True
- return
- with _manifest_lock:
- _manifest_ts = time.monotonic()
- _manifest_error = str(exc)
- logger.warning("[cloud-bridge] manifest 拉取失败: %s", exc)
- finally:
- with _manifest_lock:
- _manifest_fetching = False
- rerun = _refresh_pending
- _refresh_pending = False
- if refresh_replacement or rerun:
- _refresh_manifest_async(force=True)
-
-
-def sync_capabilities_blocking(st: Dict[str, Any]) -> None:
- """One pass over every cloud-synced kind (skills, agents, plugins). Blocking; call off-loop.
-
- Both the background manifest poll and the user-triggered sync endpoint go
- through here so no kind can be forgotten by one of the two paths.
- """
- from core.services import desktop_cloud_bundles, desktop_cloud_skills
-
- desktop_cloud_skills.sync_blocking(st)
- desktop_cloud_bundles.sync_blocking(st)
- try:
- from core.capabilities.warmup import warmup_current_account
-
- warmup_current_account(st)
- except Exception as exc: # Best effort: ordinary execution retains all checks.
- logger.warning("[caps] startup reads deferred: %s", type(exc).__name__)
-
-
-def _refresh_manifest_async(force: bool = False) -> None:
- """后台同步一次云端清单。没有定时器:``force`` 由登录 / 变更通知触发,非
- ``force`` 只是「还没有可用清单」时的有界重试(离线登录后的自愈),一旦拿到
- 清单就不再有任何后台请求。"""
- global _manifest_fetching, _refresh_pending
- st = get_state()
- if not st:
- return
- with _manifest_lock:
- if _manifest_fetching:
- _refresh_pending = _refresh_pending or force
- return
- if not force:
- if _manifest is not None and not _manifest_error:
- return
- if _manifest_ts and time.monotonic() - _manifest_ts < _MANIFEST_NEG_TTL_S:
- return
- _manifest_fetching = True
- threading.Thread(
- target=_fetch_manifest_blocking, args=(st,), name="cloud-bridge-manifest", daemon=True
- ).start()
-
-
-def get_cached_manifest() -> Optional[Dict[str, Any]]:
- """返回缓存的 manifest(可能为 None)。调用方只读,不得修改。
-
- 还没有清单时(离线登录等)触发一次有界重试;已有清单则不发任何请求,更新
- 靠登录与变更通知。不做激活判定——守门统一在公共入口(``_bridge_context``)。
- 清单只在 ``_fetch_manifest_blocking`` 里整体替换,从不原地修改,所以可以安全共享。
- """
- _refresh_manifest_async()
- with _manifest_lock:
- return _manifest
-
-
-# ── 混合能力解析(解析器裁决绑定;mcp.json 投影) ──────────────────────
-
-
-def _account_profile(st: Dict[str, Any]) -> str:
- """Profile label of the bridged account (opaque tokens get a deterministic label)."""
- from core.capabilities.ref import profile_id
- from core.services.desktop_capability_protocol import token_subject
-
- return profile_id(
- str(st["cloud_base"]), token_subject(str(st.get("token") or "")) or "opaque-subject"
- )
-
-
-def _project_managed_profile(st: Dict[str, Any], manifest: Dict[str, Any]) -> None:
- """Write the account's cloud connector references into mcp.json (managedProfiles)."""
- from core.capabilities import mcp_json
- from core.capabilities.paths import capabilities_enabled
- from core.capabilities.ref import cloud_issuer
-
- if not capabilities_enabled():
- return
- try:
- mcp_json.project_managed_profile(
- _account_profile(st),
- cloud_instance_id=cloud_issuer(str(st["cloud_base"])),
- catalog_revision=str(manifest["revision"]),
- servers=list(manifest.get("servers") or []),
- initial_enabled={
- str(s["server_id"]): bool(s.get("enabled", True))
- for s in manifest.get("servers") or []
- if isinstance(s, dict) and s.get("server_id")
- },
- )
- except mcp_json.McpJsonCorrupt as exc:
- logger.error("[cloud-bridge] mcp.json unreadable, projection skipped: %s", exc)
- except Exception as exc: # noqa: BLE001
- logger.warning("[cloud-bridge] mcp.json projection failed: %s", exc)
-
-
-def _managed_enabled(st: Dict[str, Any]) -> Dict[str, bool]:
- from core.capabilities import mcp_json
- from core.capabilities.paths import capabilities_enabled
-
- if not capabilities_enabled():
- return {}
- try:
- return mcp_json.managed_enabled(_account_profile(st))
- except mcp_json.McpJsonError as exc:
- logger.warning("[cloud-bridge] mcp.json managed flags unavailable: %s", exc)
- return {}
-
-
-def managed_connectors() -> List[Dict[str, Any]]:
- """云端下发的连接器全集(含本机停用的),供本机能力中心展示。
-
- ``_bridge_context`` 是装配用的,它按本机停用标志把关掉的过滤掉了——界面要的是
- 「有哪些」,关掉的也得列出来,开关才有地方可点。启停仍以本机 mcp.json 为准。
-
- 只读已缓存的清单,不触发任何刷新:这是界面读取路径,同步有它自己的时机。
- """
- st = get_state()
- if not st or not bridge_enabled():
- return []
- with _manifest_lock:
- manifest = _manifest
- if not manifest:
- return []
- enabled = _managed_enabled(st)
- out: List[Dict[str, Any]] = []
- for server in manifest.get("servers") or []:
- if not isinstance(server, dict):
- continue
- sid = str(server.get("server_id") or "").strip()
- if not sid:
- continue
- out.append(
- {
- "server_id": sid,
- "display_name": str(server.get("display_name") or sid),
- "description": str(server.get("description") or ""),
- "created_at": server.get("created_at"),
- "enabled": bool(enabled.get(sid, True)),
- "source_plugin": str(server.get("source_plugin") or ""),
- "icon": str(server.get("icon") or ""),
- "tools": [
- {
- "name": str(tool.get("name") or ""),
- "description": str(tool.get("description") or ""),
- }
- for tool in server.get("tools") or []
- if isinstance(tool, dict)
- ],
- }
- )
- return out
-
-
-def set_managed_connector_enabled(server_id: str, enabled: bool) -> bool:
- """记下用户对某个云端下发连接器的启停,落本机 mcp.json。
-
- 本机是这个开关的唯一记账处:装配走 ``_bridge_context()``,它就按这里的标志
- 过滤,云端不参与也不需要回写。返回 False 表示该 server_id 不是本机托管的
- 云端连接器,调用方应继续按普通目录项处理。
-
- 「数据库查询」是个伞形条目(见 ``device_catalog._db_umbrella_item``),本身没有
- 对应的 server,开关要落到它收拢的那几个成员上——否则界面上的开关点了没有任何
- 东西真的被打开。
- """
- from core.capabilities import mcp_json
- from core.capabilities.paths import capabilities_enabled
- from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
-
- st = get_state()
- if not capabilities_enabled() or not bridge_enabled() or not st:
- return False
- profile = _account_profile(st)
- if not profile:
- return False
- sid = str(server_id)
- if sid == DB_UMBRELLA_ID:
- members = [
- str(s["server_id"])
- for s in managed_connectors()
- if str(s.get("server_id")) in DB_HIDDEN_SERVERS
- ]
- if not members:
- return False
- for member in members:
- try:
- mcp_json.set_managed_enabled(profile, member, bool(enabled))
- except mcp_json.McpJsonError:
- return False
- return True
- try:
- mcp_json.set_managed_enabled(profile, sid, bool(enabled))
- except mcp_json.McpJsonError:
- return False
- return True
-
-
-_context_lock = threading.Lock()
-_context_cache: Optional[Tuple[tuple, Dict[str, Any]]] = None
-
-
-def _bridge_context() -> Optional[Dict[str, Any]]:
- """唯一守门点:桥激活且 manifest 就绪时返回上下文,否则 None。
-
- 返回 {"state": st, "servers": [含 component 的云端 server,已剔除 mcp.json 里
- 用户停用的项], "profile": 账号 profile}。同名裁决交给解析器。
- server 列表按 (账号, 清单 revision, 停用集合) 记忆化;``state`` 每次现取,
- 因为它携带轮换中的 token。
- """
- global _context_cache
- if not bridge_enabled():
- return None
- st = get_state()
- if not st:
- return None
- manifest = get_cached_manifest()
- if not manifest:
- return None
- enabled = _managed_enabled(st)
- key = (_state_fingerprint(st), str(manifest["revision"]), tuple(sorted(enabled.items())))
- with _context_lock:
- hit = _context_cache
- if hit is None or hit[0] != key:
- servers: List[Dict[str, Any]] = []
- for s in manifest.get("servers") or []:
- if not isinstance(s, dict):
- continue
- # A server with no current tool contracts cannot replace a local
- # implementation. The manifest validator already guarantees every
- # non-empty entry contains complete, cloud-supplied schemas.
- if not s.get("tools"):
- continue
- sid = str(s.get("server_id") or "").strip()
- if not sid or not enabled.get(sid, True):
- continue
- entry = dict(s)
- entry["server_id"] = sid
- entry["component"] = str(s.get("component") or sid).strip()
- servers.append(entry)
- hit = (
- key,
- {
- "servers": servers,
- "manifest_revision": str(manifest["revision"]),
- "profile": _account_profile(st),
- },
- )
- with _context_lock:
- _context_cache = hit
- if not hit[1]["servers"]:
- return None
- return {"state": st, **hit[1]}
-
-
-def _mcp_json_local_configs() -> Dict[str, dict]:
- from core.capabilities import mcp_json
- from core.capabilities.paths import capabilities_enabled
-
- if not capabilities_enabled():
- return {}
- try:
- return mcp_json.local_server_configs()
- except mcp_json.McpJsonError as exc:
- logger.warning("[cloud-bridge] mcp.json local servers unavailable: %s", exc)
- return {}
-
-
-def cloud_gateway_mcp_configs(
- enabled_mcp_ids: Optional[List[str]] = None, *, resolution_out=None, unavailable_out=None
-) -> Dict[str, dict]:
- """Desktop-only MCP config sources: cloud gateway bindings + mcp.json local servers.
-
- Only the components the resolver chose for the cloud binding get a gateway
- config; the run's enabled-id allowlist still collapses the final set.
- """
- from core.capabilities import connectors
-
- configs: Dict[str, dict] = dict(_mcp_json_local_configs())
- ctx = _bridge_context()
- res = _resolve_mcp_bindings(enabled_mcp_ids, ctx)
- if resolution_out is not None:
- resolution_out.append(res)
- requested = set(enabled_mcp_ids or [])
- for name, group in {**res.conflicts, **res.unusable}.items():
- if name in requested or any(connectors.server_id_of(c) in requested for c in group):
- from core.capabilities.errors import NameConflict, PackageMissing
-
- error = NameConflict if name in res.conflicts else PackageMissing
- if unavailable_out is not None:
- unavailable_out.update({connectors.server_id_of(c): error.code for c in group})
- continue
- raise error(
- "selected connector binding is unavailable; choose its source", runtime_name=name
- )
- chosen = res.chosen_ids()
- configs = {
- sid: cfg
- for sid, cfg in configs.items()
- if f"mcp:{connectors.MCP_JSON_PROFILE}:{sid}" in chosen
- }
- if not ctx:
- return configs
- st = ctx["state"]
- manifest_revision = ctx["manifest_revision"]
- for s in ctx["servers"]:
- # Source selection is complete; only the exact authorized cloud binding
- # chosen by the resolver may produce a gateway config.
- sid = s["server_id"]
- if f"mcp:{ctx['profile']}:{sid}" not in chosen:
- continue
- from core.services.desktop_resource_plugins import local_config
- from core.capabilities.skills import current_local_user_id
- native = local_config(s, current_local_user_id(), ctx["profile"])
- if native is not None:
- configs[sid] = native
- continue
- invoke_url = f"{st['cloud_base']}/api/v1/desktop/capability/gateway/{sid}/call"
- configs[sid] = {
- "transport": "streamable_http",
- # HttpMCPConfig requires a URL, but ManifestMCPClient never performs
- # discovery against it. The same JSON endpoint is the only remote hop.
- "url": invoke_url,
- "headers": cloud_headers(st),
- "execution_timeout": 180,
- "is_stable": False,
- "schema_source": "cloud_manifest",
- "manifest_revision": manifest_revision,
- "manifest_tools": s["tools"],
- "schema_hash": str(s["schema_hash"]),
- "gateway_invoke_url": invoke_url,
- "gateway_plugin": str(s.get("source_plugin") or ""),
- }
- return configs
-
-
-def _local_server_ids() -> Set[str]:
- """本机 DB 内全部 MCP 的 server_id。
-
- 复用 ``McpServerConfigService`` 自带的 30s 缓存与失效链路,不另建缓存。
- """
- try:
- from core.services.mcp_service import McpServerConfigService
-
- return set(McpServerConfigService.get_instance().get_all_servers(enabled_only=False))
- except Exception as exc: # noqa: BLE001
- logger.debug("[cloud-bridge] local server ids failed: %s", exc)
- return set()
-
-
-def apply_to_enabled_mcp_ids(mcp_ids: Optional[List[str]]) -> Optional[List[str]]:
- """把本轮 enabled_mcp_ids 交给解析器,按 server_id 裁决每个连接器的绑定。
-
- 候选:本机目录行(device)、云端 manifest(account)、mcp.json 本机声明。
- 同一 server_id 多个候选时,云端账号级候选默认胜出,
- 用户可用同名偏好显式改选;落选绑定记为 shadowed,可在界面
- 切换,而不是被静默顶掉。云端 manifest 尚未就绪 / 桥未激活:原样返回。
- 顺序稳定(已选项按原顺序,新增云端与 mcp.json 项追加),幂等。
- """
- if mcp_ids is None:
- return None
- ctx = _bridge_context()
- from core.capabilities import connectors
-
- local_ids = _local_server_ids()
- json_local = _mcp_json_local_declarations()
- res = _resolve_mcp_bindings(mcp_ids, ctx)
-
- chosen_ids = {connectors.server_id_of(c) for c in res.chosen.values()}
- known = local_ids | {s["server_id"] for s in (ctx or {}).get("servers", [])} | set(json_local)
- kept = [mid for mid in mcp_ids if mid in chosen_ids or mid not in known]
- existing = set(kept)
- for name in sorted(res.chosen):
- sid = connectors.server_id_of(res.chosen[name])
- if sid not in existing:
- kept.append(sid)
- existing.add(sid)
- return kept
-
-
-def _resolve_mcp_bindings(mcp_ids: Optional[List[str]], ctx):
- from core.capabilities import connectors
-
- local_ids = _local_server_ids()
- enabled = set(local_ids) if mcp_ids is None else set(mcp_ids)
- candidates = connectors.db_candidates(local_ids, enabled) + connectors.json_candidates(
- _mcp_json_local_declarations()
- )
- if ctx:
- candidates += connectors.cloud_candidates(ctx["profile"], ctx["servers"], {})
- return connectors.resolve_bindings(candidates)
-
-
-def _mcp_json_local_declarations() -> Dict[str, Dict[str, Any]]:
- from core.capabilities import mcp_json
- from core.capabilities.paths import capabilities_enabled
-
- if not capabilities_enabled():
- return {}
- try:
- return mcp_json.load().local
- except mcp_json.McpJsonError:
- return {}
-
-
-def apply_to_enabled_skill_ids(skill_ids: Optional[List[str]]) -> Optional[List[str]]:
- """把云端技能合并进本轮 enabled_skill_ids(云端为真源,见 desktop_cloud_skills)。"""
- if skill_ids is None or not bridge_active():
- return skill_ids
- from core.services import desktop_cloud_skills
-
- return desktop_cloud_skills.apply_to_enabled_skill_ids(skill_ids)
-
-
-def _clear_partial_selection():
- from core.capabilities import session_availability, registry
- if session_availability.active():
- session_availability.clear()
- registry._bump() # Runtime selection changed even though cloud intent did not.
-
-
-def retry_initial_sync(st):
- from core.capabilities import skills
- with account_scope(st):
- _clear_partial_selection()
- skills.bump_view_generation()
- _refresh_manifest_async(force=True)
-
-
-def initial_sync_status() -> Dict[str, Any]:
- """Report actual package counts without disk hashing or network probes."""
- from core.services import desktop_cloud_skills, desktop_cloud_bundles
- from core.capabilities import session_availability
- from core.capabilities.paths import revision_for_hash
-
- st = get_state()
- if not st:
- return {"ready": False, "syncing": False, "partial": False, "groups": [],
- "pending": 0, "error": "云端身份尚未就绪", "can_continue": False}
- with _manifest_lock:
- manifest_ready = _manifest is not None
- fetching = _manifest_fetching
- error = _manifest_error
- groups = {"skill": desktop_cloud_skills.status(), **desktop_cloud_bundles.status()}
- pending, completed, total = 0, 0, 0
- progress = []
- for kind, group in groups.items():
- error = error or group.get("last_error")
- rows = [row for row in group.get("installations", []) if row.get("state") != "removed"]
- done = sum(bool(row.get("content_hash")) and row.get("state") == "ready"
- and row.get("resolved_revision") == revision_for_hash(row["content_hash"])
- for row in rows)
- failed = sum(row.get("state") == "failed" or bool(row.get("last_error")) for row in rows)
- completed += done
- total += len(rows)
- pending += len(rows) - done
- progress.append({"kind": kind, "total": len(rows), "completed": done,
- "failed": failed, "manifest_ready": bool(group.get("revision"))})
- complete_manifests = all(group.get("revision") for group in groups.values())
- partial = session_availability.active(desktop_cloud_skills._profile(st))
- # Progress is read without the identity lock: package rows may wait on the
- # database, and the shell's token push must never queue behind that wait.
- require_current_account(st)
- complete = (manifest_ready and not fetching and not error and pending == 0 and complete_manifests)
- if not fetching and not complete and not partial:
- error = error or "部分能力未能同步完成,请选择重试或使用已同步能力继续"
- return {"ready": bool(complete or partial), "syncing": fetching, "partial": partial,
- "groups": progress, "completed": completed, "total": total,
- "totals_known": bool(complete_manifests), "pending": pending,
- "error": error, "can_continue": bool(manifest_ready and not fetching and not complete)}
-
-
-def bridge_status() -> Dict[str, Any]:
- """诊断视图(/v1/desktop/capability/cloud-bridge/status)。"""
- from core.services import desktop_cloud_skills
-
- st = get_state()
- with _manifest_lock:
- servers = (_manifest or {}).get("servers") or []
- err = _manifest_error
- manifest_revision = str((_manifest or {}).get("revision") or "")
- schema_ready_count = sum(
- 1 for server in servers if isinstance(server, dict) and bool(server.get("tools"))
- )
- return {
- "configured": st is not None,
- "cloud_base": (st or {}).get("cloud_base"),
- "active": bridge_active(),
- "cloud_server_count": len(servers),
- "manifest_revision": manifest_revision,
- "schema_mode": "dynamic_manifest" if manifest_revision else "none",
- "schema_ready_server_count": schema_ready_count,
- "cloud_servers": [
- {
- "server_id": s.get("server_id"),
- "component": s.get("component"),
- "display_name": s.get("display_name"),
- "origin": "cloud",
- }
- for s in servers
- if isinstance(s, dict)
- ],
- "last_error": err,
- "skills": desktop_cloud_skills.status(),
- "bundles": _bundles_status(),
- "mcp_json": _mcp_json_status(),
- }
-
-
-def _bundles_status() -> Dict[str, Any]:
- from core.services import desktop_cloud_bundles
-
- return desktop_cloud_bundles.status()
-
-
-def _mcp_json_status() -> Dict[str, Any]:
- from core.capabilities import connectors, mcp_json
- from core.capabilities.paths import capabilities_enabled
-
- if not capabilities_enabled():
- return {"enabled": False}
- try:
- doc = mcp_json.load()
- except mcp_json.McpJsonCorrupt as exc:
- return {"enabled": True, "corrupt": True, "error": str(exc)}
- res = connectors.last_resolution()
- return {
- "enabled": True,
- "generation": doc.generation,
- "local_server_count": len(doc.local),
- "managed_profiles": {
- p: len((v or {}).get("servers") or {}) for p, v in doc.managed.items()
- },
- "conflicts": sorted(res.conflicts) if res else [],
- "shadowed": sorted(res.shadowed) if res else [],
- }
-
-
-def reset_for_tests() -> None: # pragma: no cover - 仅测试用
- global _state, _state_loaded, _manifest, _manifest_ts, _manifest_error, _manifest_fetching
- global _credential_rejected, _refresh_pending
- from core.services import desktop_cloud_skills
-
- with _state_lock:
- _state = None
- _state_loaded = False
- _credential_rejected = ""
- with _manifest_lock:
- _manifest = None
- _manifest_ts = 0.0
- _manifest_error = None
- _manifest_fetching = False
- _refresh_pending = False
- desktop_cloud_skills.reset_for_tests()
diff --git a/src/backend/core/services/edition_system_config.py b/src/backend/core/services/edition_system_config.py
deleted file mode 100644
index cefd7c72..00000000
--- a/src/backend/core/services/edition_system_config.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""Community edition contributes no enterprise service configuration."""
-
-SEED_CONFIGS = []
-CONFIG_KEY_TO_ENV = {}
-
-__all__ = ["CONFIG_KEY_TO_ENV", "SEED_CONFIGS"]
diff --git a/src/backend/core/services/edition_workspace.py b/src/backend/core/services/edition_workspace.py
deleted file mode 100644
index 7b9cf653..00000000
--- a/src/backend/core/services/edition_workspace.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""Community Edition has no additional workspace volumes."""
-
-
-
-
-
-
-
-def project_directory(scope):
- return ""
-
-
-def project_directory_by_id(project_id):
- return ""
-
-
-def workspace_instructions():
- return ""
-
-
-def is_organization_path(scope, actor, path):
- return False
-
-
-
-
-def resolve_alias(path):
- return None
-
-
-def resolve_project_path(path, user_id, root, scope):
- return None
-
-
-def is_shared_scope(scope):
- return False
-
-
-def context_directory(context):
- return ""
-
-
-def source_directory(project):
- return ""
-
-
-def folder_scope_label(kind):
- return "我的空间"
-
-
-def command_instructions(scope):
- return ""
diff --git a/src/backend/core/services/kb_edition.py b/src/backend/core/services/kb_edition.py
deleted file mode 100644
index 5217dd4b..00000000
--- a/src/backend/core/services/kb_edition.py
+++ /dev/null
@@ -1,12 +0,0 @@
-"""Community knowledge-base grant policy: resources are owner-only."""
-
-from __future__ import annotations
-
-from sqlalchemy.orm import Session
-
-
-def initial_visibility_grants(db: Session, user_id: str, visibility: str) -> list[tuple[str, str]]:
- return []
-
-
-__all__ = ["initial_visibility_grants"]
diff --git a/src/backend/core/services/lark_service.py b/src/backend/core/services/lark_service.py
deleted file mode 100644
index 012509b3..00000000
--- a/src/backend/core/services/lark_service.py
+++ /dev/null
@@ -1,752 +0,0 @@
-"""Lark (Feishu) account connection service (feishu-cli plugin / lark-cli).
-
-Drives ``lark-cli auth`` **as the current user** in their dedicated backend
-HOME to complete the device-flow OAuth login (scan-to-bind), and persists
-the connection status / Lark identity summary into the ``lark_connections``
-table. The credentials themselves (user_access_token + file-based encrypted
-store, including master.key) are persisted at ``$STORAGE/lark_cache/{uid}/``
-via ``_make_lark_creds_volumes``'s per-user bind-mount, never stored in the
-DB.
-
-Structurally identical to dingtalk_service, with two differences (Lark is
-easier + Block B decision):
- 1. Device flow: ``lark-cli auth login --no-wait --json`` returns a
- structured device_code + verification_url **immediately** (no need to
- stream-parse a text box like dws); the device_code is persisted, then
- ``--device-code`` polls to completion ("poll and complete", a single
- call blocking until authorized/expired).
- 2. app_id/app_secret are **not** injected as env (with ``LARK_APP_*`` in
- the environment, lark-cli defaults to the tenant/bot token flow,
- breaking ``--as user``). Instead, before login, ``config init
- --app-secret-stdin`` seeds the app config into the per-user HOME
- (which enters the sandbox together with the credential volume).
-
-Note: the JSON shapes of the device flow / auth status are authoritative
-only per real-device testing (P0). Parsing in this module is deliberately
-lenient and all subprocess interactions degrade harmlessly — any step
-failure sets the connection to error / returns a hint, never raising a bare
-exception that breaks the front end.
-See internal design docs.
-"""
-
-from __future__ import annotations
-from core.infra.time import utc_now
-
-import asyncio
-import json
-import logging
-import os
-import re
-import shutil
-from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple
-
-from sqlalchemy.orm import Session
-
-from core.db.repository import LarkConnectionRepository
-from core.infra.proc import no_window_kwargs
-# QR rendering is identical to dingtalk's (segno SVG data-uri); reuse it directly to avoid duplicating the implementation.
-from core.services.dingtalk_service import make_qr_data_uri
-
-logger = logging.getLogger(__name__)
-
-# In-progress device-flow completion tasks (user_id → asyncio.Task). ``--device-code``
-# blocks polling until the user authorizes / it expires; run as a backend subprocess,
-# surviving across HTTP requests, updating the DB itself on completion and removing
-# itself from the map.
-_LOGIN_TASKS: "dict[str, asyncio.Task]" = {}
-_LARK_BIN = "lark-cli"
-
-# Default authorization scopes requested by the device flow: --recommend requests only the
-# recommended (auto-approvable) scopes, following least privilege; when a business command
-# hits insufficient permissions, the lark-shared skill guides the user to an incremental auth login.
-_DEFAULT_LOGIN_ARGS = ["auth", "login", "--recommend", "--no-wait", "--json"]
-
-# The four login-state fields: filled while pending, cleared on connect success/failure/disconnect.
-# Centralized as a constant to avoid missing one of the hand-copied sites.
-_CLEAR_LOGIN: Dict[str, Any] = {
- "login_verification_url": None,
- "login_verification_url_complete": None,
- "login_user_code": None,
- "login_device_code": None,
-}
-
-_EXPIRED_ERROR = "飞书登录已失效(令牌被刷新或过期),请到设置页重新扫码连接飞书"
-_NO_APP_ERROR = "飞书应用尚未配置,请联系管理员在「插件 → 飞书工作台」初始化飞书应用"
-
-# Markers in auth/login results that positively confirm "auth invalid / device code terminated".
-_AUTH_FAIL_MARKERS = (
- "expired_token", "access_denied", "invalid_grant", "invalid_device_code",
- "authorization_expired", "expired", "denied",
-)
-_PENDING_MARKERS = ("authorization_pending", "pending", "slow_down")
-
-
-# ── JSON parsing (lenient, best effort) ─────────────────────────────────────────
-
-def _loads(s: str) -> Any:
- try:
- return json.loads((s or "").strip() or "null")
- except (ValueError, TypeError):
- return None
-
-
-def parse_device_login(stdout: str) -> Dict[str, Optional[str]]:
- """``auth login --no-wait --json`` → {device_code, user_code, verification_url,
- verification_url_complete}. Field names matched leniently (Lark/OAuth device-flow conventional naming)."""
- out: Dict[str, Optional[str]] = {
- "device_code": None, "user_code": None,
- "verification_url": None, "verification_url_complete": None,
- }
- data = _loads(stdout)
- if not isinstance(data, dict):
- return out
- # Some CLIs wrap the content inside data/result
- for key in ("data", "result", "device", "deviceAuth"):
- inner = data.get(key)
- if isinstance(inner, dict):
- data = {**data, **inner}
-
- def _g(*names: str) -> Optional[str]:
- for n in names:
- for k, v in data.items():
- if k.lower().replace("_", "") == n and isinstance(v, (str, int)):
- return str(v)
- return None
-
- out["device_code"] = _g("devicecode")
- out["user_code"] = _g("usercode")
- out["verification_url"] = _g("verificationurl", "verificationuri")
- out["verification_url_complete"] = _g("verificationurlcomplete", "verificationuricomplete")
- return out
-
-
-def parse_lark_identity(stdout: str) -> Dict[str, Optional[str]]:
- """``auth status``/``auth list`` output → {lark_open_id, lark_name, tenant_key} (searched recursively)."""
- out: Dict[str, Optional[str]] = {"lark_open_id": None, "lark_name": None, "tenant_key": None}
- data = _loads(stdout)
- if data is None:
- return out
-
- def _walk(obj: Any) -> None:
- if isinstance(obj, dict):
- for k, v in obj.items():
- lk = k.lower().replace("_", "")
- if out["lark_open_id"] is None and lk in ("openid", "unionid", "userid"):
- if isinstance(v, (str, int)):
- out["lark_open_id"] = str(v)
- if out["lark_name"] is None and lk in ("name", "username", "nick", "ennname", "displayname"):
- if isinstance(v, str):
- out["lark_name"] = v
- if out["tenant_key"] is None and lk in ("tenantkey", "tenant"):
- if isinstance(v, (str, int)):
- out["tenant_key"] = str(v)
- _walk(v)
- elif isinstance(obj, list):
- for it in obj:
- _walk(it)
-
- _walk(data)
- return out
-
-
-def parse_user_identity_state(stdout: str) -> Optional[bool]:
- """``auth status`` → whether the **user** identity is usable right now;
- ``None`` when the output shape is unrecognized (caller falls back to heuristics).
-
- lark-cli reports the bot and the user identity separately::
-
- {"identities": {"bot": {"status": "ready", "available": true},
- "user": {"status": "missing", "available": false,
- "openId": "ou_…", "userName": "…",
- "tokenStatus": "expired"}}}
-
- Note that ``openId``/``userName`` **stay in the payload after the token
- expires** — they describe who last authorized, not whether the
- authorization still works. So "an identity field is present" must never be
- read as "logged in"; doing so is what let a login that expired months ago
- keep reporting 已登录 on the connection panel while the sandbox correctly
- answered "User identity: missing".
- """
- data = _loads(stdout)
- if not isinstance(data, dict):
- return None
- identities = data.get("identities")
- if not isinstance(identities, dict):
- return None
- user = identities.get("user")
- if not isinstance(user, dict):
- return None
- available = user.get("available")
- if isinstance(available, bool):
- return available
- status = str(user.get("status") or "").strip().lower()
- if not status:
- return None
- return status in ("ready", "ok", "active", "valid")
-
-
-def is_authenticated(stdout: str) -> bool:
- """``auth status`` → whether the user is logged in.
-
- Prefers the structured ``identities.user`` verdict (see
- [[parse_user_identity_state]]); only when the CLI emits a shape we do not
- recognize does it fall back to the old lenient heuristics — those also have
- to serve ``auth login --device-code`` output, which carries no
- ``identities`` block.
- """
- state = parse_user_identity_state(stdout)
- if state is not None:
- return state
- data = _loads(stdout)
- if isinstance(data, dict):
- for k, v in data.items():
- lk = k.lower().replace("_", "")
- if lk in ("authenticated", "loggedin", "isloggedin", "valid") and v is True:
- return True
- ident = parse_lark_identity(stdout)
- return bool(ident.get("lark_open_id") or ident.get("lark_name"))
-
-
-def _classify_login_result(stdout: str, stderr: str, rc: int) -> str:
- """Classify the result of ``auth login --device-code`` → ``ok`` / ``pending`` / ``error``."""
- blob = ((stdout or "") + "\n" + (stderr or "")).lower()
- if rc == 0 and is_authenticated(stdout):
- return "ok"
- if any(m in blob for m in _PENDING_MARKERS):
- return "pending"
- if any(m in blob for m in _AUTH_FAIL_MARKERS):
- return "error"
- # rc==0 but no definite identity: the caller decides whether to probe status again; here conservatively treat as pending (not yet terminal)
- return "ok" if rc == 0 else "pending"
-
-
-# ── Backend subprocess environment / invocation ─────────────────────────────────
-
-def _lark_env(user_id: str) -> Dict[str, str]:
- """Environment for running lark-cli subprocesses on the backend: per-user isolated ``HOME`` (the root of credential isolation).
-
- LARK_APP_ID/SECRET are **not** injected (with them in the env, lark-cli
- defaults to the tenant/bot token flow, breaking --as user); the app
- config lands in HOME via config init. OPENCLAW_HOME/HERMES_HOME are
- stripped — otherwise ``config init`` decides it is in an Agent context
- and refuses to run."""
- from core.sandbox.workspace.credentials import lark_home_dir
-
- home = lark_home_dir(user_id)
- try:
- home.mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] mkdir home %s failed: %s", home, exc)
- env = dict(os.environ)
- env["HOME"] = str(home)
- for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"):
- env.pop(k, None)
- return env
-
-
-async def _run_lark(
- user_id: str, args: List[str], timeout: int = 40, stdin_data: Optional[str] = None,
-) -> Tuple[str, str, int]:
- """Run one lark-cli subcommand on the backend as this user; returns (stdout, stderr, exit_code)."""
- try:
- proc = await asyncio.create_subprocess_exec(
- _LARK_BIN, *args,
- stdin=asyncio.subprocess.PIPE if stdin_data is not None else None,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- env=_lark_env(user_id),
- **no_window_kwargs(),
- )
- except FileNotFoundError:
- return "", "lark-cli binary not found on backend", 127
- try:
- out, err = await asyncio.wait_for(
- proc.communicate(input=stdin_data.encode("utf-8") if stdin_data is not None else None),
- timeout=timeout,
- )
- return out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), proc.returncode or 0
- except asyncio.TimeoutError:
- try:
- proc.kill()
- except ProcessLookupError:
- pass
- return "", f"lark-cli {' '.join(args[:2])} timeout", 124
-
-
-def _lark_app_config() -> Tuple[Optional[str], Optional[str]]:
- """Read the app's app_id / app_secret from "System Config → Lark Workspace" (deployment-level, one app shared by everyone)."""
- try:
- from core.services.system_config import SystemConfigService
-
- svc = SystemConfigService.get_instance()
- app_id = (svc.get("lark.app_id") or "").strip()
- app_secret = (svc.get("lark.app_secret") or "").strip()
- return (app_id or None), (app_secret or None)
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] read lark config failed: %s", exc)
- return None, None
-
-
-# The app base's two on-disk roots (config init writes both; copying them into the user HOME enables their auth login)
-_APP_CFG_SUB = ".lark-cli/config.json"
-_APP_STORE_SUB = ".local/share/lark-cli"
-
-
-def _app_configured() -> Optional[str]:
- """Whether the shared app has been initialized (the admin's one-click config init --new completed). Returns the app_id or None."""
- from core.sandbox.workspace.credentials import lark_app_home_dir
-
- cfg = lark_app_home_dir() / _APP_CFG_SUB
- try:
- data = json.loads(cfg.read_text(encoding="utf-8"))
- apps = data.get("apps") or []
- if apps and isinstance(apps[0], dict):
- return apps[0].get("appId")
- except Exception: # noqa: BLE001
- pass
- return None
-
-
-def _seed_user_from_shared_app(user_id: str) -> bool:
- """Copy the shared app base (config.json + master.key + appsecret_*.enc) into the user HOME.
-
- Copies only once, when the user HOME has no config.json yet (persisted
- afterwards via the bind-mount). master.key and appsecret are copied
- together so that user tokens written by this user's auth login are
- encrypted with — and decryptable by — the same key.
- User token files are never touched (those belong to each individual)."""
- import shutil
-
- from core.sandbox.workspace.credentials import lark_app_home_dir
- from core.sandbox.workspace.credentials import lark_home_dir
-
- app_home = lark_app_home_dir()
- user_home = lark_home_dir(user_id)
- user_cfg = user_home / _APP_CFG_SUB
- if user_cfg.exists():
- return True # base already present
- src_cfg = app_home / _APP_CFG_SUB
- if not src_cfg.exists():
- return False
- try:
- user_cfg.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(src_cfg, user_cfg)
- src_store = app_home / _APP_STORE_SUB
- dst_store = user_home / _APP_STORE_SUB
- dst_store.mkdir(parents=True, exist_ok=True)
- for name in os.listdir(src_store):
- if name == "master.key" or name.startswith("appsecret_"):
- shutil.copy2(src_store / name, dst_store / name)
- return True
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] seed user app from shared failed user=%s: %s", user_id, exc)
- return False
-
-
-async def _ensure_app_config(user_id: str) -> Optional[str]:
- """Ensure the user HOME has the app base. Returns an error message; None means ready.
-
- Prefer the shared app from the **admin's one-click initialization**
- (copying the base into the user HOME; the whole group shares one app);
- without a shared app, fall back to the app_id/secret entered in **System
- Config** (``config init --app-secret-stdin``); with neither → return a
- hint message (the front end guides the admin to initialize).
- """
- # 1) Shared app (admin one-click initialization)
- if _app_configured():
- if _seed_user_from_shared_app(user_id):
- return None
- # 2) app_id/secret hand-entered in System Config (fallback)
- app_id, app_secret = _lark_app_config()
- if app_id and app_secret:
- shown, _, _ = await _run_lark(user_id, ["config", "show"], timeout=15)
- if app_id in (shown or ""):
- return None
- _, err, rc = await _run_lark(
- user_id,
- ["config", "init", "--app-id", app_id, "--app-secret-stdin", "--brand", "feishu"],
- timeout=30,
- stdin_data=app_secret,
- )
- if rc != 0:
- logger.warning("[lark] config init failed user=%s rc=%s err=%s", user_id, rc, (err or "")[:200])
- return f"飞书应用配置初始化失败:{(err or '').strip()[:160] or '未知错误'}"
- return None
- return _NO_APP_ERROR
-
-
-def _update_connection(user_id: str, data: Dict[str, Any]) -> None:
- """Update the connection record with an independent DB session (for the background login task — must not borrow the request-scoped session)."""
- from core.db.engine import SessionLocal
-
- try:
- with SessionLocal() as db:
- LarkConnectionRepository(db).update(user_id, data)
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] _update_connection user=%s failed: %s", user_id, exc)
-
-
-async def _build_connected_update(user_id: str) -> Dict[str, Any]:
- """After successful login, backfill the Lark identity and return the update dict for the ``connected`` state."""
- # No ``--format json``: no shipped lark-cli accepts that flag (1.0.34 has only
- # ``--verify``; 1.0.78 spells it ``--json``), and passing it made every call return a
- # "unknown flag" error instead of the status — which is why lark_open_id / lark_name
- # stayed NULL on every connection row. Bare ``auth status`` already prints JSON on both.
- who, _, _ = await _run_lark(user_id, ["auth", "status"], timeout=20)
- ident = parse_lark_identity(who)
- return {
- "status": "connected",
- "lark_open_id": ident.get("lark_open_id"),
- "lark_name": ident.get("lark_name"),
- "tenant_key": ident.get("tenant_key"),
- **_CLEAR_LOGIN,
- "last_verified_at": utc_now(),
- "last_error": None,
- }
-
-
-async def verify_and_refresh(user_id: str) -> str:
- """Real liveness probe: ``auth status --verify`` (validates against the server,
- renewing with the refresh token along the way).
- Returns ``valid`` / ``invalid`` / ``unknown``. Network/timeout etc. draw no
- conclusion (unknown), avoiding misjudgment from a blip."""
- out, err, rc = await _run_lark(
- user_id, ["auth", "status", "--verify"], timeout=25
- )
- blob = ((out or "") + "\n" + (err or "")).strip()
- if not blob or rc == 124 or rc == 127:
- return "unknown"
- # The structured verdict is authoritative when the CLI gives us one: substring
- # markers are a poor judge here because a *healthy* status payload also carries
- # words like "expiresAt", and an expired one still carries openId/userName.
- state = parse_user_identity_state(out)
- if state is not None:
- return "valid" if state else "invalid"
- low = blob.lower()
- if any(m in low for m in _AUTH_FAIL_MARKERS) or "not authenticated" in low or "not logged in" in low:
- return "invalid"
- if is_authenticated(out):
- return "valid"
- return "unknown"
-
-
-async def _device_complete_flow(user_id: str, device_code: str) -> None:
- """Backend long-running task: ``auth login --device-code `` polls to
- completion (blocking until authorized/expired), then backfills identity.
- Manages its own DB sessions and exceptions, never raising into the event loop."""
- try:
- out, err, rc = await _run_lark(
- user_id,
- ["auth", "login", "--device-code", device_code, "--json"],
- timeout=600,
- )
- verdict = _classify_login_result(out, err, rc)
- if verdict == "ok":
- # Confirm login with another status check (fallback when --device-code succeeds but the json has no identity)
- st, _, _ = await _run_lark(user_id, ["auth", "status"], timeout=20)
- # ``rc == 0`` alone must not win: the CLI can exit 0 without ever storing a
- # user token, and that is how connection rows ended up marked ``connected``
- # with a NULL identity while the sandbox saw no token at all. A *definite*
- # negative from the structured status therefore overrides it; an
- # inconclusive status keeps the old lenient behaviour so unfamiliar CLI
- # output shapes do not start failing valid logins.
- st_state = parse_user_identity_state(st)
- if st_state is False:
- verdict = "error"
- elif st_state is True or is_authenticated(st) or rc == 0:
- _update_connection(user_id, await _build_connected_update(user_id))
- return
- else:
- verdict = "error"
- if verdict == "error":
- tail = ((err or out or "").strip())[-300:]
- _update_connection(user_id, {
- "status": "error", **_CLEAR_LOGIN,
- "last_error": f"扫码授权未完成或已过期,请重试。{('详情: ' + tail) if tail else ''}",
- })
- return
- # pending reaching here (timed out unauthorized) → set error so the user can retry
- _update_connection(user_id, {
- "status": "error", **_CLEAR_LOGIN,
- "last_error": "扫码授权超时,请重新发起连接。",
- })
- except asyncio.CancelledError:
- raise
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] device complete failed user=%s: %s", user_id, exc)
- _update_connection(user_id, {"status": "error", "last_error": f"登录异常: {exc}"})
- finally:
- _LOGIN_TASKS.pop(user_id, None)
-
-
-# ── Admin one-click initialization of the shared app (config init --new) ────────
-# The whole group shares one app, initialized only once, hence a single module-level task + state (not per-user).
-_APP_INIT_TASK: "Optional[asyncio.Task]" = None
-_app_init_state: Dict[str, Any] = {
- "status": "idle", "verification_url": None, "qr_data_uri": None, "error": None,
-}
-
-
-def _app_env() -> Dict[str, str]:
- """Environment for running the shared app's config init: HOME = the shared app HOME; Agent/LARK_APP_* interference stripped."""
- from core.sandbox.workspace.credentials import lark_app_home_dir
-
- home = lark_app_home_dir()
- try:
- home.mkdir(parents=True, exist_ok=True)
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] mkdir app home failed: %s", exc)
- env = dict(os.environ)
- env["HOME"] = str(home)
- for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"):
- env.pop(k, None)
- return env
-
-
-async def _app_init_flow() -> None:
- """Backend long-running task: ``config init --new`` stream-reads the
- ``open.feishu.cn/page/cli`` configuration URL → echoes it back for the
- admin to scan → blocks while the admin completes it in the browser →
- the app base lands in the shared HOME. Manages its own exceptions."""
- global _APP_INIT_TASK
- try:
- proc = await asyncio.create_subprocess_exec(
- _LARK_BIN, "config", "init", "--new", "--lang", "zh",
- stdin=asyncio.subprocess.DEVNULL,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.STDOUT,
- env=_app_env(),
- **no_window_kwargs(),
- )
- except FileNotFoundError:
- _app_init_state.update({"status": "error", "error": "后端未安装 lark-cli"})
- _APP_INIT_TASK = None
- return
- buf = ""
- try:
- while True:
- try:
- line = await asyncio.wait_for(proc.stdout.readline(), timeout=600)
- except asyncio.TimeoutError:
- break
- if not line:
- break
- buf += line.decode("utf-8", "replace")
- if _app_init_state.get("verification_url") is None:
- m = re.search(r"https?://open\.feishu\.cn/\S+", buf)
- if m:
- url = m.group(0).rstrip(".,;")
- _app_init_state.update({
- "status": "pending", "verification_url": url,
- "qr_data_uri": make_qr_data_uri(url), "error": None,
- })
- await asyncio.wait_for(proc.wait(), timeout=30)
- except asyncio.CancelledError:
- try:
- proc.kill()
- except ProcessLookupError:
- pass
- raise
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] app init stream failed: %s", exc)
- finally:
- _APP_INIT_TASK = None
- # Completion check: does the shared HOME now have the app configuration
- aid = _app_configured()
- if aid:
- _app_init_state.update({
- "status": "configured", "app_id": aid,
- "verification_url": None, "qr_data_uri": None, "error": None,
- })
- else:
- _app_init_state.update({
- "status": "error", "verification_url": None, "qr_data_uri": None,
- "error": "应用配置未完成或已取消,请重试。",
- })
-
-
-def _app_status_dict() -> Dict[str, Any]:
- aid = _app_configured()
- if aid:
- return {"configured": True, "app_id": aid, "status": "configured",
- "verification_url": None, "qr_data_uri": None, "error": None}
- return {"configured": False, "app_id": None,
- "status": _app_init_state.get("status", "idle"),
- "verification_url": _app_init_state.get("verification_url"),
- "qr_data_uri": _app_init_state.get("qr_data_uri"),
- "error": _app_init_state.get("error")}
-
-
-class LarkService:
- def __init__(self, db: Session):
- self.db = db
- self.repo = LarkConnectionRepository(db)
-
- def _fresh(self, user_id: str):
- """Discard the session cache and re-read — to see the latest state committed by the background task's independent session."""
- self.db.expire_all()
- return self.repo.ensure(user_id)
-
- # ── Status ───────────────────────────────────────────────────────────
- def status_dict(self, record) -> Dict[str, Any]:
- qr_target = record.login_verification_url_complete or record.login_verification_url
- return {
- "status": record.status,
- "lark_open_id": record.lark_open_id,
- "lark_name": record.lark_name,
- "tenant_key": record.tenant_key,
- "granted_scopes": list(record.granted_scopes or []),
- "verification_url": record.login_verification_url,
- "verification_url_complete": record.login_verification_url_complete,
- "qr_data_uri": make_qr_data_uri(qr_target) if record.status == "pending" else None,
- "user_code": record.login_user_code,
- "last_verified_at": record.last_verified_at.isoformat() if record.last_verified_at else None,
- "last_error": record.last_error,
- }
-
- def get_status(self, user_id: str) -> Dict[str, Any]:
- return self.status_dict(self.repo.ensure(user_id))
-
- async def probe_status(self, user_id: str) -> Dict[str, Any]:
- """Real-API liveness-probe reconciliation (corrects mismatches between the DB and actual credentials). Does not interrupt pending (login in progress)."""
- record = self._fresh(user_id)
- if record.status == "pending" and user_id in _LOGIN_TASKS:
- return self.status_dict(record)
- verdict = await verify_and_refresh(user_id)
- if verdict == "valid" and record.status != "connected":
- self.repo.update(user_id, {"status": "connected", "last_error": None})
- elif verdict == "invalid" and record.status == "connected":
- self.repo.update(user_id, {"status": "disconnected", "last_error": _EXPIRED_ERROR})
- return self.status_dict(self.repo.get(user_id))
-
- # ── Device-flow login (scan-to-bind) ─────────────────────────────────
- async def start_device_login(self, user_id: str) -> Dict[str, Any]:
- """Initiate the Lark device-flow login: first seed the app config, then
- ``auth login --no-wait`` gets the device_code / QR URL, and the
- background completion task is started. The front end presents
- qr_data_uri for the user to scan, then polls."""
- self.repo.ensure(user_id)
- # Cancel the same user's old task
- old = _LOGIN_TASKS.pop(user_id, None)
- if old is not None and not old.done():
- old.cancel()
-
- cfg_err = await _ensure_app_config(user_id)
- if cfg_err:
- self.repo.update(user_id, {"status": "error", **_CLEAR_LOGIN, "last_error": cfg_err})
- return self.status_dict(self.repo.get(user_id))
-
- out, err, rc = await _run_lark(user_id, _DEFAULT_LOGIN_ARGS, timeout=40)
- dev = parse_device_login(out)
- if rc != 0 or not (dev.get("verification_url") or dev.get("verification_url_complete")):
- tail = ((err or out or "").strip())[-200:]
- self.repo.update(user_id, {
- "status": "error", **_CLEAR_LOGIN,
- "last_error": f"发起扫码登录失败,请重试。{('详情: ' + tail) if tail else ''}",
- })
- return self.status_dict(self.repo.get(user_id))
-
- self.repo.update(user_id, {
- "status": "pending",
- "login_verification_url": dev.get("verification_url"),
- "login_verification_url_complete": dev.get("verification_url_complete"),
- "login_user_code": dev.get("user_code"),
- "login_device_code": dev.get("device_code"),
- "login_started_at": utc_now(),
- "last_error": None,
- })
- # Start the background completion task (blocking polling until authorized/expired)
- if dev.get("device_code"):
- _LOGIN_TASKS[user_id] = asyncio.create_task(
- _device_complete_flow(user_id, str(dev["device_code"]))
- )
- return self.status_dict(self._fresh(user_id))
-
- async def poll_login(self, user_id: str) -> Dict[str, Any]:
- """Poll login progress. The background task drives to connected/error;
- this reads the latest state. If the task is lost yet still pending
- (backend restart), fall back to the persisted device_code and run
- completion once more."""
- record = self._fresh(user_id)
- if record.status == "pending" and user_id not in _LOGIN_TASKS and record.login_device_code:
- _LOGIN_TASKS[user_id] = asyncio.create_task(
- _device_complete_flow(user_id, str(record.login_device_code))
- )
- return self.status_dict(record)
-
- # ── Disconnect ───────────────────────────────────────────────────────
- async def disconnect(self, user_id: str) -> Dict[str, Any]:
- """Cancel any in-progress login + backend logout + clear the backend persistent credential directory + set disconnected."""
- old = _LOGIN_TASKS.pop(user_id, None)
- if old is not None and not old.done():
- old.cancel()
- await _run_lark(user_id, ["auth", "logout"], timeout=30)
- try:
- from core.sandbox.workspace.credentials import lark_cache_dir
- from core.sandbox.workspace.credentials import purge_credential_dir
- from core.sandbox.workspace.identity import safe_user_id
-
- # Empty in place — never rmtree the root. ~/.lark-cli and
- # ~/.local/share/lark-cli under it are bind-mounted into this user's
- # live sandboxes; removing the directory orphans those mounts and the
- # next login's credentials land on an inode the sandbox cannot see
- # (see [[purge_credential_dir]]).
- if safe_user_id(user_id):
- purge_credential_dir(lark_cache_dir(user_id))
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] disconnect cache purge failed user=%s: %s", user_id, exc)
- self.repo.update(
- user_id,
- {
- "status": "disconnected",
- "lark_open_id": None,
- "lark_name": None,
- "tenant_key": None,
- "granted_scopes": [],
- **_CLEAR_LOGIN,
- "auth_bundle": None,
- "last_error": None,
- },
- )
- return self.status_dict(self.repo.get(user_id))
-
- # ── Admin: shared-app one-click initialization (config init --new) ────
- def app_status(self) -> Dict[str, Any]:
- """Query the shared app's initialization state (configured / pending+QR code / error)."""
- return _app_status_dict()
-
- async def start_app_init(self) -> Dict[str, Any]:
- """Initiate ``config init --new``: run in the background and echo the configuration QR code for the admin to scan. If already configured, report back directly."""
- global _APP_INIT_TASK
- if _app_configured():
- return _app_status_dict()
- if _APP_INIT_TASK is not None and not _APP_INIT_TASK.done():
- return _app_status_dict() # in progress, echo the current URL
- _app_init_state.update({"status": "pending", "verification_url": None,
- "qr_data_uri": None, "error": None})
- _APP_INIT_TASK = asyncio.create_task(_app_init_flow())
- # Wait up to ~15s for the configuration URL to appear
- for _ in range(30):
- await asyncio.sleep(0.5)
- if _app_init_state.get("verification_url") or _app_init_state.get("status") in ("configured", "error"):
- break
- return _app_status_dict()
-
- async def reset_app(self) -> Dict[str, Any]:
- """Clear the shared app base (for re-initialization). Does not touch users' existing login tokens."""
- global _APP_INIT_TASK
- if _APP_INIT_TASK is not None and not _APP_INIT_TASK.done():
- _APP_INIT_TASK.cancel()
- _APP_INIT_TASK = None
- try:
- import shutil
-
- from core.sandbox.workspace.credentials import lark_app_home_dir
-
- shutil.rmtree(lark_app_home_dir(), ignore_errors=True)
- except Exception as exc: # noqa: BLE001
- logger.warning("[lark] reset app home failed: %s", exc)
- _app_init_state.update({"status": "idle", "verification_url": None,
- "qr_data_uri": None, "error": None})
- return _app_status_dict()
diff --git a/src/backend/core/services/log_service.py b/src/backend/core/services/log_service.py
deleted file mode 100644
index 317ae0be..00000000
--- a/src/backend/core/services/log_service.py
+++ /dev/null
@@ -1,649 +0,0 @@
-"""Observability log writer — async, best-effort writer for the three
-tool/sub-agent/skill log tables.
-
-Writes are fire-and-forget so the SSE hot path never blocks on DB I/O.
-Failures downgrade to structured logs instead of propagating.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-import uuid
-from contextlib import contextmanager
-from contextvars import ContextVar
-from datetime import datetime, timezone
-from typing import Any, Dict, Iterator, List, Optional
-
-from sqlalchemy.orm import Session
-
-from core.db.engine import SessionLocal
-from core.db.models import (
- LocalUser,
- SkillCallLog,
- SubAgentCallLog,
- ToolCallLog,
- ToolEffectLedger,
- UserShadow,
-)
-from core.infra.background import spawn
-from core.infra.data_masking import mask_sensitive_data
-from core.infra.logging import chat_id_var, get_logger, trace_id_var, user_id_var
-
-logger = get_logger(__name__)
-
-_current_source: ContextVar[str] = ContextVar("log_current_source", default="main_agent")
-_current_subagent_log_id: ContextVar[Optional[str]] = ContextVar(
- "log_current_subagent", default=None
-)
-_current_message_id: ContextVar[str] = ContextVar("log_current_message", default="")
-
-TOOL_LOG_ENABLED = os.getenv("TOOL_CALL_LOG_ENABLED", "true").lower() == "true"
-SUBAGENT_LOG_ENABLED = os.getenv("SUBAGENT_LOG_ENABLED", "true").lower() == "true"
-SKILL_LOG_ENABLED = os.getenv("SKILL_LOG_ENABLED", "true").lower() == "true"
-# Whether sandbox-type tool calls additionally write a security audit event
-# (Security Management → Audit Logs, traceable per sandbox instance).
-# Independent of the tool-call log switch: even with tool logging off,
-# security auditing can stay on.
-SANDBOX_AUDIT_ENABLED = os.getenv("SANDBOX_AUDIT_ENABLED", "true").lower() == "true"
-
-MAX_RESULT_BYTES = int(os.getenv("LOG_MAX_RESULT_BYTES", 64 * 1024))
-MAX_STDOUT_BYTES = int(os.getenv("LOG_MAX_STDOUT_BYTES", 64 * 1024))
-
-_REDACT_FIELDS = [
- s.strip()
- for s in os.getenv(
- "LOG_REDACT_FIELDS",
- "password,token,api_key,apikey,authorization,secret,access_key",
- ).split(",")
- if s.strip()
-]
-
-def _new_id() -> str:
- return uuid.uuid4().hex
-
-
-def now_utc() -> datetime:
- return datetime.now(timezone.utc)
-
-
-def _truncate_text(text: Optional[str], limit: int) -> tuple[Optional[str], bool]:
- if text is None:
- return None, False
- if not isinstance(text, str):
- text = str(text)
- raw = text.encode("utf-8", errors="ignore")
- if len(raw) <= limit:
- return text, False
- return raw[:limit].decode("utf-8", errors="ignore") + "\n…[truncated]", True
-
-
-def _truncate_json(payload: Any, limit: int) -> tuple[Any, bool]:
- if payload is None:
- return None, False
- try:
- encoded = json.dumps(payload, ensure_ascii=False, default=str).encode(
- "utf-8", errors="ignore"
- )
- except Exception:
- return {"_repr": str(payload)[:limit]}, True
- if len(encoded) <= limit:
- return payload, False
- preview = encoded[: limit // 2].decode("utf-8", errors="ignore")
- return {"_truncated": True, "_preview": preview}, True
-
-
-def _context_ids() -> Dict[str, str]:
- return {
- "trace_id": trace_id_var.get() or "",
- "user_id": user_id_var.get() or "",
- "chat_id": chat_id_var.get() or "",
- }
-
-
-def _fetch_username(db: Session, user_id: str) -> Optional[str]:
- """The "user name" used in logs: prefer the local account display name
- (nickname → real_name), fall back to the login account username.
-
- When the audit log panel shows this column, operators should see a
- recognizable person name (e.g. "张三"), not the login account (e.g.
- "zhangsan01") — the latter would not match the display names shown
- elsewhere in the UI.
- """
- if not user_id:
- return None
- try:
- row = (
- db.query(UserShadow.username, LocalUser.nickname, LocalUser.real_name)
- .outerjoin(LocalUser, LocalUser.user_id == UserShadow.user_id)
- .filter(UserShadow.user_id == user_id)
- .one_or_none()
- )
- if not row:
- return None
- return row.nickname or row.real_name or row.username
- except Exception as exc: # noqa: BLE001
- logger.debug("username_lookup_failed", user_id=user_id, error=str(exc))
- return None
-
-
-@contextmanager
-def subagent_scope(subagent_log_id: Optional[str], source: str = "subagent") -> Iterator[None]:
- tok_id = _current_subagent_log_id.set(subagent_log_id)
- tok_src = _current_source.set(source)
- try:
- yield
- finally:
- _current_subagent_log_id.reset(tok_id)
- _current_source.reset(tok_src)
-
-
-@contextmanager
-def skill_scope(source: str = "skill") -> Iterator[None]:
- tok = _current_source.set(source)
- try:
- yield
- finally:
- _current_source.reset(tok)
-
-
-def set_current_message_id(message_id: str) -> None:
- _current_message_id.set(message_id or "")
-
-
-def current_subagent_log_id() -> Optional[str]:
- return _current_subagent_log_id.get()
-
-
-def current_source() -> str:
- return _current_source.get() or "main_agent"
-
-
-def _effective_status(declared: Optional[str], result: Any) -> str:
- """声明的状态优先,但"成功"要经得起载荷检验。
-
- 只认**顶层** error,且只认 dict / 能解析成 dict 的 JSON 串——正文里出现 error 字样的
- 正常内容(比如搜到一篇讲错误码的网页)不该被误判成故障。
- """
- status = declared or "success"
- if status != "success":
- return status
- from core.chat.tool_log import _payload_carries_error
-
- return "error" if _payload_carries_error(result) else "success"
-
-
-def _sanitize_tool_args(value: Any) -> Any:
- from core.services.tool_effect_ledger import canonical_tool_args
-
- if isinstance(value, dict):
- return canonical_tool_args(value)[2]
- return mask_sensitive_data(value, field_patterns=_REDACT_FIELDS)
-
-
-def _write_tool_call_sync(record: Dict[str, Any]) -> None:
- if not TOOL_LOG_ENABLED:
- return
- db: Session = SessionLocal()
- try:
- args_safe = _sanitize_tool_args(record.get("tool_args"))
- args_safe, _ = _truncate_json(args_safe, MAX_RESULT_BYTES)
- result_safe, result_trunc = _truncate_json(record.get("tool_result"), MAX_RESULT_BYTES)
- username = _fetch_username(db, record.get("user_id", ""))
- effect = None
- effect_id = str(record.get("effect_id") or "")
- run_id = str(record.get("run_id") or "")
- tool_call_id = str(record.get("tool_call_id") or "")
- if effect_id:
- effect = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one_or_none()
- )
- if effect is None and run_id and tool_call_id:
- effect = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.run_id == run_id,
- ToolEffectLedger.tool_call_id == tool_call_id,
- ToolEffectLedger.event_type == "intent",
- )
- .order_by(ToolEffectLedger.event_id.desc())
- .first()
- )
- # ToolCallLog stays a replaceable query projection. The pre-allocated
- # result id makes retries update the same row, while ToolEffectLedger
- # remains the append-only recovery authority.
- row_id = (
- (effect.result_id if effect is not None else None)
- or record.get("result_id")
- or record.get("id")
- or _new_id()
- )
- row = db.get(ToolCallLog, row_id) or ToolCallLog(id=row_id)
- row.effect_id = effect.effect_id if effect is not None else record.get("effect_id")
- row.trace_id = record.get("trace_id")
- row.chat_id = record.get("chat_id")
- row.message_id = record.get("message_id")
- row.user_id = record.get("user_id")
- row.user_name = username
- row.tool_name = record["tool_name"]
- row.tool_display_name = record.get("tool_display_name")
- row.tool_call_id = record.get("tool_call_id")
- row.mcp_server = record.get("mcp_server")
- row.tool_args = args_safe
- row.tool_result = result_safe
- row.result_truncated = bool(result_trunc)
- # A top-level error payload is a failed projection even when an MCP
- # adapter declared the transport call successful.
- row.status = _effective_status(record.get("status"), record.get("tool_result"))
- row.error_message = record.get("error_message")
- row.duration_ms = record.get("duration_ms")
- row.source = record.get("source") or "main_agent"
- row.subagent_log_id = record.get("subagent_log_id")
- row.skill_log_id = record.get("skill_log_id")
- row.sandbox_id = record.get("sandbox_id")
- row.started_at = record.get("started_at")
- row.created_at = record.get("created_at") or now_utc()
- db.add(row)
- db.commit()
- except Exception as exc: # noqa: BLE001
- db.rollback()
- logger.warning("tool_call_log_write_failed", error=str(exc), tool=record.get("tool_name"))
- finally:
- db.close()
-
-
-def _write_subagent_create_sync(record: Dict[str, Any]) -> None:
- if not SUBAGENT_LOG_ENABLED:
- return
- db: Session = SessionLocal()
- try:
- input_safe, _ = _truncate_json(record.get("input_messages"), MAX_RESULT_BYTES)
- username = _fetch_username(db, record.get("user_id", ""))
- row = SubAgentCallLog(
- id=record["id"],
- trace_id=record.get("trace_id"),
- chat_id=record.get("chat_id"),
- message_id=record.get("message_id"),
- user_id=record.get("user_id"),
- user_name=username,
- subagent_id=record.get("subagent_id"),
- subagent_name=record["subagent_name"],
- subagent_type=record.get("subagent_type"),
- plan_id=record.get("plan_id"),
- step_id=record.get("step_id"),
- step_index=record.get("step_index"),
- step_title=record.get("step_title"),
- model=record.get("model"),
- input_messages=input_safe,
- status="running",
- parent_subagent_log_id=record.get("parent_subagent_log_id"),
- started_at=record.get("started_at") or now_utc(),
- created_at=now_utc(),
- )
- db.add(row)
- db.commit()
- except Exception as exc: # noqa: BLE001
- db.rollback()
- logger.warning(
- "subagent_log_create_failed", error=str(exc), name=record.get("subagent_name")
- )
- finally:
- db.close()
-
-
-def _write_subagent_finish_sync(
- subagent_log_id: str,
- status: str,
- updates: Dict[str, Any],
-) -> None:
- if not SUBAGENT_LOG_ENABLED:
- return
- db: Session = SessionLocal()
- try:
- row = db.query(SubAgentCallLog).filter(SubAgentCallLog.id == subagent_log_id).one_or_none()
- if row is None:
- return
- row.status = status
- if "output_content" in updates:
- output, _ = _truncate_text(updates.get("output_content"), MAX_RESULT_BYTES)
- row.output_content = output
- if "intermediate_steps" in updates:
- steps_safe, _ = _truncate_json(updates.get("intermediate_steps"), MAX_RESULT_BYTES)
- row.intermediate_steps = steps_safe
- if "token_usage" in updates:
- row.token_usage = updates.get("token_usage")
- if "tool_calls_count" in updates:
- row.tool_calls_count = updates.get("tool_calls_count")
- if "skill_calls_count" in updates:
- row.skill_calls_count = updates.get("skill_calls_count")
- if "error_message" in updates:
- row.error_message = updates.get("error_message")
- if "duration_ms" in updates:
- row.duration_ms = updates.get("duration_ms")
- row.completed_at = now_utc()
- db.commit()
- except Exception as exc: # noqa: BLE001
- db.rollback()
- logger.warning("subagent_log_finish_failed", error=str(exc), id=subagent_log_id)
- finally:
- db.close()
-
-
-def _write_skill_call_sync(record: Dict[str, Any]) -> None:
- if not SKILL_LOG_ENABLED:
- return
- db: Session = SessionLocal()
- try:
- args_safe = mask_sensitive_data(record.get("script_args"), field_patterns=_REDACT_FIELDS)
- args_safe, _ = _truncate_json(args_safe, MAX_RESULT_BYTES)
- stdout_safe, trunc_out = _truncate_text(record.get("script_stdout"), MAX_STDOUT_BYTES)
- stderr_safe, trunc_err = _truncate_text(record.get("script_stderr"), MAX_STDOUT_BYTES)
- stdin_safe, _ = _truncate_text(record.get("script_stdin"), MAX_STDOUT_BYTES)
- username = _fetch_username(db, record.get("user_id", ""))
- row = SkillCallLog(
- id=record.get("id") or _new_id(),
- trace_id=record.get("trace_id"),
- chat_id=record.get("chat_id"),
- message_id=record.get("message_id"),
- user_id=record.get("user_id"),
- user_name=username,
- skill_id=record["skill_id"],
- skill_name=record.get("skill_name"),
- skill_version=record.get("skill_version"),
- skill_source=record.get("skill_source"),
- invocation_type=record.get("invocation_type") or "auto_load",
- script_name=record.get("script_name"),
- script_language=record.get("script_language"),
- script_args=args_safe,
- script_stdin=stdin_safe,
- script_stdout=stdout_safe,
- script_stderr=stderr_safe,
- output_truncated=bool(trunc_out or trunc_err),
- exit_code=record.get("exit_code"),
- status=record.get("status") or "success",
- error_message=record.get("error_message"),
- duration_ms=record.get("duration_ms"),
- source=record.get("source") or current_source(),
- subagent_log_id=record.get("subagent_log_id") or current_subagent_log_id(),
- started_at=record.get("started_at"),
- created_at=record.get("created_at") or now_utc(),
- )
- db.add(row)
- db.commit()
- except Exception as exc: # noqa: BLE001
- db.rollback()
- logger.warning("skill_call_log_write_failed", error=str(exc), skill=record.get("skill_id"))
- finally:
- db.close()
-
-
-# Sandbox-type tool names: these tools act directly on the sandbox (bash
-# execution, artifact transfer, file create/modify/read, search). Only these
-# tools resolve sandbox_id and write security audits; backend-only tools
-# (web / pin / data_context…) are not tagged.
-# Note: Read/Grep/Glob are attributed to the "current sandbox" when there is an
-# active sandbox session; details carry the actual path/pattern for tracing.
-_SANDBOX_TOOL_NAMES = {
- "Bash",
- "sandbox_put_artifact",
- "sandbox_get_artifact",
- "Write",
- "Edit",
- "Read",
- "Grep",
- "Glob",
-}
-
-
-async def _resolve_sandbox_id(session_id: Optional[str]) -> Optional[str]:
- """Look up the current sandbox instance id from the sandbox provider by session key (default = chat_id). Pure query; None on failure."""
- if not session_id:
- return None
- try:
- from core.sandbox import get_sandbox_provider
-
- return await get_sandbox_provider().current_sandbox_id(session_id)
- except (
- Exception
- ): # noqa: BLE001 — audit enrichment is best-effort, must never affect the main path
- return None
-
-
-# Sandbox tool → security audit action name. Audits let the security admin
-# console trace by sandbox instance, independent of tool-call logging.
-_SANDBOX_AUDIT_ACTIONS = {
- "Bash": "sandbox.bash.exec",
- "sandbox_put_artifact": "sandbox.artifact.put",
- "sandbox_get_artifact": "sandbox.artifact.get",
- "Write": "sandbox.file.write",
- "Edit": "sandbox.file.edit",
- "Read": "sandbox.file.read",
- "Grep": "sandbox.file.grep",
- "Glob": "sandbox.file.glob",
-}
-
-
-# Tool/skill status → audit-constrained success/failure/error (unlisted ones like failed/timeout → failure).
-_AUDIT_STATUS_MAP = {"success": "success", "error": "error"}
-
-
-def _audit_status(raw: Optional[str]) -> str:
- return _AUDIT_STATUS_MAP.get(raw or "success", "failure")
-
-
-def _insert_sandbox_audit(
- *,
- action: str,
- sandbox_id: Any,
- user_id: Optional[str],
- trace_id: Optional[str],
- details: Dict[str, Any],
- status: str,
- created_at: Optional[datetime],
-) -> None:
- """Write one resource=sandbox security audit row. Best-effort: failure only degrades to a log entry, never affects the main path."""
- from core.db.repository import AuditLogRepository
-
- db: Session = SessionLocal()
- try:
- # Reuse the audit repository's write (has its own add/commit/rollback; audit failure doesn't block the main path).
- AuditLogRepository(db).create(
- {
- "user_id": user_id
- or None, # empty string would trip the FK constraint; normalize to NULL
- "action": action,
- "resource_type": "sandbox",
- "resource_id": str(sandbox_id)[:64],
- "sandbox_id": str(sandbox_id)[:128],
- "details": details,
- "trace_id": trace_id,
- "status": status,
- "created_at": created_at or now_utc(),
- }
- )
- except Exception as exc: # noqa: BLE001
- logger.warning("sandbox_audit_write_failed", error=str(exc), sandbox=str(sandbox_id))
- finally:
- db.close()
-
-
-def _write_sandbox_audit_sync(record: Dict[str, Any]) -> None:
- """Write one sandbox-type tool call as a security audit event (resource=sandbox / sandbox instance id).
-
- Separate from tool-call logging: audits let "Security Management → Audit
- Logs" distinguish and filter by sandbox instance. details only summarizes
- the input args (redacted + truncated), never dumps full results.
- """
- sandbox_id = record.get("sandbox_id")
- if not sandbox_id:
- return
- tool_name = record.get("tool_name")
- action = _SANDBOX_AUDIT_ACTIONS.get(tool_name, "sandbox.tool.call")
- args_safe = _sanitize_tool_args(record.get("tool_args"))
- details: Dict[str, Any] = {"tool": tool_name}
- if isinstance(args_safe, dict):
- cmd = args_safe.get("command")
- if cmd is not None:
- preview, _ = _truncate_text(str(cmd), 2000)
- details["command"] = preview
- # File/search tools: record path and pattern (not large fields like content) to trace what was changed/read.
- for k in ("file_path", "filename", "path", "pattern", "artifact_id", "name"):
- if args_safe.get(k) is not None:
- details[k] = args_safe[k]
- if record.get("error_message"):
- details["error"], _ = _truncate_text(str(record["error_message"]), 500)
-
- _insert_sandbox_audit(
- action=action,
- sandbox_id=sandbox_id,
- user_id=record.get("user_id"),
- trace_id=record.get("trace_id"),
- details=details,
- status=_audit_status(record.get("status")),
- created_at=record.get("created_at"),
- )
-
-
-def _write_skill_sandbox_audit_sync(record: Dict[str, Any]) -> None:
- """Skill script execution (run_script) = running code in the sandbox; write one sandbox.skill.run audit.
-
- Triggered only by run_script (view / auto_load do not execute code in the
- sandbox); skipped when there is no sandbox instance.
- """
- sandbox_id = record.get("sandbox_id")
- if not sandbox_id:
- return
- details: Dict[str, Any] = {
- "skill_id": record.get("skill_id"),
- "skill_name": record.get("skill_name"),
- "script_name": record.get("script_name"),
- "script_language": record.get("script_language"),
- }
- if record.get("error_message"):
- details["error"], _ = _truncate_text(str(record["error_message"]), 500)
-
- _insert_sandbox_audit(
- action="sandbox.skill.run",
- sandbox_id=sandbox_id,
- user_id=record.get("user_id"),
- trace_id=record.get("trace_id"),
- details=details,
- status=_audit_status(record.get("status")),
- created_at=record.get("created_at"),
- )
-
-
-async def write_tool_call(record: Dict[str, Any]) -> None:
- record.setdefault("source", current_source())
- record.setdefault("subagent_log_id", current_subagent_log_id())
- ctx = _context_ids()
- record.setdefault("trace_id", ctx["trace_id"])
- record.setdefault("user_id", ctx["user_id"])
- record.setdefault("chat_id", ctx["chat_id"])
- record.setdefault("message_id", _current_message_id.get())
- # Sandbox-type tools: resolve the sandbox instance id they run in (session
- # key default = chat_id). Awaiting in place on this async path — provider
- # reads its active-session table (synchronous in-memory read, no side
- # effects) — is more reliable than passing a ContextVar across tasks.
- if record.get("tool_name") in _SANDBOX_TOOL_NAMES:
- if not record.get("sandbox_id"):
- record["sandbox_id"] = await _resolve_sandbox_id(record.get("chat_id"))
- # Security audit is independent of the tool-call log switch: write the audit first, then the (optional) tool log.
- if SANDBOX_AUDIT_ENABLED:
- try:
- await asyncio.to_thread(_write_sandbox_audit_sync, record)
- except Exception as exc: # noqa: BLE001
- logger.warning("sandbox_audit_task_failed", error=str(exc))
- if not TOOL_LOG_ENABLED:
- return
- try:
- await asyncio.to_thread(_write_tool_call_sync, record)
- except Exception as exc: # noqa: BLE001
- logger.warning("tool_call_log_task_failed", error=str(exc))
-
-
-async def start_subagent_log(record: Dict[str, Any]) -> str:
- log_id = record.get("id") or _new_id()
- record["id"] = log_id
- if not SUBAGENT_LOG_ENABLED:
- return log_id
- ctx = _context_ids()
- record.setdefault("trace_id", ctx["trace_id"])
- record.setdefault("user_id", ctx["user_id"])
- record.setdefault("chat_id", ctx["chat_id"])
- record.setdefault("message_id", _current_message_id.get())
- record.setdefault("parent_subagent_log_id", current_subagent_log_id())
- record.setdefault("started_at", now_utc())
- try:
- await asyncio.to_thread(_write_subagent_create_sync, record)
- except Exception as exc: # noqa: BLE001
- logger.warning("subagent_log_start_failed", error=str(exc))
- return log_id
-
-
-async def finish_subagent_log(
- subagent_log_id: str,
- *,
- status: str = "success",
- output_content: Optional[str] = None,
- intermediate_steps: Optional[List[Dict[str, Any]]] = None,
- token_usage: Optional[Dict[str, int]] = None,
- tool_calls_count: int = 0,
- skill_calls_count: int = 0,
- error_message: Optional[str] = None,
- duration_ms: Optional[int] = None,
-) -> None:
- if not SUBAGENT_LOG_ENABLED or not subagent_log_id:
- return
- updates = {
- "output_content": output_content,
- "intermediate_steps": intermediate_steps,
- "token_usage": token_usage,
- "tool_calls_count": tool_calls_count,
- "skill_calls_count": skill_calls_count,
- "error_message": error_message,
- "duration_ms": duration_ms,
- }
- try:
- await asyncio.to_thread(_write_subagent_finish_sync, subagent_log_id, status, updates)
- except Exception as exc: # noqa: BLE001
- logger.warning("subagent_log_finish_failed", error=str(exc))
-
-
-async def write_skill_call(record: Dict[str, Any]) -> str:
- log_id = record.get("id") or _new_id()
- record["id"] = log_id
- ctx = _context_ids()
- record.setdefault("trace_id", ctx["trace_id"])
- record.setdefault("user_id", ctx["user_id"])
- record.setdefault("chat_id", ctx["chat_id"])
- record.setdefault("message_id", _current_message_id.get())
- # Skill script execution (run_script) = running code in the sandbox → write a security audit (independent of the skill log switch).
- if SANDBOX_AUDIT_ENABLED and record.get("invocation_type") == "run_script":
- if not record.get("sandbox_id"):
- record["sandbox_id"] = await _resolve_sandbox_id(record.get("chat_id"))
- try:
- await asyncio.to_thread(_write_skill_sandbox_audit_sync, record)
- except Exception as exc: # noqa: BLE001
- logger.warning("skill_sandbox_audit_task_failed", error=str(exc))
- if not SKILL_LOG_ENABLED:
- return log_id
- try:
- await asyncio.to_thread(_write_skill_call_sync, record)
- except Exception as exc: # noqa: BLE001
- logger.warning("skill_call_log_task_failed", error=str(exc))
- return log_id
-
-
-def schedule_tool_call_write(record: Dict[str, Any]) -> None:
- spawn(write_tool_call(record), name="log_writer.tool_call")
-
-
-def schedule_skill_call_write(record: Dict[str, Any]) -> None:
- spawn(write_skill_call(record), name="log_writer.skill_call")
diff --git a/src/backend/core/services/marketplace_service.py b/src/backend/core/services/marketplace_service.py
deleted file mode 100644
index 8d97bd3d..00000000
--- a/src/backend/core/services/marketplace_service.py
+++ /dev/null
@@ -1,1400 +0,0 @@
-"""Skill Marketplace service.
-
-The skill marketplace is a set of **preloaded, curated** installable skill
-packages, shipped with the repo/image and stored under
-``extensions/skills/marketplace//`` (each directory holds the original
-SKILL.md + referenced files + a ``marketplace.json`` manifest). The built-in
-loader scans ``extensions/skills/*/SKILL.md`` with single-level matching; market
-skills live two levels deep at ``marketplace//SKILL.md`` and are NOT
-auto-loaded as built-ins — so before "installation" a market skill never
-appears in the catalog nor gets registered with the agent. Only after a
-user/admin explicitly installs it does it take effect in the DB.
-
-Installation reuses the existing ``AdminSkill`` mechanism:
-- Admin install → global skill (``owner_user_id`` empty), skill id = manifest entry_name;
-- User install → private skill (``owner_user_id`` = current user); the skill id
- gets a user-fingerprint suffix appended to guarantee global uniqueness, so
- multiple users can each install the same market skill, each with their own
- credentials.
-
-Credentials (API keys etc.): the manifest may declare ``required_secrets``;
-they are collected by the caller at install time, written to ``secrets.json``
-in that installed skill's directory, and a "credentials configuration" section
-is appended at the end of SKILL.md for the agent to read when running scripts.
-The marketplace directory itself stores NO secrets.
-
-Community publishing (marketplace_submissions): users can **submit** their
-private skills for listing in the marketplace; once an admin approves them at
-/admin, the skill appears in the market list with ``source=community`` and
-becomes installable by everyone. Submission snapshots the source skill's
-content, decoupling the listed content from the user's later edits; an admin
-rejecting an already-approved submission delists it.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import io
-import json
-import logging
-import os
-import re
-import uuid
-import zipfile
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-from core.agent_skills.binary_files import (
- BINARY_EXTENSIONS,
- JUNK_BASENAMES,
- MAX_SINGLE_FILE,
- MAX_TOTAL,
- decode_binary,
- encode_binary,
- is_binary_value,
- pack_directory,
-)
-from core.agent_skills.cache_refresh import refresh_skill_caches
-from core.agent_skills.deps_detector import detect_dependencies
-from core.agent_skills.registry import (
- SkillSpecError,
- _load_skill_metadata_from_str,
- _split_frontmatter,
-)
-from core.db.models import AdminSkill, MarketplaceSubmission
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.infra.time import utc_now
-from core.ontology.build_validator import ensure_ontology_build_valid
-from extensions.paths import SKILLS_DIR
-from fastapi import HTTPException
-from sqlalchemy.orm import Session
-from sqlalchemy.orm.attributes import flag_modified
-
-logger = logging.getLogger(__name__)
-
-# Root directory of preloaded market skill packages: ``src/backend/extensions/skills/marketplace/``.
-# Lives in the same tree as built-in skills (``extensions/skills//``), making
-# additions/removals uniform under extensions/skills; but the built-in loader's scan
-# is single-level ``glob("*/SKILL.md")`` — market skills at
-# ``marketplace//SKILL.md`` (two levels deep) are not auto-loaded as
-# built-ins, preserving the install-based model.
-MARKETPLACE_DIR = SKILLS_DIR / "marketplace"
-# Built-in default skills directory: ``extensions/skills/default//``. These
-# skills are globally resident and always available to everyone; they are also
-# shown in the skill marketplace as "built-in" entries (read from their own
-# folders, not copied into marketplace/ to avoid 4MB+ duplication).
-DEFAULT_SKILLS_DIR = SKILLS_DIR / "default"
-
-MANIFEST_NAME = "marketplace.json"
-SKILL_MD_NAME = "SKILL.md"
-SECRETS_FILENAME = "secrets.json"
-
-# The built-in default skills' "category + summary" in the marketplace (default
-# skills have no category in their frontmatter; the Chinese name comes from
-# SKILL.md's display_name). The key set defines "which default skills are listed
-# in the marketplace" — visible to everyone, marked as built-in resident.
-DEFAULT_SKILL_MARKET = {
- "pdf-editing": {"category": "文档处理", "summary": "生成 / 合并 / 拆分 / 抽页 / 填表单 / 读取 PDF 文档"},
- "capability-guide-brief": {"category": "办公效率", "summary": "速答系统能力清单与使用指引"},
- # OfficeCLI 系列(上游 iOfficeAI/OfficeCLI,Apache 2.0):靠沙盒预装的
- # ``officecli`` 单文件二进制驱动,是 Word / Excel / PPT 的**唯一**入口——
- # 自研的 word-editing / excel-editing / ppt-design 三个技能已于 2026-08 下线,
- # 能力全部由这一系列承接。三个基座(docx / xlsx / pptx)之上叠场景层,
- # 分工写在各自 SKILL.md 的 description 里,由技能选择器路由。
- "officecli-docx": {"category": "文档处理", "summary": "OfficeCLI 深度编辑 .docx:TOC / 域 / 脚注 / 公式 / 原生图表 / 批注修订 / schema 校验"},
- "officecli-xlsx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .xlsx:透视表 / 迷你图 / 条件格式 / 数据验证 / 切片器"},
- "officecli-pptx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .pptx:母版版式 / 连接线 / 形状级控制 / 逐页截图质检"},
- "officecli-word-form": {"category": "文档处理", "summary": "制作可填写 Word 表单:内容控件(SDT) + 复选框 + 邮件合并域 + 文档保护"},
- "officecli-academic-paper": {"category": "文档处理", "summary": "学术论文排版:APA / Chicago / IEEE / MLA 引用、编号公式、交叉引用、参考文献"},
- "officecli-data-dashboard": {"category": "数据分析", "summary": "CSV / 表格数据做成 Excel 数据看板:KPI 卡片 + 多图表 + 迷你图 + 条件格式"},
- "officecli-financial-model": {"category": "数据分析", "summary": "Excel 财务模型:三表 / DCF / LBO / 单位经济 / 敏感性分析,全活公式驱动"},
- "officecli-pitch-deck": {"category": "办公效率", "summary": "融资路演 PPT:种子轮 / A-C 轮 / SAFE / 可转债的投资人 deck"},
- "morph-ppt": {"category": "办公效率", "summary": "带 Morph 平滑转场动效的 PPT,自带 40 套成品设计风格包"},
- "morph-ppt-3d": {"category": "办公效率", "summary": "带 .glb 三维模型与电影级镜头运动的 Morph 动效 PPT"},
-}
-
-# Fixed skill-marketplace categories (the only allowed set): preloaded
-# manifests, community submissions, and admin-review category changes may only
-# take these 8 values; mirrored on the frontend in utils/constants.ts as
-# MARKETPLACE_CATEGORIES.
-MARKETPLACE_CATEGORIES = [
- "写作助手", "文档处理", "数据分析", "政策产业",
- "营销创意", "法务合规", "办公效率", "研发效率",
-]
-# Storage constraints / binary extension set + packing logic are unified in
-# core.agent_skills.binary_files (pack_directory / BINARY_EXTENSIONS / MAX_*);
-# admin upload and plugin import share the same single source of truth.
-
-
-# ── Manifest / listing ───────────────────────────────────────────────────────
-
-def _read_manifest(slug: str) -> Optional[Dict[str, Any]]:
- """Read and validate a single market skill's ``marketplace.json`` (tolerant; returns None if corrupted)."""
- if not slug or "/" in slug or ".." in slug:
- return None
- path = MARKETPLACE_DIR / slug / MANIFEST_NAME
- if not path.is_file():
- return None
- try:
- m = json.loads(path.read_text(encoding="utf-8"))
- except Exception as exc: # noqa: BLE001
- logger.warning("marketplace manifest broken: %s (%s)", slug, exc)
- return None
- if not isinstance(m, dict) or not m.get("entry_name"):
- return None
- m.setdefault("slug", slug)
- return m
-
-
-def market_skill_requires_secrets(slug: str) -> bool:
- """Whether a preloaded market skill declares required_secrets (public contract, reusable by other markets/callers)."""
- m = _read_manifest(slug)
- if not m:
- return False
- return bool(m.get("requires_api_key") or m.get("required_secrets"))
-
-
-def _public_meta(m: Dict[str, Any]) -> Dict[str, Any]:
- """Manifest → externally visible metadata (no skill body/file contents)."""
- return {
- "slug": m.get("slug"),
- "entry_name": m.get("entry_name"),
- "display_name": m.get("display_name") or m.get("slug"),
- "summary": m.get("summary") or "",
- "category": m.get("category") or "其他",
- "tags": list(m.get("tags") or []),
- "version": m.get("version") or "1.0.0",
- "author": m.get("author") or "skillhub",
- "icon_url": m.get("icon_url") or "",
- "source": m.get("source") or "skillhub",
- "source_url": m.get("source_url") or "",
- "downloads": int(m.get("downloads") or 0),
- "stars": int(m.get("stars") or 0),
- "featured": bool(m.get("featured")),
- "requires_api_key": bool(m.get("requires_api_key")),
- "required_secrets": list(m.get("required_secrets") or []),
- # Preloaded (filesystem) skills ship with the repo; admins cannot delete them online.
- "deletable": False,
- }
-
-
-def _submission_public_meta(sub: MarketplaceSubmission) -> Dict[str, Any]:
- """Approved community submission → external metadata isomorphic to the preloaded manifest form."""
- return {
- "slug": sub.slug,
- "entry_name": sub.slug,
- "display_name": sub.display_name or sub.slug,
- "summary": sub.summary or "",
- "category": sub.category or "社区共享",
- "tags": list(sub.tags or []),
- "version": sub.version or "1.0.0",
- "author": sub.submitter_name or sub.owner_user_id,
- "icon_url": "",
- "source": "community",
- "source_url": "",
- "downloads": 0,
- "stars": 0,
- "featured": False,
- "requires_api_key": False,
- "required_secrets": _submission_required_secrets(sub),
- # DB listing records (admin uploads / user community submissions) → admins can delete online (remove from the market).
- "deletable": True,
- }
-
-
-def _approved_submissions(db: Session) -> List[MarketplaceSubmission]:
- return (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.status == "approved")
- .order_by(MarketplaceSubmission.created_at.desc())
- .all()
- )
-
-
-def _default_skill_public_meta(slug: str) -> Optional[Dict[str, Any]]:
- """Built-in default skill (extensions/skills/default/) → market-list metadata isomorphic to the preloaded form.
-
- The Chinese name comes from SKILL.md's ``display_name``; category/summary
- come from ``DEFAULT_SKILL_MARKET``. Marked ``source='builtin'`` +
- ``builtin=True`` — these skills are globally resident and always available
- to everyone; the market shows them as "built-in" (already available) and
- they do not go through "install as a private copy" (avoiding duplicating
- 1MB+ office skills into the DB).
- """
- cfg = DEFAULT_SKILL_MARKET.get(slug)
- if cfg is None:
- return None
- md = DEFAULT_SKILLS_DIR / slug / SKILL_MD_NAME
- if not md.is_file():
- return None
- try:
- fm, _ = _split_frontmatter(md.read_text(encoding="utf-8", errors="ignore"))
- except Exception: # noqa: BLE001
- fm = {}
- name = str(fm.get("name") or slug).strip()
- tags = fm.get("tags") or []
- if isinstance(tags, str):
- tags = [t.strip() for t in tags.split(",") if t.strip()]
- return {
- "slug": slug,
- "entry_name": name,
- "display_name": str(fm.get("display_name") or name).strip(),
- "summary": cfg["summary"],
- "category": cfg["category"],
- "tags": list(tags),
- "version": str(fm.get("version") or "1.0.0"),
- "author": "内置",
- "icon_url": "",
- "source": "builtin",
- "source_url": "",
- "downloads": 0,
- "stars": 0,
- "featured": False,
- "requires_api_key": False,
- "required_secrets": [],
- "deletable": False, # built-ins ship with the repo; not deletable online
- "builtin": True, # the frontend uses this to show "built-in" and set the button to "already built-in"
- }
-
-
-def _default_skill_metas() -> List[Dict[str, Any]]:
- metas = [_default_skill_public_meta(s) for s in DEFAULT_SKILL_MARKET]
- return [m for m in metas if m]
-
-
-def list_marketplace_skills(
- db: Optional[Session] = None,
- *,
- include_disabled: bool = False,
- viewer_user_id: Optional[str] = None,
-) -> List[Dict[str, Any]]:
- """List all market skill metadata: built-in default skills + preloaded (featured first, downloads descending) + community submissions.
-
- ``include_disabled``: the admin console passes True (sees everything +
- ``market_enabled`` annotation); the user side defaults to False (only sees
- listed items, and scoped items invisible to ``viewer_user_id`` are filtered
- by visibility). See ``marketplace_listing`` for the delist switch and
- visibility scopes.
- """
- items: List[Dict[str, Any]] = []
- if MARKETPLACE_DIR.is_dir():
- for child in sorted(MARKETPLACE_DIR.iterdir()):
- if not child.is_dir():
- continue
- m = _read_manifest(child.name)
- if m:
- items.append(_public_meta(m))
- items.sort(key=lambda x: (0 if x["featured"] else 1, -x["downloads"], x["slug"]))
- # Built-in default skills come first (built-in resident, shown with priority).
- items = _default_skill_metas() + items
- if db is not None:
- items.extend(_submission_public_meta(s) for s in _approved_submissions(db))
- from core.services import marketplace_listing as ml
- items = ml.annotate_and_filter(
- db, ml.KIND_SKILL, items, id_key="slug",
- include_disabled=include_disabled, viewer_user_id=viewer_user_id,
- )
- return items
-
-
-def set_marketplace_skill_enabled(
- db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
-) -> Dict[str, Any]:
- """List/delist a market skill (controls whether it shows in the skill marketplace; installed instances are unaffected)."""
- from core.services import marketplace_listing as ml
- return ml.set_listing_enabled(db, ml.KIND_SKILL, slug, enabled, updated_by=updated_by)
-
-
-def list_categories(
- db: Optional[Session] = None,
- *,
- include_disabled: bool = False,
- viewer_user_id: Optional[str] = None,
-) -> List[str]:
- """Marketplace category list: the 8 fixed categories first (stable order), legacy leftover categories appended after."""
- seen: List[str] = list(MARKETPLACE_CATEGORIES)
- for it in list_marketplace_skills(
- db, include_disabled=include_disabled, viewer_user_id=viewer_user_id
- ):
- if it["category"] not in seen:
- seen.append(it["category"])
- return seen
-
-
-def _validate_category(category: str) -> str:
- """Validate and return a legal category; anything outside the fixed set is a straight 400."""
- category = (category or "").strip()
- if category not in MARKETPLACE_CATEGORIES:
- raise BadRequestError(
- message=f"请从固定分类中选择:{'、'.join(MARKETPLACE_CATEGORIES)}"
- )
- return category
-
-
-def _get_approved_submission(db: Session, slug: str) -> Optional[MarketplaceSubmission]:
- return (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.slug == slug, MarketplaceSubmission.status == "approved")
- .first()
- )
-
-
-def _strip_frontmatter(raw: str) -> str:
- try:
- _, body = _split_frontmatter(raw)
- except Exception: # noqa: BLE001
- body = raw
- return (body or "").strip()
-
-
-def get_marketplace_skill(slug: str, db: Optional[Session] = None) -> Dict[str, Any]:
- """Single market skill detail: metadata + file list + SKILL.md body (for the detail preview).
-
- The preloaded directory takes priority; when not found, fall back to the
- community listing (approved submission).
- """
- # Built-in default skills: read extensions/skills/default/; the body comes from SKILL.md, the file list from the directory.
- if slug in DEFAULT_SKILL_MARKET:
- meta = _default_skill_public_meta(slug)
- if meta is None:
- raise ResourceNotFoundError("marketplace_skill", slug)
- pkg_dir = DEFAULT_SKILLS_DIR / slug
- files: List[Dict[str, Any]] = []
- for p in sorted(pkg_dir.rglob("*")):
- if not p.is_file():
- continue
- rel = p.relative_to(pkg_dir).as_posix()
- if rel == SKILL_MD_NAME:
- continue
- files.append({"path": rel, "size": p.stat().st_size})
- skill_md = pkg_dir / SKILL_MD_NAME
- meta["files"] = files
- meta["instructions"] = _strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) if skill_md.is_file() else ""
- return meta
- m = _read_manifest(slug)
- if not m:
- sub = _get_approved_submission(db, slug) if db is not None else None
- if sub is None:
- raise ResourceNotFoundError("marketplace_skill", slug)
- data = _submission_public_meta(sub)
- data["files"] = [
- {"path": rel, "size": len(str(content).encode("utf-8"))}
- for rel, content in sorted((sub.extra_files or {}).items())
- if rel != REQUIRED_SECRETS_SNAPSHOT # internal sentinel, not part of the installed content
- ]
- data["instructions"] = _strip_frontmatter(sub.skill_content or "")
- return data
- pkg_dir = MARKETPLACE_DIR / slug
- data = _public_meta(m)
-
- files: List[Dict[str, Any]] = []
- for p in sorted(pkg_dir.rglob("*")):
- if not p.is_file() or p.name == MANIFEST_NAME:
- continue
- rel = p.relative_to(pkg_dir).as_posix()
- if rel == SKILL_MD_NAME: # SKILL.md is returned separately as instructions; don't duplicate it in the file list
- continue
- files.append({"path": rel, "size": p.stat().st_size})
- body = ""
- skill_md = pkg_dir / SKILL_MD_NAME
- if skill_md.is_file():
- body = _strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore"))
- data["files"] = files
- data["instructions"] = body
- return data
-
-
-# ── Installation ─────────────────────────────────────────────────────────────
-
-def _user_suffix(owner_user_id: str) -> str:
- """Stable 6-char fingerprint derived from user_id, used to deduplicate private-install skill ids."""
- return hashlib.sha1(owner_user_id.encode("utf-8")).hexdigest()[:6]
-
-
-def compute_install_id(entry_name: str, owner_user_id: Optional[str]) -> str:
- """The installed AdminSkill.skill_id: admin = original name (global); user = original name + fingerprint (private)."""
- if owner_user_id is None:
- return entry_name
- return f"{entry_name}-{_user_suffix(owner_user_id)}"
-
-
-def base_entry_name(skill_id: str, owner_user_id: Optional[str]) -> str:
- """Inverse of ``compute_install_id``: recover the underlying market entry_name from an install id.
-
- Private install ids look like ``{entry_name}-{fingerprint}``; stripping that
- user's 6-char fingerprint suffix yields the entry_name. Global ids (owner is
- None) already are the entry_name and are returned as-is. Used to merge the
- "privately installed version" and the "admin global version" onto the same
- underlying skill for deduplication — preventing the same skill from
- appearing twice in the skill library / at runtime.
- """
- if owner_user_id is None:
- return skill_id
- suffix = f"-{_user_suffix(owner_user_id)}"
- if skill_id.endswith(suffix):
- return skill_id[: -len(suffix)]
- return skill_id
-
-
-def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool:
- """Whether this market skill has been installed by the given scope (global / current user)."""
- if slug in DEFAULT_SKILL_MARKET:
- return True # built-in default skills are globally resident; considered always "installed"
- m = _read_manifest(slug)
- if not m:
- if _get_approved_submission(db, slug) is None:
- return False
- entry_name = slug
- else:
- entry_name = m["entry_name"]
- install_id = compute_install_id(entry_name, owner_user_id)
- q = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id)
- if owner_user_id is None:
- q = q.filter(AdminSkill.owner_user_id.is_(None))
- else:
- q = q.filter(AdminSkill.owner_user_id == owner_user_id)
- return q.first() is not None
-
-
-def annotate_installed(
- items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str]
-) -> List[Dict[str, Any]]:
- """Annotate each market list item with an ``installed`` flag (for the given scope). One batched query decides all."""
- if not items:
- return items
- install_ids = {
- it["slug"]: compute_install_id(it["entry_name"], owner_user_id) for it in items
- }
- q = db.query(AdminSkill.skill_id, AdminSkill.dep_status)
- if owner_user_id is None:
- q = q.filter(AdminSkill.owner_user_id.is_(None))
- else:
- q = q.filter(AdminSkill.owner_user_id == owner_user_id)
- q = q.filter(AdminSkill.skill_id.in_(list(install_ids.values())))
- status_map = {sid: dep for sid, dep in q.all()}
- # Rejected skills carry the admin's reason, surfaced to the user ("Rejected by admin: ...").
- rejected_ids = [sid for sid, dep in status_map.items() if dep == "rejected"]
- reason_map: Dict[str, Optional[str]] = {}
- if rejected_ids:
- from core.services.skill_deps_request_service import get_reject_reason
- for sid in rejected_ids:
- reason_map[sid] = get_reject_reason(db, sid)
- for it in items:
- # Built-in default skills are globally resident and always available to
- # everyone — mark them "installed/ready" directly (the frontend button
- # shows "already built-in"); no AdminSkill lookup, no install flow.
- if it.get("builtin"):
- it["installed"] = True
- it["dep_status"] = "ready"
- it["dep_reason"] = None
- continue
- iid = install_ids[it["slug"]]
- it["installed"] = iid in status_map
- # dep_status: 'installing' = dependencies installing / 'rejected' = admin declined / 'ready' = usable. None when not installed.
- it["dep_status"] = status_map.get(iid) if iid in status_map else None
- it["dep_reason"] = reason_map.get(iid)
- return items
-
-
-def _rewrite_frontmatter_name(content: str, new_name: str) -> str:
- """Rewrite the ``name:`` in the SKILL.md frontmatter to ``new_name`` (touching only that line).
-
- Replaces only the first ``name:`` line within the frontmatter block; all
- other content (including multi-line descriptions) is preserved verbatim.
- Returned unchanged when there is no frontmatter.
- """
- if not content.startswith("---"):
- return content
- end = content.find("\n---", 3)
- if end == -1:
- return content
- head = content[: end + 1]
- tail = content[end + 1 :]
- head = re.sub(r"(?m)^name:[ \t]*.*$", f"name: {new_name}", head, count=1)
- return head + tail
-
-
-def _load_package_files(pkg_dir: Path) -> Tuple[str, Dict[str, str]]:
- """Read a market skill package: returns (SKILL.md original text, {relative path: stored content}).
-
- Binary files are stored as base64 (consistent with admin upload), text as
- original text; the manifest and junk files are skipped, and per-file/total
- size caps are enforced.
- """
- skill_md = pkg_dir / SKILL_MD_NAME
- if not skill_md.is_file():
- raise BadRequestError(message="市场技能包缺少 SKILL.md")
- skill_content = skill_md.read_text(encoding="utf-8")
- extra_files = pack_directory(pkg_dir, skip_names={MANIFEST_NAME, SKILL_MD_NAME})
- return skill_content, extra_files
-
-
-def _inject_secrets(
- skill_content: str,
- extra_files: Dict[str, str],
- required_secrets: List[Dict[str, Any]],
- secrets: Dict[str, str],
-) -> str:
- """Write the user-provided credentials into secrets.json and append a "credentials configuration" section at the end of SKILL.md.
-
- Returns the SKILL.md content with the section appended. Missing required
- fields raise a 400.
- """
- cleaned: Dict[str, str] = {}
- for field in required_secrets:
- key = str(field.get("key") or "").strip()
- if not key:
- continue
- val = (secrets.get(key) or "").strip()
- if not val:
- if field.get("required"):
- label = field.get("label") or key
- raise BadRequestError(message=f"请填写必填凭据:{label}")
- continue
- cleaned[key] = val
- if not cleaned:
- return skill_content
-
- extra_files[SECRETS_FILENAME] = json.dumps(cleaned, ensure_ascii=False, indent=2)
- lines = [
- "",
- "---",
- "",
- "## 🔑 凭据配置(由安装者提供)",
- "",
- f"运行本技能所需的凭据已写入技能目录下的 `{SECRETS_FILENAME}`,包含以下字段:",
- "",
- ]
- for key in cleaned:
- lines.append(f"- `{key}`")
- lines += [
- "",
- f"执行技能脚本前,请先从 `{SECRETS_FILENAME}` 读取这些值,并以**同名环境变量**导出后再运行,例如:",
- "",
- "```bash",
- f'export {next(iter(cleaned))}="$(python3 -c "import json;print(json.load(open(\'{SECRETS_FILENAME}\'))[\'{next(iter(cleaned))}\'])")"',
- "```",
- "",
- "> 凭据仅存于本技能目录,请勿在对话中明文回显。",
- "",
- ]
- return skill_content.rstrip() + "\n" + "\n".join(lines)
-
-
-_SECRETS_SECTION_HEADER = "## 🔑 凭据配置(由安装者提供)"
-# Internal sentinel filename carrying required_secrets inside listing snapshots:
-# stored with the snapshot, popped before install/detail display, never lands in
-# the installed skill directory.
-REQUIRED_SECRETS_SNAPSHOT = "_required_secrets.json"
-
-
-def _strip_injected_secrets(skill_content: str) -> str:
- """Strip the "credentials configuration" section appended by _inject_secrets (including its preceding --- separator).
-
- Listing snapshots must be in the pre-injection form — the section instructs
- reading secrets.json, which the snapshot has already excluded; the
- installer's own credentials get a fresh section re-injected at install time.
- """
- idx = skill_content.find(_SECRETS_SECTION_HEADER)
- if idx < 0:
- return skill_content
- sep = skill_content.rfind("\n---", 0, idx)
- cut = sep if sep >= 0 else idx
- return skill_content[:cut].rstrip() + "\n"
-
-
-def _submission_required_secrets(sub: MarketplaceSubmission) -> List[Dict[str, Any]]:
- """Recover a community skill's required_secrets from the snapshot sentinel entry (empty list if absent)."""
- raw = (sub.extra_files or {}).get(REQUIRED_SECRETS_SNAPSHOT)
- if not raw:
- return []
- try:
- fields = json.loads(raw)
- return fields if isinstance(fields, list) else []
- except (TypeError, ValueError):
- return []
-
-
-def install_marketplace_skill(
- db: Session,
- slug: str,
- *,
- owner_user_id: Optional[str],
- secrets: Optional[Dict[str, str]] = None,
-) -> Dict[str, Any]:
- """Install a market skill as an ``AdminSkill`` (admin = global / user = private), returning a result dict.
-
- The preloaded directory takes priority; when the slug isn't in the preloaded
- directory, fall back to the community listing (approved submission), with
- install content taken from the submission-time snapshot.
- """
- # Built-in default skills are globally resident and always available to
- # everyone — no install needed (the frontend button is already disabled as
- # "already built-in"); this is a defensive block to avoid duplicating 1MB+
- # office skill content into the DB as a private copy.
- if slug in DEFAULT_SKILL_MARKET:
- raise BadRequestError(message="内置技能已全局可用,无需安装")
- secrets = secrets or {}
- m = _read_manifest(slug)
- if m:
- pkg_dir = MARKETPLACE_DIR / slug
- skill_content, extra_files = _load_package_files(pkg_dir)
- else:
- sub = _get_approved_submission(db, slug)
- if sub is None:
- raise ResourceNotFoundError("marketplace_skill", slug)
- m = {
- "slug": sub.slug,
- "entry_name": sub.slug,
- "display_name": sub.display_name,
- "summary": sub.summary or "",
- "user_intro": sub.user_intro or "",
- "category": sub.category or "社区共享",
- "tags": list(sub.tags or []),
- "version": sub.version or "1.0.0",
- "required_secrets": _submission_required_secrets(sub),
- }
- skill_content = sub.skill_content or ""
- extra_files = dict(sub.extra_files or {})
- # The sentinel entry only carries the credential field names; it never lands in the install directory
- extra_files.pop(REQUIRED_SECRETS_SNAPSHOT, None)
-
- entry_name = str(m["entry_name"]).strip()
- install_id = compute_install_id(entry_name, owner_user_id)
- # Always rewrite the frontmatter ``name`` to the install id: this both
- # namespaces private user installs and, along the way, normalizes illegal
- # names in market skills (containing spaces/Chinese etc.) — otherwise
- # _load_skill_metadata_from_str fails validation with "invalid skill id".
- # install_id comes from slug/entry_name and is always a legal slug.
- skill_content = _rewrite_frontmatter_name(skill_content, install_id)
-
- required_secrets = list(m.get("required_secrets") or [])
- if required_secrets:
- skill_content = _inject_secrets(
- skill_content, extra_files, required_secrets, secrets
- )
-
- # Validate SKILL.md (description required, etc.); an invalid one is a straight 400 rather than being persisted.
- try:
- meta = _load_skill_metadata_from_str(skill_content, install_id)
- except Exception as exc: # noqa: BLE001
- raise BadRequestError(message=f"市场技能 SKILL.md 不合法:{exc}")
-
- dependencies = detect_dependencies(
- {fn: c for fn, c in extra_files.items() if not is_binary_value(c)}
- )
-
- display_name = m.get("display_name") or meta.name or install_id
- description = meta.description or m.get("summary") or ""
- tags = list(m.get("tags") or meta.tags or [])
- version = m.get("version") or meta.version or "1.0.0"
- user_intro = _build_user_intro(m) or None
-
- ensure_ontology_build_valid(
- db,
- asset_type="skill",
- name=display_name or install_id,
- description=description,
- instructions=skill_content,
- tool_names=list(meta.allowed_tools or []),
- ontology_tags=list(tags),
- )
-
- now = utc_now()
- existing = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id).first()
- if existing is not None:
- # Owner conflict: cannot overwrite a public skill or another user's private skill (admin reinstalling a global one is allowed)
- if existing.owner_user_id != owner_user_id:
- if existing.owner_user_id is None:
- raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」与公共技能冲突")
- raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」已被占用")
- existing.skill_content = skill_content
- existing.display_name = display_name
- existing.description = description
- existing.user_intro = user_intro
- existing.version = version
- existing.tags = tags
- existing.allowed_tools = list(meta.allowed_tools or [])
- existing.extra_files = extra_files
- existing.dependencies = dependencies
- existing.is_enabled = True
- existing.updated_at = now
- flag_modified(existing, "tags")
- flag_modified(existing, "extra_files")
- flag_modified(existing, "dependencies")
- action = "updated"
- else:
- db.add(AdminSkill(
- skill_id=install_id,
- skill_content=skill_content,
- display_name=display_name,
- description=description,
- user_intro=user_intro,
- version=version,
- tags=tags,
- allowed_tools=list(meta.allowed_tools or []),
- extra_files=extra_files,
- dependencies=dependencies,
- is_enabled=True,
- owner_user_id=owner_user_id,
- created_at=now,
- updated_at=now,
- ))
- action = "installed"
- db.commit()
- # Strict dependency verdicts happen at the async route layer (gate_skill_deps
- # needs to actually run probes in the sandbox; this function is synchronous).
- # Here we only carry the declared dependencies out with the result; the route
- # awaits the probe → decides installing/ready.
- # Assign the installed skill a built-in icon by category (users/admins can change it later in skill editing).
- try:
- from core.services.skill_icon_service import (
- get_skill_icon,
- preset_for_category,
- set_skill_icon,
- )
- if not get_skill_icon(db, install_id):
- set_skill_icon(db, install_id, preset_for_category(m.get("category")))
- except Exception as exc: # noqa: BLE001
- logger.debug("marketplace set default icon failed: %s", exc)
- refresh_skill_caches()
- logger.info(
- "marketplace_skill_%s: slug=%s id=%s owner=%s files=%d",
- action, slug, install_id, owner_user_id or "global", len(extra_files),
- )
- return {
- "id": install_id,
- "slug": slug,
- "owner": "self" if owner_user_id else "global",
- "action": action,
- "dependencies": dependencies,
- # dep_pending / message are filled in by the route layer after awaiting gate_skill_deps (strict probe result).
- "dep_pending": False,
- "message": "技能已安装" if action == "installed" else "技能已更新",
- }
-
-
-def _build_user_intro(m: Dict[str, Any]) -> str:
- """Return only an explicitly authored user intro from marketplace metadata.
-
- The marketplace ``summary`` is card copy. Treating it as ``user_intro``
- suppresses the catalog's SKILL.md-body fallback and reduces rich skill
- documentation to a single line after installation.
- """
- return str(m.get("user_intro") or "").strip()
-
-
-# ── Admin uploads a zip → listed directly in the skill marketplace ───────────
-#
-# Skills uploaded by admins from the /admin console no longer land directly as
-# "globally enabled" AdminSkill rows; instead they are listed as a
-# status='approved' MarketplaceSubmission — appearing in the skill marketplace
-# and explicitly installable, but they do NOT enter the catalog and are not
-# auto-loaded by the agent. Shares the same table as users' "submit for
-# listing", just review-exempt (the admin IS the review authority).
-# owner_user_id uses a fixed sentinel to distinguish "admin uploads" from real
-# users' private listing submissions.
-ADMIN_UPLOAD_OWNER = "__admin_upload__"
-
-
-def parse_skill_zip(data: bytes) -> Dict[str, Any]:
- """Parse a skill zip → returns {skill_id, skill_content (original text), meta, extra_files, dependencies, skipped}.
-
- Parsing/validation only; nothing is persisted. The same unpacking logic is
- shared by admin market-listing uploads and users' private uploads.
- """
- try:
- zf = zipfile.ZipFile(io.BytesIO(data))
- except zipfile.BadZipFile:
- raise HTTPException(status_code=400, detail="Invalid zip file")
-
- from core.agent_skills.publication import normalize_file_path, safe_skill_id
-
- # Preserve ZipInfo for reads: normalized names may not exist in the archive.
- entries = {}
- for info in zf.infolist():
- raw_name = info.filename.replace("\\", "/")
- is_dir = raw_name.endswith("/")
- try:
- name = normalize_file_path(raw_name[:-1] if is_dir else raw_name)
- except ValueError as exc:
- raise HTTPException(status_code=400, detail=str(exc)) from exc
- if is_dir:
- continue
- if name in entries:
- raise HTTPException(status_code=400, detail=f"Duplicate skill file path: {name}")
- entries[name] = info
- # A file cannot also be a parent directory of another file.
- for name in entries:
- parts = name.split("/")
- if any("/".join(parts[:i]) in entries for i in range(1, len(parts))):
- raise HTTPException(status_code=400, detail=f"Conflicting skill file path: {name}")
-
- skill_md_paths = [n for n in entries if n.split("/")[-1] == "SKILL.md"]
- if not skill_md_paths:
- raise HTTPException(status_code=400, detail="No SKILL.md found in zip")
-
- skill_md_path = skill_md_paths[0]
- parts = skill_md_path.split("/")
- if len(parts) == 1:
- prefix = ""
- elif len(parts) == 2:
- prefix = parts[0] + "/"
- else:
- prefix = "/".join(parts[:-1]) + "/"
-
- try:
- raw = zf.read(entries[skill_md_path]).decode("utf-8")
- fm, _ = _split_frontmatter(raw)
- skill_id = fm.get("name", "").strip()
- if not skill_id:
- if prefix:
- skill_id = prefix.rstrip("/").split("/")[-1]
- else:
- raise HTTPException(status_code=400, detail="SKILL.md missing 'name' in frontmatter")
- except SkillSpecError as e:
- raise HTTPException(status_code=400, detail=f"Invalid SKILL.md: {e}")
-
- try:
- safe_skill_id(skill_id)
- meta = _load_skill_metadata_from_str(raw, skill_id)
- except Exception as e: # noqa: BLE001
- raise HTTPException(status_code=400, detail=f"Invalid skill: {e}")
-
- JUNK_PATH_MARKERS = ("__MACOSX/", "/.git/", "/__pycache__/", "/.svn/", "/.hg/")
- extra_files: Dict[str, str] = {}
- skipped: List[Dict[str, str]] = []
- total_size = 0
- for entry, info in entries.items():
- if entry == skill_md_path:
- continue
- if prefix and not entry.startswith(prefix):
- continue
- rel_name = entry[len(prefix):] if prefix else entry
- if not rel_name:
- continue
- base = rel_name.rsplit("/", 1)[-1]
- if base in JUNK_BASENAMES or any(m in f"/{entry}" for m in JUNK_PATH_MARKERS):
- continue
- if info.file_size > MAX_SINGLE_FILE:
- skipped.append({"file": rel_name, "reason": f"exceeds {MAX_SINGLE_FILE // (1024 * 1024)}MB single-file ceiling"})
- continue
- try:
- file_bytes = zf.read(info)
- except (KeyError, zipfile.BadZipFile):
- skipped.append({"file": rel_name, "reason": "unreadable zip entry"})
- continue
- _, ext = os.path.splitext(rel_name)
- if ext.lower() in BINARY_EXTENSIONS:
- stored = encode_binary(file_bytes)
- else:
- try:
- stored = file_bytes.decode("utf-8")
- except UnicodeDecodeError:
- stored = encode_binary(file_bytes)
- stored_size = len(stored.encode("utf-8"))
- if total_size + stored_size > MAX_TOTAL:
- skipped.append({"file": rel_name, "reason": f"would exceed {MAX_TOTAL // (1024 * 1024)}MB total ceiling"})
- continue
- total_size += stored_size
- extra_files[rel_name] = stored
-
- dependencies = detect_dependencies(
- {fn: c for fn, c in extra_files.items() if not is_binary_value(c)}
- )
- return {
- "skill_id": skill_id,
- "skill_content": raw,
- "meta": meta,
- "extra_files": extra_files,
- "dependencies": dependencies,
- "skipped": skipped,
- }
-
-
-def build_skill_zip(skill_id: str, skill_content: str, extra_files: Dict[str, str]) -> bytes:
- """Pack a DB skill into a zip byte stream (inverse of ``parse_skill_zip``).
-
- Layout is ``/SKILL.md`` + ``/``,
- consistent with the upload/import convention, so an exported package can be
- re-imported directly. base64 values in extra_files carrying the binary
- marker are restored to their original bytes.
- """
- buf = io.BytesIO()
- with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
- zf.writestr(f"{skill_id}/SKILL.md", skill_content or "")
- for rel, stored in (extra_files or {}).items():
- data = decode_binary(stored) if is_binary_value(stored) else str(stored).encode("utf-8")
- zf.writestr(f"{skill_id}/{rel}", data)
- return buf.getvalue()
-
-
-_ZIP_SKIP_DIR_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"}
-
-
-def build_skill_zip_from_dir(skill_id: str, skill_dir: Path) -> bytes:
- """Pack an on-disk skill directory (built-in/filesystem skills) into a zip byte stream.
-
- Same layout as ``build_skill_zip``; skips version-control/cache directories
- and junk files, and packs binary files as-is (no base64 round-trip,
- lossless).
- """
- buf = io.BytesIO()
- with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
- for p in sorted(skill_dir.rglob("*")):
- if not p.is_file() or p.name in JUNK_BASENAMES:
- continue
- rel = p.relative_to(skill_dir)
- if _ZIP_SKIP_DIR_PARTS.intersection(rel.parts):
- continue
- zf.write(p, f"{skill_id}/{rel.as_posix()}")
- return buf.getvalue()
-
-
-def _upsert_admin_submission(
- db: Session,
- *,
- skill_id: str,
- skill_content: str,
- display_name: str,
- summary: str,
- category: str,
- tags: List[str],
- version: str,
- extra_files: Dict[str, str],
- note: str,
- submitter_name: str,
- user_intro: Optional[str] = None,
-) -> Tuple[str, str]:
- """Upsert an approved listing record under the ADMIN_UPLOAD_OWNER sentinel, returning (slug, action).
-
- When the same skill_id already has an admin listing record, update it
- (keeping the original slug); otherwise create one. Shared by the two admin
- listing entry points: "upload zip" and "create via form".
- """
- category = _validate_category(category)
- snapshot_files = dict(extra_files)
- # Credential files are never distributed with the market snapshot (same as community listing).
- snapshot_files.pop(SECRETS_FILENAME, None)
- now = utc_now()
- existing = (
- db.query(MarketplaceSubmission)
- .filter(
- MarketplaceSubmission.skill_id == skill_id,
- MarketplaceSubmission.owner_user_id == ADMIN_UPLOAD_OWNER,
- )
- .first()
- )
- if existing is not None:
- existing.display_name = display_name
- existing.summary = summary
- existing.user_intro = str(user_intro or "").strip() or None
- existing.category = category
- existing.tags = list(tags or [])
- existing.version = version or "1.0.0"
- existing.skill_content = skill_content
- existing.extra_files = snapshot_files
- existing.note = note
- existing.status = "approved"
- existing.review_note = ""
- existing.reviewed_at = now
- existing.updated_at = now
- flag_modified(existing, "tags")
- flag_modified(existing, "extra_files")
- return existing.slug, "updated"
-
- slug = _derive_slug(db, skill_id, ADMIN_UPLOAD_OWNER)
- db.add(MarketplaceSubmission(
- submission_id=f"mkadm_{uuid.uuid4().hex[:16]}",
- slug=slug,
- skill_id=skill_id,
- owner_user_id=ADMIN_UPLOAD_OWNER,
- submitter_name=(submitter_name or "").strip(),
- display_name=display_name,
- summary=summary,
- user_intro=str(user_intro or "").strip() or None,
- category=category,
- tags=list(tags or []),
- version=version or "1.0.0",
- note=note,
- skill_content=skill_content,
- extra_files=snapshot_files,
- status="approved",
- reviewed_at=now,
- created_at=now,
- updated_at=now,
- ))
- return slug, "published"
-
-
-def publish_skill_zip_to_marketplace(
- db: Session,
- data: bytes,
- *,
- category: str,
- display_name: str = "",
- summary: str = "",
- submitter_name: str = "管理员上传",
-) -> Dict[str, Any]:
- """Admin uploads a skill zip → listed directly in the marketplace as approved (not in the catalog, not globally effective).
-
- When the same skill (by SKILL.md's name = skill_id) is uploaded again, the
- existing admin listing record is updated (keeping the original slug) rather
- than adding a new one each time.
- """
- parsed = parse_skill_zip(data)
- meta = parsed["meta"]
- name = (display_name or "").strip() or meta.name or parsed["skill_id"]
- ensure_ontology_build_valid(
- db,
- asset_type="skill",
- name=name,
- description=(summary or "").strip() or meta.description or "",
- instructions=parsed["skill_content"],
- tool_names=list(meta.allowed_tools or []),
- ontology_tags=list(meta.tags or []),
- )
- slug, action = _upsert_admin_submission(
- db,
- skill_id=parsed["skill_id"],
- skill_content=parsed["skill_content"],
- display_name=name,
- summary=(summary or "").strip() or meta.description or "",
- category=category,
- tags=list(meta.tags or []),
- version=meta.version or "1.0.0",
- extra_files=parsed["extra_files"],
- note="管理员上传,免审核直接上架",
- submitter_name=submitter_name,
- )
- db.commit()
- logger.info("marketplace_skill_admin_%s: skill=%s slug=%s", action, parsed["skill_id"], slug)
- return {
- "slug": slug,
- "skill_id": parsed["skill_id"],
- "display_name": name,
- "category": _validate_category(category),
- "stored_files": len([k for k in parsed["extra_files"] if k != SECRETS_FILENAME]),
- "skipped": parsed["skipped"],
- "action": action,
- "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新",
- }
-
-
-def publish_skill_to_marketplace(
- db: Session,
- *,
- skill_id: str,
- skill_content: str,
- display_name: str,
- description: str,
- version: str,
- tags: List[str],
- category: str,
- user_intro: Optional[str] = None,
- submitter_name: str = "管理员新建",
-) -> Dict[str, Any]:
- """Admin "create skill" form → listed in the marketplace as approved (not in the catalog, not globally effective).
-
- ``skill_content`` is the complete SKILL.md assembled from the form fields
- (valid frontmatter).
- """
- name = (display_name or "").strip() or skill_id
- try:
- meta = _load_skill_metadata_from_str(skill_content, skill_id)
- allowed_tools = list(meta.allowed_tools or [])
- parsed_description = meta.description or ""
- except Exception: # Some reviewed legacy drafts have minimal frontmatter.
- allowed_tools = []
- parsed_description = ""
- ensure_ontology_build_valid(
- db,
- asset_type="skill",
- name=name,
- description=description or parsed_description,
- instructions=skill_content,
- tool_names=allowed_tools,
- ontology_tags=list(tags or []),
- )
- slug, action = _upsert_admin_submission(
- db,
- skill_id=skill_id,
- skill_content=skill_content,
- display_name=name,
- summary=(description or "").strip(),
- category=category,
- tags=list(tags or []),
- version=version or "1.0.0",
- extra_files={},
- note="管理员新建,免审核直接上架",
- submitter_name=submitter_name,
- user_intro=user_intro,
- )
- db.commit()
- logger.info("marketplace_skill_admin_%s: skill=%s slug=%s (form)", action, skill_id, slug)
- return {
- "slug": slug,
- "skill_id": skill_id,
- "display_name": name,
- "category": _validate_category(category),
- "action": action,
- "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新",
- }
-
-
-def delete_marketplace_skill(db: Session, slug: str) -> Dict[str, Any]:
- """Admin deletes a DB listing record from the skill marketplace (admin upload / user community listing).
-
- Preloaded (filesystem) skills ship with the repo and cannot be deleted
- online — straight 400. Deletion only removes it from the market; it does not
- affect any installed skill instances (installs are content snapshots).
- """
- if _read_manifest(slug) is not None:
- raise BadRequestError(message="预置技能随仓库发布,不可在线删除")
- sub = (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.slug == slug)
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("marketplace_skill", slug)
- skill_id, owner = sub.skill_id, sub.owner_user_id
- db.delete(sub)
- db.commit()
- logger.info("marketplace_skill_deleted: slug=%s skill=%s owner=%s", slug, skill_id, owner)
- return {"slug": slug, "deleted": True}
-
-
-# ── Community listing: user submission + admin review ────────────────────────
-
-def _existing_slugs(db: Session) -> set:
- """Currently occupied market slugs: the preloaded directory + all submissions.
-
- Must include rejected rows — the slug column is table-wide UNIQUE and
- rejected submissions still occupy their slug; if derivation skipped them,
- "re-submitting after rejection" would hit the unique constraint and 500
- outright.
- """
- slugs = set()
- if MARKETPLACE_DIR.is_dir():
- slugs.update(c.name for c in MARKETPLACE_DIR.iterdir() if c.is_dir())
- rows = db.query(MarketplaceSubmission.slug).all()
- slugs.update(r[0] for r in rows)
- return slugs
-
-
-def _derive_slug(db: Session, skill_id: str, owner_user_id: str) -> str:
- """Derive the market slug from the source skill id: strip the user's own install-fingerprint suffix, then ensure global uniqueness."""
- base = skill_id
- suffix = f"-{_user_suffix(owner_user_id)}"
- if base.endswith(suffix):
- base = base[: -len(suffix)]
- base = re.sub(r"[^a-z0-9_-]+", "-", base.lower()).strip("-") or "skill"
- taken = _existing_slugs(db)
- slug = base
- n = 2
- while slug in taken:
- slug = f"{base}-{n}"
- n += 1
- return slug
-
-
-def _submission_to_dict(sub: MarketplaceSubmission, *, with_content: bool = False) -> Dict[str, Any]:
- data = {
- "submission_id": sub.submission_id,
- "slug": sub.slug,
- "skill_id": sub.skill_id,
- "owner_user_id": sub.owner_user_id,
- "submitter_name": sub.submitter_name or "",
- "display_name": sub.display_name,
- "summary": sub.summary or "",
- "user_intro": sub.user_intro or "",
- "category": sub.category or "社区共享",
- "tags": list(sub.tags or []),
- "version": sub.version or "1.0.0",
- "note": sub.note or "",
- "status": sub.status,
- "review_note": sub.review_note or "",
- "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None,
- "created_at": sub.created_at.isoformat() if sub.created_at else None,
- "file_count": len(sub.extra_files or {}),
- }
- if with_content:
- data["instructions"] = _strip_frontmatter(sub.skill_content or "")
- data["files"] = sorted((sub.extra_files or {}).keys())
- return data
-
-
-def submit_to_marketplace(
- db: Session,
- skill_id: str,
- *,
- owner_user_id: str,
- submitter_name: str = "",
- note: str = "",
- category: str = "",
- summary: str = "",
-) -> Dict[str, Any]:
- """User submits their private skill for marketplace listing (content snapshot, pending admin review)."""
- row = (
- db.query(AdminSkill)
- .filter(AdminSkill.skill_id == skill_id, AdminSkill.owner_user_id == owner_user_id)
- .first()
- )
- if row is None:
- raise ResourceNotFoundError("skill", skill_id)
-
- active = (
- db.query(MarketplaceSubmission)
- .filter(
- MarketplaceSubmission.skill_id == skill_id,
- MarketplaceSubmission.owner_user_id == owner_user_id,
- MarketplaceSubmission.status.in_(["pending", "approved"]),
- )
- .first()
- )
- if active is not None:
- state = "待审核" if active.status == "pending" else "已上架"
- raise HTTPException(status_code=409, detail=f"该技能已有{state}的上架申请")
-
- # The snapshot excludes the credentials file: when installing a market
- # skill with secrets, _inject_secrets writes the user's API key in
- # plaintext into extra_files[SECRETS_FILENAME] — that must never be
- # distributed to all installers with the listing snapshot. The credential
- # field names are kept with the snapshot as a sentinel entry (without
- # values), so installers are prompted to enter their own keys; SKILL.md is
- # correspondingly stripped of the injected section.
- snapshot_files = dict(row.extra_files or {})
- submitter_secrets = snapshot_files.pop(SECRETS_FILENAME, None)
- snapshot_content = row.skill_content
- if submitter_secrets:
- snapshot_content = _strip_injected_secrets(row.skill_content or "")
- try:
- secret_keys = sorted(json.loads(submitter_secrets).keys())
- except (TypeError, ValueError, AttributeError):
- secret_keys = []
- if secret_keys:
- # The original required/label metadata is unavailable in the installed artifact; approximate as "all required"
- snapshot_files[REQUIRED_SECRETS_SNAPSHOT] = json.dumps(
- [{"key": k, "label": k, "required": True} for k in secret_keys],
- ensure_ascii=False,
- )
-
- now = utc_now()
- sub = MarketplaceSubmission(
- submission_id=f"mksub_{uuid.uuid4().hex[:16]}",
- slug=_derive_slug(db, skill_id, owner_user_id),
- skill_id=skill_id,
- owner_user_id=owner_user_id,
- submitter_name=(submitter_name or "").strip(),
- display_name=row.display_name or skill_id,
- summary=(summary or "").strip() or (row.description or ""),
- user_intro=str(row.user_intro or "").strip() or None,
- category=_validate_category(category),
- tags=list(row.tags or []),
- version=row.version or "1.0.0",
- note=(note or "").strip(),
- skill_content=snapshot_content,
- extra_files=snapshot_files,
- status="pending",
- created_at=now,
- updated_at=now,
- )
- db.add(sub)
- db.commit()
- logger.info(
- "marketplace_submission_created: id=%s skill=%s owner=%s slug=%s",
- sub.submission_id, skill_id, owner_user_id, sub.slug,
- )
- return _submission_to_dict(sub)
-
-
-def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
- """All of the current user's listing submissions (newest → oldest)."""
- rows = (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.owner_user_id == owner_user_id)
- .order_by(MarketplaceSubmission.created_at.desc())
- .all()
- )
- return [_submission_to_dict(r) for r in rows]
-
-
-def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
- """User withdraws their own submission: pending is deleted directly; approved requires an admin to delist; rejected can be deleted and resubmitted."""
- sub = (
- db.query(MarketplaceSubmission)
- .filter(
- MarketplaceSubmission.submission_id == submission_id,
- MarketplaceSubmission.owner_user_id == owner_user_id,
- )
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("marketplace_submission", submission_id)
- if sub.status == "approved":
- raise BadRequestError(message="技能已上架,如需下架请联系管理员")
- db.delete(sub)
- db.commit()
-
-
-def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
- """Admin view: all listing submissions, filterable by status (pending first, newest → oldest)."""
- q = db.query(MarketplaceSubmission)
- if status:
- q = q.filter(MarketplaceSubmission.status == status)
- rows = q.order_by(MarketplaceSubmission.created_at.desc()).all()
- order = {"pending": 0, "approved": 1, "rejected": 2}
- rows.sort(key=lambda r: order.get(r.status, 3))
- return [_submission_to_dict(r) for r in rows]
-
-
-def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
- """Submission detail for admin review (including a SKILL.md body preview and the attachment list)."""
- sub = (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.submission_id == submission_id)
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("marketplace_submission", submission_id)
- return _submission_to_dict(sub, with_content=True)
-
-
-def review_submission(
- db: Session,
- submission_id: str,
- *,
- approve: bool,
- review_note: str = "",
- category: str = "",
-) -> Dict[str, Any]:
- """Admin review: approval means listing (appears in the market list); rejecting pending = decline, rejecting approved = delist."""
- sub = (
- db.query(MarketplaceSubmission)
- .filter(MarketplaceSubmission.submission_id == submission_id)
- .first()
- )
- if sub is None:
- raise ResourceNotFoundError("marketplace_submission", submission_id)
- # State-machine guard: approve only allows pending→approved (prevents two
- # concurrent admins / a stale list mis-click from listing a just-rejected
- # submission directly); reject allows pending (decline) and approved
- # (delist).
- if approve and sub.status != "pending":
- state = "已上架" if sub.status == "approved" else "已驳回"
- raise HTTPException(status_code=409, detail=f"该申请{state},不能重复审核通过(如需重新上架请让用户重新提交)")
- if not approve and sub.status == "rejected":
- raise HTTPException(status_code=409, detail="该申请已是驳回状态")
- sub.status = "approved" if approve else "rejected"
- sub.review_note = (review_note or "").strip()
- if approve and (category or "").strip():
- sub.category = _validate_category(category)
- sub.reviewed_at = utc_now()
- sub.updated_at = utc_now()
- db.commit()
- logger.info(
- "marketplace_submission_reviewed: id=%s slug=%s status=%s",
- sub.submission_id, sub.slug, sub.status,
- )
- return _submission_to_dict(sub)
diff --git a/src/backend/core/services/mcp_marketplace_service.py b/src/backend/core/services/mcp_marketplace_service.py
deleted file mode 100644
index f75952f7..00000000
--- a/src/backend/core/services/mcp_marketplace_service.py
+++ /dev/null
@@ -1,1772 +0,0 @@
-"""MCP marketplace lifecycle: submit, review, publish, install, and suspend.
-
-Marketplace rows are credential-free reviewed snapshots. Installation creates
-or updates an ``AdminMcpServer`` owned by the installing user (or global for an
-admin install), encrypting install-time header values before persistence.
-"""
-
-from __future__ import annotations
-from core.infra.time import utc_now
-
-import json
-import re
-import uuid
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
-
-from core.config.mcp_marketplace_catalog import CURATED_MCP_MARKET_ITEMS
-from core.db.models import (
- AdminMcpServer,
- McpMarketInstallation,
- McpMarketItem,
- McpMarketSubmission,
- McpMarketVersion,
-)
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.services import marketplace_listing as ml
-from core.services.mcp_management_service import (
- auth_schema_from_headers,
- decrypt_mcp_headers,
- encrypt_mcp_headers,
- materialize_mcp_http_connection,
- mcp_oauth_bundle_storage_key,
- mcp_query_secret_storage_key,
- mcp_url_override_storage_key,
- probe_mcp_connectivity,
- refresh_mcp_caches,
- tool_snapshot_hash,
- validate_remote_mcp_url,
-)
-from sqlalchemy.orm import Session
-
-MCP_MARKET_CATEGORIES = [
- "信息检索",
- "数据分析",
- "内容创作",
- "办公协作",
- "研发工具",
- "业务系统",
- "自动化",
- "通用工具",
-]
-_SECRET_QUERY_TERMS = ("key", "token", "secret", "password", "credential")
-
-
-def ensure_curated_market_items(db: Session) -> List[str]:
- """Seed credential-free official templates once, without resurrecting deletions."""
- seeded: List[str] = []
- updated = False
- now = utc_now()
- for definition in CURATED_MCP_MARKET_ITEMS:
- slug = str(definition["slug"])
- # Soft-deleted rows deliberately count as existing: an administrator's
- # removal choice must survive application restarts.
- existing_item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
- if existing_item:
- if existing_item.deleted_at is None:
- existing_version = (
- db.query(McpMarketVersion)
- .filter(McpMarketVersion.version_id == existing_item.latest_version_id)
- .first()
- )
- if existing_version and existing_version.approved_by == "system-curated":
- schema = list(definition.get("auth_schema") or [])
- auth_config = _normalize_auth_config(definition.get("auth_config"), schema)
- notice = dict(existing_version.listing_notice or {})
- notice.update(
- {
- "discovery_mode": "per_install",
- "install_notice": str(definition.get("install_notice") or ""),
- "docs_url": str(definition.get("docs_url") or ""),
- }
- )
- description = str(definition.get("description") or "")
- user_intro = str(definition.get("user_intro") or "")
- icon = str(definition.get("icon") or "")
- changed = (
- list(existing_version.auth_schema or []) != schema
- or dict(existing_version.auth_config or {}) != auth_config
- or dict(existing_version.listing_notice or {}) != notice
- or (existing_item.description or "") != description
- or (existing_item.user_intro or "") != user_intro
- or (existing_item.icon or "") != icon
- )
- if changed:
- existing_version.auth_schema = schema
- existing_version.auth_config = auth_config
- existing_version.listing_notice = notice
- existing_item.description = description
- existing_item.user_intro = user_intro
- existing_item.icon = icon
- existing_item.updated_at = now
- server_ids = [
- row.server_id
- for row in db.query(McpMarketInstallation)
- .filter(McpMarketInstallation.slug == slug)
- .all()
- ]
- if server_ids:
- (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id.in_(server_ids))
- .update({AdminMcpServer.icon: icon}, synchronize_session=False)
- )
- updated = True
- continue
- tools = list(definition.get("tools") or [])
- listing_notice = {
- "discovery_mode": "per_install",
- "install_notice": str(definition.get("install_notice") or ""),
- "docs_url": str(definition.get("docs_url") or ""),
- }
- version_id = f"mcpver_curated_{slug}"
- item = McpMarketItem(
- slug=slug,
- display_name=str(definition["display_name"]),
- description=str(definition.get("description") or ""),
- user_intro=str(definition.get("user_intro") or ""),
- category=_validate_category(str(definition.get("category") or "通用工具")),
- tags=list(definition.get("tags") or []),
- icon=str(definition.get("icon") or ""),
- publisher_id=None,
- publisher_name=str(definition.get("publisher_name") or "HugAgentOS"),
- source="admin",
- latest_version_id=version_id,
- status="active",
- status_reason=None,
- last_verified_at=None,
- created_at=now,
- updated_at=now,
- )
- db.add(item)
- db.add(
- McpMarketVersion(
- version_id=version_id,
- slug=slug,
- version=_normalize_version(str(definition.get("version") or "1.0.0")),
- transport=str(definition.get("transport") or "streamable_http"),
- url=str(definition["url"]),
- auth_schema=list(definition.get("auth_schema") or []),
- auth_config=_normalize_auth_config(
- definition.get("auth_config"),
- list(definition.get("auth_schema") or []),
- ),
- tools_json=tools,
- tool_hash=tool_snapshot_hash(tools),
- listing_notice=listing_notice,
- source_server_id=None,
- approved_by="system-curated",
- approved_at=now,
- created_at=now,
- )
- )
- seeded.append(slug)
- if not seeded and not updated:
- return []
- db.commit()
- for slug in seeded:
- ml.set_listing_enabled(db, ml.KIND_MCP, slug, True, updated_by="system-curated")
- return seeded
-
-
-def _validate_category(category: str) -> str:
- value = (category or "").strip()
- if value not in MCP_MARKET_CATEGORIES:
- raise BadRequestError(message=f"MCP 市场分类必须是:{', '.join(MCP_MARKET_CATEGORIES)}")
- return value
-
-
-def _normalize_auth_config(
- auth_config: Any,
- auth_schema: List[Dict[str, Any]],
-) -> Dict[str, Any]:
- """Return a backwards-compatible none/token/oauth2 authentication contract."""
- config = dict(auth_config or {}) if isinstance(auth_config, dict) else {}
- normalized: List[Dict[str, Any]] = []
- seen: set[str] = set()
- for raw in config.get("methods") or []:
- if not isinstance(raw, dict):
- continue
- method = dict(raw)
- method_id = str(method.get("id") or method.get("type") or "").strip()
- method_type = str(method.get("type") or "").strip()
- if not method_id or method_id in seen or method_type not in {"none", "token", "oauth2"}:
- continue
- method["id"] = method_id
- method["type"] = method_type
- method["label"] = str(method.get("label") or method_type)
- if method_type == "oauth2":
- method["client_registration"] = str(
- method.get("client_registration") or "dynamic_or_manual"
- )
- method["scopes"] = [str(scope) for scope in (method.get("scopes") or []) if str(scope)]
- normalized.append(method)
- seen.add(method_id)
- if not normalized:
- inferred = "token" if auth_schema else "none"
- normalized = [
- {"id": inferred, "type": inferred, "label": "Token" if auth_schema else "无需认证"}
- ]
- default_method = str(config.get("default_method") or normalized[0]["id"])
- if default_method not in {method["id"] for method in normalized}:
- default_method = str(normalized[0]["id"])
- credential_mode = str(config.get("credential_mode") or "auto")
- if credential_mode not in {"auto", "installer", "admin"}:
- credential_mode = "auto"
- return {
- "default_method": default_method,
- "methods": normalized,
- "credential_mode": credential_mode,
- }
-
-
-def _endpoint_identity(url: str) -> tuple[str, str, int | None]:
- """Scheme/host/port of an endpoint, ignoring path, query, and credentials."""
- parsed = urlparse((url or "").strip())
- scheme = (parsed.scheme or "").lower()
- default_port = 443 if scheme == "https" else 80 if scheme == "http" else None
- return scheme, (parsed.hostname or "").lower(), parsed.port or default_port
-
-
-def _installs_admin_published_endpoint(
- item: McpMarketItem,
- version: McpMarketVersion,
- runtime_url: str,
-) -> bool:
- """Whether an install still targets the exact endpoint an admin put on the shelf.
-
- Admin publishing already vets the endpoint and may legitimately point at the
- deployment's own network (the built-in MCP container, a LAN service). The
- SSRF boundary exists for endpoints the *installer* supplies, so it must key
- on the endpoint's provenance rather than on who is installing: an unchanged
- admin-published host stays trusted for everyone who installs it, while a
- URL-target credential that redirects the install elsewhere is checked
- strictly, whoever submits it.
- """
- if item.source != "admin":
- return False
- return _endpoint_identity(runtime_url) == _endpoint_identity(version.url)
-
-
-def _credential_free_connection(row: AdminMcpServer) -> tuple[str, List[Dict[str, Any]]]:
- """Strip likely query-string secrets and expose only their install-time schema."""
- parsed = urlparse(row.url or "")
- public_query: List[tuple[str, str]] = []
- auth_schema = auth_schema_from_headers(row.headers)
- known_keys = {str(field.get("key") or "") for field in auth_schema}
- for name, value in parse_qsl(parsed.query, keep_blank_values=True):
- normalized = re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper()
- looks_secret = any(term in name.lower() for term in _SECRET_QUERY_TERMS)
- credential_key = f"QUERY_{normalized or 'SECRET'}"
- if looks_secret and value:
- if credential_key not in known_keys:
- auth_schema.append(
- {
- "key": credential_key,
- "label": name,
- "target": "query",
- "name": name,
- "required": True,
- "secret": True,
- }
- )
- known_keys.add(credential_key)
- continue
- public_query.append((name, value))
- template_url = urlunparse(parsed._replace(query=urlencode(public_query)))
- return template_url, auth_schema
-
-
-def _normalize_version(version: str) -> str:
- value = (version or "1.0.0").strip()
- if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z._+-]{0,49}", value):
- raise BadRequestError(message="版本号格式无效")
- return value
-
-
-def _new_slug(db: Session, display_name: str, source_server_id: str) -> str:
- ascii_slug = re.sub(r"[^a-z0-9]+", "-", display_name.lower()).strip("-")
- base = (ascii_slug or f"mcp-{source_server_id[-12:].lower()}")[:100]
- slug = base
- suffix = 2
- while (
- db.query(McpMarketItem.slug).filter(McpMarketItem.slug == slug).first()
- or db.query(McpMarketSubmission.slug)
- .filter(
- McpMarketSubmission.slug == slug,
- McpMarketSubmission.status.in_(["pending", "approved"]),
- McpMarketSubmission.deleted_at.is_(None),
- )
- .first()
- ):
- slug = f"{base[:115]}-{suffix}"
- suffix += 1
- return slug
-
-
-def _source_slug(db: Session, row: AdminMcpServer, owner_user_id: str) -> str:
- previous = (
- db.query(McpMarketSubmission.slug)
- .filter(
- McpMarketSubmission.source_server_id == row.server_id,
- McpMarketSubmission.owner_user_id == owner_user_id,
- McpMarketSubmission.deleted_at.is_(None),
- )
- .order_by(McpMarketSubmission.created_at.desc())
- .first()
- )
- return str(previous[0]) if previous else _new_slug(db, row.display_name, row.server_id)
-
-
-def _version_for_item(db: Session, item: McpMarketItem) -> McpMarketVersion:
- version = (
- db.query(McpMarketVersion)
- .filter(McpMarketVersion.version_id == item.latest_version_id)
- .first()
- )
- if not version:
- raise ResourceNotFoundError("mcp_market_version", item.latest_version_id)
- return version
-
-
-def _installed_slug_set(db: Session, owner_user_id: Optional[str]) -> set[str]:
- query = db.query(McpMarketInstallation.slug).filter(
- McpMarketInstallation.status.in_(["active", "suspended"])
- )
- query = (
- query.filter(McpMarketInstallation.owner_user_id.is_(None))
- if owner_user_id is None
- else query.filter(McpMarketInstallation.owner_user_id == owner_user_id)
- )
- return {str(row[0]) for row in query.all()}
-
-
-def _item_dict(
- db: Session,
- item: McpMarketItem,
- version: McpMarketVersion,
- *,
- installed: bool = False,
-) -> Dict[str, Any]:
- auth_schema = list(version.auth_schema or [])
- auth_config = _normalize_auth_config(version.auth_config, auth_schema)
- requires_auth = any(method["type"] != "none" for method in auth_config["methods"])
- credentials_managed_by_admin = bool(
- _admin_managed_credential_headers(
- db,
- item,
- version,
- auth_method=str(auth_config["default_method"]),
- )
- )
- if auth_config["credential_mode"] == "auto":
- auth_config["credential_mode"] = "admin" if credentials_managed_by_admin else "installer"
- supports_admin_credentials = bool(_admin_credential_source(db, item, version))
- return {
- "slug": item.slug,
- "display_name": item.display_name,
- "description": item.description or "",
- "summary": item.description or "",
- "user_intro": item.user_intro or "",
- "category": item.category,
- "tags": list(item.tags or []),
- "icon": item.icon or "",
- "publisher_name": item.publisher_name or "",
- "source": item.source,
- "version": version.version,
- "version_id": version.version_id,
- "transport": version.transport,
- "url_origin": _safe_origin(version.url),
- "auth_schema": auth_schema,
- "auth_config": auth_config,
- "requires_auth": requires_auth,
- "credentials_managed_by_admin": credentials_managed_by_admin,
- "supports_admin_credentials": supports_admin_credentials,
- "requires_user_credentials": requires_auth and not credentials_managed_by_admin,
- "tools": list(version.tools_json or []),
- "tool_count": len(version.tools_json or []),
- "tool_hash": version.tool_hash,
- "listing_notice": dict(version.listing_notice or {}),
- "status": item.status,
- "status_reason": item.status_reason or "",
- "last_verified_at": item.last_verified_at.isoformat() if item.last_verified_at else None,
- "installed": installed,
- "deletable": True,
- "created_at": item.created_at.isoformat() if item.created_at else None,
- "updated_at": item.updated_at.isoformat() if item.updated_at else None,
- }
-
-
-def _safe_origin(url: str) -> str:
- from urllib.parse import urlparse
-
- parsed = urlparse(url or "")
- return f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else ""
-
-
-def list_market_items(
- db: Session,
- *,
- owner_user_id: Optional[str],
- viewer_user_id: Optional[str],
- include_disabled: bool = False,
-) -> Dict[str, Any]:
- query = db.query(McpMarketItem).filter(McpMarketItem.deleted_at.is_(None))
- if not include_disabled:
- query = query.filter(McpMarketItem.status == "active")
- rows = query.order_by(McpMarketItem.updated_at.desc(), McpMarketItem.slug).all()
- installed = _installed_slug_set(db, owner_user_id)
- items = [
- _item_dict(db, row, _version_for_item(db, row), installed=row.slug in installed)
- for row in rows
- ]
- items = ml.annotate_and_filter(
- db,
- ml.KIND_MCP,
- items,
- id_key="slug",
- include_disabled=include_disabled,
- viewer_user_id=viewer_user_id,
- )
- categories = [
- category
- for category in MCP_MARKET_CATEGORIES
- if any(i["category"] == category for i in items)
- ]
- return {"items": items, "categories": categories}
-
-
-def get_market_item(
- db: Session,
- slug: str,
- *,
- owner_user_id: Optional[str],
- viewer_user_id: Optional[str],
- admin: bool = False,
-) -> Dict[str, Any]:
- item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
- if not item or (not admin and item.status != "active"):
- raise ResourceNotFoundError("mcp_market_item", slug)
- if not admin:
- ml.ensure_item_visible(db, ml.KIND_MCP, slug, viewer_user_id, resource="mcp_market_item")
- data = _item_dict(
- db,
- item,
- _version_for_item(db, item),
- installed=slug in _installed_slug_set(db, owner_user_id),
- )
- state = ml.annotate_and_filter(
- db,
- ml.KIND_MCP,
- [data],
- id_key="slug",
- include_disabled=admin,
- viewer_user_id=viewer_user_id,
- )
- if not state:
- raise ResourceNotFoundError("mcp_market_item", slug)
- return state[0]
-
-
-async def submit_to_marketplace(
- db: Session,
- source_server_id: str,
- *,
- owner_user_id: str,
- submitter_name: str,
- category: str,
- version: str,
- summary: str = "",
- note: str = "",
- tags: Optional[List[str]] = None,
-) -> Dict[str, Any]:
- row = (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == source_server_id,
- AdminMcpServer.owner_user_id == owner_user_id,
- )
- .first()
- )
- if not row:
- raise ResourceNotFoundError("private_mcp_server", source_server_id)
- if row.transport not in ("streamable_http", "sse") or not row.url:
- raise BadRequestError(message="只有远程 HTTP/SSE MCP 可以申请上架")
- installed_from_market = (
- db.query(McpMarketInstallation.install_id)
- .filter(McpMarketInstallation.server_id == row.server_id)
- .first()
- )
- if (row.extra_config or {}).get("market_slug") or installed_from_market:
- raise BadRequestError(message="从 MCP 市场安装的实例不能再次申请上架")
- pending = (
- db.query(McpMarketSubmission)
- .filter(
- McpMarketSubmission.source_server_id == source_server_id,
- McpMarketSubmission.owner_user_id == owner_user_id,
- McpMarketSubmission.status == "pending",
- McpMarketSubmission.deleted_at.is_(None),
- )
- .first()
- )
- if pending:
- raise BadRequestError(message="该 MCP 已有待审核的上架申请")
-
- await validate_remote_mcp_url(row.url, require_https=False)
- ok, error = await probe_mcp_connectivity(row, db)
- if not ok:
- raise BadRequestError(message=f"MCP 上架检测失败:{error}")
- tools = list(row.tools_json or [])
- if not tools:
- raise BadRequestError(message="MCP 未发现任何工具,不能申请上架")
- template_url, auth_schema = _credential_free_connection(row)
- now = utc_now()
- submission = McpMarketSubmission(
- submission_id=f"mcpsub_{uuid.uuid4().hex}",
- slug=_source_slug(db, row, owner_user_id),
- source_server_id=row.server_id,
- owner_user_id=owner_user_id,
- submitter_name=submitter_name or "",
- display_name=row.display_name,
- description=(summary or row.description or "").strip(),
- user_intro=row.user_intro,
- category=_validate_category(category),
- tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())),
- icon=row.icon,
- version=_normalize_version(version),
- transport=row.transport,
- url=template_url,
- auth_schema=auth_schema,
- auth_config=_normalize_auth_config(None, auth_schema),
- tools_json=tools,
- tool_hash=tool_snapshot_hash(tools),
- note=(note or "").strip(),
- status="pending",
- created_at=now,
- updated_at=now,
- )
- db.add(submission)
- db.commit()
- db.refresh(submission)
- return _submission_dict(submission)
-
-
-def _submission_dict(row: McpMarketSubmission, *, detail: bool = False) -> Dict[str, Any]:
- data: Dict[str, Any] = {
- "submission_id": row.submission_id,
- "slug": row.slug,
- "source_server_id": row.source_server_id,
- "owner_user_id": row.owner_user_id,
- "submitter_name": row.submitter_name or "",
- "display_name": row.display_name,
- "description": row.description or "",
- "category": row.category,
- "tags": list(row.tags or []),
- "icon": row.icon or "",
- "version": row.version,
- "transport": row.transport,
- "url_origin": _safe_origin(row.url),
- "auth_schema": list(row.auth_schema or []),
- "auth_config": _normalize_auth_config(row.auth_config, list(row.auth_schema or [])),
- "tool_count": len(row.tools_json or []),
- "tool_hash": row.tool_hash,
- "listing_notice": dict(row.listing_notice or {}),
- "note": row.note or "",
- "status": row.status,
- "review_note": row.review_note or "",
- "reviewed_by": row.reviewed_by,
- "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None,
- "created_at": row.created_at.isoformat() if row.created_at else None,
- }
- if detail:
- data["user_intro"] = row.user_intro or ""
- data["url"] = row.url
- data["tools"] = list(row.tools_json or [])
- return data
-
-
-def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
- rows = (
- db.query(McpMarketSubmission)
- .filter(
- McpMarketSubmission.owner_user_id == owner_user_id,
- McpMarketSubmission.deleted_at.is_(None),
- )
- .order_by(McpMarketSubmission.created_at.desc())
- .all()
- )
- return [_submission_dict(row) for row in rows]
-
-
-def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
- row = (
- db.query(McpMarketSubmission)
- .filter(
- McpMarketSubmission.submission_id == submission_id,
- McpMarketSubmission.owner_user_id == owner_user_id,
- McpMarketSubmission.deleted_at.is_(None),
- )
- .first()
- )
- if not row:
- raise ResourceNotFoundError("mcp_market_submission", submission_id)
- if row.status != "pending":
- raise BadRequestError(message="只有待审核申请可以撤回")
- row.status = "withdrawn"
- row.deleted_at = utc_now()
- row.updated_at = utc_now()
- db.commit()
-
-
-def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
- query = db.query(McpMarketSubmission).filter(McpMarketSubmission.deleted_at.is_(None))
- if status:
- query = query.filter(McpMarketSubmission.status == status)
- return [
- _submission_dict(row) for row in query.order_by(McpMarketSubmission.created_at.desc()).all()
- ]
-
-
-def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
- row = (
- db.query(McpMarketSubmission)
- .filter(
- McpMarketSubmission.submission_id == submission_id,
- McpMarketSubmission.deleted_at.is_(None),
- )
- .first()
- )
- if not row:
- raise ResourceNotFoundError("mcp_market_submission", submission_id)
- return _submission_dict(row, detail=True)
-
-
-def _publish_snapshot(
- db: Session,
- *,
- slug: str,
- display_name: str,
- description: str,
- user_intro: Optional[str],
- category: str,
- tags: List[str],
- icon: Optional[str],
- publisher_id: Optional[str],
- publisher_name: str,
- source: str,
- version: str,
- transport: str,
- url: str,
- auth_schema: List[Dict[str, Any]],
- auth_config: Optional[Dict[str, Any]],
- tools: List[Dict[str, Any]],
- source_server_id: Optional[str],
- approved_by: Optional[str],
-) -> McpMarketItem:
- if (
- db.query(McpMarketVersion)
- .filter(McpMarketVersion.slug == slug, McpMarketVersion.version == version)
- .first()
- ):
- raise BadRequestError(message=f"市场条目 {slug} 已存在版本 {version},请提升版本号")
- now = utc_now()
- # Include a soft-deleted row so a later reviewed version can resurrect the
- # same stable marketplace slug without colliding on the primary key.
- item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
- version_id = f"mcpver_{uuid.uuid4().hex}"
- if item is None:
- item = McpMarketItem(
- slug=slug,
- display_name=display_name,
- description=description,
- user_intro=user_intro,
- category=_validate_category(category),
- tags=tags,
- icon=icon,
- publisher_id=publisher_id,
- publisher_name=publisher_name,
- source=source,
- latest_version_id=version_id,
- status="active",
- last_verified_at=now,
- created_at=now,
- updated_at=now,
- )
- db.add(item)
- db.flush()
- else:
- item.deleted_at = None
- item.display_name = display_name
- item.description = description
- item.user_intro = user_intro
- item.category = _validate_category(category)
- item.tags = tags
- item.icon = icon
- item.publisher_id = publisher_id
- item.publisher_name = publisher_name
- item.latest_version_id = version_id
- item.status = "active"
- item.status_reason = None
- item.last_verified_at = now
- item.updated_at = now
- db.add(
- McpMarketVersion(
- version_id=version_id,
- slug=slug,
- version=_normalize_version(version),
- transport=transport,
- url=url,
- auth_schema=auth_schema,
- auth_config=_normalize_auth_config(auth_config, auth_schema),
- tools_json=tools,
- tool_hash=tool_snapshot_hash(tools),
- source_server_id=source_server_id,
- approved_by=approved_by,
- approved_at=now,
- created_at=now,
- )
- )
- return item
-
-
-def _next_revalidated_version(db: Session, slug: str, current_version: str) -> str:
- """Return a collision-free version for an administrator-accepted snapshot."""
- semantic = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", current_version)
- if semantic:
- major, minor, patch = (int(value) for value in semantic.groups())
- candidate = f"{major}.{minor}.{patch + 1}"
- while (
- db.query(McpMarketVersion.version_id)
- .filter(
- McpMarketVersion.slug == slug,
- McpMarketVersion.version == candidate,
- )
- .first()
- ):
- patch += 1
- candidate = f"{major}.{minor}.{patch + 1}"
- return candidate
-
- base = re.sub(r"[^0-9A-Za-z._+-]", "-", current_version).strip("-.") or "1.0.0"
- sequence = 1
- while True:
- candidate = f"{base[:32]}+recheck.{sequence}"
- if not (
- db.query(McpMarketVersion.version_id)
- .filter(
- McpMarketVersion.slug == slug,
- McpMarketVersion.version == candidate,
- )
- .first()
- ):
- return candidate
- sequence += 1
-
-
-def _version_probe_candidate(item: McpMarketItem, version: McpMarketVersion) -> AdminMcpServer:
- """Build a credential-free probe target from an immutable reviewed version."""
- return AdminMcpServer(
- server_id=f"market_revalidate_{uuid.uuid4().hex}",
- display_name=item.display_name,
- description=item.description or "",
- user_intro=item.user_intro,
- transport=version.transport,
- command=None,
- args=[],
- url=version.url,
- env_vars={},
- env_inherit=[],
- headers={},
- is_stable=False,
- is_enabled=False,
- sort_order=0,
- extra_config={},
- tools_json=list(version.tools_json or []),
- icon=item.icon,
- owner_user_id=None,
- source_plugin=None,
- created_by="market-revalidation",
- )
-
-
-def _accept_revalidated_snapshot(
- db: Session,
- item: McpMarketItem,
- version: McpMarketVersion,
- tools: List[Dict[str, Any]],
-) -> McpMarketVersion:
- """Publish live drift as a new immutable version after explicit admin review."""
- next_version = _next_revalidated_version(db, item.slug, version.version)
- _publish_snapshot(
- db,
- slug=item.slug,
- display_name=item.display_name,
- description=item.description or "",
- user_intro=item.user_intro,
- category=item.category,
- tags=list(item.tags or []),
- icon=item.icon,
- publisher_id=item.publisher_id,
- publisher_name=item.publisher_name or "",
- source=item.source,
- version=next_version,
- transport=version.transport,
- url=version.url,
- auth_schema=list(version.auth_schema or []),
- auth_config=dict(version.auth_config or {}),
- tools=tools,
- source_server_id=version.source_server_id,
- approved_by="admin-revalidation",
- )
- db.flush()
- accepted = _version_for_item(db, item)
- if version.source_server_id:
- (
- db.query(McpMarketInstallation)
- .filter(
- McpMarketInstallation.slug == item.slug,
- McpMarketInstallation.server_id == version.source_server_id,
- McpMarketInstallation.version_id == version.version_id,
- )
- .update(
- {
- McpMarketInstallation.version_id: accepted.version_id,
- McpMarketInstallation.updated_at: utc_now(),
- },
- synchronize_session=False,
- )
- )
- db.commit()
- refresh_mcp_caches()
- return accepted
-
-
-async def review_submission(
- db: Session,
- submission_id: str,
- *,
- approve: bool,
- review_note: str = "",
- category: str = "",
- reviewed_by: Optional[str] = None,
-) -> Dict[str, Any]:
- row = (
- db.query(McpMarketSubmission)
- .filter(
- McpMarketSubmission.submission_id == submission_id,
- McpMarketSubmission.deleted_at.is_(None),
- )
- .first()
- )
- if not row:
- raise ResourceNotFoundError("mcp_market_submission", submission_id)
- if (approve and row.status != "pending") or (
- not approve and row.status not in ("pending", "approved")
- ):
- raise BadRequestError(message="该申请当前状态不可审核")
- now = utc_now()
- if approve:
- source = (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == row.source_server_id,
- AdminMcpServer.owner_user_id == row.owner_user_id,
- )
- .first()
- )
- if not source:
- raise BadRequestError(message="申请人的原始 MCP 已删除,不能通过审核")
- source_template_url, source_auth_schema = _credential_free_connection(source)
- if source.transport != row.transport or source_template_url.strip() != row.url.strip():
- raise BadRequestError(message="MCP 连接地址或传输方式已变化,请申请人重新提交")
- if source_auth_schema != list(row.auth_schema or []):
- raise BadRequestError(message="MCP 认证参数已变化,请申请人重新提交")
- await validate_remote_mcp_url(source.url or "", require_https=False)
- ok, error = await probe_mcp_connectivity(source, db)
- if not ok:
- db.rollback()
- raise BadRequestError(message=f"MCP 审核复检失败:{error}")
- if tool_snapshot_hash(source.tools_json or []) != row.tool_hash:
- db.rollback()
- raise BadRequestError(message="MCP 工具或参数结构已变化,请申请人重新提交")
- _publish_snapshot(
- db,
- slug=row.slug,
- display_name=row.display_name,
- description=row.description,
- user_intro=row.user_intro,
- category=category or row.category,
- tags=list(row.tags or []),
- icon=row.icon,
- publisher_id=row.owner_user_id,
- publisher_name=row.submitter_name,
- source="community",
- version=row.version,
- transport=row.transport,
- url=row.url,
- auth_schema=list(row.auth_schema or []),
- auth_config=_normalize_auth_config(row.auth_config, list(row.auth_schema or [])),
- tools=list(row.tools_json or []),
- source_server_id=row.source_server_id,
- approved_by=reviewed_by,
- )
- row.status = "approved"
- else:
- row.status = "rejected"
- item = db.query(McpMarketItem).filter(McpMarketItem.slug == row.slug).first()
- if item:
- item.deleted_at = now
- item.updated_at = now
- ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, False, updated_by=reviewed_by)
- row.review_note = (review_note or "").strip()
- row.reviewed_by = reviewed_by
- row.reviewed_at = now
- row.updated_at = now
- db.commit()
- if approve:
- ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, True, updated_by=reviewed_by)
- return _submission_dict(row, detail=True)
-
-
-async def publish_admin_server(
- db: Session,
- source_server_id: str,
- *,
- category: str,
- version: str,
- summary: str = "",
- tags: Optional[List[str]] = None,
- publisher_name: str = "管理员",
- link_as_installation: bool = True,
- auth_schema_override: Optional[List[Dict[str, Any]]] = None,
- auth_config_override: Optional[Dict[str, Any]] = None,
- allow_deferred_discovery: bool = False,
-) -> Dict[str, Any]:
- row = (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == source_server_id,
- AdminMcpServer.owner_user_id.is_(None),
- )
- .first()
- )
- if not row:
- raise ResourceNotFoundError("admin_mcp_server", source_server_id)
- if row.transport not in ("streamable_http", "sse") or not row.url:
- raise BadRequestError(message="MCP 市场仅支持远程 HTTP/SSE 服务;stdio 请通过插件市场分发")
- if (row.extra_config or {}).get("market_slug"):
- raise BadRequestError(message="从 MCP 市场安装的实例不能重复发布")
- await validate_remote_mcp_url(
- row.url,
- allow_private_network=True,
- require_https=False,
- )
- ok, error = await probe_mcp_connectivity(row, db)
- if not ok and not allow_deferred_discovery:
- raise BadRequestError(message=f"MCP 发布检测失败:{error}")
- tools = list(row.tools_json or [])
- if not tools and not allow_deferred_discovery:
- raise BadRequestError(message="MCP 未发现任何工具,不能发布")
- existing = (
- db.query(McpMarketItem)
- .join(McpMarketVersion, McpMarketVersion.version_id == McpMarketItem.latest_version_id)
- .filter(McpMarketVersion.source_server_id == source_server_id)
- .first()
- )
- slug = existing.slug if existing else _new_slug(db, row.display_name, source_server_id)
- template_url, inferred_auth_schema = _credential_free_connection(row)
- stored_market_config = dict(row.extra_config or {})
- auth_schema = list(
- auth_schema_override
- if auth_schema_override is not None
- else stored_market_config.get("market_auth_schema") or inferred_auth_schema
- )
- auth_config = _normalize_auth_config(
- (
- auth_config_override
- if auth_config_override is not None
- else stored_market_config.get("market_auth_config")
- ),
- auth_schema,
- )
- item = _publish_snapshot(
- db,
- slug=slug,
- display_name=row.display_name,
- description=(summary or row.description or "").strip(),
- user_intro=row.user_intro,
- category=category,
- tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())),
- icon=row.icon,
- publisher_id=None,
- publisher_name=publisher_name,
- source="admin",
- version=version,
- transport=row.transport,
- url=template_url,
- auth_schema=auth_schema,
- auth_config=auth_config,
- tools=tools,
- source_server_id=row.server_id,
- approved_by=publisher_name,
- )
- db.flush()
- if allow_deferred_discovery:
- market_version = _version_for_item(db, item)
- market_version.listing_notice = {
- **dict(market_version.listing_notice or {}),
- "discovery_mode": "per_install",
- "install_notice": "该 MCP 的完整工具清单将在用户完成认证后动态发现。",
- }
- # Existing global MCPs are linked as an installation. A server created
- # through the marketplace-first flow remains a disabled source template
- # until an administrator explicitly installs it from the market.
- if link_as_installation:
- installation = (
- db.query(McpMarketInstallation)
- .filter(McpMarketInstallation.server_id == row.server_id)
- .first()
- )
- if installation is None:
- db.add(
- McpMarketInstallation(
- install_id=f"mcpins_{uuid.uuid4().hex}",
- slug=item.slug,
- version_id=item.latest_version_id,
- server_id=row.server_id,
- owner_user_id=None,
- status="active",
- installed_by="admin",
- created_at=utc_now(),
- updated_at=utc_now(),
- )
- )
- else:
- installation.slug = item.slug
- installation.version_id = item.latest_version_id
- installation.status = "active"
- installation.updated_at = utc_now()
- else:
- for installation in (
- db.query(McpMarketInstallation)
- .filter(McpMarketInstallation.server_id == row.server_id)
- .all()
- ):
- db.delete(installation)
- row.is_enabled = False
- row.extra_config = {
- **dict(row.extra_config or {}),
- "market_source_only": True,
- "market_source_slug": item.slug,
- }
- row.headers = encrypt_mcp_headers(decrypt_mcp_headers(row.headers))
- db.commit()
- ml.set_listing_enabled(db, ml.KIND_MCP, item.slug, True, updated_by=publisher_name)
- refresh_mcp_caches()
- return _item_dict(db, item, _version_for_item(db, item), installed=link_as_installation)
-
-
-def _credential_storage_key(field: Dict[str, Any]) -> str:
- target = str(field.get("target") or "header")
- name = str(field.get("name") or field.get("key") or "").strip()
- if target == "query":
- return mcp_query_secret_storage_key(name)
- if target == "url":
- return mcp_url_override_storage_key()
- return name
-
-
-def _admin_credential_source(
- db: Session,
- item: McpMarketItem,
- version: McpMarketVersion,
-) -> Optional[AdminMcpServer]:
- if item.source != "admin" or not version.source_server_id:
- return None
- return (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == version.source_server_id,
- AdminMcpServer.owner_user_id.is_(None),
- )
- .first()
- )
-
-
-def _admin_managed_credential_headers(
- db: Session,
- item: McpMarketItem,
- version: McpMarketVersion,
- *,
- auth_method: str,
-) -> Dict[str, str]:
- """Return an admin publisher's credentials without exposing them to marketplace APIs.
-
- Marketplace versions remain credential-free. For an admin-published MCP,
- the original global/source ``AdminMcpServer`` is the encrypted credential
- authority. If it still has every required field for the selected token
- method, user installations may copy those values into their own encrypted
- server row without asking the user to enter the same token again.
- """
- auth_config = _normalize_auth_config(version.auth_config, list(version.auth_schema or []))
- if auth_config["credential_mode"] == "installer":
- return {}
- selected = next(
- (method for method in auth_config["methods"] if method["id"] == auth_method),
- None,
- )
- if selected is None or selected["type"] != "token":
- return {}
- fields = [
- field
- for field in (version.auth_schema or [])
- if not field.get("methods") or auth_method in field.get("methods", [])
- ]
- if not fields:
- return {}
- source = _admin_credential_source(db, item, version)
- if source is None:
- return {}
-
- stored = decrypt_mcp_headers(source.headers)
- source_query = dict(parse_qsl(urlparse(source.url or "").query, keep_blank_values=True))
- managed: Dict[str, str] = {}
- for field in fields:
- storage_key = _credential_storage_key(field)
- value = str(stored.get(storage_key) or "")
- if not value and str(field.get("target") or "header") == "query":
- query_name = str(field.get("name") or field.get("key") or "").strip()
- value = str(source_query.get(query_name) or "")
- if value:
- managed[storage_key] = value
- elif field.get("required", True):
- return {}
- return managed
-
-
-def _materialize_credential_value(field: Dict[str, Any], value: str) -> str:
- prefix = str(field.get("prefix") or "")
- if prefix and not value.lower().startswith(prefix.lower()):
- return f"{prefix}{value}"
- return value
-
-
-def _installation_secrets(
- version: McpMarketVersion,
- credentials: Dict[str, str],
- existing_headers: Dict[str, str] | None = None,
- *,
- auth_method: str,
- oauth_bundle: Optional[Dict[str, Any]] = None,
-) -> Dict[str, str]:
- stored: Dict[str, str] = {}
- missing: List[str] = []
- for field in version.auth_schema or []:
- method_ids = [str(value) for value in (field.get("methods") or []) if str(value)]
- if method_ids and auth_method not in method_ids:
- continue
- key = str(field.get("key") or "").strip()
- storage_key = _credential_storage_key(field)
- if not key or not storage_key:
- continue
- value = str(credentials.get(key) or "").strip()
- if value:
- stored[storage_key] = _materialize_credential_value(field, value)
- elif storage_key in (existing_headers or {}):
- stored[storage_key] = str((existing_headers or {})[storage_key])
- elif field.get("required", True):
- missing.append(str(field.get("label") or key))
- if missing:
- raise BadRequestError(message=f"请填写安装凭据:{', '.join(missing)}")
- oauth_key = mcp_oauth_bundle_storage_key()
- if oauth_bundle:
- stored[oauth_key] = json.dumps(oauth_bundle, ensure_ascii=False, separators=(",", ":"))
- elif auth_method == "oauth2" and oauth_key in (existing_headers or {}):
- stored[oauth_key] = str((existing_headers or {})[oauth_key])
- return stored
-
-
-async def install_market_item(
- db: Session,
- slug: str,
- *,
- owner_user_id: Optional[str],
- installed_by: Optional[str],
- credentials: Optional[Dict[str, str]] = None,
- auth_method: Optional[str] = None,
- oauth_bundle: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]:
- item = (
- db.query(McpMarketItem)
- .filter(
- McpMarketItem.slug == slug,
- McpMarketItem.deleted_at.is_(None),
- McpMarketItem.status == "active",
- )
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- if owner_user_id is not None:
- if slug in ml.get_disabled_ids(db, ml.KIND_MCP):
- raise ResourceNotFoundError("mcp_market_item", slug)
- ml.ensure_item_visible(db, ml.KIND_MCP, slug, owner_user_id, resource="mcp_market_item")
- version = _version_for_item(db, item)
- auth_config = _normalize_auth_config(version.auth_config, list(version.auth_schema or []))
- selected_method = str(auth_method or auth_config["default_method"])
- selected = next(
- (method for method in auth_config["methods"] if method["id"] == selected_method),
- None,
- )
- if selected is None:
- raise BadRequestError(message="不支持所选 MCP 认证方式")
- if selected["type"] == "oauth2" and not oauth_bundle:
- raise BadRequestError(message="请先完成 OAuth 登录")
- if selected["type"] != "oauth2" and oauth_bundle:
- raise BadRequestError(message="当前认证方式不能写入 OAuth 凭据")
- existing = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug)
- existing = (
- existing.filter(McpMarketInstallation.owner_user_id.is_(None))
- if owner_user_id is None
- else existing.filter(McpMarketInstallation.owner_user_id == owner_user_id)
- ).first()
- existing_server = (
- db.query(AdminMcpServer).filter(AdminMcpServer.server_id == existing.server_id).first()
- if existing
- else None
- )
- if existing_server is None and owner_user_id is None and version.source_server_id:
- source_candidate = (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == version.source_server_id,
- AdminMcpServer.owner_user_id.is_(None),
- )
- .first()
- )
- if source_candidate and (source_candidate.extra_config or {}).get("market_source_only"):
- existing_server = source_candidate
- old_headers = decrypt_mcp_headers(existing_server.headers) if existing_server else {}
- managed_headers = _admin_managed_credential_headers(
- db,
- item,
- version,
- auth_method=selected_method,
- )
- fallback_headers = {**managed_headers, **old_headers}
- headers = _installation_secrets(
- version,
- dict(credentials or {}),
- fallback_headers,
- auth_method=selected_method,
- oauth_bundle=oauth_bundle,
- )
- runtime_url, _ = materialize_mcp_http_connection(version.url, headers)
- await validate_remote_mcp_url(
- runtime_url,
- allow_private_network=_installs_admin_published_endpoint(item, version, runtime_url),
- require_https=False,
- )
-
- now = utc_now()
- server_id = existing_server.server_id if existing_server else f"mmcp_{uuid.uuid4().hex[:20]}"
- candidate = AdminMcpServer(
- server_id=server_id,
- display_name=item.display_name,
- description=item.description or "",
- user_intro=item.user_intro,
- transport=version.transport,
- command=None,
- args=[],
- url=version.url,
- env_vars={},
- env_inherit=[],
- headers=headers,
- is_stable=False,
- is_enabled=True,
- sort_order=0,
- extra_config={
- **(
- {
- key: value
- for key, value in dict(existing_server.extra_config or {}).items()
- if key not in {"market_source_only", "market_source_slug"}
- }
- if existing_server
- else {}
- ),
- "market_slug": slug,
- "market_version": version.version,
- "listing_notice": dict(version.listing_notice or {}),
- "auth_method": selected_method,
- },
- tools_json=list(version.tools_json or []),
- icon=item.icon,
- owner_user_id=owner_user_id,
- source_plugin=None,
- created_by=installed_by,
- created_at=existing_server.created_at if existing_server else now,
- updated_at=now,
- )
-
- ok, error = await probe_mcp_connectivity(candidate, db)
- if not ok:
- raise BadRequestError(message=f"MCP 安装检测失败:{error}")
- discovery_mode = str((version.listing_notice or {}).get("discovery_mode") or "reviewed")
- if discovery_mode == "per_install":
- candidate.extra_config = {
- **dict(candidate.extra_config or {}),
- "discovery_mode": discovery_mode,
- }
- else:
- actual_hash = tool_snapshot_hash(candidate.tools_json or [])
- if actual_hash != version.tool_hash:
- item.status = "changed"
- item.status_reason = "远程工具或参数结构已变化,等待管理员重新审核"
- item.updated_at = now
- db.commit()
- raise BadRequestError(message="远程 MCP 工具清单已发生变化,已暂停安装并等待重新审核")
-
- server = existing_server or AdminMcpServer(server_id=server_id)
- for field in (
- "display_name",
- "description",
- "user_intro",
- "transport",
- "command",
- "args",
- "url",
- "env_vars",
- "env_inherit",
- "is_stable",
- "is_enabled",
- "sort_order",
- "extra_config",
- "tools_json",
- "icon",
- "owner_user_id",
- "source_plugin",
- "created_by",
- "updated_at",
- ):
- setattr(server, field, getattr(candidate, field))
- server.headers = encrypt_mcp_headers(headers)
- if not existing_server:
- server.created_at = now
- db.add(server)
- db.flush()
- if existing:
- previous_version_id = existing.version_id
- existing.version_id = version.version_id
- existing.status = "active"
- existing.installed_by = installed_by
- existing.updated_at = now
- action = "updated" if previous_version_id != version.version_id else "existing"
- else:
- existing = McpMarketInstallation(
- install_id=f"mcpins_{uuid.uuid4().hex}",
- slug=slug,
- version_id=version.version_id,
- server_id=server.server_id,
- owner_user_id=owner_user_id,
- status="active",
- installed_by=installed_by,
- created_at=now,
- updated_at=now,
- )
- db.add(existing)
- action = "installed"
- db.commit()
- refresh_mcp_caches()
- return {
- "install_id": existing.install_id,
- "server_id": server.server_id,
- "slug": slug,
- "version": version.version,
- "action": action,
- }
-
-
-def set_market_enabled(
- db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
-) -> Dict[str, Any]:
- item = (
- db.query(McpMarketItem)
- .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- return ml.set_listing_enabled(db, ml.KIND_MCP, slug, enabled, updated_by=updated_by)
-
-
-def update_market_item(
- db: Session,
- slug: str,
- *,
- display_name: Optional[str] = None,
- description: Optional[str] = None,
- user_intro: Optional[str] = None,
- category: Optional[str] = None,
- tags: Optional[List[str]] = None,
- icon: Optional[str] = None,
- requires_auth: Optional[bool] = None,
- credential_mode: Optional[str] = None,
- managed_credentials: Optional[Dict[str, str]] = None,
-) -> Dict[str, Any]:
- """Edit marketplace presentation metadata and its administrator-owned auth policy.
-
- Tool schemas, transport, endpoint templates, and version numbers remain
- immutable. Authentication can be disabled, delegated to each installer,
- or backed by the administrator source server's encrypted credentials.
- Secret values are never persisted in marketplace rows or returned by this
- function. Presentation fields are propagated to existing installations.
- """
- item = (
- db.query(McpMarketItem)
- .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- version = _version_for_item(db, item)
- previous_auth_schema = list(version.auth_schema or [])
-
- if display_name is not None:
- normalized_name = display_name.strip()
- if not normalized_name:
- raise BadRequestError(message="MCP 市场名称不能为空")
- item.display_name = normalized_name
- if description is not None:
- item.description = description.strip()
- if user_intro is not None:
- item.user_intro = user_intro.strip() or None
- if category is not None:
- item.category = _validate_category(category)
- if tags is not None:
- item.tags = list(dict.fromkeys(str(tag).strip() for tag in tags if str(tag).strip()))
- if icon is not None:
- item.icon = icon.strip() or None
-
- auth_schema = list(version.auth_schema or [])
- auth_config = _normalize_auth_config(version.auth_config, auth_schema)
- if requires_auth is False:
- auth_schema = []
- auth_config = _normalize_auth_config(
- {
- "default_method": "none",
- "methods": [{"id": "none", "type": "none", "label": "无需认证"}],
- "credential_mode": "installer",
- },
- auth_schema,
- )
- elif requires_auth is True and not any(
- method["type"] != "none" for method in auth_config["methods"]
- ):
- auth_schema = [
- {
- "key": "Authorization",
- "label": "Token / Authorization",
- "target": "header",
- "required": True,
- "secret": True,
- "prefix": "Bearer ",
- }
- ]
- auth_config = _normalize_auth_config(
- {
- "default_method": "token",
- "methods": [{"id": "token", "type": "token", "label": "Token"}],
- "credential_mode": credential_mode or "installer",
- },
- auth_schema,
- )
-
- if credential_mode is not None:
- if credential_mode not in {"installer", "admin"}:
- raise BadRequestError(message="MCP 凭据提供方式无效")
- if credential_mode == "admin" and not auth_schema:
- raise BadRequestError(message="请先勾选需要 Token/Auth")
- auth_config["credential_mode"] = credential_mode
-
- version.auth_schema = auth_schema
- version.auth_config = auth_config
-
- admin_source = None
- if item.source == "admin" and version.source_server_id:
- admin_source = (
- db.query(AdminMcpServer)
- .filter(
- AdminMcpServer.server_id == version.source_server_id,
- AdminMcpServer.owner_user_id.is_(None),
- )
- .first()
- )
- if admin_source is not None:
- admin_source.extra_config = {
- **dict(admin_source.extra_config or {}),
- "market_auth_schema": auth_schema,
- "market_auth_config": auth_config,
- }
-
- if auth_config["credential_mode"] == "admin":
- if item.source != "admin" or not version.source_server_id:
- raise BadRequestError(message="只有管理员发布的 MCP 才能统一配置 Token")
- if admin_source is None:
- raise BadRequestError(message="管理员发布源已不存在,无法统一配置 Token")
- source_headers = decrypt_mcp_headers(admin_source.headers)
- supplied = dict(managed_credentials or {})
- allowed_keys = {str(field.get("key") or "") for field in auth_schema}
- unknown_keys = sorted(key for key in supplied if key not in allowed_keys)
- if unknown_keys:
- raise BadRequestError(message=f"未知的 MCP 认证字段:{', '.join(unknown_keys)}")
- for field in auth_schema:
- key = str(field.get("key") or "").strip()
- value = str(supplied.get(key) or "").strip()
- if value:
- source_headers[_credential_storage_key(field)] = _materialize_credential_value(
- field, value
- )
- admin_source.headers = encrypt_mcp_headers(source_headers)
- managed_headers = _admin_managed_credential_headers(
- db,
- item,
- version,
- auth_method=str(auth_config["default_method"]),
- )
- if not managed_headers:
- raise BadRequestError(message="请填写管理员统一 Token/Auth")
- else:
- managed_headers = {}
-
- item.updated_at = utc_now()
-
- installation_server_ids = [
- str(row[0])
- for row in db.query(McpMarketInstallation.server_id)
- .filter(McpMarketInstallation.slug == slug)
- .all()
- ]
- installations: List[AdminMcpServer] = []
- if installation_server_ids:
- installations = (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id.in_(installation_server_ids))
- .all()
- )
- for server in installations:
- server.display_name = item.display_name
- server.description = item.description or ""
- server.user_intro = item.user_intro
- server.icon = item.icon
- server.updated_at = item.updated_at
- if requires_auth is False:
- existing_headers = decrypt_mcp_headers(server.headers)
- for field in previous_auth_schema:
- existing_headers.pop(_credential_storage_key(field), None)
- server.headers = encrypt_mcp_headers(existing_headers)
- elif managed_headers:
- existing_headers = decrypt_mcp_headers(server.headers)
- existing_headers.update(managed_headers)
- server.headers = encrypt_mcp_headers(existing_headers)
-
- db.commit()
- refresh_mcp_caches()
- result = get_market_item(
- db,
- slug,
- owner_user_id=None,
- viewer_user_id=None,
- admin=True,
- )
- result["updated_installations"] = len(installations)
- return result
-
-
-def delete_market_item(
- db: Session, slug: str, *, updated_by: Optional[str] = None
-) -> Dict[str, Any]:
- item = (
- db.query(McpMarketItem)
- .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- version = _version_for_item(db, item)
- if version.source_server_id:
- source = (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id == version.source_server_id)
- .first()
- )
- if source and (source.extra_config or {}).get("market_source_only"):
- db.delete(source)
- item.deleted_at = utc_now()
- item.updated_at = utc_now()
- db.commit()
- ml.set_listing_enabled(db, ml.KIND_MCP, slug, False, updated_by=updated_by)
- return {"slug": slug, "deleted": True}
-
-
-def set_suspended(
- db: Session,
- slug: str,
- *,
- suspended: bool,
- reason: str = "",
-) -> Dict[str, Any]:
- item = (
- db.query(McpMarketItem)
- .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- now = utc_now()
- item.status = "suspended" if suspended else "active"
- item.status_reason = (reason or "").strip() if suspended else None
- item.updated_at = now
- affected = 0
- installs = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug).all()
- if suspended:
- server_ids = []
- for install in installs:
- install.status = "suspended"
- install.updated_at = now
- server_ids.append(install.server_id)
- if server_ids:
- affected = (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id.in_(server_ids))
- .update({AdminMcpServer.is_enabled: False}, synchronize_session=False)
- )
- else:
- server_ids = []
- for install in installs:
- if install.status != "suspended":
- continue
- install.status = "active"
- install.updated_at = now
- server_ids.append(install.server_id)
- if server_ids:
- affected = (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id.in_(server_ids))
- .update({AdminMcpServer.is_enabled: True}, synchronize_session=False)
- )
- db.commit()
- refresh_mcp_caches()
- return {
- "slug": slug,
- "status": item.status,
- "disabled_installations": affected if suspended else 0,
- "restored_installations": affected if not suspended else 0,
- }
-
-
-async def revalidate_market_item(
- db: Session,
- slug: str,
- *,
- accept_changes: bool = False,
-) -> Dict[str, Any]:
- item = (
- db.query(McpMarketItem)
- .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- version = _version_for_item(db, item)
- if str((version.listing_notice or {}).get("discovery_mode") or "") == "per_install":
- # Curated templates have no shared provider credential, so their live
- # tools are intentionally discovered with each installer's credential.
- # Revalidation checks only a fixed public endpoint; private endpoint
- # templates are validated when the user installs them.
- has_endpoint_override = any(
- str(field.get("target") or "") == "url" for field in version.auth_schema or []
- )
- if not has_endpoint_override:
- try:
- await validate_remote_mcp_url(version.url, require_https=False)
- item.status = "active"
- item.status_reason = None
- except BadRequestError as exc:
- item.status = "suspended"
- item.status_reason = f"官方端点安全复检失败:{exc}"
- else:
- item.status = "active"
- item.status_reason = "个人端点将在安装时使用用户凭据验证"
- item.last_verified_at = utc_now()
- item.updated_at = utc_now()
- db.commit()
- return _item_dict(db, item, version, installed=False)
- source = (
- db.query(AdminMcpServer)
- .filter(AdminMcpServer.server_id == version.source_server_id)
- .first()
- if version.source_server_id
- else None
- )
- if source:
- source_template_url, source_auth_schema = _credential_free_connection(source)
- if (
- source.transport != version.transport
- or source_template_url.strip() != version.url.strip()
- ):
- item.status = "changed"
- item.status_reason = "原始 MCP 的连接地址或传输方式已变化,需发布新版本"
- item.last_verified_at = utc_now()
- item.updated_at = utc_now()
- db.commit()
- return _item_dict(db, item, version, installed=False)
- auth_policy = _normalize_auth_config(version.auth_config, list(version.auth_schema or []))
- if auth_policy["credential_mode"] == "auto" and source_auth_schema != list(
- version.auth_schema or []
- ):
- item.status = "changed"
- item.status_reason = "原始 MCP 的认证参数已变化,需发布新版本"
- item.last_verified_at = utc_now()
- item.updated_at = utc_now()
- db.commit()
- return _item_dict(db, item, version, installed=False)
- try:
- await validate_remote_mcp_url(
- version.url,
- allow_private_network=item.source == "admin",
- require_https=False,
- )
- except BadRequestError as exc:
- set_suspended(db, slug, suspended=True, reason=f"远程地址安全复检失败:{exc}")
- return _item_dict(db, item, version, installed=False)
- probe_target = source or _version_probe_candidate(item, version)
- ok, error = await probe_mcp_connectivity(probe_target, db if source else None)
- discovered_tools = list(probe_target.tools_json or [])
- if not ok:
- item.status = "changed"
- prefix = "复检连接失败" if source else "原始 MCP 连接已不存在,版本端点复检失败"
- item.status_reason = f"{prefix}:{error}"
- elif not discovered_tools:
- item.status = "changed"
- item.status_reason = "复检连接成功,但远程 MCP 未返回任何工具,暂不接受新快照"
- else:
- actual_hash = tool_snapshot_hash(discovered_tools)
- if actual_hash == version.tool_hash:
- item.status = "active"
- item.status_reason = None
- elif accept_changes and item.source == "admin":
- previous_version = version.version
- accepted = _accept_revalidated_snapshot(db, item, version, discovered_tools)
- result = _item_dict(db, item, accepted, installed=False)
- result["snapshot_accepted"] = True
- result["previous_version"] = previous_version
- return result
- else:
- item.status = "changed"
- item.status_reason = "远程工具或参数结构已变化,需发布新版本并重新审核"
- item.last_verified_at = utc_now()
- item.updated_at = utc_now()
- db.commit()
- return _item_dict(db, item, version, installed=False)
diff --git a/src/backend/core/services/mcp_oauth_service.py b/src/backend/core/services/mcp_oauth_service.py
deleted file mode 100644
index ec465578..00000000
--- a/src/backend/core/services/mcp_oauth_service.py
+++ /dev/null
@@ -1,529 +0,0 @@
-"""Provider-neutral OAuth 2.1 flow for remote MCP marketplace installs.
-
-The implementation delegates MCP protected-resource discovery, authorization
-server metadata, PKCE, DCR, resource indicators, token exchange, and refresh to
-the official Python MCP SDK. Marketplace metadata remains credential-free;
-the resulting bundle is encrypted inside the concrete installation only.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import json
-import time
-import uuid
-from dataclasses import dataclass, field
-from typing import Any, Dict, Optional
-from urllib.parse import urlencode, urlparse
-
-from core.config.settings import settings
-from core.db.engine import SessionLocal
-from core.db.models import AdminMcpServer, McpMarketItem, McpMarketVersion
-from core.infra.crypto import decrypt_secret, encrypt_secret
-from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.infra.ephemeral import get_ephemeral_state
-from core.services.mcp_management_service import (
- decrypt_mcp_headers,
- encrypt_mcp_headers,
- mcp_oauth_bundle_storage_key,
- probe_mcp_connectivity,
- validate_remote_mcp_url,
-)
-from mcp.client.auth import OAuthClientProvider, TokenStorage
-from mcp.shared.auth import (
- OAuthClientInformationFull,
- OAuthClientMetadata,
- OAuthMetadata,
- OAuthToken,
- ProtectedResourceMetadata,
-)
-from sqlalchemy.orm import Session
-
-_FLOW_TTL_SECONDS = 10 * 60
-# Authorization-URL discovery costs several sequential round trips (protected
-# resource metadata, authorization server metadata, dynamic registration), so a
-# slow link must keep the flow polling instead of failing the install outright.
-_DISCOVERY_GRACE_SECONDS = 5.0
-
-
-class OAuthBundleStorage(TokenStorage):
- """SDK token storage backed by an encrypted MCP installation row."""
-
- def __init__(self, bundle: Optional[Dict[str, Any]] = None, *, server_id: Optional[str] = None):
- self.bundle: Dict[str, Any] = dict(bundle or {})
- self.server_id = server_id
-
- async def get_tokens(self) -> OAuthToken | None:
- raw = self.bundle.get("tokens")
- return OAuthToken.model_validate(raw) if isinstance(raw, dict) else None
-
- async def set_tokens(self, tokens: OAuthToken) -> None:
- previous = self.bundle.get("tokens") if isinstance(self.bundle.get("tokens"), dict) else {}
- if not tokens.refresh_token and previous.get("refresh_token"):
- tokens = tokens.model_copy(update={"refresh_token": previous["refresh_token"]})
- self.bundle["tokens"] = tokens.model_dump(mode="json", exclude_none=True)
- self.bundle["expires_at"] = (
- time.time() + int(tokens.expires_in) if tokens.expires_in is not None else None
- )
- self._persist()
-
- async def get_client_info(self) -> OAuthClientInformationFull | None:
- raw = self.bundle.get("client_info")
- return OAuthClientInformationFull.model_validate(raw) if isinstance(raw, dict) else None
-
- async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
- self.bundle["client_info"] = client_info.model_dump(mode="json", exclude_none=True)
- self._persist()
-
- def capture_context(self, provider: OAuthClientProvider) -> Dict[str, Any]:
- context = provider.context
- self.bundle["server_url"] = context.server_url
- self.bundle["client_metadata"] = context.client_metadata.model_dump(
- mode="json", exclude_none=True
- )
- self.bundle["oauth_metadata"] = (
- context.oauth_metadata.model_dump(mode="json", exclude_none=True)
- if context.oauth_metadata
- else None
- )
- self.bundle["protected_resource_metadata"] = (
- context.protected_resource_metadata.model_dump(mode="json", exclude_none=True)
- if context.protected_resource_metadata
- else None
- )
- self.bundle["auth_server_url"] = context.auth_server_url
- self.bundle["protocol_version"] = context.protocol_version
- self._persist()
- return dict(self.bundle)
-
- def _persist(self) -> None:
- if not self.server_id:
- return
- db = SessionLocal()
- try:
- row = (
- db.query(AdminMcpServer).filter(AdminMcpServer.server_id == self.server_id).first()
- )
- if not row:
- return
- headers = decrypt_mcp_headers(row.headers)
- headers[mcp_oauth_bundle_storage_key()] = json.dumps(
- self.bundle, ensure_ascii=False, separators=(",", ":")
- )
- row.headers = encrypt_mcp_headers(headers)
- db.commit()
- finally:
- db.close()
-
-
-def build_oauth_provider(
- server_url: str,
- bundle: Dict[str, Any],
- *,
- storage: Optional[OAuthBundleStorage] = None,
- redirect_handler=None,
- callback_handler=None,
-) -> OAuthClientProvider:
- """Rehydrate an SDK provider for first login or automatic refresh."""
- actual_storage = storage or OAuthBundleStorage(bundle)
- metadata_raw = bundle.get("client_metadata") or {
- "redirect_uris": ["http://localhost/oauth/callback"],
- "client_name": "HugAgentOS MCP Client",
- "grant_types": ["authorization_code", "refresh_token"],
- "response_types": ["code"],
- }
- provider = OAuthClientProvider(
- server_url=server_url,
- client_metadata=OAuthClientMetadata.model_validate(metadata_raw),
- storage=actual_storage,
- redirect_handler=redirect_handler,
- callback_handler=callback_handler,
- timeout=300.0,
- )
-
- async def _validate_oauth_request(request) -> None:
- # OAuth discovery metadata is controlled by the remote resource. Apply
- # the same DNS/IP SSRF boundary to every SDK-generated request, not only
- # to the original MCP URL.
- await validate_remote_mcp_url(str(request.url), require_https=False)
-
- provider.mcp_request_hook = _validate_oauth_request
- context = provider.context
- if isinstance(bundle.get("oauth_metadata"), dict):
- context.oauth_metadata = OAuthMetadata.model_validate(bundle["oauth_metadata"])
- if isinstance(bundle.get("protected_resource_metadata"), dict):
- context.protected_resource_metadata = ProtectedResourceMetadata.model_validate(
- bundle["protected_resource_metadata"]
- )
- context.auth_server_url = bundle.get("auth_server_url")
- context.protocol_version = bundle.get("protocol_version")
- expires_at = bundle.get("expires_at")
- context.token_expiry_time = float(expires_at) if expires_at else None
- return provider
-
-
-@dataclass
-class OAuthInstallFlow:
- flow_id: str
- slug: str
- owner_user_id: Optional[str]
- installed_by: Optional[str]
- auth_method: str
- credentials: Dict[str, str]
- callback_url: str
- client_id: str = ""
- client_secret: str = ""
- created_at: float = field(default_factory=time.time)
- status: str = "starting"
- authorization_url: str = ""
- error: str = ""
- result: Dict[str, Any] = field(default_factory=dict)
- callback_code: str = ""
- callback_state: Optional[str] = None
- ready: asyncio.Event = field(default_factory=asyncio.Event)
- callback_received: asyncio.Event = field(default_factory=asyncio.Event)
-
-
-_flows: Dict[str, OAuthInstallFlow] = {}
-
-
-def _flow_key(flow_id: str) -> str:
- return f"mcp:oauth:flow:{flow_id}"
-
-
-def _callback_key(flow_id: str) -> str:
- return f"mcp:oauth:callback:{flow_id}"
-
-
-async def _store_flow_status(flow: OAuthInstallFlow) -> None:
- payload = {
- "flow_id": flow.flow_id,
- "slug": flow.slug,
- "owner_user_id": flow.owner_user_id,
- "status": flow.status,
- "authorization_url": flow.authorization_url,
- "error": flow.error,
- "result": dict(flow.result or {}),
- }
- await get_ephemeral_state().put(
- _flow_key(flow.flow_id),
- json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
- ttl=_FLOW_TTL_SECONDS,
- )
-
-
-def public_oauth_callback_url(request: Any) -> str:
- """Build a trusted browser callback without reflecting an arbitrary Host header."""
- configured = settings.server.mcp_oauth_public_base_url
- if configured:
- parsed = urlparse(configured)
- if (
- parsed.scheme not in {"http", "https"}
- or not parsed.netloc
- or parsed.username
- or parsed.password
- or parsed.query
- or parsed.fragment
- ):
- raise BadRequestError(message="MCP_OAUTH_PUBLIC_BASE_URL 配置无效")
- return f"{configured}/v1/mcp-market/oauth/callback"
-
- origin = str(request.headers.get("origin") or "").rstrip("/")
- parsed_origin = urlparse(origin)
- if not origin or parsed_origin.scheme not in {"http", "https"} or not parsed_origin.netloc:
- raise BadRequestError(message="OAuth 登录必须从浏览器发起")
- request_host = str(request.headers.get("host") or request.url.netloc).lower()
- if settings.server.is_prod and parsed_origin.netloc.lower() != request_host:
- raise BadRequestError(
- message="跨域部署必须配置 MCP_OAUTH_PUBLIC_BASE_URL(例如 http(s)://app.example.com/api)"
- )
- return f"{origin}/api/v1/mcp-market/oauth/callback"
-
-
-def _purge_expired_flows() -> None:
- cutoff = time.time() - _FLOW_TTL_SECONDS
- for flow_id, flow in list(_flows.items()):
- if flow.created_at < cutoff:
- _flows.pop(flow_id, None)
-
-
-def _get_version(db: Session, slug: str) -> tuple[McpMarketItem, McpMarketVersion]:
- item = (
- db.query(McpMarketItem)
- .filter(
- McpMarketItem.slug == slug,
- McpMarketItem.deleted_at.is_(None),
- McpMarketItem.status == "active",
- )
- .first()
- )
- if not item:
- raise ResourceNotFoundError("mcp_market_item", slug)
- version = (
- db.query(McpMarketVersion)
- .filter(McpMarketVersion.version_id == item.latest_version_id)
- .first()
- )
- if not version:
- raise ResourceNotFoundError("mcp_market_version", item.latest_version_id)
- return item, version
-
-
-async def start_oauth_install(
- db: Session,
- *,
- slug: str,
- owner_user_id: Optional[str],
- installed_by: Optional[str],
- auth_method: str,
- credentials: Dict[str, str],
- callback_url_base: str,
- client_id: str = "",
- client_secret: str = "",
-) -> Dict[str, str]:
- """Start a browser OAuth install and wait only until the authorize URL is known."""
- from core.services import mcp_marketplace_service as market
-
- _purge_expired_flows()
- _, version = _get_version(db, slug)
- auth_config = market._normalize_auth_config(
- version.auth_config, list(version.auth_schema or [])
- )
- method = next((row for row in auth_config["methods"] if row["id"] == auth_method), None)
- if not method or method["type"] != "oauth2":
- raise BadRequestError(message="该 MCP 不支持所选 OAuth 认证方式")
- if method.get("client_id_required") and not client_id.strip():
- raise BadRequestError(message="该 OAuth 服务需要 Client ID")
- if method.get("client_secret_required") and not client_secret.strip():
- raise BadRequestError(message="该 OAuth 服务需要 Client Secret")
- flow_id = f"mcpoauth_{uuid.uuid4().hex}"
- callback_url = f"{callback_url_base}?{urlencode({'flow_id': flow_id})}"
- flow = OAuthInstallFlow(
- flow_id=flow_id,
- slug=slug,
- owner_user_id=owner_user_id,
- installed_by=installed_by,
- auth_method=auth_method,
- credentials=dict(credentials or {}),
- callback_url=callback_url,
- client_id=client_id.strip(),
- client_secret=client_secret.strip(),
- )
- _flows[flow_id] = flow
- await _store_flow_status(flow)
- asyncio.create_task(_run_flow(flow), name=f"mcp-oauth-{flow_id}")
- with contextlib.suppress(asyncio.TimeoutError):
- await asyncio.wait_for(flow.ready.wait(), timeout=_DISCOVERY_GRACE_SECONDS)
- if flow.status == "failed":
- raise BadRequestError(message=flow.error or "无法启动 OAuth 登录")
- return {
- "flow_id": flow.flow_id,
- "authorization_url": flow.authorization_url,
- "status": flow.status,
- }
-
-
-async def _run_flow(flow: OAuthInstallFlow) -> None:
- from core.services import mcp_marketplace_service as market
-
- db = SessionLocal()
- try:
- item, version = _get_version(db, flow.slug)
- auth_config = market._normalize_auth_config(
- version.auth_config, list(version.auth_schema or [])
- )
- method = next(row for row in auth_config["methods"] if row["id"] == flow.auth_method)
- headers = market._installation_secrets(
- version,
- flow.credentials,
- {},
- auth_method=flow.auth_method,
- )
- runtime_url, _ = market.materialize_mcp_http_connection(version.url, headers)
- await validate_remote_mcp_url(runtime_url, require_https=False)
-
- metadata = OAuthClientMetadata(
- redirect_uris=[flow.callback_url],
- token_endpoint_auth_method=("client_secret_post" if flow.client_secret else "none"),
- grant_types=["authorization_code", "refresh_token"],
- response_types=["code"],
- scope=" ".join(method.get("scopes") or []) or None,
- client_name="HugAgentOS MCP Client",
- )
- storage = OAuthBundleStorage(
- {
- "server_url": runtime_url,
- "client_metadata": metadata.model_dump(mode="json", exclude_none=True),
- }
- )
- if flow.client_id:
- await storage.set_client_info(
- OAuthClientInformationFull(
- **metadata.model_dump(mode="json", exclude_none=True),
- client_id=flow.client_id,
- client_secret=flow.client_secret or None,
- )
- )
-
- async def redirect_handler(url: str) -> None:
- flow.authorization_url = url
- flow.status = "waiting_for_user"
- await _store_flow_status(flow)
- flow.ready.set()
-
- async def callback_handler() -> tuple[str, str | None]:
- deadline = time.monotonic() + 300.0
- while time.monotonic() < deadline:
- if flow.callback_received.is_set():
- if flow.error:
- raise RuntimeError(flow.error)
- return flow.callback_code, flow.callback_state
- raw = await get_ephemeral_state().get(_callback_key(flow.flow_id))
- if raw:
- payload = json.loads(raw)
- callback_error = str(payload.get("error") or "")
- if callback_error:
- raise RuntimeError(callback_error)
- encrypted_code = str(payload.get("code") or "")
- callback_code = decrypt_secret(encrypted_code) if encrypted_code else ""
- return callback_code or "", payload.get("state")
- await asyncio.sleep(0.5)
- raise TimeoutError("OAuth 登录回调超时")
-
- provider = build_oauth_provider(
- runtime_url,
- storage.bundle,
- storage=storage,
- redirect_handler=redirect_handler,
- callback_handler=callback_handler,
- )
- candidate = AdminMcpServer(
- server_id=f"oauth_probe_{uuid.uuid4().hex[:16]}",
- display_name=item.display_name,
- description=item.description or "",
- transport=version.transport,
- url=version.url,
- headers=headers,
- args=[],
- env_vars={},
- env_inherit=[],
- is_stable=False,
- is_enabled=False,
- extra_config={},
- )
- ok, error = await probe_mcp_connectivity(
- candidate,
- db,
- oauth_provider=provider,
- timeout_seconds=300.0,
- )
- if not ok:
- raise RuntimeError(f"OAuth MCP 连接失败:{error}")
- if flow.error:
- raise RuntimeError(flow.error)
- persisted = await get_ephemeral_state().get(_flow_key(flow.flow_id))
- if persisted:
- persisted_status = json.loads(persisted)
- if persisted_status.get("status") == "failed":
- raise RuntimeError(str(persisted_status.get("error") or "OAuth 登录已取消"))
- bundle = storage.capture_context(provider)
- flow.status = "installing"
- await _store_flow_status(flow)
- flow.result = await market.install_market_item(
- db,
- flow.slug,
- owner_user_id=flow.owner_user_id,
- installed_by=flow.installed_by,
- credentials=flow.credentials,
- auth_method=flow.auth_method,
- oauth_bundle=bundle,
- )
- flow.status = "completed"
- await _store_flow_status(flow)
- except BaseException as exc: # noqa: BLE001
- db.rollback()
- flow.status = "failed"
- flow.error = str(exc)
- await _store_flow_status(flow)
- finally:
- flow.ready.set()
- db.close()
-
-
-async def complete_callback(
- flow_id: str,
- *,
- code: str = "",
- state: Optional[str] = None,
- error: str = "",
-) -> None:
- flow = _flows.get(flow_id)
- public_raw = await get_ephemeral_state().get(_flow_key(flow_id))
- if not flow and not public_raw:
- raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
- await get_ephemeral_state().put(
- _callback_key(flow_id),
- json.dumps(
- {
- "code": encrypt_secret(code) if code else "",
- "state": state,
- "error": error,
- },
- ensure_ascii=False,
- separators=(",", ":"),
- ),
- ttl=_FLOW_TTL_SECONDS,
- )
- if flow:
- flow.callback_code = code
- flow.callback_state = state
- flow.error = error
- flow.status = "failed" if error else "processing_callback"
- flow.callback_received.set()
- await _store_flow_status(flow)
- else:
- payload = json.loads(public_raw)
- payload["status"] = "failed" if error else "processing_callback"
- payload["error"] = error
- await get_ephemeral_state().put(
- _flow_key(flow_id),
- json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
- ttl=_FLOW_TTL_SECONDS,
- )
-
-
-async def cancel_flow(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]:
- """Idempotently stop an OAuth flow owned by the current installer."""
- status = await get_flow_status(flow_id, owner_user_id=owner_user_id)
- if status["status"] in {"completed", "failed"}:
- return status
- await complete_callback(flow_id, error="OAuth 登录已取消")
- return await get_flow_status(flow_id, owner_user_id=owner_user_id)
-
-
-async def get_flow_status(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]:
- raw = await get_ephemeral_state().get(_flow_key(flow_id))
- if not raw:
- raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
- payload = json.loads(raw)
- if payload.get("owner_user_id") != owner_user_id:
- raise ResourceNotFoundError("mcp_oauth_flow", flow_id)
- return {
- "flow_id": flow_id,
- "status": str(payload.get("status") or "starting"),
- "authorization_url": str(payload.get("authorization_url") or ""),
- "error": str(payload.get("error") or ""),
- "result": dict(payload.get("result") or {}),
- }
-
-
-__all__ = [
- "OAuthBundleStorage",
- "build_oauth_provider",
- "cancel_flow",
- "complete_callback",
- "get_flow_status",
- "public_oauth_callback_url",
- "start_oauth_install",
-]
diff --git a/src/backend/core/services/mcp_tool_metadata.py b/src/backend/core/services/mcp_tool_metadata.py
deleted file mode 100644
index acebc62c..00000000
--- a/src/backend/core/services/mcp_tool_metadata.py
+++ /dev/null
@@ -1,19 +0,0 @@
-"""Capture the same public MCP schema for discovery, hashing and desktop delivery."""
-import hashlib
-import json
-from core.services.desktop_capability_protocol import public_tool_schema
-
-
-def snapshot_tool(tool):
- raw = getattr(tool, "_tool", tool)
- if hasattr(raw, "model_dump"):
- data = raw.model_dump(by_alias=True, mode="json")
- else:
- data = {"name": tool.name, "description": getattr(tool, "description", ""),
- "inputSchema": getattr(tool, "inputSchema", None) or getattr(tool, "input_schema", {})}
- return public_tool_schema(data)
-
-
-def tool_snapshot_hash(tools):
- normalized = sorted([public_tool_schema(t) for t in tools or []], key=lambda t: t["name"])
- return hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
diff --git a/src/backend/core/services/mcp_tool_schema_backfill.py b/src/backend/core/services/mcp_tool_schema_backfill.py
deleted file mode 100644
index 7ecb8275..00000000
--- a/src/backend/core/services/mcp_tool_schema_backfill.py
+++ /dev/null
@@ -1,110 +0,0 @@
-"""Capture real tool schemas for MCP servers that only know their tools by name.
-
-A plugin manifest declares its tools for display — ``{name, description}`` with no
-parameters. Probing the running server is what fills ``AdminMcpServer.tools_json``
-with each tool's ``inputSchema``, and that column is the desktop's only source of
-tool parameters: the cloud ships those entries verbatim in the capability manifest
-and the desktop builds the model-facing tool from them. A server left with
-display-only entries therefore offers the model parameter-less tools, so arguments
-have nowhere to go and every call arrives blank.
-
-Backfill runs where the servers are reachable (cloud startup, and after a plugin
-install or upgrade rewrites the manifest), never on the per-message path.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import List, Optional
-
-logger = logging.getLogger(__name__)
-
-# Probing is a network round trip per server; keep startup bounded while still
-# repairing a whole installation in one pass.
-_MAX_CONCURRENCY = 4
-_PROBE_TIMEOUT_S = 20.0
-
-
-def _needs_backfill(tools_json) -> bool:
- from core.llm.mcp_manager import has_usable_schema
-
- if not isinstance(tools_json, list) or not tools_json:
- return False
- return any(not has_usable_schema(item) for item in tools_json)
-
-
-async def backfill_missing_tool_schemas(server_ids: Optional[List[str]] = None) -> int:
- """Probe enabled MCP servers whose stored tools carry no schema.
-
- Returns the number of servers whose ``tools_json`` was refreshed. Failures are
- logged and skipped: an unreachable server keeps its display-only entries, and
- the manifest client withholds those tools rather than offering broken ones.
- """
- from core.db.engine import SessionLocal
- from core.db.models import AdminMcpServer
- from core.services.mcp_management_service import probe_mcp_connectivity
-
- from core.capabilities import device_catalog
-
- from core.capabilities.paths import capabilities_enabled
- if capabilities_enabled() and not server_ids:
- from core.plugins.local.legacy_migration import migrate as migrate_plugins
- from core.services.local_management_migration import migrate as migrate_skills
- await asyncio.to_thread(migrate_plugins)
- await asyncio.to_thread(migrate_skills)
-
- if device_catalog.active() and not server_ids:
- # 混合模式下工具 schema 随云端能力清单下发,不靠探活取。业务库里留下的
- # 本机插件行指向 compose 服务名,桌面端根本连不上——照探只会每次启动都
- # 刷一轮「无法连接远端 MCP」,修不好任何东西。
- logger.debug("[mcp-schema] hybrid desktop: schemas come from the cloud manifest, skipping probe")
- return 0
-
- db = SessionLocal()
- try:
- from core.services.manager_bundle_upgrade import refresh
- refresh(db)
- query = db.query(AdminMcpServer).filter(AdminMcpServer.is_enabled.is_(True))
- if server_ids:
- query = query.filter(AdminMcpServer.server_id.in_(list(server_ids)))
- rows = [row for row in query.all() if _needs_backfill(row.tools_json)]
- if not rows:
- return 0
-
- semaphore = asyncio.Semaphore(_MAX_CONCURRENCY)
- repaired: List[str] = []
-
- async def _probe(row) -> None:
- async with semaphore:
- try:
- ok, detail = await probe_mcp_connectivity(
- row, db, timeout_seconds=_PROBE_TIMEOUT_S
- )
- except Exception as exc: # noqa: BLE001 - one server must not stop the pass
- logger.warning(
- "[mcp-schema] probe raised for '%s': %s", row.server_id, exc
- )
- return
- if ok and not _needs_backfill(row.tools_json):
- repaired.append(row.server_id)
- else:
- logger.warning(
- "[mcp-schema] '%s' still has tools without a schema: %s",
- row.server_id,
- detail,
- )
-
- await asyncio.gather(*(_probe(row) for row in rows))
- if repaired:
- db.commit()
- logger.info(
- "[mcp-schema] tool schemas captured for %d server(s): %s",
- len(repaired),
- ", ".join(sorted(repaired)),
- )
- else:
- db.rollback()
- return len(repaired)
- finally:
- db.close()
diff --git a/src/backend/core/services/prompt_version_service.py b/src/backend/core/services/prompt_version_service.py
deleted file mode 100644
index f8a85395..00000000
--- a/src/backend/core/services/prompt_version_service.py
+++ /dev/null
@@ -1,889 +0,0 @@
-"""Prompt version pool service.
-
-Stores multiple versions of system / code_exec / distillation / sub-agent prompts in
-ContentBlock(id="prompt_versions") and provides activation + CRUD.
-
-Design:
-- Single ContentBlock row holds {active: {kind: version_id}, versions: [...]}.
-- Each version has (kind, id) as its composite key.
-- `get_active_version(kind)` returns the currently active version for a kind,
- or None if the DB is empty — callers then fall back to filesystem.
-- Shipped prompt files under prompts/prompt_text/ and extensions/agents/default/
- serve as seeds and as the last-resort fallback when the DB is unreachable.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-from dataclasses import dataclass
-from datetime import datetime, timezone
-from pathlib import Path
-from threading import Lock
-from typing import Any, Dict, List, Optional
-
-from core.content.content_blocks import DEFAULT_PROMPT_VERSIONS, PROMPT_VERSIONS_BLOCK_ID
-from core.db.engine import SessionLocal
-from core.db.models import ContentBlock
-from extensions.paths import DEFAULT_AGENTS_DIR
-from sqlalchemy.orm import Session
-
-logger = logging.getLogger(__name__)
-
-
-#: 每个版本池默认版本的 id。内置 kind 与自定义 kind 共用这一个约定。
-DEFAULT_VERSION_ID = "default"
-
-
-@dataclass(frozen=True)
-class KindSpec:
- """一个内置 kind 的全部差异都集中在这里,别处不再按 kind 分支。
-
- ``layout`` 决定目录里的 md 文件怎么变成 parts:
-
- - ``concat``:目录下每个 ``*.system.md`` 是一段,拼起来构成一份提示词
- - ``independent``:每个文件是一份互不相干的独立提示词,按 part_id 单取
- - ``single``:整个 kind 只有 ``.system.md`` 一个文件
- """
-
- subdir: tuple[str, ...]
- layout: str
- label: str
- name: str
- desc: str
- visible: bool = True
- preview: str | None = None
- #: 兜底 md 的绝对目录。可选模块的提示词随模块代码放(不进 CE),不在 prompt_text/ 下。
- root: Path | None = None
-
-
-#: 内置 kind:各自对应一段固定的运行时装配位置,有文件系统兜底,不可删。
-#: 新增一个内置 kind = 这张表加一行 + 放好 md 文件,不必再改别处。
-KIND_SPECS: Dict[str, KindSpec] = {
- "desktop": KindSpec(("desktop",), "independent", "桌面端", "default - 桌面端提示词", "本机环境、文件、项目与工具规则;仅桌面端装配", preview="desktop-preview"),
- "system": KindSpec(
- ("default", "system"),
- "concat",
- "系统提示词",
- "default - 标准系统提示词",
- "当前默认系统提示词,包含 role / constraints / tools / workflow / format",
- ),
- "turbo": KindSpec(
- ("turbo",),
- "single",
- "极速模式",
- "default - 极速模式",
- "极速模式(快速查询)系统提示词:仅联网搜索/网页抓取/知识库检索三类工具,"
- "1-2 轮并行调用后直接作答",
- ),
- "plan_mode": KindSpec(
- ("plan_mode",),
- "single",
- "计划模式",
- "default - 计划模式",
- "Plan 模式下用于拆解用户任务为可执行步骤的 sub-agent 系统提示词",
- ),
- "plan_tool": KindSpec(
- ("plan_tool",),
- "single",
- "任务计划清单",
- "default - 任务计划清单",
- "顶层对话注册 update_plan 工具时追加的计划清单说明",
- ),
- "code_exec": KindSpec(
- ("code_exec", "system"),
- "concat",
- "代码执行",
- "default - 代码执行 (沙盒)",
- "Lab 代码执行模式的系统提示词(沙盒环境、工具能力、执行规范等)",
- ),
- "distillation": KindSpec(
- ("distillation",),
- "independent",
- "蒸馏",
- "default - 技能蒸馏",
- "从对话轨迹蒸馏出可复用技能的系统提示词",
- ),
- "subagents": KindSpec(
- ("subagents",),
- "independent",
- "平台默认子智能体",
- "default - 平台默认子智能体",
- "探索员、执行员和审查员三个平台内置角色的独立系统提示词",
- visible=False,
- root=DEFAULT_AGENTS_DIR,
- ),
-}
-
-#: 核心内置 kind。可选模块启用时经 :func:`register_kind` 往 KIND_SPECS 追加自己的 kind,
-#: 运行时判断一律查 KIND_SPECS,这个元组只列核心那批。
-BUILTIN_KINDS = tuple(KIND_SPECS)
-
-
-def register_kind(key: str, spec: KindSpec) -> None:
- """可选模块注册自己的内置 kind(模块关闭时不注册,Config 台就不出现这个 tab)。
-
- 须在启动播种(``seed_from_filesystem``)之前调用,即放在模块的 ``setup()`` 里。
- """
- KIND_SPECS[key] = spec
-
-#: 兼容别名。历史上这个名字既是"内置清单"也是"合法性白名单";自定义 kind 出现后
-#: 两者分家了——校验一律走 :func:`is_valid_kind`,这里只保留内置那批。
-VALID_KINDS = BUILTIN_KINDS
-
-
-def list_custom_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]:
- """管理员自建的 kind(``[{"key","label"}]``)。
-
- 存在版本池同一个 ContentBlock 的 ``custom_kinds`` 下——它们和内置 kind 共用
- versions/active 两张表,差别只在于没有文件系统兜底、可以删。
- 「对话模式」就是靠这个扩展的:管理员在提示词管理新开一个 tab,模式那边就能绑它。
- """
- payload = _load_payload(db)
- out: List[Dict[str, str]] = []
- for item in payload.get("custom_kinds") or []:
- if not isinstance(item, dict):
- continue
- key = str(item.get("key") or "").strip()
- if key and key not in KIND_SPECS:
- out.append({"key": key, "label": str(item.get("label") or key)})
- return out
-
-
-def all_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]:
- """内置 + 自定义的完整 kind 清单,供管理端渲染 tab 与模式绑定下拉。"""
- items = [{"key": k, "label": s.label, "builtin": True, "visible": s.visible, "preview": s.preview, "description": s.desc} for k, s in KIND_SPECS.items()]
- items += [{**c, "builtin": False} for c in list_custom_kinds(db)]
- return items
-
-
-def is_valid_kind(kind: str, db: Optional[Session] = None) -> bool:
- if kind in KIND_SPECS:
- return True
- return any(c["key"] == kind for c in list_custom_kinds(db))
-
-
-def create_custom_kind(
- key: str, label: str, db: Optional[Session] = None, updated_by: str = "admin"
-) -> Dict[str, str]:
- """新开一个 kind(tab),并给它建一个空的 ``default`` 版本并激活。
-
- 没有初始版本的 kind 在管理端会是个点进去什么都没有的空 tab,所以这里顺手建上,
- 管理员进去直接就能写正文。
- """
- cleaned = "".join(
- ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "_" for ch in (key or "").strip().lower()
- ).strip("-_")
- if not cleaned:
- raise ValueError("kind key required")
- if cleaned in KIND_SPECS:
- raise ValueError(f"「{cleaned}」是内置 kind,不能重复创建")
- payload = _clone(_load_payload(db))
- customs = payload.setdefault("custom_kinds", [])
- if any(str((c or {}).get("key")) == cleaned for c in customs):
- raise ValueError(f"kind「{cleaned}」已存在")
- customs.append({"key": cleaned, "label": (label or cleaned).strip()[:60]})
-
- # 空的初始版本:一个可编辑的 part,内容留空由管理员填。
- versions = payload.setdefault("versions", [])
- now = datetime.now(timezone.utc).isoformat()
- versions.append(
- {
- "kind": cleaned,
- "id": "default",
- "name": (label or cleaned).strip()[:60],
- "description": "",
- "parts": [{"part_id": cleaned, "display_name": label or cleaned, "content": ""}],
- "created_at": now,
- "updated_at": now,
- "updated_by": updated_by,
- }
- )
- payload.setdefault("active", {})[cleaned] = "default"
- _save_payload(payload, db=db, updated_by=updated_by)
- return {"key": cleaned, "label": (label or cleaned).strip()[:60]}
-
-
-def delete_custom_kind(key: str, db: Optional[Session] = None) -> bool:
- """删掉一个自定义 kind 及其全部版本。内置 kind 拒绝删除。
-
- 注意:绑了这个 kind 的对话模式会退回默认提示词装配(``render_kind_segment(fs_fallback=False)``
- 取不到就返回空串,装配侧按"没配提示词"处理),不会让对话起不来。
- """
- if key in KIND_SPECS:
- raise ValueError("内置 kind 不可删除")
- payload = _clone(_load_payload(db))
- customs = payload.get("custom_kinds") or []
- remaining = [c for c in customs if str((c or {}).get("key")) != key]
- if len(remaining) == len(customs):
- return False
- payload["custom_kinds"] = remaining
- payload["versions"] = [v for v in (payload.get("versions") or []) if v.get("kind") != key]
- (payload.get("active") or {}).pop(key, None)
- _save_payload(payload, db=db)
- return True
-
-# Process-local payload cache, checked against the DB revision on every read.
-_payload_cache: Optional[Dict[str, Any]] = None
-_payload_cache_key: Optional[tuple] = None
-_payload_cache_lock = Lock()
-
-# kind -> (目录签名, parts)。签名变了才重读磁盘,见 _read_fs_parts。
-_fs_parts_cache: Dict[str, tuple] = {}
-_fs_parts_cache_lock = Lock()
-
-
-# ── Helpers ─────────────────────────────────────────────────────────────────
-
-
-def _backend_root() -> Path:
- # src/backend
- return Path(__file__).resolve().parents[2]
-
-
-def _fs_dir(kind: str) -> Path:
- spec = KIND_SPECS.get(kind)
- if spec is None:
- raise ValueError(f"unknown kind: {kind}")
- if spec.root is not None:
- return spec.root
- return _backend_root().joinpath("prompts", "prompt_text", *spec.subdir)
-
-
-def _fs_signature(dirp: Path) -> tuple:
- """目录里每个 md 的 (文件名, mtime, 大小)。用它判断要不要重读磁盘。"""
- try:
- names = sorted(
- [p.name for p in dirp.glob("*.system.md")]
- + [p.relative_to(dirp).as_posix() for p in dirp.glob("*/system.md")]
- )
- except OSError:
- return ()
- sig: List[tuple] = []
- for name in names:
- try:
- st = os.stat(dirp / name)
- except OSError:
- continue
- sig.append((name, st.st_mtime_ns, st.st_size))
- return tuple(sig)
-
-
-def _read_fs_parts(kind: str) -> List[Dict[str, Any]]:
- """Read on-disk markdown into a parts[] list.
-
- 布局由 :data:`KIND_SPECS` 的 ``layout`` 决定(concat / independent / single)。
- concat 版的 sort_order 取文件名的数字前缀(``05_x`` → 5),新增文件落到它本该
- 在的位置,不挤动相邻段。
-
- 结果按目录内容签名缓存:这些 md 是只读的部署产物,而本函数落在每轮对话的
- 提示词兜底路径和管理台的每次 GET 上,不做缓存就是反复读同一批文件。
- """
- dirp = _fs_dir(kind)
- sig = _fs_signature(dirp)
- with _fs_parts_cache_lock:
- cached = _fs_parts_cache.get(kind)
- if cached is not None and cached[0] == sig:
- return _clone(cached[1])
-
- parts = _build_fs_parts(kind, dirp, [name for name, _, _ in sig])
- with _fs_parts_cache_lock:
- _fs_parts_cache[kind] = (sig, parts)
- return _clone(parts)
-
-
-def _prompt_part_id(filename: str) -> str:
- """Map a flat prompt file or an agent package prompt to its stable part ID."""
- if filename.endswith("/system.md"):
- return Path(filename).parent.name
- return filename.removesuffix(".system.md")
-
-
-def _build_fs_parts(kind: str, dirp: Path, files: List[str]) -> List[Dict[str, Any]]:
- layout = KIND_SPECS[kind].layout
-
- if layout == "single":
- fname = f"{kind}.system.md"
- if fname not in files:
- return []
- return [
- {
- "part_id": kind,
- "display_name": kind,
- "content": (dirp / fname).read_text(encoding="utf-8"),
- "sort_order": 0,
- "is_enabled": True,
- }
- ]
-
- if layout == "independent":
- # 每个文件是一份独立提示词(按 part_id 单取,见 render_active_prompt_part),
- # 不像 concat 那样拼成一份。skill_distiller 固定排头,兼容旧数据。
- if kind == "distillation":
- files = sorted(files, key=lambda f: (f != "skill_distiller.system.md", f))
- return [
- {
- "part_id": _prompt_part_id(fname),
- "display_name": _prompt_part_id(fname),
- "content": (dirp / fname).read_text(encoding="utf-8"),
- "sort_order": idx * 10,
- "is_enabled": True,
- }
- for idx, fname in enumerate(files)
- ]
-
- parts: List[Dict[str, Any]] = []
- for idx, fname in enumerate(files):
- name = _prompt_part_id(fname)
- prefix = name.split("_", 1)[0]
- parts.append(
- {
- "part_id": f"system/{name}",
- "display_name": name,
- "content": (dirp / fname).read_text(encoding="utf-8"),
- "sort_order": int(prefix) if prefix.isdigit() else idx * 10,
- "is_enabled": True,
- }
- )
- return parts
-
-
-def _default_version_for_kind(kind: str, version_id: Optional[str] = None) -> Dict[str, Any]:
- """Build a new version dict from filesystem for a given kind."""
- spec = KIND_SPECS.get(kind)
- if spec is None:
- raise ValueError(f"unknown kind: {kind}")
-
- now = datetime.now(timezone.utc).isoformat()
- return {
- "id": version_id or DEFAULT_VERSION_ID,
- "kind": kind,
- "name": spec.name,
- "description": spec.desc,
- "parts": _read_fs_parts(kind),
- "created_at": now,
- "updated_at": now,
- }
-
-
-# ── Payload load / save ─────────────────────────────────────────────────────
-
-
-def _load_payload(db: Optional[Session] = None) -> Dict[str, Any]:
- """Read the committed pool revision before reusing a worker-local payload.
-
- Workers do not share Python memory. Local invalidation alone leaves other
- workers serving old versions indefinitely, including on read-modify-write.
- Query columns directly so an existing ORM identity-map entry cannot hide a
- newer payload. The payload and its revision are loaded in the same query.
- """
- global _payload_cache, _payload_cache_key
- own_session = db is None
- if own_session:
- db = SessionLocal()
- try:
- revision = (
- db.query(ContentBlock.updated_at)
- .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID)
- .first()
- )
- cache_key = (db.get_bind(), tuple(revision) if revision is not None else None)
- with _payload_cache_lock:
- if _payload_cache is not None and _payload_cache_key == cache_key:
- return _clone(_payload_cache)
-
- row = (
- db.query(ContentBlock.updated_at, ContentBlock.payload)
- .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID)
- .first()
- )
- payload = (
- row.payload
- if row is not None and isinstance(row.payload, dict)
- else _clone(DEFAULT_PROMPT_VERSIONS)
- )
- # A write may have committed between the two SELECTs. Cache the revision
- # returned with the payload, never the earlier revision probe.
- cache_key = (db.get_bind(), (row.updated_at,) if row is not None else None)
- with _payload_cache_lock:
- _payload_cache = payload
- _payload_cache_key = cache_key
- return _clone(payload)
- finally:
- if own_session:
- db.close()
-
-
-def _save_payload(
- payload: Dict[str, Any],
- *,
- db: Optional[Session] = None,
- updated_by: str = "system",
-) -> None:
- own_session = db is None
- if own_session:
- db = SessionLocal()
- try:
- row = db.query(ContentBlock).filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID).first()
- now = datetime.now(timezone.utc)
- if row:
- row.payload = payload
- row.updated_at = now
- row.updated_by = updated_by
- else:
- row = ContentBlock(
- id=PROMPT_VERSIONS_BLOCK_ID,
- payload=payload,
- updated_at=now,
- updated_by=updated_by,
- )
- db.add(row)
- db.commit()
- finally:
- if own_session:
- db.close()
-
- invalidate_cache()
-
-
-def invalidate_cache() -> None:
- """Drop this worker's cache; other workers detect the DB revision on read."""
- global _payload_cache, _payload_cache_key
- with _payload_cache_lock:
- _payload_cache = None
- _payload_cache_key = None
-
-
-def _clone(obj: Any) -> Any:
- import copy
-
- return copy.deepcopy(obj)
-
-
-# ── Public API ──────────────────────────────────────────────────────────────
-
-
-def list_versions(kind: Optional[str] = None, db: Optional[Session] = None) -> List[Dict[str, Any]]:
- payload = _load_payload(db)
- versions = payload.get("versions") or []
- active = payload.get("active") or {}
- items = []
- for v in versions:
- if kind and v.get("kind") != kind:
- continue
- items.append(
- {
- "id": v.get("id"),
- "kind": v.get("kind"),
- "name": v.get("name") or v.get("id"),
- "description": v.get("description") or "",
- "parts_count": len(v.get("parts") or []),
- "is_active": active.get(v.get("kind")) == v.get("id"),
- "created_at": v.get("created_at"),
- "updated_at": v.get("updated_at"),
- }
- )
- return items
-
-
-def get_version(
- kind: str, version_id: str, db: Optional[Session] = None
-) -> Optional[Dict[str, Any]]:
- if not is_valid_kind(kind, db):
- return None
- payload = _load_payload(db)
- active = payload.get("active") or {}
- for v in payload.get("versions") or []:
- if v.get("kind") == kind and v.get("id") == version_id:
- data = _clone(v)
- data["is_active"] = active.get(kind) == version_id
- return data
- return None
-
-
-def get_active_version(kind: str, db: Optional[Session] = None) -> Optional[Dict[str, Any]]:
- """Return the active version for a kind, or None if not found / DB empty."""
- if not is_valid_kind(kind, db):
- return None
- payload = _load_payload(db)
- active_id = (payload.get("active") or {}).get(kind)
- if not active_id:
- return None
- for v in payload.get("versions") or []:
- if v.get("kind") == kind and v.get("id") == active_id:
- return _clone(v)
- return None
-
-
-def upsert_version(
- kind: str,
- version_id: str,
- *,
- name: Optional[str] = None,
- description: Optional[str] = None,
- parts: Optional[List[Dict[str, Any]]] = None,
- from_id: Optional[str] = None,
- db: Optional[Session] = None,
- updated_by: str = "admin",
-) -> Dict[str, Any]:
- if not is_valid_kind(kind, db):
- raise ValueError(f"invalid kind: {kind}")
- if not version_id:
- raise ValueError("version_id required")
-
- payload = _clone(_load_payload(db))
- versions: List[Dict[str, Any]] = payload.setdefault("versions", [])
- now = datetime.now(timezone.utc).isoformat()
-
- existing = next(
- (v for v in versions if v.get("kind") == kind and v.get("id") == version_id), None
- )
-
- # If cloning, look up source (optional)
- source_parts: List[Dict[str, Any]] = []
- if from_id:
- src = next((v for v in versions if v.get("kind") == kind and v.get("id") == from_id), None)
- if src:
- source_parts = _clone(src.get("parts") or [])
- if name is None:
- name = f"{src.get('name') or from_id} 副本"
- if description is None:
- description = src.get("description") or ""
-
- if existing:
- if name is not None:
- existing["name"] = name
- if description is not None:
- existing["description"] = description
- if parts is not None:
- existing["parts"] = _normalize_parts(parts)
- existing["updated_at"] = now
- saved = existing
- else:
- # On create: if from_id is passed and parts is empty (empty list / not passed),
- # always treat it as a clone.
- # Only explicitly passing non-empty parts overrides the clone's source content.
- if parts and len(parts) > 0:
- final_parts = _normalize_parts(parts)
- elif from_id and source_parts:
- final_parts = source_parts
- else:
- final_parts = _normalize_parts(parts) if parts is not None else []
- saved = {
- "id": version_id,
- "kind": kind,
- "name": name or version_id,
- "description": description or "",
- "parts": final_parts,
- "created_at": now,
- "updated_at": now,
- }
- versions.append(saved)
-
- _save_payload(payload, db=db, updated_by=updated_by)
- return saved
-
-
-def delete_version(kind: str, version_id: str, db: Optional[Session] = None) -> None:
- if not is_valid_kind(kind, db):
- raise ValueError(f"invalid kind: {kind}")
- payload = _clone(_load_payload(db))
- active = payload.get("active") or {}
- if active.get(kind) == version_id:
- raise ValueError("cannot delete the currently active version; activate another first")
-
- versions: List[Dict[str, Any]] = payload.get("versions") or []
- new_list = [v for v in versions if not (v.get("kind") == kind and v.get("id") == version_id)]
- if len(new_list) == len(versions):
- raise KeyError(f"version not found: {kind}/{version_id}")
- payload["versions"] = new_list
- _save_payload(payload, db=db)
-
-
-def activate_version(kind: str, version_id: str, db: Optional[Session] = None) -> None:
- if not is_valid_kind(kind, db):
- raise ValueError(f"invalid kind: {kind}")
- payload = _clone(_load_payload(db))
- versions: List[Dict[str, Any]] = payload.get("versions") or []
- if not any(v.get("kind") == kind and v.get("id") == version_id for v in versions):
- raise KeyError(f"version not found: {kind}/{version_id}")
- payload.setdefault("active", {})[kind] = version_id
- _save_payload(payload, db=db)
- # Downstream prompt builders cache by (active_id, updated_at); poke their cache too.
- try:
- from prompts.prompt_runtime import invalidate_prompt_cache
-
- invalidate_prompt_cache()
- except Exception:
- pass
-
-
-def _normalize_parts(parts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
- result: List[Dict[str, Any]] = []
- for idx, p in enumerate(parts or []):
- pid = (p.get("part_id") or "").strip()
- if not pid:
- continue
- result.append(
- {
- "part_id": pid,
- "display_name": p.get("display_name") or pid.split("/")[-1],
- "content": p.get("content") or "",
- "sort_order": int(
- p.get("sort_order") if p.get("sort_order") is not None else idx * 10
- ),
- "is_enabled": bool(p.get("is_enabled", True)),
- }
- )
- # Stable sort by sort_order
- result.sort(key=lambda x: x["sort_order"])
- return result
-
-
-# ── Seeding ─────────────────────────────────────────────────────────────────
-
-
-def seed_from_filesystem(
- *,
- db: Optional[Session] = None,
-) -> Dict[str, Any]:
- """Create default versions from filesystem if not already present."""
- payload = _clone(_load_payload(db))
- versions: List[Dict[str, Any]] = payload.setdefault("versions", [])
- active: Dict[str, str] = payload.setdefault("active", {})
- changed = False
-
- def exists(kind: str, vid: str) -> bool:
- return any(v.get("kind") == kind and v.get("id") == vid for v in versions)
-
- added: List[str] = []
-
- # ── One-time migration: rename legacy system/v4 → system/default ──
- # Keeps the user's existing edits; the name change aligns the in-code
- # default version id with other kinds (code_exec/default, distillation/default, …).
- v4_row = next((v for v in versions if v.get("kind") == "system" and v.get("id") == "v4"), None)
- default_row = next(
- (v for v in versions if v.get("kind") == "system" and v.get("id") == "default"), None
- )
- if v4_row and not default_row:
- v4_row["id"] = "default"
- # Refresh display name only if it still looks like the factory label
- if v4_row.get("name", "").startswith("v4 - 标准版"):
- v4_row["name"] = "default - 标准系统提示词"
- if active.get("system") == "v4":
- active["system"] = "default"
- added.append("system/v4 → system/default (renamed)")
- changed = True
-
- # ── One-time migration: extract system/90_plan_mode from every system
- # version into a new plan_mode/default version (if not already seeded).
- # After extraction, the system/90_plan_mode part is REMOVED from system
- # versions so the main agent prompt no longer duplicates it.
- if not exists("plan_mode", "default"):
- extracted_content: Optional[str] = None
- for v in versions:
- if v.get("kind") != "system":
- continue
- new_parts: List[Dict[str, Any]] = []
- for p in v.get("parts") or []:
- if (p.get("part_id") or "").strip() == "system/90_plan_mode":
- if extracted_content is None and (p.get("content") or "").strip():
- extracted_content = p["content"]
- changed = True
- continue
- new_parts.append(p)
- v["parts"] = new_parts
- if extracted_content:
- now = datetime.now(timezone.utc).isoformat()
- versions.append(
- {
- "id": "default",
- "kind": "plan_mode",
- "name": "default - 计划模式(从 v4 迁移)",
- "description": "由历史 v4 system/90_plan_mode 片段迁移而来的 plan_mode 默认版本",
- "parts": [
- {
- "part_id": "plan_mode",
- "display_name": "plan_mode",
- "content": extracted_content,
- "sort_order": 0,
- "is_enabled": True,
- }
- ],
- "created_at": now,
- "updated_at": now,
- }
- )
- active["plan_mode"] = "default"
- added.append("plan_mode/default (migrated)")
- changed = True
-
- # 每个内置 kind 的默认版本:没有就按文件系统建;已有就把**新增**的 md 文件补进去。
- # 补这一步是因为默认版本只在首次冷启动时从文件系统建一次,之后新加的 *.system.md
- # 再也进不了已建库的部署。已有 part 的正文原样保留,管理员改过的不会被覆盖。
- for kind in list(KIND_SPECS):
- if not exists(kind, DEFAULT_VERSION_ID):
- default_row = _default_version_for_kind(kind, DEFAULT_VERSION_ID)
- versions.append(default_row)
- added.append(f"{kind}/{DEFAULT_VERSION_ID}")
- changed = True
- else:
- default_row = next(
- v for v in versions if v.get("kind") == kind and v.get("id") == DEFAULT_VERSION_ID
- )
- existing_pids = {
- (p.get("part_id") or "").strip() for p in default_row.get("parts") or []
- }
- for fs_part in _read_fs_parts(kind):
- if fs_part["part_id"] not in existing_pids:
- default_row.setdefault("parts", []).append(fs_part)
- added.append(f"{kind}/{DEFAULT_VERSION_ID}:{fs_part['part_id']}")
- changed = True
- if kind not in active:
- active[kind] = DEFAULT_VERSION_ID
- changed = True
-
- if added or changed:
- _save_payload(payload, db=db, updated_by="system_seed")
- if added:
- logger.info("[prompt_version_service] seeded: %s", ", ".join(added))
- return {"added": added, "active": active}
-
-
-# ── Assembled prompt helpers (used by runtime callers) ──────────────────────
-
-
-def render_active_prompt(kind: str, db: Optional[Session] = None) -> Optional[str]:
- """Concatenate enabled parts of the active version into a single string.
-
- Returns None if no active version exists in DB (caller should fall back).
- """
- v = get_active_version(kind, db=db)
- if not v:
- return None
- parts = v.get("parts") or []
- chunks: List[str] = []
- for p in parts:
- if not p.get("is_enabled", True):
- continue
- content = (p.get("content") or "").strip()
- if content:
- chunks.append(content)
- return "\n\n".join(chunks) if chunks else None
-
-
-def render_active_prompt_part(
- kind: str, part_id: str, db: Optional[Session] = None
-) -> Optional[str]:
- """Return a single named part of the active version (independent prompts).
-
- Used by distillation and platform-subagent prompts where each part is a standalone
- system prompt (skill_distiller / session_digest / colleague_distiller /
- personal_distiller) rather than a concatenation segment.
-
- Back-compat: an active version that predates the multi-part layout holds
- exactly one part (the legacy skill_distiller prompt) — only a request for
- ``skill_distiller`` may fall back to it; other part_ids must not silently
- receive the wrong prompt.
- Returns None if no active version / no match (caller falls back to fs).
- """
- v = get_active_version(kind, db=db)
- if not v:
- return None
- parts = v.get("parts") or []
- for p in parts:
- if p.get("part_id") == part_id:
- content = (p.get("content") or "").strip()
- return content or None
- if len(parts) == 1 and part_id == "skill_distiller":
- content = (parts[0].get("content") or "").strip()
- return content or None
- return None
-
-
-def _skipped_parts(kind: str) -> set:
- try:
- from core.config.local_mode import local_mode_enabled
-
- if local_mode_enabled():
- from prompts.desktop_workspace import SKIP_PARTS
- return SKIP_PARTS.get(kind) or set()
- except Exception:
- logger.debug("local-mode check failed while assembling %s", kind, exc_info=True)
- return set()
-
-
-def render_kind_segment(
- kind: str, db: Optional[Session] = None, *, fs_fallback: bool = True
-) -> str:
- """某个 kind 装配好的正文:DB 激活版本 → 文件系统默认,都没有就返回空串。
-
- 这是「一个 kind 的正文怎么取」的唯一实现——代码执行段、极速模式、任务计划清单
- 以及 Config 管理台 ``/v1/admin/prompts/preview`` 的预览都走它,预览与智能体
- 实际看到的因此不会漂移。
-
- 文件系统这一步读的就是播种用的那批 md,所以尚未播种的部署渲染出来的,与它之后
- 存进库里的是同一份。``fs_fallback=False`` 用于「对话模式」绑定的 kind:绑了个
- 空 kind 就该退回默认装配,而不是套用别的 kind 的话术。
- """
- skip = _skipped_parts(kind)
- try:
- rendered = _render_active_parts(kind, db=db, skip=skip)
- if rendered:
- return rendered
- except Exception:
- logger.debug("render %s active prompt failed", kind, exc_info=True)
- if not fs_fallback or kind not in KIND_SPECS:
- return ""
- chunks = [
- (p.get("content") or "").strip()
- for p in _read_fs_parts(kind)
- if p.get("is_enabled", True)
- and (p.get("content") or "").strip()
- and (p.get("part_id") or "") not in skip
- ]
- return "\n\n".join(chunks)
-
-
-def _render_active_parts(kind: str, *, db: Optional[Session], skip: set) -> Optional[str]:
- """激活版本的正文,按 ``skip`` 剔除若干段。``skip`` 为空时等同 render_active_prompt。"""
- if not skip:
- return render_active_prompt(kind, db=db)
- version = get_active_version(kind, db=db)
- if not version:
- return None
- chunks = [
- (p.get("content") or "").strip()
- for p in (version.get("parts") or [])
- if p.get("is_enabled", True)
- and (p.get("content") or "").strip()
- and (p.get("part_id") or "").strip() not in skip
- ]
- return "\n\n".join(chunks) or None
-
-
-#: 极速模式在库和文件都取不到时的最后兜底——它替换掉整份系统提示词,返回空串会让
-#: 智能体裸奔,所以这一段必须留在代码里。
-_TURBO_LAST_RESORT = (
- "你是极速查询助手。收到问题后,最多发起 1-2 轮工具调用(同一轮内可并行"
- "调用多个检索工具),拿到结果立即用中文简洁作答,并标注信息来源。"
- "不要使用超出检索范围的工具,不要反复迭代。"
-)
-
-
-def render_turbo_system_prompt(db: Optional[Session] = None) -> str:
- """极速模式(快速查询)的整份系统提示词。
-
- 极速模式不做常规装配:智能体只带检索类工具,默认提示词的工具/流程段都不适用。
- """
- return render_kind_segment("turbo", db=db) or _TURBO_LAST_RESORT
-
-
-def effective_parts(kind: str, db: Optional[Session] = None) -> List[Dict[str, Any]]:
- """Read the active version, with whole-version filesystem fallback only."""
- try:
- version = get_active_version(kind, db=db)
- except Exception:
- logger.warning("Prompt version unavailable for %s; using filesystem", kind, exc_info=True)
- version = None
- return version.get("parts", []) if version is not None else _read_fs_parts(kind)
diff --git a/src/backend/core/services/session_operations/admission.py b/src/backend/core/services/session_operations/admission.py
deleted file mode 100644
index 3ad4ee85..00000000
--- a/src/backend/core/services/session_operations/admission.py
+++ /dev/null
@@ -1,262 +0,0 @@
-"""One durable admission path for streaming and complete agent responses."""
-
-import asyncio
-from dataclasses import dataclass
-
-import core.services.session_operations.admission_errors as chat_admission
-import core.services.session_operations.agent_targets as chat_agent_targets
-import core.services.session_operations.invocation as chat_invocation
-import core.services.session_operations.request_context as chat_request_context
-import core.services.session_operations.session_context as chat_session_context
-import core.chat.context as chat_context
-import core.db.engine as db_engine
-import core.services as chat_services
-from core.chat.schemas import ChatRequest
-from core.auth.backend import UserContext
-from core.infra.logging import get_logger, trace_id_var
-from core.services.chat_sequencer import ChatBusyError, ChatSequencer
-from fastapi import HTTPException
-from orchestration import chat_run_executor
-from sqlalchemy.orm import Session
-
-logger = get_logger(__name__)
-
-
-@dataclass(frozen=True)
-class StartedResponse:
- run_id: str
- message_id: str
- chat_id: str
- user_message_id: str
-
-
-async def _read_preferences(request: ChatRequest, user_id: str):
- # Sessions cannot cross threads; each bounded read owns its connection.
- def capabilities():
- with db_engine.SessionLocal() as db:
- return chat_context.resolve_enabled_capabilities(
- db, user_id, request.enabled_skills, request.enabled_agents, request.enabled_mcps
- )
-
- def preferences():
- with db_engine.SessionLocal() as db:
- return chat_services.UserService(db).get_user_settings(user_id)
-
- return await asyncio.gather(asyncio.to_thread(capabilities), asyncio.to_thread(preferences))
-
-
-def _check_project(db: Session, request: ChatRequest, user_id: str) -> None:
- """A chat bound to a project always runs in that project, whichever client sends to it."""
- from core.services.project_access import require_visible_project
-
- if request.chat_id:
- pair = chat_services.ChatService(db).get_session_with_access(request.chat_id, user_id)
- if pair is not None and pair[0].project_id:
- request.project_id = pair[0].project_id
- if request.project_id:
- require_visible_project(db, user_id, request.project_id)
-
-
-def _link_attachments(db: Session, request: ChatRequest, user_id: str) -> None:
- ids = [item.file_id for item in request.attachments if item.file_id]
- if not ids:
- return
- from core.db.models import Artifact
-
- db.query(Artifact).filter(
- Artifact.artifact_id.in_(ids),
- Artifact.user_id == user_id,
- Artifact.chat_id.is_(None),
- ).update({"chat_id": request.chat_id}, synchronize_session="fetch")
- db.commit()
-
-
-async def _start_response_run(
- request: ChatRequest, user: UserContext, db: Session
-) -> StartedResponse:
- from core.services.agent_api_service import (
- begin_agent_api_call,
- bind_agent_api_run,
- fail_agent_api_call,
- prepare_agent_api_request,
- )
-
- # Scope and API-session provenance come only from the authenticated key.
- # prepare logs its own rejected scoped requests before it raises.
- from core.services.project_api_service import prepare_project_api_request
-
- request = prepare_project_api_request(db, user, request)
- request, scope = prepare_agent_api_request(db, user, request)
- if user.public_chat_visitor:
- from core.services.public_chat_service import validate_visitor_attachments
-
- validate_visitor_attachments(db, user, request.attachments)
- call_id = (
- begin_agent_api_call(db, scope, request.stream, trace_id=trace_id_var.get())
- if scope
- else None
- )
- try:
- chat_session_context._ensure_main_model_configured()
- user_id = chat_context.resolve_db_user_id(
- db, chat_session_context._authenticated_user_id(user)
- )
- from core.services.evaluation_admission import authorize
-
- await authorize(request, user_id, agent_scoped=bool(scope))
- if scope:
- # Admission already verified this exact DB-backed agent. Never
- # enumerate the owner's catalog or parse delegation commands here.
- from core.db.models import UserAgent
-
- agent = db.get(UserAgent, scope["agent_id"])
- if agent is None:
- raise HTTPException(status_code=403, detail={"code": "agent_unavailable"})
- agent_name = agent.name
- request._resolved_agent_profile = "local"
- execution_message, command = request.message, None
- else:
- request, agent_name, execution_message, command = (
- chat_agent_targets._resolve_chat_agent_targets(db, request, user_id)
- )
- request = chat_invocation._resolve_explicit_capability_invocation(db, request, user_id)
- message = chat_context.build_effective_user_message(
- execution_message,
- request.quoted_follow_up,
- chat_request_context._resolve_reference_block(db, request, user_id),
- )
- provider_id = chat_session_context._resolve_selected_model_provider_id(db, request, user_id)
- model_name = chat_session_context._resolve_actual_chat_model_name(request, provider_id)
- (skills, agents, mcps), settings = await _read_preferences(request, user_id)
- if user.public_chat_visitor:
- from core.services.public_chat_service import visitor_approval_mode
-
- settings["tool_approval_mode"] = visitor_approval_mode(
- db, user, settings.get("tool_approval_mode")
- )
- _check_project(db, request, user_id)
- service = chat_services.ChatService(db)
- chat_session_context._ensure_chat_session(
- service,
- request.chat_id,
- user_id,
- request.message,
- agent_id=request.agent_id,
- agent_name=agent_name,
- plan_chat=request.plan_chat,
- workflow_chat=request.workflow_chat,
- site_chat=request.site_chat,
- project_id=request.project_id,
- )
- context = chat_request_context._build_ctx(
- request,
- user_id,
- skills,
- agents,
- mcps,
- memory_enabled=bool(settings.get("memory_enabled", False)) and not scope,
- memory_write_enabled=bool(settings.get("memory_write_enabled", False)) and not scope,
- reranker_enabled=bool(settings.get("reranker_enabled", False)),
- model_provider_id=provider_id,
- actual_model_name=model_name,
- ontology_enabled=bool(settings.get("ontology_enabled", False)) and not scope,
- ontology_pack_ids=settings.get("ontology_pack_ids") or None,
- approval_mode=settings.get("tool_approval_mode"),
- )
- if user.api_key_project_id:
- from core.services.project_api_service import make_project_api_scope
-
- context["project_api_scope"] = make_project_api_scope(user, request.chat_id)
- payload = request.model_dump(exclude_none=True)
- if user.api_key_project_id:
- payload["project_api_scope"] = make_project_api_scope(user, request.chat_id)
- # Authentication owns provenance; never accept a client-supplied origin.
- payload["request_origin"] = "api" if user.api_key_id else "cloud"
- if user.api_key_id:
- payload["api_key_id"] = str(user.api_key_id)
- if command:
- explicit_command = {
- "agent_id": command.agent_id,
- "agent_name": command.agent_name,
- "task": command.task,
- }
- context["explicit_subagent_command"] = explicit_command
- payload["explicit_subagent_command"] = explicit_command
- if provider_id:
- payload["model_provider_id"] = provider_id
- else:
- payload.pop("model_provider_id", None)
- if scope:
- context["agent_api_scope"] = dict(scope)
- payload["agent_api_scope"] = dict(scope)
-
- try:
- with ChatSequencer(db).launching_main_run(
- chat_id=request.chat_id,
- user_id=user_id,
- user_content=request.message,
- model=model_name,
- user_extra_data=chat_request_context._build_user_extra_data(request, provider_id),
- request_payload=payload,
- ) as accepted:
- if user.public_chat_visitor:
- from core.db.models import ChatSession
-
- session = db.get(ChatSession, request.chat_id)
- if session and not (session.extra_data or {}).get("public_chat_title_set"):
- session.title = (
- request.message.strip().split("\n", 1)[0][:80] or session.title
- )
- session.extra_data = {
- **(session.extra_data or {}),
- "public_chat_title_set": True,
- }
- db.flush()
- # No history or attachment mutation before writer admission.
- messages = chat_session_context._load_session_messages(
- service, request.chat_id, user_id
- )
- from core.chat.plan_progress import clear_plan_progress
-
- clear_plan_progress(request.chat_id)
- _link_attachments(db, request, user_id)
- if call_id:
- bind_agent_api_run(db, call_id, accepted.run.run_id)
- run = await chat_run_executor.start_run(
- accepted_run=accepted.run,
- chat_id=request.chat_id,
- user_id=user_id,
- session_messages=messages,
- effective_user_message=message,
- raw_user_message=request.message,
- context=context,
- model_name=model_name,
- )
- except ChatBusyError as exc:
- raise chat_admission.chat_busy_http_exception(exc) from exc
- # Title work is auxiliary: it starts after first-message admission and
- # never waits for the agent answer or blocks a successfully launched run.
- from core.services.chat_titles import schedule_first_title
-
- try:
- schedule_first_title(request.chat_id, user_id, run.run_id)
- except Exception as title_error:
- logger.warning("chat_title_schedule_failed: %s", type(title_error).__name__)
- return StartedResponse(run.run_id, run.message_id, request.chat_id, accepted.user_message.message_id)
- except Exception as exc:
- if call_id:
- status = exc.status_code if isinstance(exc, HTTPException) else 500
- # Logs carry a bounded machine code, never raw model/tool errors or secrets.
- detail = exc.detail if isinstance(exc, HTTPException) else None
- code = detail.get("code") if isinstance(detail, dict) else None
- fail_agent_api_call(db, call_id, status, code or "request_rejected")
- if isinstance(exc, HTTPException):
- raise
- logger.exception("agent_response_start_failed", chat_id=request.chat_id)
- raise HTTPException(
- status_code=500, detail=chat_context.resolve_user_facing_error(exc)
- ) from exc
- finally:
- # Both HTTP transports must release the request transaction before
- # waiting on a worker that writes the same chat rows.
- chat_session_context._release_request_session(db)
diff --git a/src/backend/core/services/skill_draft_service.py b/src/backend/core/services/skill_draft_service.py
deleted file mode 100644
index 80193c60..00000000
--- a/src/backend/core/services/skill_draft_service.py
+++ /dev/null
@@ -1,192 +0,0 @@
-"""Skill draft approve/reject — the single implementation shared by the AdminApp review endpoint and ConfigApp colleague-distillation save.
-
-Extracted from api/routes/v1/admin_skill_drafts.py::approve_draft with behavior kept
-identical; adds owner_user_id (whether the artifact is global or belongs to a specific
-user) and edited_content (minor tweaks before saving).
-The "immutable after approved/rejected" constraint is enforced here, naturally
-preventing the two consoles from double-persisting.
-"""
-
-from __future__ import annotations
-from core.infra.time import utc_now
-
-import logging
-import uuid
-from datetime import datetime
-from typing import Optional, Tuple
-
-from fastapi import HTTPException
-from sqlalchemy.orm import Session
-
-from core.db.models import AdminSkill, AdminSkillDraft
-from core.ontology.build_validator import ensure_ontology_build_valid
-
-logger = logging.getLogger(__name__)
-
-
-def get_draft_or_404(db: Session, draft_id: str) -> AdminSkillDraft:
- d = db.query(AdminSkillDraft).filter(AdminSkillDraft.draft_id == draft_id).first()
- if not d:
- raise HTTPException(status_code=404, detail="draft_not_found")
- return d
-
-
-def approve_draft(
- db: Session,
- draft_id: str,
- *,
- enable_immediately: bool = False,
- reviewer_comment: Optional[str] = None,
- reviewer_id: Optional[str] = None,
- owner_user_id: Optional[str] = None,
- edited_content: Optional[str] = None,
- publish_to_marketplace: bool = False,
- category: Optional[str] = None,
-) -> Tuple[AdminSkillDraft, str]:
- """Approve a draft: materialize the skill artifact per decision. Returns (draft, skill_id).
-
- - decision='new_skill' with ``publish_to_marketplace=True`` → **publish to the skill
- marketplace** (submitted as approved, effective only once a user installs it). Must
- specify ``category`` (one of the marketplace's 8 fixed categories); in this mode
- ``enable_immediately`` / ``owner_user_id`` do not apply (marketplace items have no
- enable/ownership concept). — AdminApp pending-draft review takes this path.
- - decision='new_skill' with ``publish_to_marketplace=False`` → directly create a formal
- skill (optionally enable immediately / specify owner as a private skill). — ConfigApp
- colleague-distillation "confirm persist" takes this path.
- - decision='patch' → patch an existing formal skill in place (in-place update of a live skill).
- - edited_content non-empty → overwrite the draft content with it before persisting (only pending is editable)
- """
- d = get_draft_or_404(db, draft_id)
- if d.review_status != "pending":
- raise HTTPException(status_code=400, detail="draft_not_pending")
-
- if edited_content is not None and edited_content.strip():
- d.skill_content = edited_content
-
- skill_id = d.proposed_skill_id
- validation_target = None
- if d.decision == "patch":
- target_id = d.patch_target_id or skill_id
- validation_target = (
- db.query(AdminSkill).filter(AdminSkill.skill_id == target_id).first()
- )
- if not validation_target:
- raise HTTPException(status_code=400, detail=f"patch_target_missing:{target_id}")
- ensure_ontology_build_valid(
- db,
- asset_type="skill",
- name=d.display_name
- or (validation_target.display_name if validation_target else None)
- or skill_id,
- description=d.description
- or (validation_target.description if validation_target else None)
- or "",
- instructions=d.skill_content or "",
- tool_names=list(
- d.allowed_tools
- or (validation_target.allowed_tools if validation_target else None)
- or []
- ),
- ontology_tags=list(
- d.tags or (validation_target.tags if validation_target else None) or []
- ),
- )
-
- if d.decision == "new_skill" and publish_to_marketplace:
- # Approved new skill goes to the skill marketplace (not directly into the formal store). The marketplace publish commits internally.
- from core.services import marketplace_service
-
- result = marketplace_service.publish_skill_to_marketplace(
- db,
- skill_id=skill_id,
- skill_content=d.skill_content,
- display_name=d.display_name or skill_id,
- description=d.description or "",
- version=d.version or "0.1.0",
- tags=list(d.tags or []),
- category=category or "",
- submitter_name=(
- "同事蒸馏审核" if d.draft_kind == "colleague" else "蒸馏审核"
- ),
- )
- skill_id = result["skill_id"]
- elif d.decision == "new_skill":
- existing: Optional[AdminSkill] = (
- db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first()
- )
- if existing:
- skill_id = f"{skill_id}-{uuid.uuid4().hex[:4]}"
- created_by = (
- "colleague_distill" if d.draft_kind == "colleague" else "distiller_approved"
- )
- skill = AdminSkill(
- skill_id=skill_id,
- skill_content=d.skill_content,
- display_name=d.display_name or skill_id,
- description=d.description or "",
- version=d.version or "0.1.0",
- tags=list(d.tags or []),
- allowed_tools=list(d.allowed_tools or []),
- extra_files={},
- is_enabled=bool(enable_immediately),
- owner_user_id=str(owner_user_id) if owner_user_id else None,
- created_at=utc_now(),
- updated_at=utc_now(),
- created_by=created_by,
- )
- db.add(skill)
- elif d.decision == "patch":
- target_id = d.patch_target_id or skill_id
- target = validation_target
- assert target is not None
- target.skill_content = d.skill_content
- if d.display_name:
- target.display_name = d.display_name
- if d.description:
- target.description = d.description
- if d.version:
- target.version = d.version
- if d.tags:
- target.tags = list(d.tags)
- if d.allowed_tools:
- target.allowed_tools = list(d.allowed_tools)
- target.updated_at = utc_now()
- skill_id = target_id
- else:
- raise HTTPException(status_code=400, detail=f"unsupported_decision:{d.decision}")
-
- d.review_status = "approved"
- d.reviewer_comment = reviewer_comment
- if reviewer_id:
- d.reviewer_id = reviewer_id
- d.reviewed_at = utc_now()
- d.updated_at = utc_now()
-
- db.commit()
- try:
- from core.agent_skills.cache_refresh import refresh_skill_caches
- refresh_skill_caches()
- except Exception as exc:
- logger.warning("skill_draft_service: cache refresh failed (%s)", exc)
- return d, skill_id
-
-
-def reject_draft(
- db: Session,
- draft_id: str,
- *,
- rejected_reason: str,
- reviewer_id: Optional[str] = None,
-) -> AdminSkillDraft:
- d = get_draft_or_404(db, draft_id)
- if d.review_status != "pending":
- raise HTTPException(status_code=400, detail="draft_not_pending")
-
- d.review_status = "rejected"
- d.rejected_reason = rejected_reason
- if reviewer_id:
- d.reviewer_id = reviewer_id
- d.reviewed_at = utc_now()
- d.updated_at = utc_now()
- db.commit()
- return d
diff --git a/src/backend/core/services/system_config.py b/src/backend/core/services/system_config.py
deleted file mode 100644
index a6afd2ef..00000000
--- a/src/backend/core/services/system_config.py
+++ /dev/null
@@ -1,655 +0,0 @@
-"""Central service configuration service (DB-driven, cached).
-
-Manages external service configs (DB query, KB, industry, file parser) stored
-in the system_configs table. Thread-safe singleton with a short TTL cache so
-admin changes take effect within seconds without a restart.
-
-Falls back to os.getenv() when DB has no value for a given key.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import threading
-import time
-from typing import Optional
-
-from core.db.engine import SessionLocal
-from core.db.models import SystemConfig
-from core.services.edition_system_config import CONFIG_KEY_TO_ENV as EDITION_CONFIG_KEY_TO_ENV
-from core.services.edition_system_config import SEED_CONFIGS as EDITION_SEED_CONFIGS
-
-logger = logging.getLogger(__name__)
-
-_CACHE_TTL_SECONDS = 30.0
-
-_PROVIDER_SCOPED_KNOWLEDGE_FIELDS = ("url", "api_key", "allowed_dataset_ids")
-
-# ── Seed definitions (single source of truth) ────────────────────────────────
-# (config_key, default_value, display_name, description, group_key, is_secret)
-SEED_CONFIGS: list[tuple[str, str | None, str, str, str, bool]] = [
- # query_database
- (
- "query_database.url",
- None,
- "数据库查询接口 URL",
- "NL2SQL 数据库查询服务地址",
- "query_database",
- False,
- ),
- ("query_database.timeout", "40", "超时时间(秒)", "请求超时时间", "query_database", False),
- ("query_database.retry_times", "1", "重试次数", "请求失败重试次数", "query_database", False),
- (
- "query_database.max_output_tokens",
- "45000",
- "最大输出 Token",
- "数据库查询最大输出 Token 数",
- "query_database",
- False,
- ),
- (
- "database_query.capability_enabled",
- "true",
- "数据库查询能力总开关",
- "控制统一「数据库查询」能力是否在能力中心可用。由数据库工具页维护,不写入 catalog.json。",
- "query_database",
- False,
- ),
- (
- "knowledge_base.detail_max_chars",
- "50000",
- "详情最大字符数",
- "知识库文档详情最大字符数",
- "knowledge_base",
- False,
- ),
- # file_parser
- ("file_parser.api_url", None, "文件解析 API URL", "PDF/文档解析服务地址", "file_parser", False),
- ("file_parser.timeout", "60", "超时时间(秒)", "文件解析请求超时", "file_parser", False),
- ("file_parser.lang_list", "ch", "语言列表", "OCR 语言列表", "file_parser", False),
- ("file_parser.backend", "pipeline", "解析后端", "解析后端引擎", "file_parser", False),
- ("file_parser.parse_method", "auto", "解析方法", "auto / ocr / txt", "file_parser", False),
- (
- "file_parser.formula_enable",
- "true",
- "启用公式识别",
- "是否启用公式识别",
- "file_parser",
- False,
- ),
- ("file_parser.table_enable", "true", "启用表格识别", "是否启用表格识别", "file_parser", False),
- # internet_search
- (
- "internet_search.engine",
- "tavily",
- "搜索引擎",
- "互联网搜索引擎 (tavily / baidu / langsearch)",
- "internet_search",
- False,
- ),
- (
- "internet_search.tavily_api_key",
- None,
- "Tavily API Key",
- "Tavily 互联网搜索服务密钥",
- "internet_search",
- True,
- ),
- (
- "internet_search.baidu_api_key",
- None,
- "百度搜索 API Key",
- "百度千帆 AppBuilder 搜索服务密钥",
- "internet_search",
- True,
- ),
- (
- "internet_search.langsearch_api_key",
- None,
- "LangSearch API Key",
- "LangSearch Web Search API 服务密钥",
- "internet_search",
- True,
- ),
- # sandbox / code execution. The admin toggle is the **sole authority** (no env fallback):
- # explicitly defaults to "true"; admins can turn it off in "System config", which writes "false" and persists.
- (
- "sandbox.code_capability_enable",
- "true",
- "主智能体代码执行能力",
- "开启后主/计划/批量/子智能体默认具备文件工具+沙箱+我的空间访问;关闭则仅 Lab "
- "入口可用。本开关为唯一控制源,保存后 ≤30s 生效、无需重启。",
- "sandbox",
- False,
- ),
- # ontology —— independent enterprise governance gate for plugin imports.
- # Unlike the user's ontology preference, this switch cannot be bypassed by
- # turning off ontology validation in personal settings.
- (
- "ontology.force_plugin_import_build_validation",
- "false",
- "强制构建时本体校验",
- "插件导入或安装时必须通过激活 Domain Pack 的构建校验;个人本体开关不能关闭此门禁。",
- "ontology",
- False,
- ),
- # context —— the single compaction trigger ratio. Read per turn by
- # compaction_service.resolve_trigger_ratio and applied by every phase (mid-turn at the ReAct step
- # boundary, pre-turn before a turn samples, post-turn background pre-warm) — it is no longer an
- # "in-turn only" knob competing with CHAT_COMPACT_TRIGGER_RATIO. Admin config takes precedence over
- # env; takes effect on new conversations after saving, no restart needed.
- # ⚠️ Grouped under context, not chat: the chat group is in service_configs._HIDDEN_GROUPS
- # (chat.user_model_switch_enabled is exclusively managed by the model management panel to prevent a double entry point),
- # so attaching to the chat group would hide the whole group in the system config UI.
- (
- "chat.compress_in_turn_ratio",
- "0.90",
- "轮内压缩触发比例",
- "上下文占用超过「模型窗口 × 该值」时触发压缩(0.5~0.90)。计数优先用模型返回的"
- "真实 token 用量,拿不到时才退回字节估算,两者都不含模型输出长度——输出余量已由"
- "「窗口 95% 可用」统一预留。调大=更晚压缩(上下文更完整);调小=更早压缩。"
- "该比例对轮内、轮前、轮末三个时机统一生效。",
- "context",
- False,
- ),
- # dingtalk —— DingTalk workbench Custom App (enterprise self-built app) credentials. Once AppKey/AppSecret are filled,
- # DingTalk login and the dws command inside the sandbox execute as that self-built app, bypassing DingTalk's co-creation-era
- # "CLI data access" org toggle and self-controlling scope; leave empty to use the dws default shared app (org admin must
- # enable CLI data access first). Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart.
- (
- "dingtalk.client_id",
- None,
- "钉钉 AppKey (client-id)",
- "企业自建钉钉应用的 AppKey。在 open-dev.dingtalk.com 创建企业内部应用后获取。",
- "dingtalk",
- True,
- ),
- (
- "dingtalk.client_secret",
- None,
- "钉钉 AppSecret (client-secret)",
- "企业自建钉钉应用的 AppSecret,与 AppKey 配对。",
- "dingtalk",
- True,
- ),
- (
- "dingtalk.trusted_domains",
- "*.dingtalk.com",
- "受信域名白名单",
- "dws bearer token 允许发送的域名,逗号分隔,默认锁死 *.dingtalk.com 防 token 外泄。",
- "dingtalk",
- False,
- ),
- # firecrawl —— deployment-level credentials for the firecrawl web scraping/retrieval CLI. The firecrawl CLI is preinstalled
- # in the sandbox; at runtime these two items are injected as FIRECRAWL_API_KEY / FIRECRAWL_API_URL env vars into
- # every user's sandbox commands (one shared config for all users, mirroring the DingTalk dws Custom App approach).
- # - Cloud version: fill only api_key (starts with fc-).
- # - Self-hosted: fill api_url pointing to the self-built instance (the CLI skips cloud auth once it sees api_url); if the
- # self-built instance also needs a key, fill both.
- # Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart.
- (
- "firecrawl.api_key",
- None,
- "Firecrawl API Key",
- "firecrawl 云版 API Key(fc- 开头),在 firecrawl.dev 注册获取。用自托管实例时可留空。",
- "firecrawl",
- True,
- ),
- (
- "firecrawl.api_url",
- None,
- "Firecrawl 自托管地址",
- "自托管 firecrawl 实例地址(如 http://firecrawl:3002)。填了即走自建实例、CLI 跳过云端鉴权;用云版时留空。",
- "firecrawl",
- False,
- ),
- # lark —— Feishu workbench app (enterprise self-built app) credentials. Once AppID/AppSecret are filled, the lark-cli inside
- # the sandbox uses that self-built app as its OAuth app: admin configures once, and each user completes the device-flow
- # auth login to operate Feishu as themselves. The app config is seeded by the backend into each user's lark HOME
- # (config init), not injected as a global env var — this avoids lark-cli defaulting to the tenant/bot token flow when
- # LARK_APP_* is in the environment, which would break --as user. Sole authority = admin config, ≤30s after saving, no restart.
- (
- "lark.app_id",
- None,
- "飞书 App ID",
- "企业自建飞书应用的 App ID。在 open.feishu.cn 创建企业自建应用后获取。",
- "lark",
- True,
- ),
- (
- "lark.app_secret",
- None,
- "飞书 App Secret",
- "企业自建飞书应用的 App Secret,与 App ID 配对。",
- "lark",
- True,
- ),
- # auth —— login and session. Number of days a session is kept after the user checks "remember login state" (checked by
- # default) on the login page. Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart;
- # if unchecked, the default session duration (SESSION_TTL_HOURS) is used.
- (
- "auth.remember_me_days",
- "30",
- "记住登录状态时长(天)",
- "用户在登录页勾选「记住登录状态」后,登录会话保持的天数(默认 30 天)。"
- "未勾选则使用默认会话时长。",
- "auth",
- False,
- ),
- # turbo —— 极速模式(检索速查)。工具集独立于能力目录(catalog)的启停状态:
- # 极速模式装配的 MCP 以本组配置为唯一来源,管理台改动 ≤30s 生效、无需重启。
- (
- "turbo.mcp_server_ids",
- "retrieve_dataset_content,internet_search,web_fetch",
- "极速模式可用工具",
- "极速模式下装配的 MCP 工具集合(逗号分隔的 server id)。"
- "该集合不受「能力目录」中 MCP/技能/智能体/插件启停的影响,是极速模式工具面的唯一来源。",
- "turbo",
- False,
- ),
- (
- "turbo.skill_ids",
- "",
- "极速模式可用技能",
- "极速模式下装配的技能集合(逗号分隔的 skill id)。"
- "极速模式没有 Bash / 沙箱 / 文件写入,选纯文本类技能(写作模板、话术规范等)效果最好;"
- "需要代码执行的技能只会被读到说明、无法执行。同样不受「能力目录」启停影响。",
- "turbo",
- False,
- ),
- (
- "turbo.plugin_ids",
- "",
- "极速模式可用插件",
- "极速模式下装配的插件集合(逗号分隔的 install_id)。"
- "插件是「技能 + 工具」的能力包,选中后其全部工具与技能一并在极速模式生效;"
- "与上面的 MCP 工具集合合并去重,同样不受「能力目录」启停影响。",
- "turbo",
- False,
- ),
- (
- "turbo.manual_invoke_enable",
- "true",
- "允许手动呼唤能力",
- "开启后,用户在极速模式对话框中通过斜杠命令呼唤技能/插件、@ 呼唤子智能体时,"
- "被呼唤的能力将临时装配到本次请求(仅显式呼唤时可用,不会把全量能力清单注入提示词)。"
- "关闭后极速模式严格只用上面的工具集合。",
- "turbo",
- False,
- ),
- (
- "turbo.max_iters",
- "4",
- "迭代轮数上限",
- "极速模式单次回答的最大 ReAct 迭代轮数(含最终作答轮)。"
- "上限越小响应越快;默认 4 对应「最多 1-2 轮并行检索 + 直接作答」的产品契约。",
- "turbo",
- False,
- ),
-] + EDITION_SEED_CONFIGS
-
-# config_key → env var name mapping
-_CONFIG_KEY_TO_ENV: dict[str, str] = {
- "query_database.url": "QUERY_DATABASE_URL",
- "query_database.timeout": "QUERY_DATABASE_TIMEOUT_SECONDS",
- "query_database.retry_times": "QUERY_DATABASE_RETRY_TIMES",
- "query_database.max_output_tokens": "QUERY_DATABASE_MAX_OUTPUT_TOKENS",
- "knowledge_base.detail_max_chars": "KB_DETAIL_CONTENT_MAX_CHARS",
- "file_parser.api_url": "FILE_PARSER_API_URL",
- "file_parser.timeout": "FILE_PARSER_TIMEOUT",
- "file_parser.lang_list": "FILE_PARSER_LANG_LIST",
- "file_parser.backend": "FILE_PARSER_BACKEND",
- "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD",
- "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE",
- "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE",
- "internet_search.engine": "INTERNET_SEARCH_ENGINE",
- "internet_search.tavily_api_key": "TAVILY_API_KEY",
- "internet_search.baidu_api_key": "BAIDU_API_KEY",
- "internet_search.langsearch_api_key": "LANGSEARCH_API_KEY",
- # Note: sandbox.code_capability_enable deliberately does **not** map to an env var —— the admin toggle is the sole
- # authority (plan B), explicitly seeded to "true", and env CODE_CAPABILITY_ENABLED is retired.
- **EDITION_CONFIG_KEY_TO_ENV,
-}
-
-# Reverse mapping for env-fallback lookups
-_ENV_TO_CONFIG_KEY: dict[str, str] = {v: k for k, v in _CONFIG_KEY_TO_ENV.items()}
-
-
-def get_config_key_for_env(env_var: str) -> Optional[str]:
- """Return the system-config key that backs ``env_var``, if any.
-
- Public lookup helper so callers (e.g. ``runtime_env``) don't need to import
- the internal reverse map.
- """
- return _ENV_TO_CONFIG_KEY.get(env_var)
-
-
-class SystemConfigService:
- """Thread-safe singleton that resolves service configs from DB with env fallback."""
-
- _instance: Optional["SystemConfigService"] = None
- _lock = threading.Lock()
-
- def __init__(self) -> None:
- self._cache: dict[str, Optional[str]] = {}
- self._cache_meta: dict[str, dict] = {} # full row metadata
- self._cache_ts: float = 0.0
- self._cache_lock = threading.Lock()
-
- @classmethod
- def get_instance(cls) -> "SystemConfigService":
- if cls._instance is None:
- with cls._lock:
- if cls._instance is None:
- cls._instance = cls()
- return cls._instance
-
- # ── get / get_group ─────────────────────────────────────────────
-
- def get(self, key: str, default: str | None = None) -> str | None:
- """Get a config value. DB first, then env fallback, then default."""
- self._maybe_refresh()
- if key in self._cache and self._cache[key] is not None:
- return self._cache[key]
- # env fallback
- env_key = _CONFIG_KEY_TO_ENV.get(key)
- if env_key:
- env_val = os.getenv(env_key)
- if env_val is not None:
- return env_val.strip()
- return default
-
- def get_group(self, group_key: str) -> dict[str, str]:
- """Return all config key→value pairs for a group."""
- self._maybe_refresh()
- result: dict[str, str] = {}
- for key, meta in self._cache_meta.items():
- if meta.get("group_key") == group_key:
- val = self.get(key)
- if val is not None:
- result[key] = val
- return result
-
- def get_all_configs(self) -> list[dict]:
- """Return all config rows as dicts (for API responses)."""
- self._maybe_refresh()
- return list(self._cache_meta.values())
-
- def get_group_configs(self, group_key: str) -> list[dict]:
- """Return config rows for a specific group."""
- self._maybe_refresh()
- return [m for m in self._cache_meta.values() if m.get("group_key") == group_key]
-
- # ── set ─────────────────────────────────────────────────────────
-
- def set(self, key: str, value: str | None, updated_by: str = "admin") -> None:
- """Update a config value in DB."""
- try:
- db = SessionLocal()
- try:
- row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first()
- if row is None:
- logger.warning("[SystemConfigService] key %s not found, skipping set", key)
- return
- row.config_value = value
- row.updated_by = updated_by
- db.commit()
- finally:
- db.close()
- except Exception as exc:
- logger.error("[SystemConfigService] set(%s) failed: %s", key, exc)
- raise
- self.invalidate_cache()
-
- def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None:
- """Batch update multiple config values. Each item: {key, value}.
-
- For secret fields, masked values (containing '****') are skipped to
- prevent the frontend from accidentally overwriting real secrets with
- their masked representation.
- """
- try:
- db = SessionLocal()
- try:
- for item in items:
- key = item.get("key", "").strip()
- value = item.get("value")
- if not key:
- continue
- row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first()
- if row is None:
- continue
- # Skip masked values for secret fields
- if row.is_secret and isinstance(value, str) and "****" in value:
- continue
- row.config_value = value if value != "" else None
- row.updated_by = updated_by
- db.commit()
- finally:
- db.close()
- except Exception as exc:
- logger.error("[SystemConfigService] bulk_set failed: %s", exc)
- raise
- self.invalidate_cache()
-
- # ── cache management ──────────────────────────────────────────
-
- def invalidate_cache(self) -> None:
- with self._cache_lock:
- self._cache.clear()
- self._cache_meta.clear()
- self._cache_ts = 0.0
-
- def _maybe_refresh(self) -> None:
- now = time.monotonic()
- if now - self._cache_ts < _CACHE_TTL_SECONDS and self._cache_meta:
- return
- with self._cache_lock:
- if now - self._cache_ts < _CACHE_TTL_SECONDS and self._cache_meta:
- return
- self._load_from_db()
- self._cache_ts = time.monotonic()
-
- def _ensure_seed_rows(self, db) -> None:
- """Insert any missing seed config rows. Idempotent."""
- rows_by_key = {row.config_key: row for row in db.query(SystemConfig).all()}
- existing_keys = set(rows_by_key)
- inserted_keys: set[str] = set()
- inserted = 0
- for key, val, display, desc, group, secret in SEED_CONFIGS:
- if key not in existing_keys:
- row = SystemConfig(
- config_key=key,
- config_value=val,
- display_name=display,
- description=desc,
- group_key=group,
- is_secret=secret,
- )
- db.add(row)
- rows_by_key[key] = row
- inserted_keys.add(key)
- inserted += 1
- migrated = self._migrate_legacy_knowledge_config(rows_by_key, inserted_keys)
- if inserted or migrated:
- db.commit()
- logger.info(
- "[SystemConfigService] Inserted %d missing seed row(s), migrated %d legacy KB value(s)",
- inserted,
- migrated,
- )
-
- @staticmethod
- def _migrate_legacy_knowledge_config(
- rows_by_key: dict[str, SystemConfig],
- inserted_keys: set[str],
- ) -> int:
- """Move the old shared KB connection into one provider exactly once.
-
- Older releases stored every external backend in the shared
- knowledge_base.url/api_key/allowed_dataset_ids rows. Those rows cannot
- prove which dropdown option they belonged to, and copying them into the
- currently selected provider can reproduce the original cross-provider
- leak. Treat them as Dify compatibility values and leave FastGPT/WeKnora
- empty until each provider is configured explicitly.
-
- inserted_keys is the one-time guard: after scoped rows exist, later
- legacy edits can never overwrite provider-specific values.
- """
- migrated = 0
- for field in _PROVIDER_SCOPED_KNOWLEDGE_FIELDS:
- destination_key = f"knowledge_base.dify.{field}"
- if destination_key not in inserted_keys:
- continue
- destination = rows_by_key.get(destination_key)
- legacy = rows_by_key.get(f"knowledge_base.{field}")
- legacy_value = getattr(legacy, "config_value", None)
- if (
- destination is not None
- and not getattr(destination, "config_value", None)
- and legacy_value not in (None, "")
- ):
- destination.config_value = legacy_value
- migrated += 1
- return migrated
-
- def _load_from_db(self) -> None:
- new_cache: dict[str, Optional[str]] = {}
- new_meta: dict[str, dict] = {}
- try:
- db = SessionLocal()
- try:
- self._ensure_seed_rows(db)
- rows = db.query(SystemConfig).all()
- for row in rows:
- new_cache[row.config_key] = row.config_value
- new_meta[row.config_key] = {
- "config_key": row.config_key,
- "config_value": row.config_value,
- "display_name": row.display_name,
- "description": row.description,
- "group_key": row.group_key,
- "is_secret": row.is_secret,
- "updated_at": row.updated_at.isoformat() if row.updated_at else None,
- "updated_by": row.updated_by,
- }
- finally:
- db.close()
- except Exception as exc:
- logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", exc)
- return
-
- self._cache = new_cache
- self._cache_meta = new_meta
-
- # ── env overlay for MCP sub-processes ─────────────────────────
-
- def get_service_env_overlay(self) -> dict[str, str]:
- """Return env-var style dict for injecting into MCP sub-processes.
-
- Maps DB configs to the env var names that MCP servers already read via os.getenv().
- Only includes keys that have a non-None value (from DB or env fallback).
- """
- overlay: dict[str, str] = {}
- for config_key, env_key in _CONFIG_KEY_TO_ENV.items():
- val = self.get(config_key)
- if val is not None:
- overlay[env_key] = val
- return overlay
-
-
-def code_capability_enabled() -> bool:
- """Master switch for the main agent's code execution capability (single source of truth at runtime).
-
- Sole control source = the DB value of Config console "System config" (plan B, env fallback retired);
- seeded explicitly to "true". Saving in the console calls invalidate_cache; takes effect ≤30s without restart.
- agent_factory and the admin_prompts preview call the same source, ensuring the console display matches runtime.
- """
- try:
- val = SystemConfigService.get_instance().get("sandbox.code_capability_enable", "false")
- except Exception: # noqa: BLE001 — conservatively disable on config-layer errors
- return False
- return str(val).strip().lower() in ("1", "true", "yes", "on")
-
-
-#: 极速模式检索工具集的代码内兜底(配置层异常/为空时使用)。
-DEFAULT_TURBO_MCP_SERVER_IDS = frozenset(
- {"retrieve_dataset_content", "internet_search", "web_fetch"}
-)
-
-
-def turbo_mcp_server_ids() -> frozenset[str]:
- """极速模式装配的 MCP server 集合(运行时唯一真源)。
-
- 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.mcp_server_ids``
- (逗号分隔)。刻意不与能力目录(catalog)的启停求交集——极速模式的工具面
- 是独立的产品契约。配置异常或解析为空时回退到内置检索三件套。
- """
- default = ",".join(sorted(DEFAULT_TURBO_MCP_SERVER_IDS))
- try:
- raw = SystemConfigService.get_instance().get("turbo.mcp_server_ids", default) or default
- except Exception: # noqa: BLE001 — conservatively fall back to the built-in trio
- return DEFAULT_TURBO_MCP_SERVER_IDS
- ids = frozenset(part.strip() for part in str(raw).split(",") if part.strip())
- return ids or DEFAULT_TURBO_MCP_SERVER_IDS
-
-
-def turbo_skill_ids() -> tuple[str, ...]:
- """极速模式装配的技能集合(``AdminSkill.skill_id`` / 内置技能 id 列表)。
-
- 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.skill_ids``(逗号
- 分隔)。与用户本轮显式呼唤的技能合并去重。默认空 —— 不选就没有技能,
- 极速模式的系统提示词里也就不出现技能清单。
- """
- try:
- raw = SystemConfigService.get_instance().get("turbo.skill_ids", "") or ""
- except Exception: # noqa: BLE001 — 配置层异常时按「没配技能」处理
- return ()
- return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip()))
-
-
-def turbo_plugin_ids() -> tuple[str, ...]:
- """极速模式装配的插件集合(``InstalledPlugin.install_id`` 列表)。
-
- 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.plugin_ids``(逗号
- 分隔)。插件是「技能 + 工具」的能力包,装配时由 agent_factory 展开成其组件
- 技能 / MCP,与 ``turbo.mcp_server_ids`` 合并去重。默认空 —— 不选就没有插件。
- """
- try:
- raw = SystemConfigService.get_instance().get("turbo.plugin_ids", "") or ""
- except Exception: # noqa: BLE001 — 配置层异常时按「没配插件」处理
- return ()
- return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip()))
-
-
-def turbo_manual_invoke_enabled() -> bool:
- """极速模式下是否允许显式呼唤(斜杠技能 / @子智能体 / 插件)临时装配。"""
- try:
- val = SystemConfigService.get_instance().get("turbo.manual_invoke_enable", "true")
- except Exception: # noqa: BLE001
- return True
- return str(val).strip().lower() in ("1", "true", "yes", "on")
-
-
-def turbo_max_iters() -> int:
- """极速模式的 ReAct 迭代硬上限(最小 2,防止配错把回答轮也掐掉)。"""
- try:
- val = SystemConfigService.get_instance().get("turbo.max_iters", "4")
- return max(2, int(str(val).strip()))
- except Exception: # noqa: BLE001
- return 4
-
-
-def auto_plan_entry_enabled() -> bool:
- """Master switch for the main agent's in-conversation plan tracker (the update_plan tool).
-
- Control source = the Config console "System config" DB value ``chat.auto_plan_entry_enable``,
- defaulting to "true" (enabled by default). Ops can set it to false to disable the capability, ≤30s, no restart.
- Conservatively disable on config-layer errors (don't register the tool, fall back to manually clicking "Plan mode").
- """
- try:
- val = SystemConfigService.get_instance().get("chat.auto_plan_entry_enable", "true")
- except Exception: # noqa: BLE001 — conservatively disable on config-layer errors
- return False
- return str(val).strip().lower() in ("1", "true", "yes", "on")
diff --git a/src/backend/core/services/tool_effect_ledger.py b/src/backend/core/services/tool_effect_ledger.py
deleted file mode 100644
index 03bb6d0d..00000000
--- a/src/backend/core/services/tool_effect_ledger.py
+++ /dev/null
@@ -1,1808 +0,0 @@
-"""Durable intent-before-invocation gateway for tool effects.
-
-``ToolEffectLedger`` is append-only and authoritative. The companion lease
-row is only a mutable execution/recovery claim. All public execution paths go
-through :class:`ToolEffectGateway`; adapters are never called until an Intent
-has committed and an invocation guard has rechecked ownership.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import inspect
-import json
-import logging
-import re
-import time
-import uuid
-from contextlib import suppress
-from contextvars import ContextVar
-from dataclasses import dataclass
-from datetime import datetime, timedelta, timezone
-from typing import Any, Awaitable, Callable, Mapping, Optional, cast
-
-from sqlalchemy import and_, case, or_
-from sqlalchemy.exc import IntegrityError, OperationalError
-from sqlalchemy.orm import Session, aliased, sessionmaker
-
-from core.db.engine import SessionLocal
-from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger
-from core.infra.data_masking import mask_sensitive_data
-from core.services.run_journal import (
- LIVE_STATUSES,
- RunJournal,
- RunLeaseLost,
- RunNotFound,
-)
-
-logger = logging.getLogger(__name__)
-
-RECOVERY_POLICIES = ("replay_safe", "reconcile", "never_replay")
-# needs_attention 的现场值得多留一阵子再回收,保留期是普通终态的这个倍数——从
-# 同一个 retention_days 推导,不额外引入一个可独立漂移的阈值。
-_ATTENTION_RETENTION_FACTOR = 4
-_REDACT_FIELDS = [
- "password",
- "token",
- "api_key",
- "apikey",
- "authorization",
- "secret",
- "access_key",
- "cookie",
-]
-_WITHHOLD_FIELDS = {"command", "content", "stdin", "script", "code"}
-_EMBEDDED_SECRET_PATTERNS = (
- re.compile(r"(?i)(authorization\s*:\s*bearer\s+)[^\s'\";]+"),
- re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}"),
- re.compile(
- r"(?i)([?&](?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)=)[^\s]+"
- ),
- re.compile(
- r"(?i)((?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)\s*[=:]\s*)[^\s'\";]+"
- ),
-)
-
-
-class ToolEffectError(RuntimeError):
- pass
-
-
-class ToolIntentCommitError(ToolEffectError):
- """The intent could not be durably committed; invocation is forbidden."""
-
-
-class ToolOutcomeUnknown(ToolEffectError):
- """The ledger proves intent but cannot prove the external outcome."""
-
-
-def find_tool_outcome_unknown(exc: BaseException) -> Optional[ToolOutcomeUnknown]:
- """Unwrap AgentScope/anyio ExceptionGroup layers without losing safety meaning."""
- pending = [exc]
- seen: set[int] = set()
- while pending:
- current = pending.pop()
- if id(current) in seen:
- continue
- seen.add(id(current))
- if isinstance(current, ToolOutcomeUnknown):
- return current
- nested = getattr(current, "exceptions", None)
- if isinstance(nested, tuple):
- pending.extend(item for item in nested if isinstance(item, BaseException))
- if isinstance(current.__cause__, BaseException):
- pending.append(current.__cause__)
- if isinstance(current.__context__, BaseException):
- pending.append(current.__context__)
- return None
-
-
-class ToolEffectLeaseLost(ToolEffectError):
- pass
-
-
-class ToolIdempotencyConflict(ToolEffectError):
- """A caller reused one explicit key for different logical input."""
-
-
-@dataclass(frozen=True)
-class ReconciliationResult:
- outcome: str
- result: Any = None
-
- @classmethod
- def applied(cls, result: Any) -> "ReconciliationResult":
- return cls("applied", result)
-
- @classmethod
- def not_applied(cls) -> "ReconciliationResult":
- return cls("not_applied")
-
- @classmethod
- def unknown(cls) -> "ReconciliationResult":
- return cls("unknown")
-
-
-@dataclass(frozen=True)
-class ToolPolicy:
- policy: str
- replayer: Optional[Callable[["ToolIntent"], Awaitable[Any] | Any]] = None
- reconciler: Optional[
- Callable[["ToolIntent"], Awaitable[ReconciliationResult | Any] | Any]
- ] = None
-
-
-@dataclass(frozen=True)
-class ToolIntent:
- effect_id: str
- result_id: str
- run_id: str
- operation_seq: int
- tool_name: str
- tool_call_id: str
- args_hash: str
- redacted_args: Mapping[str, Any]
- idempotency_key: str
- recovery_policy: str
- created_at: Optional[datetime] = None
-
-
-@dataclass(frozen=True)
-class IntentDecision:
- action: str
- intent: ToolIntent
- result: Any = None
- result_state: str = "success"
-
- @property
- def effect_id(self) -> str:
- return self.intent.effect_id
-
-
-@dataclass(frozen=True)
-class ToolExecutionOutcome:
- result: Any
- effect_id: str
- result_id: str
- result_state: str
- invoked: bool
-
-
-@dataclass(frozen=True)
-class RecoverySweepDecision:
- effect_id: str
- run_id: str
- action: str
-
-
-@dataclass(frozen=True)
-class ToolInvocationContext:
- effect_id: str
- result_id: str
- idempotency_key: str
- run_id: str
- fencing_owner: str
-
-
-CURRENT_TOOL_EFFECT: ContextVar[Optional[ToolInvocationContext]] = ContextVar(
- "jx_current_tool_effect", default=None
-)
-
-
-class ToolRecoveryRegistry:
- """Explicit adapter contracts. Unknown tools fail closed as never-replay."""
-
- def __init__(self) -> None:
- self._policies: dict[str, ToolPolicy] = {}
-
- def register(
- self,
- tool_name: str,
- policy: str,
- *,
- replayer: Optional[Callable[[ToolIntent], Awaitable[Any] | Any]] = None,
- reconciler: Optional[
- Callable[[ToolIntent], Awaitable[ReconciliationResult | Any] | Any]
- ] = None,
- ) -> None:
- if policy not in RECOVERY_POLICIES:
- raise ValueError(f"invalid recovery policy: {policy}")
- if policy == "replay_safe" and reconciler is not None:
- raise ValueError("replay_safe does not accept a reconciler")
- if policy == "never_replay" and (
- replayer is not None or reconciler is not None
- ):
- raise ValueError("never_replay does not accept recovery adapters")
- self._policies[str(tool_name)] = ToolPolicy(policy, replayer, reconciler)
-
- def resolve(self, tool_name: str) -> ToolPolicy:
- return self._policies.get(str(tool_name), ToolPolicy("never_replay"))
-
-
-async def _default_replayer(intent: ToolIntent) -> Any:
- from orchestration.tool_effect_recovery import replay_tool_intent
-
- return await replay_tool_intent(intent)
-
-
-async def _scheduled_task_reconciler(intent: ToolIntent) -> ReconciliationResult:
- from orchestration.tool_effect_recovery import reconcile_scheduled_task_intent
-
- return cast(ReconciliationResult, await reconcile_scheduled_task_intent(intent))
-
-
-def build_default_tool_recovery_registry() -> ToolRecoveryRegistry:
- registry = ToolRecoveryRegistry()
- for name in (
- "Read",
- "read_project_instructions",
- "list_sites",
- "Glob",
- "Grep",
- "view_text_file",
- "get_data_context",
- "read_image",
- "read_artifact",
- "channel_read_attachment",
- "sandbox_get_artifact",
- ):
- registry.register(name, "replay_safe", replayer=_default_replayer)
- for name in (
- "Write",
- "save_project_instructions",
- "Edit",
- "Bash",
- "sandbox_put_artifact",
- "pin_to_workspace",
- "run_job",
- "choose_design",
- "call_subagent",
- "update_plan",
- "load_plugin",
- ):
- registry.register(name, "never_replay")
- for name in (
- "create_scheduled_task",
- "update_scheduled_task",
- "delete_scheduled_task",
- ):
- registry.register(
- name,
- "reconcile",
- replayer=_default_replayer,
- reconciler=_scheduled_task_reconciler,
- )
- return registry
-
-
-DEFAULT_TOOL_RECOVERY_REGISTRY = build_default_tool_recovery_registry()
-
-
-def _utcnow() -> datetime:
- return datetime.now(timezone.utc)
-
-
-def _aware(value: Optional[datetime]) -> Optional[datetime]:
- if value is None:
- return None
- if value.tzinfo is None:
- return value.replace(tzinfo=timezone.utc)
- return value.astimezone(timezone.utc)
-
-
-def _embedded_redact(value: str) -> str:
- cleaned = value
- for pattern in _EMBEDDED_SECRET_PATTERNS:
- cleaned = pattern.sub(r"\1[REDACTED]", cleaned)
- return cleaned
-
-
-def _sanitize_value(value: Any, *, key: str = "") -> Any:
- lower = key.lower()
- if isinstance(value, Mapping):
- return {str(k): _sanitize_value(v, key=str(k)) for k, v in value.items()}
- if isinstance(value, (list, tuple)):
- return [_sanitize_value(item, key=key) for item in value]
- if isinstance(value, str):
- if lower in _WITHHOLD_FIELDS:
- digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
- return {"_withheld": True, "sha256": digest, "length": len(value)}
- return _embedded_redact(value)
- return value
-
-
-def canonical_tool_args(args: Mapping[str, Any]) -> tuple[str, str, Mapping[str, Any]]:
- canonical = json.dumps(
- args or {}, sort_keys=True, separators=(",", ":"), default=str
- )
- args_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
- by_field = mask_sensitive_data(dict(args or {}), field_patterns=_REDACT_FIELDS)
- return canonical, args_hash, _sanitize_value(by_field)
-
-
-def tool_args_hash(args: Mapping[str, Any]) -> str:
- return canonical_tool_args(args)[1]
-
-
-def stable_idempotency_key(run_id: str, tool_name: str, invocation_id: str) -> str:
- raw = f"{run_id}\x00{tool_name}\x00{invocation_id}".encode("utf-8")
- return "tool:" + hashlib.sha256(raw).hexdigest()
-
-
-def _resolved_idempotency_key(
- *,
- run_id: str,
- principal_scope: str,
- tool_name: str,
- tool_call_id: str,
- explicit_key: Optional[str],
-) -> tuple[str, bool]:
- caller_key = str(explicit_key or "").strip()
- if caller_key:
- scoped = f"{principal_scope}\x00{caller_key}".encode("utf-8")
- return "caller:" + hashlib.sha256(scoped).hexdigest(), True
- if not tool_call_id:
- raise ToolIntentCommitError(
- "tool_call_id or explicit idempotency_key is required"
- )
- return stable_idempotency_key(run_id, tool_name, tool_call_id), False
-
-
-class ToolEffectJournal:
- """Small synchronous transaction boundary behind the async gateway."""
-
- def __init__(
- self,
- session_factory: sessionmaker = SessionLocal,
- *,
- clock: Callable[[], datetime] = _utcnow,
- ) -> None:
- self._sessions = session_factory
- self._clock = clock
- self._runs = RunJournal(session_factory, clock=clock)
-
- @staticmethod
- def _intent_from_row(row: ToolEffectLedger) -> ToolIntent:
- return ToolIntent(
- effect_id=row.effect_id,
- result_id=row.result_id,
- run_id=row.run_id,
- operation_seq=int(row.operation_seq),
- tool_name=row.tool_name,
- tool_call_id=row.tool_call_id or "",
- args_hash=row.args_hash,
- redacted_args=dict(row.redacted_args or {}),
- idempotency_key=row.idempotency_key or "",
- recovery_policy=row.recovery_policy,
- created_at=_aware(row.created_at),
- )
-
- @staticmethod
- def _terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]:
- return (
- db.query(ToolEffectLedger)
- .filter(ToolEffectLedger.terminal_effect_id == effect_id)
- .one_or_none()
- )
-
- @staticmethod
- def _decision_from_terminal(
- intent: ToolIntent, terminal: ToolEffectLedger
- ) -> IntentDecision:
- if terminal.event_type == "unknown_outcome":
- return IntentDecision("unknown", intent, result_state="unknown")
- return IntentDecision(
- "known",
- intent,
- terminal.result_payload,
- "success" if terminal.event_type == "result" else "error",
- )
-
- @staticmethod
- def _validate_identity(
- row: ToolEffectLedger, *, tool_name: str, args_hash: str
- ) -> None:
- if row.tool_name != tool_name or row.args_hash != args_hash:
- raise ToolIdempotencyConflict(
- "idempotency_key already belongs to different tool arguments"
- )
-
- def _append_event(
- self,
- db: Session,
- run: ChatRun,
- intent: ToolEffectLedger,
- *,
- owner: str,
- event_type: str,
- operation_type: str,
- phase: str,
- safety: str,
- result: Any = None,
- error: Optional[str] = None,
- terminal: bool = False,
- ) -> None:
- receipt = self._runs._append_locked(
- db,
- run,
- owner=owner,
- operation_type=operation_type,
- phase=phase,
- safety=safety,
- payload={"effect_id": intent.effect_id, "result_id": intent.result_id},
- )
- db.add(
- ToolEffectLedger(
- effect_id=intent.effect_id,
- result_id=intent.result_id,
- run_id=intent.run_id,
- operation_seq=receipt.operation_seq,
- event_type=event_type,
- tool_name=intent.tool_name,
- tool_call_id=intent.tool_call_id,
- args_hash=intent.args_hash,
- redacted_args=intent.redacted_args,
- recovery_policy=intent.recovery_policy,
- result_payload=result,
- error_message=error,
- terminal_effect_id=intent.effect_id if terminal else None,
- )
- )
-
- def _mark_unknown_locked(
- self,
- db: Session,
- run: ChatRun,
- intent: ToolEffectLedger,
- lease: ToolEffectLease,
- *,
- owner: str,
- claim_owner: str,
- reason: str,
- ) -> None:
- if self._terminal_row(db, intent.effect_id) is not None:
- return
- if lease.run_owner != owner or lease.claim_owner != claim_owner:
- raise ToolEffectLeaseLost(intent.effect_id)
- self._append_event(
- db,
- run,
- intent,
- owner=owner,
- event_type="unknown_outcome",
- operation_type="tool_outcome_unknown",
- phase="needs_attention",
- safety="manual_review",
- error=reason,
- terminal=True,
- )
- run.status = "needs_attention"
- run.failure_reason = reason
- run.lease_owner = None
- run.lease_expires_at = None
- run.updated_at = self._clock()
- lease.claim_owner = None
- lease.lease_expires_at = None
- lease.updated_at = self._clock()
-
- def _existing_decision(
- self,
- *,
- key: str,
- caller_run_id: str,
- run_owner: str,
- claim_owner: str,
- tool_name: str,
- args_hash: str,
- lease_seconds: int,
- ) -> IntentDecision:
- with self._sessions() as db:
- intent_row = (
- db.query(ToolEffectLedger)
- .filter(ToolEffectLedger.idempotency_key == key)
- .one()
- )
- self._validate_identity(
- intent_row, tool_name=tool_name, args_hash=args_hash
- )
- intent = self._intent_from_row(intent_row)
- caller_run = db.get(ChatRun, caller_run_id)
- original_run = db.get(ChatRun, intent.run_id)
- if caller_run is None or original_run is None:
- raise ToolIntentCommitError(f"run does not exist: {caller_run_id}")
- if original_run.user_id != caller_run.user_id:
- raise ToolIdempotencyConflict(
- "idempotency_key belongs to a different principal"
- )
- terminal = self._terminal_row(db, intent.effect_id)
- if terminal is not None and caller_run_id == intent.run_id:
- return self._decision_from_terminal(intent, terminal)
- now = _aware(self._clock())
- caller_expiry = _aware(caller_run.lease_expires_at)
- if (
- caller_run.status not in LIVE_STATUSES
- or caller_run.lease_owner != run_owner
- or caller_expiry is None
- or now is None
- or caller_expiry <= now
- ):
- raise ToolIntentCommitError(f"run lease is not owned: {caller_run_id}")
- if terminal is not None:
- return self._decision_from_terminal(intent, terminal)
- if intent.run_id != caller_run_id:
- return IntentDecision("wait", intent)
-
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == intent.effect_id)
- .with_for_update()
- .one()
- )
- run = self._runs._locked_owned_run(db, caller_run_id, run_owner)
- terminal = self._terminal_row(db, intent.effect_id)
- if terminal is not None:
- return self._decision_from_terminal(intent, terminal)
- lease_expiry = _aware(lease.lease_expires_at)
- if (
- lease.run_owner == run_owner
- and lease_expiry is not None
- and lease_expiry > now
- ):
- if lease.claim_owner == claim_owner:
- return IntentDecision(
- "reconcile"
- if intent.recovery_policy == "reconcile"
- else "execute",
- intent,
- )
- return IntentDecision("wait", intent)
-
- previous_owner = lease.run_owner
- previous_claim = lease.claim_owner
- previous_expiry = lease.lease_expires_at
- affected = (
- db.query(ToolEffectLease)
- .filter(
- ToolEffectLease.effect_id == intent.effect_id,
- ToolEffectLease.run_owner == previous_owner,
- ToolEffectLease.claim_owner == previous_claim,
- ToolEffectLease.lease_expires_at == previous_expiry,
- or_(
- ToolEffectLease.lease_expires_at.is_(None),
- ToolEffectLease.lease_expires_at <= self._clock(),
- ),
- )
- .update(
- {
- ToolEffectLease.run_owner: run_owner,
- ToolEffectLease.claim_owner: claim_owner,
- ToolEffectLease.lease_expires_at: self._clock()
- + timedelta(seconds=max(1, lease_seconds)),
- ToolEffectLease.updated_at: self._clock(),
- },
- synchronize_session=False,
- )
- )
- if not affected:
- db.rollback()
- return IntentDecision("wait", intent)
- db.expire_all()
- lease = db.get(ToolEffectLease, intent.effect_id)
- self._append_event(
- db,
- run,
- intent_row,
- owner=run_owner,
- event_type="recovery_claim",
- operation_type="tool_effect_recovery_claimed",
- phase=f"tool_intent_{intent.recovery_policy}",
- safety=intent.recovery_policy,
- )
- if intent.recovery_policy == "never_replay":
- self._mark_unknown_locked(
- db,
- run,
- intent_row,
- lease,
- owner=run_owner,
- claim_owner=claim_owner,
- reason="non-replayable tool intent has no committed result",
- )
- db.commit()
- return IntentDecision("unknown", intent)
- db.commit()
- return IntentDecision(
- "reconcile" if intent.recovery_policy == "reconcile" else "execute",
- intent,
- )
-
- def begin_intent(
- self,
- *,
- run_id: str,
- owner: str,
- claim_owner: str,
- tool_call_id: str,
- tool_name: str,
- args: Mapping[str, Any],
- recovery_policy: str,
- idempotency_key: Optional[str] = None,
- lease_seconds: int = 300,
- _conflict_retries: int = 4,
- ) -> IntentDecision:
- if recovery_policy not in RECOVERY_POLICIES:
- raise ValueError(f"invalid recovery policy: {recovery_policy}")
- _canonical, args_hash, redacted = canonical_tool_args(args)
- with self._sessions() as db:
- caller_run = db.get(ChatRun, run_id)
- if caller_run is None:
- raise ToolIntentCommitError(f"run does not exist: {run_id}")
- principal_scope = caller_run.user_id
- key, _explicit = _resolved_idempotency_key(
- run_id=run_id,
- principal_scope=str(principal_scope),
- tool_name=tool_name,
- tool_call_id=tool_call_id,
- explicit_key=idempotency_key
- or str(args.get("idempotency_key") or args.get("request_id") or "").strip(),
- )
- try:
- with self._sessions() as db:
- found = (
- db.query(ToolEffectLedger.event_id)
- .filter(ToolEffectLedger.idempotency_key == key)
- .first()
- )
- if found is not None:
- return self._existing_decision(
- key=key,
- caller_run_id=run_id,
- run_owner=owner,
- claim_owner=claim_owner,
- tool_name=tool_name,
- args_hash=args_hash,
- lease_seconds=lease_seconds,
- )
-
- with self._sessions() as db:
- run = self._runs._locked_owned_run(db, run_id, owner)
- found = (
- db.query(ToolEffectLedger.event_id)
- .filter(ToolEffectLedger.idempotency_key == key)
- .first()
- )
- if found is not None:
- db.rollback()
- return self._existing_decision(
- key=key,
- caller_run_id=run_id,
- run_owner=owner,
- claim_owner=claim_owner,
- tool_name=tool_name,
- args_hash=args_hash,
- lease_seconds=lease_seconds,
- )
- effect_id = f"eff_{uuid.uuid4().hex}"
- result_id = f"toolres_{uuid.uuid4().hex}"
- receipt = self._runs._append_locked(
- db,
- run,
- owner=owner,
- operation_type="tool_intent_committed",
- phase=f"tool_intent_{recovery_policy}",
- safety=recovery_policy,
- payload={
- "effect_id": effect_id,
- "result_id": result_id,
- "tool_name": tool_name,
- "tool_call_id": tool_call_id,
- "args_hash": args_hash,
- },
- )
- row = ToolEffectLedger(
- effect_id=effect_id,
- result_id=result_id,
- run_id=run_id,
- operation_seq=receipt.operation_seq,
- event_type="intent",
- tool_name=tool_name,
- tool_call_id=tool_call_id,
- args_hash=args_hash,
- redacted_args=dict(redacted),
- idempotency_key=key,
- recovery_policy=recovery_policy,
- )
- db.add(row)
- db.add(
- ToolEffectLease(
- effect_id=effect_id,
- run_id=run_id,
- run_owner=owner,
- claim_owner=claim_owner,
- lease_expires_at=self._clock()
- + timedelta(seconds=max(1, lease_seconds)),
- updated_at=self._clock(),
- )
- )
- run.updated_at = self._clock()
- db.commit()
- return IntentDecision("execute", self._intent_from_row(row))
- except ToolEffectError:
- raise
- except (IntegrityError, OperationalError) as exc:
- with self._sessions() as db:
- found = (
- db.query(ToolEffectLedger.event_id)
- .filter(ToolEffectLedger.idempotency_key == key)
- .first()
- )
- if found is not None:
- return self._existing_decision(
- key=key,
- caller_run_id=run_id,
- run_owner=owner,
- claim_owner=claim_owner,
- tool_name=tool_name,
- args_hash=args_hash,
- lease_seconds=lease_seconds,
- )
- if _conflict_retries > 0:
- return self.begin_intent(
- run_id=run_id,
- owner=owner,
- claim_owner=claim_owner,
- tool_call_id=tool_call_id,
- tool_name=tool_name,
- args=args,
- recovery_policy=recovery_policy,
- idempotency_key=idempotency_key,
- lease_seconds=lease_seconds,
- _conflict_retries=_conflict_retries - 1,
- )
- raise ToolIntentCommitError(str(exc)) from exc
- except (RunNotFound, RunLeaseLost) as exc:
- raise ToolIntentCommitError(str(exc)) from exc
- except Exception as exc: # noqa: BLE001
- raise ToolIntentCommitError(str(exc)) from exc
-
- def acquire_invocation_claim(
- self,
- effect_id: str,
- *,
- run_owner: str,
- claim_owner: str,
- lease_seconds: int = 300,
- ) -> None:
- """Validate and extend one invocation claim in a short atomic transaction."""
- now = self._clock()
- expires = now + timedelta(seconds=max(1, lease_seconds))
- with self._sessions() as db:
- live_run = db.query(ChatRun.run_id).filter(
- ChatRun.status.in_(LIVE_STATUSES),
- ChatRun.lease_owner == run_owner,
- ChatRun.lease_expires_at > now,
- )
- affected = (
- db.query(ToolEffectLease)
- .filter(
- ToolEffectLease.effect_id == effect_id,
- ToolEffectLease.run_owner == run_owner,
- ToolEffectLease.claim_owner == claim_owner,
- ToolEffectLease.lease_expires_at > now,
- ToolEffectLease.run_id.in_(live_run),
- )
- .update(
- {
- ToolEffectLease.lease_expires_at: expires,
- ToolEffectLease.updated_at: now,
- },
- synchronize_session=False,
- )
- )
- if not affected or self._terminal_row(db, effect_id) is not None:
- db.rollback()
- raise ToolEffectLeaseLost(effect_id)
- db.commit()
-
- def renew_invocation_claim(
- self,
- effect_id: str,
- *,
- run_owner: str,
- claim_owner: str,
- lease_seconds: int,
- ) -> bool:
- if not self._runs.renew(
- self._intent_run_id(effect_id),
- owner=run_owner,
- lease_seconds=lease_seconds,
- ):
- return False
- try:
- self.acquire_invocation_claim(
- effect_id,
- run_owner=run_owner,
- claim_owner=claim_owner,
- lease_seconds=lease_seconds,
- )
- return True
- except ToolEffectLeaseLost:
- return False
-
- def renew_recovery_claim(
- self,
- effect_id: str,
- *,
- recovery_owner: str,
- lease_seconds: int,
- ) -> bool:
- return self.renew_invocation_claim(
- effect_id,
- run_owner=recovery_owner,
- claim_owner=recovery_owner,
- lease_seconds=lease_seconds,
- )
-
- def _intent_run_id(self, effect_id: str) -> str:
- with self._sessions() as db:
- row = (
- db.query(ToolEffectLedger.run_id)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one()
- )
- return str(row[0])
-
- def abandon_invocation_claim(
- self,
- effect_id: str,
- *,
- run_owner: str,
- claim_owner: str,
- ) -> None:
- """Expire a cancelled/ambiguous call so policy recovery can take over."""
- now = self._clock()
- with self._sessions() as db:
- db.query(ToolEffectLease).filter(
- ToolEffectLease.effect_id == effect_id,
- ToolEffectLease.run_owner == run_owner,
- ToolEffectLease.claim_owner == claim_owner,
- ).update(
- {
- ToolEffectLease.lease_expires_at: now,
- ToolEffectLease.updated_at: now,
- },
- synchronize_session=False,
- )
- db.commit()
-
- def _commit_result_locked(
- self,
- db: Session,
- intent: ToolEffectLedger,
- lease: ToolEffectLease,
- *,
- owner: str,
- result: Any,
- failed: bool,
- reconciled: bool,
- ) -> Any:
- terminal = self._terminal_row(db, intent.effect_id)
- if terminal is not None:
- if terminal.event_type == "unknown_outcome":
- raise ToolOutcomeUnknown(intent.effect_id)
- return terminal.result_payload
- if lease.run_owner != owner:
- raise ToolEffectLeaseLost(intent.effect_id)
- run = self._runs._locked_owned_run(db, intent.run_id, owner)
- self._append_event(
- db,
- run,
- intent,
- owner=owner,
- event_type="failure" if failed else "result",
- operation_type="tool_failure_committed"
- if failed
- else "tool_result_committed",
- phase="tool_result_committed",
- safety="reconciled" if reconciled else intent.recovery_policy,
- result=result,
- error=str(result)[:4000] if failed else None,
- terminal=True,
- )
- lease.claim_owner = None
- lease.lease_expires_at = None
- lease.updated_at = self._clock()
- run.updated_at = self._clock()
- return result
-
- def commit_result(
- self,
- effect_id: str,
- *,
- run_owner: str,
- claim_owner: str,
- result: Any,
- failed: bool = False,
- reconciled: bool = False,
- _conflict_retries: int = 4,
- ) -> Any:
- with self._sessions() as db:
- intent = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one()
- )
- terminal = self._terminal_row(db, effect_id)
- if terminal is not None:
- if terminal.event_type == "unknown_outcome":
- raise ToolOutcomeUnknown(effect_id)
- return terminal.result_payload
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == effect_id)
- .with_for_update()
- .one()
- )
- now = _aware(self._clock())
- lease_expiry = _aware(lease.lease_expires_at)
- if (
- lease.run_owner != run_owner
- or lease.claim_owner != claim_owner
- or lease_expiry is None
- or now is None
- or lease_expiry <= now
- ):
- # A concurrent retry can observe the lease only after the
- # winning commit cleared its claim. Re-check the durable
- # terminal decision before reporting lease loss: identical
- # result commits are idempotent even when the first pre-lock
- # terminal read raced with the winner.
- terminal = self._terminal_row(db, effect_id)
- if terminal is not None:
- if terminal.event_type == "unknown_outcome":
- raise ToolOutcomeUnknown(effect_id)
- return terminal.result_payload
- raise ToolEffectLeaseLost(effect_id)
- value = self._commit_result_locked(
- db,
- intent,
- lease,
- owner=run_owner,
- result=result,
- failed=failed,
- reconciled=reconciled,
- )
- try:
- db.commit()
- except (IntegrityError, OperationalError) as exc:
- db.rollback()
- terminal = self.terminal_decision(effect_id)
- if terminal is not None:
- if terminal.action == "unknown":
- raise ToolOutcomeUnknown(effect_id)
- return terminal.result
- if _conflict_retries > 0:
- return self.commit_result(
- effect_id,
- run_owner=run_owner,
- claim_owner=claim_owner,
- result=result,
- failed=failed,
- reconciled=reconciled,
- _conflict_retries=_conflict_retries - 1,
- )
- raise ToolEffectError(str(exc)) from exc
- return value
-
- def mark_unknown(
- self,
- effect_id: str,
- *,
- owner: str,
- claim_owner: str,
- reason: str,
- ) -> None:
- with self._sessions() as db:
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == effect_id)
- .with_for_update()
- .one()
- )
- intent = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one()
- )
- run = self._runs._locked_owned_run(db, intent.run_id, owner)
- self._mark_unknown_locked(
- db,
- run,
- intent,
- lease,
- owner=owner,
- claim_owner=claim_owner,
- reason=reason,
- )
- db.commit()
-
- def terminal_decision(self, effect_id: str) -> Optional[IntentDecision]:
- with self._sessions() as db:
- intent = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one_or_none()
- )
- if intent is None:
- return None
- terminal = self._terminal_row(db, effect_id)
- return (
- self._decision_from_terminal(self._intent_from_row(intent), terminal)
- if terminal is not None
- else None
- )
-
- def pending_effect_ids(self) -> list[str]:
- with self._sessions() as db:
- terminal = aliased(ToolEffectLedger)
- rows = (
- db.query(ToolEffectLedger.effect_id)
- .filter(
- ToolEffectLedger.event_type == "intent",
- ~db.query(terminal.event_id)
- .filter(terminal.terminal_effect_id == ToolEffectLedger.effect_id)
- .exists(),
- )
- .order_by(ToolEffectLedger.created_at, ToolEffectLedger.event_id)
- .all()
- )
- return [row.effect_id for row in rows]
-
- def prune_settled(self, *, retention_days: int = 7) -> int:
- """Drop per-run recovery/observability facts of long-terminal runs.
-
- Covers the tool effect ledger and leases plus the other append-only
- per-run journals (harness usage attempts, harness events, run
- operations); without this they grow without bound.
-
- The run's own recovery columns go the same way. ``recovery_snapshot``
- carries the whole conversation as of that turn, so a long chat stores
- roughly its own history once per turn — the single largest thing in a
- desktop database. Only crash recovery reads it, and only for runs that
- are still live and still replayable, which is exactly what the cutoff
- and the settled filter below exclude.
-
- ``needs_attention`` is kept ``_ATTENTION_RETENTION_FACTOR`` times longer
- rather than exempted, and that applies to every tier above. Exempting it
- means never: nothing in the backend transitions a run out of that status,
- so an exemption pins its journals — and its snapshot — for good. In
- production every record that had outlived the retention window belonged
- to such a run.
- """
- from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt
-
- now = _aware(self._clock())
- cutoff = now - timedelta(days=retention_days)
- attention_cutoff = now - timedelta(days=retention_days * _ATTENTION_RETENTION_FACTOR)
- with self._sessions() as db:
-
- earliest_cutoff = max(cutoff, attention_cutoff)
-
- def _prunable(run_col, stamp):
- """Row whose run is terminal and past that run's retention tier.
-
- ``stamp < earliest_cutoff`` is implied by the EXISTS (it is the
- loosest of the tiers) and stated anyway: it is the only
- predicate on these tables an index can serve, and without it the
- planner has nothing to range-scan on.
- """
- tier = case(
- (ChatRun.status == "needs_attention", attention_cutoff),
- else_=cutoff,
- )
- return and_(
- stamp < earliest_cutoff,
- db.query(ChatRun.run_id)
- .filter(
- ChatRun.run_id == run_col,
- ChatRun.status.notin_(LIVE_STATUSES),
- stamp < tier,
- )
- .exists(),
- )
-
- deleted = (
- db.query(ToolEffectLedger)
- .filter(_prunable(ToolEffectLedger.run_id, ToolEffectLedger.created_at))
- .delete(synchronize_session=False)
- )
- db.query(ToolEffectLease).filter(
- _prunable(ToolEffectLease.run_id, ToolEffectLease.updated_at)
- ).delete(synchronize_session=False)
- for model in (HarnessUsageAttempt, HarnessEventLog, ChatRunOperation):
- deleted += (
- db.query(model)
- .filter(_prunable(model.run_id, model.created_at))
- .delete(synchronize_session=False)
- )
- deleted += (
- db.query(ChatRun)
- .filter(
- ChatRun.status.notin_(LIVE_STATUSES),
- ChatRun.completed_at < earliest_cutoff,
- ChatRun.completed_at
- < case(
- (ChatRun.status == "needs_attention", attention_cutoff),
- else_=cutoff,
- ),
- ChatRun.recovery_snapshot.isnot(None),
- )
- .update(
- {ChatRun.recovery_snapshot: None, ChatRun.request_payload: None},
- synchronize_session=False,
- )
- )
- db.commit()
- return deleted
-
- def settle_terminal_run_intent(self, effect_id: str) -> Optional[ToolIntent]:
- """Close an orphan intent without replay after its owning run is terminal.
-
- Cancellation/failure is a hard replay boundary: the user's stopped run
- must never perform a delayed external action. We still append an
- ``unknown_outcome`` fact so the intent cannot remain pending forever.
- """
- with self._sessions() as db:
- intent = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .with_for_update()
- .one_or_none()
- )
- if intent is None or self._terminal_row(db, effect_id) is not None:
- return None
- run = (
- db.query(ChatRun)
- .filter(ChatRun.run_id == intent.run_id)
- .with_for_update()
- .one_or_none()
- )
- if run is None or run.status in (*LIVE_STATUSES, "needs_attention"):
- return None
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == effect_id)
- .with_for_update()
- .one()
- )
- terminal_status = str(run.status)
- self._append_event(
- db,
- run,
- intent,
- owner="system:terminal-settlement",
- event_type="unknown_outcome",
- operation_type="tool_outcome_unknown_after_terminal_run",
- phase=str(run.run_phase or terminal_status),
- safety="manual_review",
- error=f"run became {terminal_status} before the tool outcome committed",
- terminal=True,
- )
- # _append_event advances only the operation ledger. Preserve the
- # authoritative terminal verdict and merely close the effect claim.
- run.status = terminal_status
- lease.claim_owner = None
- lease.lease_expires_at = None
- lease.updated_at = self._clock()
- run.updated_at = self._clock()
- db.commit()
- return self._intent_from_row(intent)
-
- def claim_recovery_intent(
- self,
- effect_id: str,
- *,
- recovery_owner: str,
- lease_seconds: int = 120,
- ) -> Optional[ToolIntent]:
- with self._sessions() as db:
- intent = (
- db.query(ToolEffectLedger)
- .filter(
- ToolEffectLedger.effect_id == effect_id,
- ToolEffectLedger.event_type == "intent",
- )
- .one_or_none()
- )
- if intent is None or self._terminal_row(db, effect_id) is not None:
- return None
- now = self._clock()
- from core.services.api_key_runs import authorize_run_execution
-
- if not authorize_run_execution(db, intent.run_id, now):
- return None
- expires = now + timedelta(seconds=max(1, lease_seconds))
- run_affected = (
- db.query(ChatRun)
- .filter(
- ChatRun.run_id == intent.run_id,
- ChatRun.status.in_((*LIVE_STATUSES, "needs_attention")),
- or_(
- ChatRun.lease_owner.is_(None),
- ChatRun.lease_expires_at.is_(None),
- ChatRun.lease_expires_at <= now,
- ChatRun.lease_owner == recovery_owner,
- ),
- )
- .update(
- {
- ChatRun.status: "running",
- ChatRun.lease_owner: recovery_owner,
- ChatRun.lease_expires_at: expires,
- ChatRun.updated_at: now,
- },
- synchronize_session=False,
- )
- )
- if not run_affected:
- db.rollback()
- return None
- lease_affected = (
- db.query(ToolEffectLease)
- .filter(
- ToolEffectLease.effect_id == effect_id,
- or_(
- ToolEffectLease.lease_expires_at.is_(None),
- ToolEffectLease.lease_expires_at <= now,
- ToolEffectLease.claim_owner == recovery_owner,
- ),
- )
- .update(
- {
- ToolEffectLease.run_owner: recovery_owner,
- ToolEffectLease.claim_owner: recovery_owner,
- ToolEffectLease.lease_expires_at: expires,
- ToolEffectLease.updated_at: now,
- },
- synchronize_session=False,
- )
- )
- if not lease_affected or self._terminal_row(db, effect_id) is not None:
- db.rollback()
- return None
- db.expire_all()
- run = db.get(ChatRun, intent.run_id)
- self._append_event(
- db,
- run,
- intent,
- owner=recovery_owner,
- event_type="recovery_claim",
- operation_type="tool_effect_recovery_claimed",
- phase=f"tool_intent_{intent.recovery_policy}",
- safety=intent.recovery_policy,
- )
- db.commit()
- return self._intent_from_row(intent)
-
- def commit_recovered_result(
- self,
- effect_id: str,
- result: Any,
- *,
- recovery_owner: str,
- ) -> Any:
- value = self.commit_result(
- effect_id,
- run_owner=recovery_owner,
- claim_owner=recovery_owner,
- result=result,
- reconciled=True,
- )
- with self._sessions() as db:
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == effect_id)
- .with_for_update()
- .one()
- )
- run = (
- db.query(ChatRun)
- .filter(ChatRun.run_id == lease.run_id)
- .with_for_update()
- .one()
- )
- if lease.run_owner != recovery_owner or run.lease_owner != recovery_owner:
- raise ToolEffectLeaseLost(effect_id)
- run.lease_owner = None
- run.lease_expires_at = None
- run.updated_at = self._clock()
- db.commit()
- return value
-
- def release_recovery_claim(
- self,
- effect_id: str,
- *,
- recovery_owner: str,
- retry_seconds: int = 30,
- ) -> None:
- """Defer a failed adapter without letting RunJournal resume past the pending effect."""
- with self._sessions() as db:
- lease = (
- db.query(ToolEffectLease)
- .filter(ToolEffectLease.effect_id == effect_id)
- .with_for_update()
- .one()
- )
- if lease.run_owner != recovery_owner or lease.claim_owner != recovery_owner:
- raise ToolEffectLeaseLost(effect_id)
- run = (
- db.query(ChatRun)
- .filter(ChatRun.run_id == lease.run_id)
- .with_for_update()
- .one()
- )
- if run.lease_owner != recovery_owner:
- raise ToolEffectLeaseLost(effect_id)
- retry_at = self._clock() + timedelta(seconds=max(1, retry_seconds))
- lease.claim_owner = None
- lease.lease_expires_at = retry_at
- lease.updated_at = self._clock()
- run.lease_expires_at = retry_at
- run.updated_at = self._clock()
- db.commit()
-
-
-class ToolEffectGateway:
- """The single execution state machine used by production middleware and tests."""
-
- def __init__(
- self,
- journal: ToolEffectJournal,
- registry: ToolRecoveryRegistry,
- *,
- poll_interval: float = 0.05,
- lease_seconds: int = 300,
- ) -> None:
- self.journal = journal
- self.registry = registry
- self.poll_interval = poll_interval
- self.lease_seconds = lease_seconds
-
- @staticmethod
- async def _call(handler, intent: ToolIntent): # noqa: ANN001, ANN205
- value = handler(intent) if handler is not None else None
- return await value if inspect.isawaitable(value) else value
-
- @staticmethod
- def _reconciliation(value: Any) -> ReconciliationResult:
- if isinstance(value, ReconciliationResult):
- return value
- return (
- ReconciliationResult.applied(value)
- if value is not None
- else ReconciliationResult.unknown()
- )
-
- def _start_heartbeat(
- self,
- effect_id: str,
- *,
- run_owner: str,
- claim_owner: str,
- ) -> tuple[asyncio.Task, asyncio.Event]:
- lost = asyncio.Event()
- interval = max(0.05, min(30.0, self.lease_seconds / 3))
-
- async def _beat() -> None:
- while True:
- await asyncio.sleep(interval)
- try:
- ok = await asyncio.to_thread(
- self.journal.renew_invocation_claim,
- effect_id,
- run_owner=run_owner,
- claim_owner=claim_owner,
- lease_seconds=self.lease_seconds,
- )
- except Exception: # noqa: BLE001 - lost heartbeat means lost ownership
- ok = False
- if not ok:
- lost.set()
- return
-
- return asyncio.create_task(_beat()), lost
-
- @staticmethod
- async def _stop_heartbeat(task: asyncio.Task) -> None:
- task.cancel()
- with suppress(asyncio.CancelledError):
- await task
-
- async def execute_outcome(
- self,
- *,
- run_id: str,
- owner: str,
- tool_call_id: str,
- tool_name: str,
- args: Mapping[str, Any],
- invoke: Callable[[], Awaitable[Any] | Any],
- idempotency_key: Optional[str] = None,
- classify_failure: Optional[Callable[[Any], bool]] = None,
- ) -> ToolExecutionOutcome:
- claim_owner = f"tool:{uuid.uuid4().hex}"
- policy = self.registry.resolve(tool_name)
- while True:
- decision = await asyncio.to_thread(
- self.journal.begin_intent,
- run_id=run_id,
- owner=owner,
- claim_owner=claim_owner,
- tool_call_id=tool_call_id,
- tool_name=tool_name,
- args=args,
- recovery_policy=policy.policy,
- idempotency_key=idempotency_key,
- lease_seconds=self.lease_seconds,
- )
- if decision.action == "known":
- return ToolExecutionOutcome(
- decision.result,
- decision.intent.effect_id,
- decision.intent.result_id,
- decision.result_state,
- False,
- )
- if decision.action == "unknown":
- raise ToolOutcomeUnknown(decision.effect_id)
- if decision.action == "wait":
- await asyncio.sleep(self.poll_interval)
- terminal = await asyncio.to_thread(
- self.journal.terminal_decision, decision.effect_id
- )
- if terminal is None:
- continue
- if terminal.action == "unknown":
- raise ToolOutcomeUnknown(decision.effect_id)
- return ToolExecutionOutcome(
- terminal.result,
- terminal.intent.effect_id,
- terminal.intent.result_id,
- terminal.result_state,
- False,
- )
- if decision.action not in ("execute", "reconcile"):
- raise ToolEffectError(
- f"unsupported tool intent action: {decision.action}"
- )
-
- await asyncio.to_thread(
- self.journal.acquire_invocation_claim,
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- lease_seconds=self.lease_seconds,
- )
- heartbeat, claim_lost = self._start_heartbeat(
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- )
- token = CURRENT_TOOL_EFFECT.set(
- ToolInvocationContext(
- decision.intent.effect_id,
- decision.intent.result_id,
- decision.intent.idempotency_key,
- decision.intent.run_id,
- claim_owner,
- )
- )
- try:
- if decision.action == "reconcile":
- if policy.reconciler is None:
- self.journal.mark_unknown(
- decision.effect_id,
- owner=owner,
- claim_owner=claim_owner,
- reason="reconcile policy has no reconciliation adapter",
- )
- raise ToolOutcomeUnknown(decision.effect_id)
- reconciliation = self._reconciliation(
- await self._call(policy.reconciler, decision.intent)
- )
- if claim_lost.is_set():
- raise ToolEffectLeaseLost(decision.effect_id)
- if reconciliation.outcome == "applied":
- result = await asyncio.to_thread(
- self.journal.commit_result,
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- result=reconciliation.result,
- reconciled=True,
- )
- return ToolExecutionOutcome(
- result,
- decision.intent.effect_id,
- decision.intent.result_id,
- "success",
- False,
- )
- if reconciliation.outcome != "not_applied":
- self.journal.mark_unknown(
- decision.effect_id,
- owner=owner,
- claim_owner=claim_owner,
- reason="reconciliation could not determine external outcome",
- )
- raise ToolOutcomeUnknown(decision.effect_id)
- value = invoke()
- result = await value if inspect.isawaitable(value) else value
- if claim_lost.is_set():
- raise ToolEffectLeaseLost(decision.effect_id)
- failed = (
- bool(classify_failure(result))
- if classify_failure is not None
- else False
- )
- # A returned error payload is still a definitive tool outcome.
- # Persist it and give the original error back to the model so
- # it can correct the request or explain an authorization issue.
- # ToolOutcomeUnknown is reserved for invocations that never
- # produced a result at all (exceptions, disconnects, crashes).
- result = await asyncio.to_thread(
- self.journal.commit_result,
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- result=result,
- failed=failed,
- )
- return ToolExecutionOutcome(
- result,
- decision.intent.effect_id,
- decision.intent.result_id,
- "error" if failed else "success",
- True,
- )
- except asyncio.CancelledError:
- with suppress(Exception):
- self.journal.abandon_invocation_claim(
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- )
- raise
- except ToolOutcomeUnknown:
- with suppress(Exception):
- self.journal.abandon_invocation_claim(
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- )
- raise
- except Exception as exc:
- with suppress(Exception):
- self.journal.abandon_invocation_claim(
- decision.effect_id,
- run_owner=owner,
- claim_owner=claim_owner,
- )
- raise ToolOutcomeUnknown(decision.effect_id) from exc
- finally:
- await self._stop_heartbeat(heartbeat)
- CURRENT_TOOL_EFFECT.reset(token)
-
- async def execute(self, **kwargs) -> Any: # noqa: ANN003
- return (await self.execute_outcome(**kwargs)).result
-
-
-async def recover_incomplete_tool_effects(
- *,
- journal: Optional[ToolEffectJournal] = None,
- registry: Optional[ToolRecoveryRegistry] = None,
- lease_seconds: int = 120,
- heartbeat_interval: Optional[float] = None,
-) -> list[RecoverySweepDecision]:
- """Recover each orphan independently; one broken adapter cannot starve later work."""
-
- journal = journal or ToolEffectJournal()
- registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY
- from core.harness.usage import (
- UsageAttempt,
- attempt_status_for_exception,
- record_usage_safely,
- )
- from core.services.harness_ledger import HarnessUsageLedger
-
- usage_ledger = HarnessUsageLedger(journal._sessions)
-
- async def _recorded_recovery_call(handler, intent, phase): # noqa: ANN001
- started = time.monotonic()
- status = "failed"
- try:
- result = await ToolEffectGateway._call(handler, intent)
- status = "success"
- return result
- except asyncio.CancelledError:
- status = "cancelled"
- raise
- except Exception as exc:
- status = attempt_status_for_exception(exc)
- raise
- finally:
- try:
- await record_usage_safely(
- usage_ledger,
- UsageAttempt(
- run_id=intent.run_id,
- kind="tool",
- operation_name=intent.tool_name,
- effect_id=intent.effect_id,
- status=status,
- latency_ms=int((time.monotonic() - started) * 1_000),
- metadata={"recovery_phase": phase},
- ),
- )
- except Exception: # noqa: BLE001
- logger.debug("tool recovery usage persistence failed", exc_info=True)
-
- decisions: list[RecoverySweepDecision] = []
- for effect_id in await asyncio.to_thread(journal.pending_effect_ids):
- terminal_intent = journal.settle_terminal_run_intent(effect_id)
- if terminal_intent is not None:
- decisions.append(
- RecoverySweepDecision(
- effect_id, terminal_intent.run_id, "terminal_unknown"
- )
- )
- continue
- recovery_owner = f"tool-recovery:{uuid.uuid4().hex}"
- intent = journal.claim_recovery_intent(
- effect_id,
- recovery_owner=recovery_owner,
- lease_seconds=lease_seconds,
- )
- if intent is None:
- continue
- policy = registry.resolve(intent.tool_name)
- claim_lost = asyncio.Event()
-
- async def _heartbeat() -> None:
- interval = heartbeat_interval
- if interval is None:
- interval = max(0.05, min(30.0, lease_seconds / 3))
- while True:
- await asyncio.sleep(interval)
- try:
- ok = await asyncio.to_thread(
- journal.renew_recovery_claim,
- effect_id,
- recovery_owner=recovery_owner,
- lease_seconds=lease_seconds,
- )
- except Exception: # noqa: BLE001 - lost heartbeat means lost ownership
- ok = False
- if not ok:
- claim_lost.set()
- return
-
- heartbeat = asyncio.create_task(_heartbeat())
- try:
- if intent.recovery_policy == "never_replay":
- journal.mark_unknown(
- effect_id,
- owner=recovery_owner,
- claim_owner=recovery_owner,
- reason="non-replayable tool intent has no committed result",
- )
- decisions.append(
- RecoverySweepDecision(effect_id, intent.run_id, "needs_attention")
- )
- continue
- if intent.recovery_policy == "replay_safe":
- if policy.replayer is None:
- raise ToolEffectError("replay_safe policy has no replay adapter")
- result = await _recorded_recovery_call(
- policy.replayer, intent, "replay"
- )
- if claim_lost.is_set():
- raise ToolEffectLeaseLost(effect_id)
- journal.commit_recovered_result(
- effect_id, result, recovery_owner=recovery_owner
- )
- decisions.append(
- RecoverySweepDecision(effect_id, intent.run_id, "replayed")
- )
- continue
-
- if policy.reconciler is None:
- raise ToolEffectError("reconcile policy has no reconciliation adapter")
- reconciliation = ToolEffectGateway._reconciliation(
- await _recorded_recovery_call(policy.reconciler, intent, "reconcile")
- )
- if claim_lost.is_set():
- raise ToolEffectLeaseLost(effect_id)
- if reconciliation.outcome == "applied":
- result = reconciliation.result
- elif (
- reconciliation.outcome == "not_applied" and policy.replayer is not None
- ):
- result = await _recorded_recovery_call(
- policy.replayer, intent, "replay_after_reconcile"
- )
- else:
- journal.mark_unknown(
- effect_id,
- owner=recovery_owner,
- claim_owner=recovery_owner,
- reason="reconciliation could not determine external outcome",
- )
- decisions.append(
- RecoverySweepDecision(effect_id, intent.run_id, "needs_attention")
- )
- continue
- if claim_lost.is_set():
- raise ToolEffectLeaseLost(effect_id)
- journal.commit_recovered_result(
- effect_id, result, recovery_owner=recovery_owner
- )
- decisions.append(
- RecoverySweepDecision(effect_id, intent.run_id, "reconciled")
- )
- except Exception: # noqa: BLE001 - release this claim and continue the sweep
- try:
- journal.release_recovery_claim(effect_id, recovery_owner=recovery_owner)
- except ToolEffectError:
- pass
- decisions.append(RecoverySweepDecision(effect_id, intent.run_id, "retry"))
- finally:
- heartbeat.cancel()
- with suppress(asyncio.CancelledError):
- await heartbeat
- return decisions
-
-
-__all__ = [
- "CURRENT_TOOL_EFFECT",
- "DEFAULT_TOOL_RECOVERY_REGISTRY",
- "IntentDecision",
- "ReconciliationResult",
- "RecoverySweepDecision",
- "ToolEffectError",
- "ToolEffectGateway",
- "ToolEffectJournal",
- "ToolEffectLeaseLost",
- "ToolExecutionOutcome",
- "ToolIdempotencyConflict",
- "ToolIntent",
- "ToolIntentCommitError",
- "ToolInvocationContext",
- "ToolOutcomeUnknown",
- "ToolPolicy",
- "ToolRecoveryRegistry",
- "build_default_tool_recovery_registry",
- "canonical_tool_args",
- "find_tool_outcome_unknown",
- "recover_incomplete_tool_effects",
- "stable_idempotency_key",
- "tool_args_hash",
-]
diff --git a/src/backend/core/services/user_agent_base.py b/src/backend/core/services/user_agent_base.py
deleted file mode 100644
index 60a71db6..00000000
--- a/src/backend/core/services/user_agent_base.py
+++ /dev/null
@@ -1,854 +0,0 @@
-"""Edition-neutral business logic for personal and administrator sub-agents."""
-
-from __future__ import annotations
-from core.infra.time import utc_now
-
-import logging
-import re
-import uuid
-from datetime import datetime
-from decimal import Decimal
-from typing import Any, Dict, List, Optional
-
-from core.db.models import UserAgent
-from core.db.repository import AuditLogRepository, UserAgentRepository
-from core.ontology.build_validator import ensure_ontology_build_valid
-from sqlalchemy.orm import Session
-
-logger = logging.getLogger(__name__)
-
-
-def _desktop_store() -> bool:
- """Whether this process keeps a desktop capability file store (agents projected there)."""
- from core.capabilities.paths import capabilities_enabled
-
- return capabilities_enabled()
-
-
-MAX_USER_AGENTS = 20
-DEFAULT_AGENT_VERSION = "V1.0"
-MAX_CHANGE_HISTORY = 30
-NON_VERSIONED_FIELDS = {"is_enabled"}
-VERSIONED_FIELDS = {
- "name": "名称",
- "description": "简介",
- "system_prompt": "角色设定",
- "welcome_message": "开场白",
- "suggested_questions": "推荐问题",
- "mcp_server_ids": "绑定工具",
- "skill_ids": "绑定技能",
- "plugin_ids": "绑定插件",
- "kb_ids": "绑定知识库",
- "model_provider_id": "模型",
- "temperature": "温度",
- "max_tokens": "最大输出长度",
- "max_iters": "最大推理轮次",
- "timeout": "超时时间",
- "is_enabled": "启用状态",
-}
-
-
-class UserAgentBaseService:
- """Service for user agent CRUD and permission checks."""
-
- def __init__(self, db: Session):
- self.db = db
- self.repo = UserAgentRepository(db)
-
- # ── Queries ──────────────────────────────────────────────────────
-
- def list_for_user(self, user_id: str) -> List[Dict[str, Any]]:
- agents = self.repo.list_for_user(user_id)
- rows = [self._serialize(a) for a in agents]
- if _desktop_store():
- # Desktop: the bridged account's stored agents join the device's own;
- # a display-name clash is decided by the resolver, never by order.
- from core.capabilities import agents as caps_agents
-
- return caps_agents.merge_visible(user_id, rows)
- return rows
-
- def list_admin(self) -> List[Dict[str, Any]]:
- agents = self.repo.list_admin()
- return [self._serialize(a) for a in agents]
-
- def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]:
- agent = self.repo.get_by_id(agent_id)
- if not agent:
- stored = self._stored_account_agent(agent_id, user_id)
- if stored is not None:
- return stored.to_serialized()
- raise LookupError(f"Agent {agent_id} not found")
- if user_id and not self._is_accessible(agent, user_id):
- raise PermissionError("No access to this agent")
- return self._serialize(agent)
-
- def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent:
- """Return the ORM object (for direct use by workflow/factory).
-
- On a desktop the bridged account's stored agents are a second, equally
- authoritative source: an ``AgentDefinition`` with the same attribute
- surface is returned for them.
- """
- agent = self.repo.get_by_id(agent_id)
- if not agent:
- stored = self._stored_account_agent(agent_id, user_id)
- if stored is not None:
- return stored # type: ignore[return-value]
- raise LookupError(f"Agent {agent_id} not found")
- if user_id and not self._is_accessible(agent, user_id):
- raise PermissionError("No access to this agent")
- return agent
-
- @staticmethod
- def _stored_account_agent(agent_id: str, user_id: Optional[str] = None):
- if not _desktop_store():
- return None
- from core.capabilities import agents as caps_agents
- from core.capabilities.skills import account_authorized_for
-
- if not account_authorized_for(user_id):
- return None
- return caps_agents.account_definition(agent_id)
-
- def _project_to_store(self, agent: UserAgent) -> None:
- if not _desktop_store():
- return
- from core.capabilities import agents as caps_agents
-
- caps_agents.publish_local_agent(self._serialize(agent))
-
- @staticmethod
- def _remove_from_store(agent_id: str) -> None:
- if not _desktop_store():
- return
- from core.capabilities import agents as caps_agents
-
- caps_agents.remove_local_agent(agent_id)
-
- # ── Mutations ────────────────────────────────────────────────────
-
- def create(
- self,
- user_id: Optional[str],
- operator_name: Optional[str],
- owner_type: str,
- data: Dict[str, Any],
- scope_id: Optional[str] = None,
- ) -> Dict[str, Any]:
- data = dict(data)
- incoming_extra = dict(data.get("extra_config") or {})
- ontology_tags = list(
- data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or []
- )
- incoming_extra["ontology_tags"] = ontology_tags
- data["extra_config"] = incoming_extra
- ensure_ontology_build_valid(
- self.db,
- asset_type="subagent",
- name=str(data.get("name") or ""),
- description=str(data.get("description") or ""),
- instructions=str(data.get("system_prompt") or ""),
- mcp_server_ids=list(data.get("mcp_server_ids") or []),
- skill_ids=list(data.get("skill_ids") or []),
- plugin_ids=list(data.get("plugin_ids") or []),
- ontology_tags=ontology_tags,
- )
- self._validate_create_scope(user_id, owner_type, scope_id)
-
- agent_id = f"ua_{uuid.uuid4().hex[:16]}"
- created_at = self._now_iso()
- creation_history = [
- {
- "version": DEFAULT_AGENT_VERSION,
- "timestamp": created_at,
- "content": "创建了子智能体",
- "operator_name": operator_name or user_id or "未知用户",
- "details": [],
- }
- ]
- extra_config = self._merge_extra_config(
- current_extra=None,
- incoming_extra=incoming_extra,
- version=DEFAULT_AGENT_VERSION,
- change_history=creation_history,
- )
-
- record = {
- "agent_id": agent_id,
- "owner_type": owner_type,
- **self._owner_fields(user_id, owner_type, scope_id),
- "created_by": user_id,
- **data,
- "extra_config": extra_config,
- }
- agent = self.repo.create(record)
- self._project_to_store(agent)
- self._audit(
- user_id,
- "agent.create",
- agent_id,
- {"owner_type": owner_type, "name": data.get("name")},
- )
- return self._serialize(agent)
-
- def update(
- self,
- agent_id: str,
- user_id: Optional[str],
- operator_name: Optional[str],
- owner_type: str,
- data: Dict[str, Any],
- ) -> Dict[str, Any]:
- agent = self.repo.get_by_id(agent_id)
- if not agent:
- raise LookupError(f"Agent {agent_id} not found")
- self._check_ownership(agent, user_id, owner_type)
-
- data = dict(data)
- current_extra = dict(agent.extra_config or {})
- incoming_extra = dict(data.get("extra_config") or {})
- ontology_tags = list(
- data.pop(
- "ontology_tags",
- incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []),
- )
- or []
- )
- incoming_extra["ontology_tags"] = ontology_tags
- if "extra_config" in data or ontology_tags != list(
- current_extra.get("ontology_tags") or []
- ):
- data["extra_config"] = incoming_extra
- ensure_ontology_build_valid(
- self.db,
- asset_type="subagent",
- name=str(data.get("name", agent.name) or ""),
- description=str(data.get("description", agent.description) or ""),
- instructions=str(data.get("system_prompt", agent.system_prompt) or ""),
- mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []),
- skill_ids=list(data.get("skill_ids", agent.skill_ids) or []),
- plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []),
- ontology_tags=ontology_tags,
- )
-
- changed_fields = self._collect_changed_fields(agent, data)
- versioned_fields = [field for field in changed_fields if field not in NON_VERSIONED_FIELDS]
- changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields]
- next_version = self._read_version(current_extra)
- change_history = self._read_change_history(current_extra)
-
- if changed_labels:
- change_summary = self._build_change_summary(changed_fields, data)
- change_details = self._build_change_details(agent, changed_fields, data)
- entry_version = next_version
- if versioned_fields:
- next_version = self._increment_version(next_version)
- entry_version = next_version
- change_history.append(
- {
- "version": entry_version,
- "timestamp": self._now_iso(),
- "content": change_summary,
- "operator_name": operator_name or user_id or "未知用户",
- "details": change_details,
- }
- )
- change_history = change_history[-MAX_CHANGE_HISTORY:]
-
- payload = dict(data)
- payload["extra_config"] = self._merge_extra_config(
- current_extra=current_extra,
- incoming_extra=incoming_extra,
- version=next_version,
- change_history=change_history,
- )
-
- agent = self.repo.update(agent_id, payload)
- self._project_to_store(agent)
- audit_details = {"fields": list(data.keys())}
- if changed_labels:
- audit_details["change_summary"] = change_summary
- audit_details["version"] = next_version
- self._audit(user_id, "agent.update", agent_id, audit_details)
- return self._serialize(agent)
-
- def delete(
- self,
- agent_id: str,
- user_id: Optional[str],
- owner_type: str,
- ) -> bool:
- agent = self.repo.get_by_id(agent_id)
- if not agent:
- raise LookupError(f"Agent {agent_id} not found")
- self._check_ownership(agent, user_id, owner_type)
-
- ok = self.repo.delete(agent_id)
- if ok:
- self._remove_from_store(agent_id)
- self._audit(user_id, "agent.delete", agent_id)
- return ok
-
- def toggle_enabled(self, agent_id: str) -> Dict[str, Any]:
- agent = self.repo.get_by_id(agent_id)
- if not agent:
- raise LookupError(f"Agent {agent_id} not found")
- new_val = not agent.is_enabled
- agent = self.repo.update(agent_id, {"is_enabled": new_val})
- self._project_to_store(agent)
- return self._serialize(agent)
-
- # ── Available resources ──────────────────────────────────────────
-
- def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]:
- """Return MCP servers, skills, plugins, and KB spaces bindable to agents.
-
- Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists
- and instead bound as a whole via the ``plugins`` list (plugin =
- installable/removable unit, expanded at runtime into its skills+tools).
- owner_user_id is used to include that user's private plugins and MCPs.
-
- The MCP list intentionally contains capabilities the user has personally
- switched off. A sub-agent binding is an explicit, narrower capability
- grant and therefore may opt into one of those tools without turning it
- on for the user's main agent. Deployment-global MCPs disabled by an
- administrator remain unavailable.
- """
- from core.db.models import AdminMcpServer, InstalledPlugin, KBSpace
- from sqlalchemy import or_
-
- # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ──
- # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins.
- try:
- pq = self.db.query(InstalledPlugin)
- if owner_user_id:
- pq = pq.filter(
- or_(
- InstalledPlugin.owner_user_id == owner_user_id,
- InstalledPlugin.owner_user_id.is_(None),
- )
- )
- else:
- pq = pq.filter(InstalledPlugin.owner_user_id.is_(None))
- plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all()
- except Exception as exc: # noqa: BLE001
- logger.debug("Failed to list installed plugins: %s", exc)
- plugin_rows = []
-
- plugin_skill_ids: set = set()
- plugin_mcp_ids: set = set()
- # The same plugin may exist both as a global version (install_id=slug@global)
- # and as the user's private version (slug@) — same name → duplicate
- # display (the user sees two "定时任务管理" entries). Dedupe by slug and
- # show only one: prefer the user's private version (its components carry
- # the user's fingerprint and match their sandbox credentials); fall back
- # to the global version when there is no private one.
- # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all**
- # rows (including the deduped-away one), so the loose skill/MCP lists
- # can fully exclude both versions' components.
- _plugin_by_slug: Dict[str, Dict[str, Any]] = {}
- for p in plugin_rows:
- cids = p.component_ids or {}
- s_ids = list(cids.get("skills") or [])
- m_ids = list(cids.get("mcp") or [])
- plugin_skill_ids.update(s_ids)
- plugin_mcp_ids.update(m_ids)
- slug = (p.install_id or "").rsplit("@", 1)[0]
- is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id
- existing = _plugin_by_slug.get(slug)
- if existing is None or (is_owned and not existing["_owned"]):
- _plugin_by_slug[slug] = {
- "id": p.install_id,
- "name": p.name,
- "description": p.description or "",
- "skill_count": len(s_ids),
- "mcp_count": len(m_ids),
- "_owned": is_owned,
- }
- plugin_list: List[Dict[str, Any]] = [
- {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values()
- ]
-
- # Built-in plugin MCPs can also be present in the static catalog without
- # a source_plugin DB row. They still belong under the plugin selector,
- # not the loose MCP selector.
- try:
- from core.plugins.management import builtin_plugin_component_ids
-
- _, builtin_plugin_mcp_ids = builtin_plugin_component_ids()
- plugin_mcp_ids.update(builtin_plugin_mcp_ids)
- except Exception as exc: # noqa: BLE001
- logger.debug("Failed to load built-in plugin MCP ids: %s", exc)
-
- # Resolve the user's personal on/off layer once so each MCP option can
- # explain whether it is already enabled for the main agent. This flag
- # is display metadata only; disabled options remain bindable here.
- enabled_mcp_ids: Optional[set[str]] = None
- if owner_user_id:
- try:
- from core.config.catalog_resolver import resolve_all_runtime_enabled
-
- _skills, _agents, resolved_mcps = resolve_all_runtime_enabled(
- self.db, owner_user_id
- )
- if resolved_mcps is not None:
- enabled_mcp_ids = set(resolved_mcps)
- except Exception as exc: # noqa: BLE001
- logger.debug("Failed to resolve user MCP enablement: %s", exc)
-
- # ── MCP tools (exclude plugin-sourced + owner isolation) ──
- # Owner isolation: private entries (owner_user_id non-null) are visible
- # only to their owner. Otherwise private copies produced by other users
- # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list →
- # duplicates of "定时任务" etc. Empty owner = globally shared entry,
- # visible to everyone.
- _mcp_owner_ok = (
- or_(
- AdminMcpServer.owner_user_id.is_(None),
- AdminMcpServer.owner_user_id == owner_user_id,
- )
- if owner_user_id
- else AdminMcpServer.owner_user_id.is_(None)
- )
- mcp_servers = (
- self.db.query(AdminMcpServer)
- .filter(
- _mcp_owner_ok,
- # Authoritative exclusion: any plugin-sourced MCP (source_plugin
- # nonnull) never enters the loose list — it is bound as a whole via
- # the plugins list instead. More robust than checking only
- # plugin_mcp_ids (doesn't depend on the plugin row still
- # existing/being visible), and also blocks orphaned plugin MCP rows.
- AdminMcpServer.source_plugin.is_(None),
- )
- .order_by(AdminMcpServer.sort_order)
- .all()
- )
- mcp_list: List[Dict[str, Any]] = []
- seen_mcp_ids: set = set()
- for server in mcp_servers:
- # A global false is an administrator lock. A private false is the
- # owner's personal off state and remains eligible for an explicit
- # sub-agent binding.
- if server.owner_user_id is None and not server.is_enabled:
- continue
- if server.server_id in plugin_mcp_ids:
- continue
- mcp_list.append(
- {
- "id": server.server_id,
- "name": server.display_name,
- "description": server.description,
- "enabled": (
- server.server_id in enabled_mcp_ids
- if enabled_mcp_ids is not None
- else bool(server.is_enabled)
- ),
- }
- )
- seen_mcp_ids.add(server.server_id)
-
- # Some umbrella/built-in MCP entries are catalog-defined and do not
- # necessarily have a same-named AdminMcpServer row. Include every
- # administrator-enabled catalog item so the selector is complete.
- try:
- from core.config.catalog_runtime import get_runtime_catalog
-
- runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False)
- for item in runtime_catalog.get("mcp") or []:
- item_id = str(item.get("id") or "").strip()
- if (
- not item_id
- or item_id in seen_mcp_ids
- or item_id in plugin_mcp_ids
- or not bool(item.get("enabled", True))
- ):
- continue
- mcp_list.append(
- {
- "id": item_id,
- "name": item.get("name") or item_id,
- "description": item.get("description") or item.get("desc") or "",
- "enabled": (
- item_id in enabled_mcp_ids
- if enabled_mcp_ids is not None
- else bool(item.get("enabled", True))
- ),
- }
- )
- seen_mcp_ids.add(item_id)
- except Exception as exc: # noqa: BLE001
- logger.debug("Failed to load catalog MCP resources: %s", exc)
-
- # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ──
- # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all**
- # materialized skills on disk — including other users' private skills
- # and plugin skills materialized by other users' plugin installs.
- # Excluding via plugin_skill_ids alone (components of plugins visible to
- # the current user only) would miss some, letting other users' plugin
- # skills sneak into "bindable skills". So compute two **authoritative**
- # exclusion sets directly from AdminSkill and filter both sources
- # uniformly:
- # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner);
- # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user).
- from core.db.models import AdminSkill
-
- all_plugin_skill_ids: set = set()
- foreign_private_skill_ids: set = set()
- try:
- for row in self.db.query(
- AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id
- ).all():
- if row.source_plugin:
- all_plugin_skill_ids.add(row.skill_id)
- if row.owner_user_id and row.owner_user_id != owner_user_id:
- foreign_private_skill_ids.add(row.skill_id)
- except Exception as exc:
- logger.debug("Failed to precompute skill exclusion sets: %s", exc)
-
- _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids
-
- skill_list: List[Dict[str, Any]] = []
- try:
- _skill_owner_ok = (
- or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id)
- if owner_user_id
- else AdminSkill.owner_user_id.is_(None)
- )
- db_skills = (
- self.db.query(AdminSkill)
- .filter(
- AdminSkill.is_enabled == True,
- _skill_owner_ok,
- )
- .order_by(AdminSkill.updated_at.desc())
- .all()
- )
- seen_ids = set()
- for s in db_skills:
- if s.skill_id in _excluded_skill_ids:
- continue
- skill_list.append(
- {"id": s.skill_id, "name": s.display_name, "description": s.description or ""}
- )
- seen_ids.add(s.skill_id)
- except Exception:
- seen_ids = set()
-
- try:
- from core.agent_skills.loader import get_skill_loader
-
- loader = get_skill_loader()
- for sid, meta in loader.load_all_metadata().items():
- if sid not in seen_ids and sid not in _excluded_skill_ids:
- skill_list.append(
- {
- "id": sid,
- "name": getattr(meta, "name", sid),
- "description": getattr(meta, "description", ""),
- }
- )
- except Exception as exc:
- logger.debug("Failed to load filesystem skills: %s", exc)
-
- # KB spaces
- kb_list: List[Dict[str, Any]] = []
- try:
- kb_spaces = (
- self.db.query(KBSpace)
- .filter(
- KBSpace.deleted_at.is_(None),
- )
- .order_by(KBSpace.created_at.desc())
- .all()
- )
- kb_list = [
- {"id": s.kb_id, "name": s.name, "description": s.description or ""}
- for s in kb_spaces
- ]
- except Exception:
- pass
-
- return {
- "mcp_servers": mcp_list,
- "skills": skill_list,
- "plugins": plugin_list,
- "kb_spaces": kb_list,
- "ontology_tags": self._ontology_tag_options(),
- }
-
- def _ontology_tag_options(self) -> List[Dict[str, Any]]:
- """Controlled labels that activate a sub-agent ontology workflow at runtime."""
- from core.services.ontology_service import OntologyService
-
- return OntologyService(self.db).list_asset_tag_options("subagent")
-
- # ── Helpers ───────────────────────────────────────────────────────
-
- def _validate_create_scope(
- self, user_id: Optional[str], owner_type: str, scope_id: Optional[str]
- ) -> None:
- if owner_type == "user":
- if not user_id:
- raise ValueError("user_id required for user agents")
- if self.repo.count_user_agents(user_id) >= MAX_USER_AGENTS:
- raise ValueError(f"Maximum {MAX_USER_AGENTS} agents per user reached")
- return
- if owner_type == "admin":
- return
- raise ValueError(f"Unsupported agent owner type: {owner_type}")
-
- @staticmethod
- def _owner_fields(
- user_id: Optional[str], owner_type: str, scope_id: Optional[str]
- ) -> Dict[str, Any]:
- return {"user_id": user_id if owner_type == "user" else None}
-
- def _is_accessible(self, agent: UserAgent, user_id: str) -> bool:
- if agent.owner_type == "admin" and agent.is_enabled:
- return True
- if agent.owner_type == "user" and agent.user_id == user_id:
- return True
- return False
-
- def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None:
- # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes
- if owner_type == "admin":
- if agent.owner_type != "admin":
- raise PermissionError("Admin can only modify admin agents")
- return
- if agent.owner_type == "user" and agent.user_id == user_id:
- return
- raise PermissionError("You can only modify your own agents")
-
- @classmethod
- def _serialize(cls, agent: UserAgent) -> Dict[str, Any]:
- extra_config = agent.extra_config or {}
- return {
- "agent_id": agent.agent_id,
- "owner_type": agent.owner_type,
- "user_id": agent.user_id,
- **cls._serialize_scope(agent),
- "name": agent.name,
- "avatar": agent.avatar,
- "description": agent.description,
- "system_prompt": agent.system_prompt,
- "welcome_message": agent.welcome_message,
- "suggested_questions": agent.suggested_questions or [],
- "mcp_server_ids": agent.mcp_server_ids or [],
- "skill_ids": agent.skill_ids or [],
- "plugin_ids": agent.plugin_ids or [],
- "kb_ids": agent.kb_ids or [],
- "model_provider_id": agent.model_provider_id,
- "temperature": float(agent.temperature) if agent.temperature is not None else None,
- "max_tokens": agent.max_tokens,
- "max_iters": agent.max_iters,
- "timeout": agent.timeout,
- "is_enabled": agent.is_enabled,
- "sort_order": agent.sort_order,
- "source_market_slug": agent.source_market_slug,
- "ontology_tags": list(extra_config.get("ontology_tags") or []),
- "extra_config": extra_config,
- "version": cls._read_version(extra_config),
- "change_history": cls._read_change_history(extra_config),
- "created_at": agent.created_at.isoformat() if agent.created_at else None,
- "updated_at": agent.updated_at.isoformat() if agent.updated_at else None,
- "created_by": agent.created_by,
- }
-
- @staticmethod
- def _serialize_scope(agent: UserAgent) -> Dict[str, Any]:
- return {}
-
- def _audit(
- self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None
- ) -> None:
- try:
- audit_repo = AuditLogRepository(self.db)
- audit_repo.create(
- {
- "user_id": user_id,
- "action": action,
- "resource_type": "user_agent",
- "resource_id": resource_id,
- "details": details or {},
- "status": "success",
- }
- )
- except Exception as exc:
- logger.warning("Audit log failed: %s", exc)
-
- @staticmethod
- def _normalize_value(value: Any) -> Any:
- if isinstance(value, Decimal):
- return float(value)
- if isinstance(value, list):
- return list(value)
- return value
-
- @staticmethod
- def _now_iso() -> str:
- return utc_now().replace(microsecond=0).isoformat()
-
- @classmethod
- def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]:
- fields: List[str] = []
- for field in VERSIONED_FIELDS:
- if field not in data:
- continue
- old_value = cls._normalize_value(getattr(agent, field, None))
- new_value = cls._normalize_value(data.get(field))
- if old_value != new_value:
- fields.append(field)
- return fields
-
- @staticmethod
- def _read_version(extra_config: Optional[Dict[str, Any]]) -> str:
- if not isinstance(extra_config, dict):
- return DEFAULT_AGENT_VERSION
- raw = extra_config.get("version")
- return UserAgentBaseService._normalize_version(raw if isinstance(raw, str) else "")
-
- @staticmethod
- def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]:
- if not isinstance(extra_config, dict):
- return []
- raw_history = extra_config.get("change_history")
- if not isinstance(raw_history, list):
- return []
-
- history: List[Dict[str, str]] = []
- for item in raw_history:
- if not isinstance(item, dict):
- continue
- timestamp = item.get("timestamp")
- content = item.get("content")
- version = item.get("version")
- operator_name = item.get("operator_name")
- details = item.get("details")
- if not isinstance(timestamp, str) or not isinstance(content, str):
- continue
- history.append(
- {
- "timestamp": timestamp,
- "content": content,
- "version": UserAgentBaseService._normalize_version(
- version if isinstance(version, str) else ""
- ),
- "operator_name": (
- operator_name
- if isinstance(operator_name, str) and operator_name.strip()
- else "未知用户"
- ),
- "details": UserAgentBaseService._normalize_change_details(details),
- }
- )
- return history
-
- @staticmethod
- def _increment_version(version: str) -> str:
- normalized = UserAgentBaseService._normalize_version(version)
- match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized)
- if not match:
- return "V1.1"
- major, minor = (int(part) for part in match.groups())
- return f"V{major}.{minor + 1}"
-
- @staticmethod
- def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str:
- if changed_fields == ["is_enabled"]:
- return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体"
-
- changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields]
- if not changed_labels:
- return "更新了智能体配置"
- if len(changed_labels) <= 3:
- return f"修改了{'、'.join(changed_labels)}"
- preview = "、".join(changed_labels[:3])
- return f"修改了{preview}等{len(changed_labels)}项"
-
- @classmethod
- def _build_change_details(
- cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any]
- ) -> List[Dict[str, str]]:
- details: List[Dict[str, str]] = []
- for field in changed_fields:
- old_value = cls._stringify_detail_value(field, getattr(agent, field, None))
- new_value = cls._stringify_detail_value(field, data.get(field))
- details.append(
- {
- "field": VERSIONED_FIELDS[field],
- "before": old_value,
- "after": new_value,
- }
- )
- return details
-
- @staticmethod
- def _stringify_detail_value(field: str, value: Any) -> str:
- if field == "is_enabled":
- return "启用" if bool(value) else "关闭"
- if value is None:
- return "未填写"
- if isinstance(value, list):
- return "、".join(str(item) for item in value) if value else "未填写"
- if isinstance(value, bool):
- return "是" if value else "否"
- text = str(value).strip()
- return text if text else "未填写"
-
- @staticmethod
- def _normalize_change_details(details: Any) -> List[Dict[str, str]]:
- if not isinstance(details, list):
- return []
- normalized: List[Dict[str, str]] = []
- for item in details:
- if not isinstance(item, dict):
- continue
- field = item.get("field")
- before = item.get("before")
- after = item.get("after")
- if not isinstance(field, str):
- continue
- normalized.append(
- {
- "field": field,
- "before": before if isinstance(before, str) else "未填写",
- "after": after if isinstance(after, str) else "未填写",
- }
- )
- return normalized
-
- @staticmethod
- def _normalize_version(version: str) -> str:
- if not isinstance(version, str) or not version.strip():
- return DEFAULT_AGENT_VERSION
- raw = version.strip()
- if re.match(r"^[Vv]\d+\.\d+$", raw):
- return f"V{raw[1:]}"
- legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw)
- if legacy_patch:
- major, minor, patch = (int(part) for part in legacy_patch.groups())
- return f"V{major}.{minor + patch}"
- legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw)
- if legacy_minor:
- major, minor = (int(part) for part in legacy_minor.groups())
- return f"V{major}.{minor}"
- return DEFAULT_AGENT_VERSION
-
- @staticmethod
- def _merge_extra_config(
- current_extra: Optional[Dict[str, Any]],
- incoming_extra: Optional[Dict[str, Any]],
- *,
- version: str,
- change_history: List[Dict[str, str]],
- ) -> Dict[str, Any]:
- merged = dict(current_extra or {})
- if isinstance(incoming_extra, dict):
- merged.update(incoming_extra)
- merged["version"] = version
- merged["change_history"] = change_history
- return merged
diff --git a/src/backend/core/spaces/edition_access.py b/src/backend/core/spaces/edition_access.py
deleted file mode 100644
index 7f5a7637..00000000
--- a/src/backend/core/spaces/edition_access.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""Personal file mounts for the community distribution."""
-workspace_mount_names = ("myspace", "skills")
-
-def team_grants(db, actor):
- return ()
-
-def session_entrypoint(entrypoint, user_id):
- return list(entrypoint)
-
-def project_location(db, project):
- return None
-
-def matches_scope(project, scope):
- return scope.kind == project.kind and scope.root_folder_id == project.linked_folder_id
-
-def owner_space(column, owner):
- raise ValueError("Unsupported directory owner")
diff --git a/src/backend/core/spaces/edition_revocation.py b/src/backend/core/spaces/edition_revocation.py
deleted file mode 100644
index d84eaa20..00000000
--- a/src/backend/core/spaces/edition_revocation.py
+++ /dev/null
@@ -1,3 +0,0 @@
-"""The community distribution has no dedicated account computers."""
-def revoke_computers(actor):
- return None
diff --git a/src/backend/core/vision/__init__.py b/src/backend/core/vision/__init__.py
deleted file mode 100644
index 06cf1624..00000000
--- a/src/backend/core/vision/__init__.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""视觉桥(Vision Bridge):让纯文本主模型「看得见」图片。
-
-图片 → 多模态模型 → 结构化文字证据 → 注入主模型上下文。主模型全程只吃文本。
-
-对外入口:
-
-- :func:`core.vision.service.is_available` —— 当前是否配了可用的视觉模型
-- :func:`core.vision.service.get_vision_bridge` —— 识别单张/多张图
-- :func:`core.vision.render.render_evidence` —— 证据 → 可注入文本(带不可信输入围栏)
-- :func:`core.vision.service.model_supports_vision` —— 某个模型配置是否原生多模态
-- :func:`core.vision.service.resolve_vision_mode` —— 本轮模型看图方式:native / bridge / none
-"""
-
-from core.vision.render import render_evidence, render_many, render_unavailable
-from core.vision.schema import VisionEvidence, parse_evidence
-from core.vision.service import (
- VISION_ROLE_KEY,
- VisionBridge,
- VisionMode,
- VisionResult,
- get_vision_bridge,
- is_available,
- model_supports_vision,
- resolve_vision_config,
- resolve_vision_mode,
- sniff_mime,
-)
-
-__all__ = [
- "VISION_ROLE_KEY",
- "VisionBridge",
- "VisionEvidence",
- "VisionMode",
- "VisionResult",
- "get_vision_bridge",
- "is_available",
- "model_supports_vision",
- "parse_evidence",
- "render_evidence",
- "render_many",
- "render_unavailable",
- "resolve_vision_config",
- "resolve_vision_mode",
- "sniff_mime",
-]
diff --git a/src/backend/extensions/README.md b/src/backend/extensions/README.md
index 3aaf8df2..e1cb4e6d 100644
--- a/src/backend/extensions/README.md
+++ b/src/backend/extensions/README.md
@@ -19,7 +19,7 @@ policy and iteration limits. Core runtime code applies the policy against the
parent's actual grants. Active database prompt parts (`subagents` kind) take
precedence over the shipped Markdown. The files remain the seed and fallback.
-Lifecycle code stays in `core/agent_skills/`, `core/plugins/` and
+Lifecycle code stays in `core/capabilities/skills/`, `core/capabilities/plugins/` and
`core/capabilities/`. Agent execution stays in `core/llm/` and `orchestration/`.
User installations, credentials and mutable state belong to the configured
runtime stores, never this directory.
diff --git a/src/backend/extensions/mcp_servers/_packaging.py b/src/backend/extensions/mcp_servers/_packaging.py
index 4675d086..a5b3a3ba 100644
--- a/src/backend/extensions/mcp_servers/_packaging.py
+++ b/src/backend/extensions/mcp_servers/_packaging.py
@@ -28,7 +28,7 @@
def read_artifact_bytes(file_id: str) -> Optional[bytes]:
"""从共享产物库按 file_id 读回字节(local: 读 path;oss: download_bytes)。"""
try:
- from core.artifacts import store
+ from core.files.artifacts import store
meta = store.get_artifact(file_id)
if not meta:
@@ -39,7 +39,7 @@ def read_artifact_bytes(file_id: str) -> Optional[bytes]:
return fh.read()
key = meta.get("storage_key")
if key:
- from core.storage import get_storage
+ from core.files.storage import get_storage
return bytes(get_storage().download_bytes(key))
except Exception as exc: # noqa: BLE001
@@ -129,7 +129,7 @@ def is_plugin_root(root: Path) -> bool:
却能从后台上传导入——同一个包两条通路结论不同。所以直接问导入器。
"""
try:
- from core.plugins.packaging.importer import detect_manifest
+ from core.capabilities.plugins.packaging.importer import detect_manifest
detect_manifest(root)
return True
diff --git a/src/backend/extensions/mcp_servers/_serve.py b/src/backend/extensions/mcp_servers/_serve.py
index 039d4769..cbbf74d5 100644
--- a/src/backend/extensions/mcp_servers/_serve.py
+++ b/src/backend/extensions/mcp_servers/_serve.py
@@ -1,4 +1,4 @@
-"""Shared launcher for FastMCP servers.
+"""Shared launcher for MCPServer servers.
Every MCP server's ``main()`` calls ``run(mcp, default_port=NNNN)``.
Picks transport from ``--transport`` (``stdio`` for local debug, the
@@ -13,14 +13,14 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
- from mcp.server.fastmcp import FastMCP
+ from mcp.server.mcpserver import MCPServer
def _streamable_http_bind_host() -> str:
return os.getenv("MCP_BIND_HOST", "0.0.0.0").strip() or "0.0.0.0"
-def run(mcp: "FastMCP", default_port: int) -> None:
+def run(mcp: "MCPServer", default_port: int) -> None:
parser = argparse.ArgumentParser(description=mcp.name)
parser.add_argument(
"--transport",
@@ -31,22 +31,16 @@ def run(mcp: "FastMCP", default_port: int) -> None:
args = parser.parse_args()
if args.transport == "streamable-http":
- mcp.settings.port = args.port
- # Compose needs all-interface binding inside its private container
- # network. The no-Docker local/desktop launcher explicitly supplies
- # MCP_BIND_HOST=127.0.0.1 so these internal control-plane ports are not
- # exposed to the user's LAN (and do not trigger an avoidable Windows
- # firewall prompt).
- mcp.settings.host = _streamable_http_bind_host()
- # MCP's default DNS-rebinding allow-list is localhost only. Backend
- # reaches us via the docker DNS name (e.g. ``mcp:9108``); rebinding
- # protection isn't relevant on a private network with no browser
- # traffic.
from mcp.server.transport_security import TransportSecuritySettings
- mcp.settings.transport_security = TransportSecuritySettings(
- enable_dns_rebinding_protection=False,
+ host = _streamable_http_bind_host()
+ mcp.run(
+ "streamable-http", host=host, port=args.port,
+ transport_security=TransportSecuritySettings(
+ enable_dns_rebinding_protection=True,
+ allowed_hosts=["127.0.0.1:*", "localhost:*", "[::1]:*", "mcp:*", "hugagent-mcp:*"],
+ allowed_origins=["http://127.0.0.1:*", "http://localhost:*"],
+ ),
)
- mcp.run("streamable-http")
else:
asyncio.run(mcp.run_stdio_async())
diff --git a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py
index b2ac18ba..070b16b7 100644
--- a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py
@@ -10,6 +10,9 @@
"""
from __future__ import annotations
+from features.marketplace import agent_catalog
+from features.marketplace import agent_installation
+from features.marketplace import agent_submissions
import logging
from typing import Any, Dict, List, Optional, Tuple
@@ -25,12 +28,10 @@
# 绑定类字段一次最多绑多少项,防止模型把整库 id 一股脑塞进来。
_MAX_BINDINGS = 100
-
# ── 通用 ────────────────────────────────────────────────────────────────
def _no_user() -> Dict[str, Any]:
return {"ok": False, "message": "❌ 无法确定用户身份(缺 X-Current-User-Id 头),拒绝操作。"}
-
def _invalidate_user_cache(user_id: Optional[str]) -> None:
"""清掉该用户的 30s 能力解析缓存。
@@ -40,13 +41,12 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None:
CATALOG_MUTATING_TOOLS 白名单。本调用作为进程内一致性的兜底保留,无害。
"""
try:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(str(user_id) if user_id else None)
except Exception as exc: # noqa: BLE001
logger.debug("agent_manager: invalidate_capability_cache failed (%s)", exc)
-
def _require_cap(db, user_id: str, cap: str) -> Optional[Dict[str, Any]]:
"""能力位校验。缺权限返回错误 dict,否则 None。"""
try:
@@ -62,24 +62,21 @@ def _require_cap(db, user_id: str, cap: str) -> Optional[Dict[str, Any]]:
return {"ok": False, "message": f"❌ 权限校验失败:{exc}"}
return None
-
def _svc(db):
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
return UserAgentService(db)
-
def valid_categories() -> List[str]:
"""子智能体市场的固定分类(**与技能市场的 8 类不同**)。
从单一真相源读,别在工具描述里硬写一份——分类调整时两边会漂移,
模型照着过期的列表猜,每次上架都要失败一轮。
"""
- from core.services.agent_market_categories import AGENT_MARKETPLACE_CATEGORIES
+ from features.marketplace.agent_categories import AGENT_MARKETPLACE_CATEGORIES
return list(AGENT_MARKETPLACE_CATEGORIES)
-
def _clean_ids(raw: Optional[List[str]], field: str) -> Tuple[Optional[List[str]], Optional[str]]:
"""绑定 id 数组归一化:去空白/去重/保序/限量。返回 (清洗后, 错误信息)。"""
if raw is None:
@@ -96,7 +93,6 @@ def _clean_ids(raw: Optional[List[str]], field: str) -> Tuple[Optional[List[str]
return None, f"❌ {field} 一次最多绑 {_MAX_BINDINGS} 项(收到 {len(out)} 项)。"
return out, None
-
def _binding_updates(
raw: Dict[str, Optional[List[str]]], *, fill_missing: bool
) -> Tuple[Dict[str, List[str]], Optional[str]]:
@@ -115,7 +111,6 @@ def _binding_updates(
data[field] = cleaned or []
return data, None
-
def _clean_max_iters(value: Optional[int]) -> Tuple[Optional[int], Optional[str]]:
"""max_iters 校验。None 表示"没传",返回 (None, None) 让调用方跳过。"""
if value is None:
@@ -128,7 +123,6 @@ def _clean_max_iters(value: Optional[int]) -> Tuple[Optional[int], Optional[str]
return None, "❌ max_iters 取值范围 1–100。"
return mi, None
-
def _slim(a: Dict[str, Any]) -> Dict[str, Any]:
"""把 service 的完整序列化裁成智能体读得动的摘要(别把 change_history 等整包回吐)。"""
return {
@@ -148,7 +142,6 @@ def _slim(a: Dict[str, Any]) -> Dict[str, Any]:
"updated_at": a.get("updated_at"),
}
-
def _my_agents(db, user_id: str) -> List[Dict[str, Any]]:
"""只取"我自己建的/装的"子智能体。
@@ -164,7 +157,6 @@ def _my_agents(db, user_id: str) -> List[Dict[str, Any]]:
and not str(a.get("agent_id") or "").startswith(_BUILTIN_PREFIX)
]
-
def _resolve_agent(db, user_id: str, ref: str) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]]]:
"""agent_ref → (唯一命中, 候选)。先精确 agent_id,再按名称模糊匹配。仅本人的子智能体。"""
ref = (ref or "").strip()
@@ -178,7 +170,6 @@ def _resolve_agent(db, user_id: str, ref: str) -> Tuple[Optional[Dict[str, Any]]
return cands[0], []
return None, cands[:10]
-
def _need_clarify(cands: List[Dict[str, Any]]) -> Dict[str, Any]:
return {
"ok": False,
@@ -187,23 +178,22 @@ def _need_clarify(cands: List[Dict[str, Any]]) -> Dict[str, Any]:
"candidates": [{"agent_id": c.get("agent_id"), "name": c.get("name")} for c in cands],
}
-
# ── ① 搜索子智能体市场(只读)──────────────────────────────────────────
def search_agent_market(*, user_id: str, query: str = "", category: str = "") -> Dict[str, Any]:
if not user_id:
return _no_user()
from core.db.engine import SessionLocal
- from core.services import agent_market_service
+
q = (query or "").strip().lower()
cat = (category or "").strip()
with SessionLocal() as db:
# 与用户端市场接口同一套可见范围过滤(scoped 条目仅授权者可见)
- items = agent_market_service.list_marketplace_agents(db, viewer_user_id=user_id)
+ items = agent_catalog.list_marketplace_agents(db, viewer_user_id=user_id)
# list_marketplace_agents 只标 market_enabled/visibility,不标 installed——不补这一下,
# 每条都会显示"未安装",模型就会去重装用户已经有的智能体(一次代价高昂的克隆)。
# 插件侧的 list_plugins 自带该标注,两边口径要一致。
- agent_market_service.annotate_installed(items, db, user_id)
+ agent_installation.annotate_installed(items, db, user_id)
def _match(it: Dict[str, Any]) -> bool:
if cat and str(it.get("category") or "") != cat:
@@ -235,7 +225,6 @@ def _match(it: Dict[str, Any]) -> bool:
)
return {"ok": True, "count": len(slim), "agents": slim, "message": msg}
-
# ── ② 从市场安装(私有,写)────────────────────────────────────────────
def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]:
if not user_id:
@@ -244,7 +233,7 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]:
if not slug:
return {"ok": False, "message": "❌ 请提供要安装的子智能体 slug。"}
from core.db.engine import SessionLocal
- from core.services import agent_market_service
+
with SessionLocal() as db:
cap_err = _require_cap(db, user_id, "can_add_agent")
@@ -252,13 +241,13 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]:
return cap_err
# 可见范围守卫:对该用户不可见的 scoped 条目按不存在处理(与用户端安装接口一致)。
# 不要包 try——守卫出错就该报错,静默跳过等于对不可见条目失败开放。
- from core.auth.marketplace_visibility import is_item_visible
- from core.services import marketplace_listing as ml
+ from features.marketplace.edition import is_item_visible
+ from features.marketplace import edition as ml
if not is_item_visible(db, ml.KIND_AGENT, slug, user_id):
return {"ok": False, "message": f"❌ 子智能体市场里找不到「{slug}」。"}
try:
- res = agent_market_service.install_marketplace_agent(
+ res = agent_installation.install_marketplace_agent(
db, slug, owner_user_id=user_id, operator_name=user_id
)
except Exception as exc: # noqa: BLE001
@@ -285,7 +274,6 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]:
),
}
-
# ── ③ 列出可绑能力(只读,创建/修改前必调)──────────────────────────────
def list_bindable_capabilities(*, user_id: str) -> Dict[str, Any]:
if not user_id:
@@ -329,7 +317,6 @@ def _pick(items: Any) -> List[Dict[str, Any]]:
),
}
-
# ── ④ 从零创建(写)────────────────────────────────────────────────────
def create_agent(
*,
@@ -412,7 +399,6 @@ def create_agent(
),
}
-
def _friendly_error(action: str, exc: Exception) -> str:
"""把 service 层异常转成人话。本体校验失败要说清楚是本体没过,而不是甩一句失败。"""
text = str(exc) or exc.__class__.__name__
@@ -425,7 +411,6 @@ def _friendly_error(action: str, exc: Exception) -> str:
return f"❌ {action}失败:找不到该子智能体(可能已被删除)。"
return f"❌ {action}失败:{text}"
-
# ── ⑤ 列出我的子智能体(只读)──────────────────────────────────────────
def list_my_agents(*, user_id: str) -> Dict[str, Any]:
if not user_id:
@@ -442,7 +427,6 @@ def list_my_agents(*, user_id: str) -> Dict[str, Any]:
)
return {"ok": True, "count": len(agents), "agents": agents, "message": msg}
-
# ── ⑥ 修改(字段级部分更新,写)────────────────────────────────────────
def edit_agent(
*,
@@ -544,7 +528,6 @@ def edit_agent(
),
}
-
# ── ⑦ 删除(写)────────────────────────────────────────────────────────
def delete_agent(*, user_id: str, agent_ref: str) -> Dict[str, Any]:
if not user_id:
@@ -576,7 +559,6 @@ def delete_agent(*, user_id: str, agent_ref: str) -> Dict[str, Any]:
"message": f"✅ 已删除子智能体「{display}」(agent_id={deleted_id})。",
}
-
# ── ⑧ 申请上架市场(写)────────────────────────────────────────────────
def submit_agent_to_market(
*, user_id: str, agent_id: str, category: str = "", summary: str = "", note: str = ""
@@ -587,7 +569,7 @@ def submit_agent_to_market(
if not agent_id:
return {"ok": False, "message": "❌ 请提供要上架的 agent_id(取自 list_my_agents)。"}
from core.db.engine import SessionLocal
- from core.services import agent_market_service
+
with SessionLocal() as db:
cap_err = _require_cap(db, user_id, "can_add_agent")
@@ -612,7 +594,7 @@ def submit_agent_to_market(
),
}
try:
- sub = agent_market_service.submit_to_marketplace(
+ sub = agent_submissions.submit_to_marketplace(
db,
str(hit.get("agent_id")),
owner_user_id=user_id,
diff --git a/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py b/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py
index dbf93650..a77565ba 100644
--- a/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py
@@ -13,11 +13,11 @@
from typing import Any, Dict, List, Optional
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers.agent_manager_mcp import impl
-mcp = FastMCP("hugagent-agent-manager")
+mcp = MCPServer("hugagent-agent-manager")
_HDR_USER = "x-current-user-id"
diff --git a/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py b/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py
index 942c315f..6d7eb789 100644
--- a/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py
@@ -69,7 +69,7 @@ def _commit_effect_receipt(
def _svc(db):
- from core.services.automation_service import AutomationService
+ from features.automation.service import AutomationService
return AutomationService(db)
@@ -88,7 +88,7 @@ def _resolve_ref(db, user_id: str, ref: str) -> Tuple[Optional[Any], List[Any]]:
返回 (task, candidates):命中唯一 → (task, []);多命中 → (None, candidates);无 → (None, [])。
"""
- from core.db.models import ScheduledTask
+ from features.automation.models import ScheduledTask
ref = (ref or "").strip()
if not ref:
@@ -113,8 +113,8 @@ def _resolve_ref(db, user_id: str, ref: str) -> Tuple[Optional[Any], List[Any]]:
def _task_brief(task) -> Dict[str, Any]:
- from core.services.automation_service import AutomationService
- from core.services.delivery_targets import resolve_delivery_targets, describe_targets
+ from features.automation.service import AutomationService
+ from features.automation.delivery_targets import resolve_delivery_targets, describe_targets
d = AutomationService.task_to_dict(task)
targets = resolve_delivery_targets(task.extra_data)
@@ -157,7 +157,7 @@ def list_conversations(*, user_id: str) -> Dict[str, Any]:
if not user_id:
return {"ok": False, "message": "❌ 无法确定用户身份。"}
from core.db.engine import SessionLocal
- from core.services.channel_service import list_owner_conversations
+ from integrations.channels.service import list_owner_conversations
with SessionLocal() as db:
convs = list_owner_conversations(db, user_id, limit=100)
@@ -179,7 +179,7 @@ def _resolve_create_targets(
deliver_to:空 → 按来源自动;"inapp" → 仅站内;否则当作 conversation_id(须本人拥有的渠道会话)。
"""
- from core.services.delivery_targets import build_delivery_targets
+ from features.automation.delivery_targets import build_delivery_targets
val = (deliver_to or "").strip()
if not val:
@@ -231,7 +231,7 @@ def create_task(
return {"ok": False, "message": f"❌ cron 表达式无效:{cron_expression}"}
from core.db.engine import SessionLocal
- from core.services.delivery_targets import describe_targets
+ from features.automation.delivery_targets import describe_targets
with SessionLocal() as db:
try:
@@ -250,7 +250,7 @@ def create_task(
targets, err = _resolve_create_targets(user_id, deliver_to, channel_origin, db)
if err:
return {"ok": False, "message": err}
- from core.services.automation_execution import execution_metadata
+ from features.automation.execution import execution_metadata
import pytz
try:
pytz.timezone(timezone)
@@ -307,7 +307,7 @@ def list_tasks(*, user_id: str, status: str = "active") -> Dict[str, Any]:
def get_task(*, user_id: str, task_ref: str) -> Dict[str, Any]:
from core.db.engine import SessionLocal
- from core.services.automation_service import AutomationService, SUMMARY_LIMIT_BRIEF
+ from features.automation.service import AutomationService, SUMMARY_LIMIT_BRIEF
with SessionLocal() as db:
task, cands = _resolve_ref(db, user_id, task_ref)
diff --git a/src/backend/extensions/mcp_servers/automation_task_mcp/server.py b/src/backend/extensions/mcp_servers/automation_task_mcp/server.py
index a97429ba..0b4696f7 100644
--- a/src/backend/extensions/mcp_servers/automation_task_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/automation_task_mcp/server.py
@@ -13,11 +13,11 @@
from typing import Any, Dict, Optional
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers.automation_task_mcp import impl
-mcp = FastMCP("hugagent-automation-task")
+mcp = MCPServer("hugagent-automation-task")
_HDR_USER = "x-current-user-id"
_HDR_CHANNEL = "x-channel-id"
@@ -115,7 +115,7 @@ async def list_scheduled_tasks(
用户问"我有哪些定时任务/查看我的定时任务/有哪些计划任务"时调用。
参数 status:"active"(默认,仅生效中) / "paused" / "all"(全部)。
"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.list_tasks(user_id=_user(ctx), status=status)
@@ -131,7 +131,7 @@ async def get_scheduled_task(
task_ref 可传 task_id 或任务名称(名称模糊匹配;匹配到多个会返回候选让你向用户确认)。
"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.get_task(user_id=_user(ctx), task_ref=task_ref)
@@ -153,7 +153,7 @@ async def update_scheduled_task(
task_ref 传 task_id 或名称(多命中会要求澄清)。
tool_effect_id 由 ToolGateway 内部注入,模型不要填写。
"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.update_task(
@@ -169,7 +169,7 @@ async def update_scheduled_task(
@mcp.tool()
async def pause_scheduled_task(task_ref: str, execution_location: str = "", ctx: Context | None = None) -> Dict[str, Any]:
"""暂停一个定时任务(暂停后到点不再触发,可随后恢复)。task_ref=task_id 或名称。"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.pause_task(user_id=_user(ctx), task_ref=task_ref)
@@ -178,7 +178,7 @@ async def pause_scheduled_task(task_ref: str, execution_location: str = "", ctx:
@mcp.tool()
async def resume_scheduled_task(task_ref: str, execution_location: str = "", ctx: Context | None = None) -> Dict[str, Any]:
"""恢复一个被暂停的定时任务。task_ref=task_id 或名称。"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.resume_task(user_id=_user(ctx), task_ref=task_ref)
@@ -196,7 +196,7 @@ async def delete_scheduled_task(
【铁律】未成功调用本工具拿到 ✅ 前不要声称已删除。匹配到多个任务时必须先向用户确认,禁止猜删。
tool_effect_id 由 ToolGateway 内部注入,模型不要填写。
"""
- from core.services.automation_execution import instance_location
+ from features.automation.execution import instance_location
if execution_location and execution_location != instance_location():
return {"ok": False, "message": "执行位置不匹配,请选择对应的本机或云端任务"}
return impl.delete_task(
diff --git a/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py b/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py
index a78de5d7..bd1f4b8a 100644
--- a/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py
@@ -3,18 +3,18 @@
import uuid
import base64
from core.infra.http_errors import response_error
-from mcp.server.fastmcp.utilities.types import Image
+from mcp.server.mcpserver.utilities.types import Image
from typing import Any
import httpx
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers._ports import PORTS
from extensions.mcp_servers._serve import run
-mcp = FastMCP("browser-automation")
+mcp = MCPServer("browser-automation")
async def call(ctx, action, resource_id=None, params=None, checkpoint_id=None):
headers = ctx.request_context.request.headers
- from core.llm.mcp_invocation import verify
+ from core.llm.mcp.invocation import verify
claims = verify(headers, "browser_runtime")
plugin_id = claims.get("plugin", "")
slug = plugin_id.split(":", 2)[2] if plugin_id else "browser-automation"
diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py b/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py
index f762d106..592c0d07 100755
--- a/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py
+++ b/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py
@@ -7,7 +7,7 @@
async def check():
tools = await mcp.list_tools()
- schema = tools[0].inputSchema
+ schema = tools[0].input_schema
assert "queries" in schema["properties"]
assert "cn_only" not in schema["properties"]
assert schema["properties"]["max_results"]["maximum"] == 8
diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py b/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py
index 1d0b5a22..3229df15 100755
--- a/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py
@@ -11,7 +11,9 @@
import anyio
import httpx
-from core.config.runtime_env import get_runtime_value
+from orchestration.configuration import register_configuration
+register_configuration()
+from core.config.store.runtime_env import get_runtime_value
from .merge import merge_sources
from .models import SearchRequest
diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/server.py b/src/backend/extensions/mcp_servers/internet_search_mcp/server.py
index 8e800893..055be6ef 100755
--- a/src/backend/extensions/mcp_servers/internet_search_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/internet_search_mcp/server.py
@@ -5,44 +5,31 @@
import anyio
import httpx
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from pydantic import Field
from .impl import MAX_INFLIGHT, SearchService
from .models import Depth, Query, ResultLimit, SearchRequest, Topic
-# FastMCP enters lifespan per MCP session. Share one service across overlapping
-# sessions on this server's event loop; the last session closes its client.
-_service: SearchService | None = None
-_sessions = 0
-
-
@asynccontextmanager
-async def lifespan(server: FastMCP):
- global _service, _sessions
- if _service is None:
- _service = SearchService(
- httpx.AsyncClient(limits=httpx.Limits(max_connections=MAX_INFLIGHT))
- )
- service = _service
- _sessions += 1
+async def lifespan(server: MCPServer):
+ service = SearchService(
+ httpx.AsyncClient(limits=httpx.Limits(max_connections=MAX_INFLIGHT))
+ )
try:
yield service
finally:
- _sessions -= 1
- if _sessions == 0:
- _service = None
- with anyio.CancelScope(shield=True):
- await service.client.aclose()
+ with anyio.move_on_after(5, shield=True):
+ await service.client.aclose()
-class SearchMCP(FastMCP):
- """Validate the raw MCP boundary before FastMCP drops unknown arguments."""
+class SearchMCP(MCPServer):
+ """Validate the raw MCP boundary before MCPServer drops unknown arguments."""
- async def call_tool(self, name: str, arguments: dict[str, Any]):
+ async def call_tool(self, name: str, arguments: dict[str, Any], context=None):
if name == "internet_search":
SearchRequest.model_validate(arguments)
- return await super().call_tool(name, arguments)
+ return await super().call_tool(name, arguments, context)
mcp = SearchMCP("hugagent-internet-search", lifespan=lifespan)
diff --git a/src/backend/extensions/mcp_servers/management_registration.py b/src/backend/extensions/mcp_servers/management_registration.py
index 8e40b137..a1a01742 100644
--- a/src/backend/extensions/mcp_servers/management_registration.py
+++ b/src/backend/extensions/mcp_servers/management_registration.py
@@ -2,10 +2,10 @@
import asyncio
from typing import Literal
from pydantic import BaseModel, ConfigDict
-from mcp.server.fastmcp import Context
+from mcp.server.mcpserver import Context
from mcp.types import ToolAnnotations
-from core.services.management_contract import MANAGERS, tools
-from core.services.management_errors import management_error
+from core.capabilities.plugins.management.contract import MANAGERS, tools
+from core.capabilities.plugins.management.errors import management_error
class ArtifactSource(BaseModel):
@@ -18,7 +18,7 @@ def register(mcp, manager, user_from_context):
kind = MANAGERS[manager]
async def call(action, ctx, **arguments):
- from core.services import cloud_management as service
+ from core.capabilities.plugins.management import cloud as service
try:
return await asyncio.to_thread(getattr(service, action), user_from_context(ctx), kind, **arguments)
except Exception as exc:
diff --git a/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py
index de4c1b45..1f6d66ac 100644
--- a/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py
@@ -1,6 +1,7 @@
"""Cloud marketplace operations for plugin-manager; installation lifecycle lives in cloud_management."""
from __future__ import annotations
+from features.marketplace import plugin_market
import logging
from typing import Any, Dict, List, Optional, Tuple
@@ -16,7 +17,7 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None:
见 agent_manager_mcp/impl.py 同名函数的说明——两个 MCP 面对的是同一个跨进程缓存问题。
"""
try:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(str(user_id) if user_id else None)
except Exception as exc: # noqa: BLE001
@@ -63,13 +64,13 @@ def search_plugin_market(*, user_id: str, query: str = "", category: str = "") -
if not user_id:
return _no_user()
from core.db.engine import SessionLocal
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
q = (query or "").strip().lower()
cat = (category or "").strip()
with SessionLocal() as db:
# include_disabled=False:用户侧只看得到已发布的条目,与前端插件市场同口径。
- items = plugin_service.list_plugins(db, user_id, include_disabled=False) or []
+ items = plugin_market.list_plugins(db, user_id, include_disabled=False) or []
def _match(it: Dict[str, Any]) -> bool:
if cat and str(it.get("category") or "") != cat:
@@ -109,7 +110,7 @@ def get_plugin_info(*, user_id: str, slug: str) -> Dict[str, Any]:
if not slug:
return {"ok": False, "message": "❌ 请提供插件 slug。"}
from core.db.engine import SessionLocal
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
with SessionLocal() as db:
detail = None
@@ -168,7 +169,7 @@ def install_plugin(
if not slug:
return {"ok": False, "message": "❌ 请提供要安装的插件 slug。"}
from core.db.engine import SessionLocal
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
with SessionLocal() as db:
cap_err = _require_cap(db, user_id, "can_import_plugin")
diff --git a/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py b/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py
index 23bbd74c..766ffe3c 100644
--- a/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py
@@ -11,11 +11,11 @@
from typing import Any, Dict, Optional
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers.plugin_manager_mcp import impl
-mcp = FastMCP("hugagent-plugin-manager")
+mcp = MCPServer("hugagent-plugin-manager")
_HDR_USER = "x-current-user-id"
diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py
index b46bbdd7..88747295 100644
--- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py
@@ -6,18 +6,8 @@
import os
from typing import Any, Dict, List
-from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility
-from core.kb.external_retrieval import (
- MAX_RETRIEVE_TOKENS,
- RETRIEVE_MAX_CONCURRENCY,
- RETRIEVE_REQUEST_TIMEOUT_SECONDS,
- RETRIEVE_TOTAL_TIMEOUT_SECONDS,
- DatasetRetrievalTimeoutError,
- DatasetRetrievalUnavailableError,
- list_external_datasets,
- retrieve_dataset_content,
- retrieve_dataset_content_async,
-)
+from features.knowledge.edition import get_accessible_local_kb_ids, is_shared_visibility
+from features.knowledge.edition import MAX_RETRIEVE_TOKENS, RETRIEVE_MAX_CONCURRENCY, RETRIEVE_REQUEST_TIMEOUT_SECONDS, RETRIEVE_TOTAL_TIMEOUT_SECONDS, DatasetRetrievalTimeoutError, DatasetRetrievalUnavailableError, list_external_datasets, retrieve_dataset_content, retrieve_dataset_content_async
from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import (
LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS,
LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS,
@@ -36,7 +26,7 @@ def _list_local_datasets(
private_items: List[Dict[str, Any]] = []
try:
from core.db.engine import SessionLocal
- from core.db.models import KBDocument, KBSpace
+ from features.knowledge.models import KBDocument, KBSpace
allowed = {item.strip() for item in (allowed_kb_ids or "").split(",") if item.strip()}
user_id = (current_user_id or "").strip() or os.getenv("CURRENT_USER_ID", "").strip()
@@ -79,7 +69,7 @@ def list_all_datasets(
allowed_kb_ids: str | None = None,
current_user_id: str | None = None,
) -> Dict[str, Any]:
- from core.kb.external_provider import is_enabled
+ from features.knowledge.edition import is_enabled
local_shared, private_items = _list_local_datasets(
allowed_kb_ids=allowed_kb_ids, current_user_id=current_user_id
diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py
index d37ecff3..5c921487 100644
--- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py
+++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py
@@ -8,8 +8,10 @@
import time
from typing import Any
-from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility
-from core.config.runtime_env import get_runtime_value
+from features.knowledge.edition import get_accessible_local_kb_ids, is_shared_visibility
+from orchestration.configuration import register_configuration
+register_configuration()
+from core.config.store.runtime_env import get_runtime_value
def _read_int_env(name: str, default: int) -> int:
@@ -60,7 +62,7 @@ def _fetch_parent_contents(parent_ids: list[str]) -> dict[str, str]:
return {}
try:
from core.db.engine import SessionLocal
- from core.db.models import KBChunk
+ from features.knowledge.models import KBChunk
db = SessionLocal()
try:
@@ -91,7 +93,7 @@ def _build_runtime_local_kb_section() -> str:
kb_names: dict[str, str] = {}
try:
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
db = SessionLocal()
try:
@@ -156,7 +158,7 @@ def _remaining_stage_timeout() -> float:
if not allowed:
try:
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
with SessionLocal() as _db:
if user_id:
@@ -197,7 +199,7 @@ def _remaining_stage_timeout() -> float:
private_ids: list[str] = list(search_kb_ids)
try:
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
with SessionLocal() as _db:
rows = (
@@ -223,7 +225,8 @@ def _remaining_stage_timeout() -> float:
# ── Embed query ──────────────────────────────────────────────────────────
try:
- from core.kb.kb_vector import embed_text, hybrid_search
+ from core.models.embedding import embed_text
+ from features.knowledge.kb_vector import hybrid_search
query_vec = embed_text(query, timeout=_remaining_stage_timeout())
except LocalKnowledgeBaseTimeoutError:
@@ -253,7 +256,7 @@ def _remaining_stage_timeout() -> float:
kb_meta: list[dict[str, str]] = []
try:
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
with SessionLocal() as _db:
spaces = (
@@ -289,7 +292,7 @@ def _remaining_stage_timeout() -> float:
)
if (_reranker_flag or "").lower() in ("true", "1"):
try:
- from core.kb.kb_vector import is_reranker_configured, rerank
+ from core.models.reranking import is_reranker_configured, rerank
if is_reranker_configured() and top_hits:
contents = [hit.get("content", "") for hit in top_hits]
@@ -390,7 +393,7 @@ def _attach_assets(results: list) -> None:
if not results:
return
try:
- from core.kb.kb_assets import fetch_assets_for_chunks
+ from features.knowledge.kb_assets import fetch_assets_for_chunks
grouped = fetch_assets_for_chunks([r.get("id", "") for r in results])
except Exception as exc:
@@ -413,7 +416,7 @@ def retrieve_public_local_kb(
) -> dict:
"""Search only accessible shared local spaces through the public tool."""
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
user_id = (current_user_id or _get_current_user_id()).strip()
if not user_id:
diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py
index 23051db8..a2a1cc12 100755
--- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py
@@ -13,7 +13,7 @@
import logging
from typing import Any, Dict, Optional
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from .descriptions import (
_BASE_LOCAL_KB_TOOL_DESCRIPTION,
@@ -32,7 +32,19 @@
tool_timeout_payload,
)
-mcp = FastMCP("hugagent-retrieve-dataset-content")
+class RetrievalMCP(MCPServer):
+ async def list_tools(self):
+ tools = await super().list_tools()
+ try:
+ from extensions.mcp_servers.retrieve_dataset_content_mcp.wiki_impl import wiki_supported
+ if wiki_supported():
+ return tools
+ except Exception as exc:
+ _LOGGER.warning("wiki capability probe failed, hiding wiki tools: %s", exc)
+ return [tool for tool in tools if tool.name not in _WIKI_TOOL_NAMES]
+
+
+mcp = RetrievalMCP("hugagent-retrieve-dataset-content")
_LOGGER = logging.getLogger(__name__)
@@ -97,7 +109,7 @@ async def retrieve_dataset_content(
current_user_id = _get_header(ctx, _HDR_CURRENT_USER_ID)
try:
- from core.kb.external_provider import is_enabled
+ from features.knowledge.edition import is_enabled
from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import retrieve_public_local_kb
if not await asyncio.to_thread(is_enabled):
@@ -457,35 +469,6 @@ async def wiki_fetch_source(
return await _run_wiki("wiki_fetch_source", call, empty={"items": []})
-# ── 按后端动态暴露工具 ───────────────────────────────────────────────────────
-#
-# FastMCP 在 __init__ 里就把 self.list_tools 绑进了 lowlevel handler,所以事后
-# 改 mcp.list_tools 不生效——必须对 lowlevel server 重新注册一次(装饰器是
-# 覆盖写 request_handlers,不是追加)。
-
-
-_ORIGINAL_LIST_TOOLS = mcp.list_tools
-
-
-async def _list_tools_filtered():
- """平台没有 Wiki 源时隐藏 Wiki 工具;公有检索按配置选择后端。
-
- 每次 list_tools 都重新判定,所以切换知识库后端、或新建了一个开 Wiki 的自建库
- 之后,无需重启 mcp 容器。
- """
- tools = await _ORIGINAL_LIST_TOOLS()
- try:
- from extensions.mcp_servers.retrieve_dataset_content_mcp.wiki_impl import wiki_supported
-
- if wiki_supported():
- return tools
- except Exception as exc:
- _LOGGER.warning("wiki capability probe failed, hiding wiki tools: %s", exc)
- return [tool for tool in tools if tool.name not in _WIKI_TOOL_NAMES]
-
-
-mcp._mcp_server.list_tools()(_list_tools_filtered)
-
def main() -> None:
from extensions.mcp_servers import _serve
diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py
index 84b2c8a6..88d224f3 100644
--- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py
+++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py
@@ -54,7 +54,7 @@ def _accessible_local_kb_ids(current_user_id: Optional[str]) -> List[str]:
if not user_id:
return []
try:
- from core.auth.kb_permissions import get_accessible_local_kb_ids
+ from features.knowledge.edition import get_accessible_local_kb_ids
from core.db.engine import SessionLocal
with SessionLocal() as db:
@@ -68,7 +68,7 @@ def _assert_access(
kb_id: str, *, allowed_kb_ids: Optional[str], current_user_id: Optional[str]
) -> None:
"""自建库要过权限;外接库的白名单由其实现模块把关。"""
- from core.kb.wiki_router import is_local_kb
+ from features.knowledge.wiki_router import is_local_kb
if not is_local_kb(kb_id):
return
@@ -89,7 +89,7 @@ def _wiki_module(
权限校验和模块解析绑在一起,是为了让「先鉴权再取数」成为唯一写法——分开写
迟早会有某个工具漏掉那一行。
"""
- from core.kb.wiki_router import wiki_module_for
+ from features.knowledge.wiki_router import wiki_module_for
_assert_access(kb_id, allowed_kb_ids=allowed_kb_ids, current_user_id=current_user_id)
module = wiki_module_for(kb_id)
@@ -101,7 +101,7 @@ def _wiki_module(
def wiki_supported() -> bool:
"""平台上是否存在任何 Wiki 源(决定工具是否暴露在清单里)。"""
try:
- from core.kb.wiki_router import supports_wiki
+ from features.knowledge.wiki_router import supports_wiki
return bool(supports_wiki())
except Exception:
@@ -125,7 +125,7 @@ def resolve_dataset_id(
if explicit:
return explicit
- from core.kb.wiki_router import wiki_capable_local_kb_ids
+ from features.knowledge.wiki_router import wiki_capable_local_kb_ids
local_candidates = _split_ids(allowed_kb_ids) or _accessible_local_kb_ids(current_user_id)
wiki_locals = wiki_capable_local_kb_ids(local_candidates)
@@ -137,7 +137,7 @@ def resolve_dataset_id(
return external[0]
try:
- from core.kb.external_provider import list_collections
+ from features.knowledge.edition import list_collections
for item in list_collections() or []:
caps = item.get("capabilities") or {}
diff --git a/src/backend/extensions/mcp_servers/site_publish_mcp/server.py b/src/backend/extensions/mcp_servers/site_publish_mcp/server.py
index fbd11d5d..dc2e7468 100644
--- a/src/backend/extensions/mcp_servers/site_publish_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/site_publish_mcp/server.py
@@ -5,13 +5,13 @@
from typing import Any, Dict, Literal, Optional
-from core.services.application_schema import ApplicationPayload, ToolDefinition
+from features.hosted_apps.schema import ApplicationPayload, ToolDefinition
from pydantic import BaseModel
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers.site_publish_mcp import impl
-mcp = FastMCP("hugagent-site-publish")
+mcp = MCPServer("hugagent-site-publish")
_HDR_USER = "x-current-user-id"
_HDR_CHAT = "x-chat-id"
diff --git a/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py
index d29a84cc..e1f2ce53 100644
--- a/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py
+++ b/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py
@@ -1,6 +1,9 @@
"""Cloud marketplace operations for skill-manager; installation lifecycle lives in cloud_management."""
from __future__ import annotations
+from features.marketplace import skill_catalog
+from features.marketplace import skill_installation
+from features.marketplace import skill_submissions
import logging
from typing import Any, Dict, List, Optional, Tuple
@@ -18,7 +21,7 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None:
重新拉 ``GET /v1/catalog``(跑在 backend 进程)顺带完成,见 api/routes/v1/catalog.py。
本调用作为进程内一致性的兜底保留,无害。"""
try:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(str(user_id) if user_id else None)
except Exception as exc: # noqa: BLE001
@@ -43,13 +46,13 @@ def search_marketplace(*, user_id: str, query: str = "", category: str = "") ->
if not user_id:
return _no_user()
from core.db.engine import SessionLocal
- from core.services import marketplace_service
+
q = (query or "").strip().lower()
cat = (category or "").strip()
with SessionLocal() as db:
# 与用户端市场接口同一套可见范围过滤(scoped 条目仅授权者可见)
- items = marketplace_service.list_marketplace_skills(db, viewer_user_id=user_id)
+ items = skill_catalog.list_marketplace_skills(db, viewer_user_id=user_id)
def _match(it: Dict[str, Any]) -> bool:
if cat and str(it.get("category") or "") != cat:
@@ -93,19 +96,19 @@ def install_from_marketplace(
if not slug:
return {"ok": False, "message": "❌ 请提供要安装的技能 slug。"}
from core.db.engine import SessionLocal
- from core.services import marketplace_service
+
with SessionLocal() as db:
cap_err = _require_cap(db, user_id, "can_add_skill")
if cap_err:
return cap_err
# 可见范围守卫:对该用户不可见的 scoped 条目按不存在处理(与用户端安装接口一致)
- from core.auth.marketplace_visibility import is_item_visible
- from core.services import marketplace_listing as ml
+ from features.marketplace.edition import is_item_visible
+ from features.marketplace import edition as ml
if not is_item_visible(db, ml.KIND_SKILL, slug, user_id):
return {"ok": False, "message": f"❌ 技能市场里找不到「{slug}」。"}
try:
- res = marketplace_service.install_marketplace_skill(
+ res = skill_installation.install_marketplace_skill(
db, slug, owner_user_id=user_id, secrets=secrets or {}
)
except Exception as exc: # noqa: BLE001
@@ -133,14 +136,14 @@ def submit_to_marketplace(
if not skill_id:
return {"ok": False, "message": "❌ 请提供要上架的私有技能 skill_id(先用 list_skills 查看)。"}
from core.db.engine import SessionLocal
- from core.services import marketplace_service
+
with SessionLocal() as db:
cap_err = _require_cap(db, user_id, "can_add_skill")
if cap_err:
return cap_err
try:
- res = marketplace_service.submit_to_marketplace(
+ res = skill_submissions.submit_to_marketplace(
db,
skill_id,
owner_user_id=user_id,
diff --git a/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py b/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py
index 5d7c3e87..9f36e588 100644
--- a/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py
@@ -11,11 +11,11 @@
from typing import Any, Dict, Optional
-from mcp.server.fastmcp import Context, FastMCP
+from mcp.server.mcpserver import Context, MCPServer
from extensions.mcp_servers.skill_manager_mcp import impl
-mcp = FastMCP("hugagent-skill-manager")
+mcp = MCPServer("hugagent-skill-manager")
_HDR_USER = "x-current-user-id"
diff --git a/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py b/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py
index 3ab61e75..fbfd7c05 100644
--- a/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py
+++ b/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py
@@ -1,12 +1,6 @@
"""Developer-owned execution policy for the space MCP contract."""
-from core.llm.tool_permissions import (
- DOMAIN_APPROVAL,
- DOMAIN_DENY,
- WRITE,
- PermissionIntent,
- ToolPermissionSpec,
-)
+from core.sandbox.permissions.dispatch_models import DOMAIN_APPROVAL, DOMAIN_DENY, WRITE, PermissionIntent, ToolPermissionSpec
from .server import ARGUMENTS, READ_TOOL, WRITE_TOOL
diff --git a/src/backend/extensions/mcp_servers/space_files_mcp/server.py b/src/backend/extensions/mcp_servers/space_files_mcp/server.py
index 22fbc76c..9287d3c9 100644
--- a/src/backend/extensions/mcp_servers/space_files_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/space_files_mcp/server.py
@@ -4,9 +4,9 @@
import json
from typing import Literal
-from core.services.desktop_capability_protocol import canonical_hash
-from core.spaces.access import resolve_grants
-from core.spaces.files import SpaceFiles
+from core.capabilities.manifest_protocol import canonical_hash
+from core.files.spaces.access import resolve_grants
+from core.files.spaces.files import SpaceFiles
from fastapi import HTTPException
from pydantic import BaseModel, ConfigDict, Field, ValidationError
@@ -71,7 +71,7 @@ def manifest():
def resolve(user, name, schema_hash):
- from core.services.desktop_capability_protocol import CapabilityManifestStaleError
+ from core.capabilities.manifest_protocol import CapabilityManifestStaleError
if not user or str(user).startswith("api_ak_") or name not in ARGUMENTS:
return None
@@ -87,7 +87,7 @@ def resolve(user, name, schema_hash):
def execute(user, tool_name, arguments):
from core.db.engine import SessionLocal
- from core.storage import get_storage
+ from core.files.storage import get_storage
try:
args = ARGUMENTS[tool_name].model_validate(arguments)
diff --git a/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py b/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py
index 0671867d..5799a100 100644
--- a/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py
+++ b/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py
@@ -4,13 +4,13 @@
from typing import Annotated, Any, Literal
import httpx
-from mcp.server import FastMCP
+from mcp.server.mcpserver import MCPServer
from pydantic import Field
from .impl import fetch_url
logger = logging.getLogger(__name__)
-mcp = FastMCP("hugagent-web-fetch")
+mcp = MCPServer("hugagent-web-fetch")
@mcp.tool()
diff --git a/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json b/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json
index 9c5680a7..bfcadab0 100644
--- a/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json
+++ b/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json
@@ -3,7 +3,7 @@
"mcpServers": {
"agent_manager": {
"type": "streamable-http",
- "url": "http://mcp:9115/mcp/"
+ "url": "http://mcp:9115/mcp"
}
}
}
diff --git a/src/backend/extensions/plugins/marketplace/automation/mcp.json b/src/backend/extensions/plugins/marketplace/automation/mcp.json
index 551c583c..73c1613b 100644
--- a/src/backend/extensions/plugins/marketplace/automation/mcp.json
+++ b/src/backend/extensions/plugins/marketplace/automation/mcp.json
@@ -3,7 +3,7 @@
"mcpServers": {
"automation_task": {
"type": "streamable-http",
- "url": "http://mcp:9108/mcp/"
+ "url": "http://mcp:9108/mcp"
}
}
}
diff --git a/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json b/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json
index 9987f5b2..fe9f3283 100644
--- a/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json
+++ b/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json
@@ -3,7 +3,7 @@
"mcpServers": {
"plugin_manager": {
"type": "streamable-http",
- "url": "http://mcp:9116/mcp/"
+ "url": "http://mcp:9116/mcp"
}
}
}
diff --git a/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py b/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py
index 7e8fc8df..70923748 100644
--- a/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py
+++ b/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""打包前自检一个插件目录的结构是否合法(导入前跑一遍,省得来回试错)。
-规则对齐后端导入器 core/plugins/packaging/importer.py:
+规则对齐后端导入器 core/capabilities/plugins/packaging/importer.py:
- 包根必须有 plugin.json(或 .claude-plugin/plugin.json);
- plugin.json 必须是合法 JSON 且含非空 name;
- name 必须匹配 ^[a-z0-9_-]{1,100}$(会被用来生成安装 id);
diff --git a/src/backend/extensions/plugins/marketplace/sites/mcp.json b/src/backend/extensions/plugins/marketplace/sites/mcp.json
index 4e51449e..cf976440 100644
--- a/src/backend/extensions/plugins/marketplace/sites/mcp.json
+++ b/src/backend/extensions/plugins/marketplace/sites/mcp.json
@@ -3,7 +3,7 @@
"mcpServers": {
"site_publish": {
"type": "streamable-http",
- "url": "http://mcp:9113/mcp/"
+ "url": "http://mcp:9113/mcp"
}
}
}
diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json b/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json
index 177f6ee8..e246f89d 100644
--- a/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json
+++ b/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json
@@ -3,7 +3,7 @@
"mcpServers": {
"skill_manager": {
"type": "streamable-http",
- "url": "http://mcp:9112/mcp/"
+ "url": "http://mcp:9112/mcp"
}
}
}
diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md
index fa5ef921..a27941c8 100644
--- a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md
+++ b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md
@@ -25,7 +25,7 @@ allowed_tools: Read, Bash # 可选,声明技能用到的工具
---
```
-- 后端技能引擎(core/agent_skills/registry.py)对 `name` 的校验是 `^[a-z0-9_-]{1,63}$`——
+- 后端技能引擎(core/capabilities/skills/registry.py)对 `name` 的校验是 `^[a-z0-9_-]{1,63}$`——
**允许下划线**(与 Anthropic 原版只许连字符不同)。含空格/中文/大写的 name 会在落库时被
自动规范化,但建议一开始就写规范。
- `description` 缺失或为空会直接落库失败。
diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py
index 8b98473f..09fd81b9 100644
--- a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py
+++ b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""快速校验一个技能目录的 SKILL.md 是否合法(落库前自检)。
-规则对齐后端技能引擎 core/agent_skills/registry.py(**不是** Anthropic 原版规则):
+规则对齐后端技能引擎 core/capabilities/skills/registry.py(**不是** Anthropic 原版规则):
- 必须有 `---` 包裹的 YAML frontmatter;
- frontmatter 必须有 `name` 与 `description`;
- name 必须匹配 ^[a-z0-9_-]{1,63}$(小写字母/数字/下划线/连字符,1–63 字符);
diff --git a/src/backend/extensions/skills/marketplace/data-mask/config.yaml b/src/backend/extensions/skills/marketplace/data-mask/config.yaml
index b3ec2ec6..d95bc389 100644
--- a/src/backend/extensions/skills/marketplace/data-mask/config.yaml
+++ b/src/backend/extensions/skills/marketplace/data-mask/config.yaml
@@ -1,83 +1,83 @@
-# 数据脱敏工具配置文件
-# 支持自定义:单位后缀、白名单列、黑名单列、脱敏排除规则
-
-# ── 自定义单位后缀 ──
-# 将自动追加到内置单位列表(万元、亿美元...)之后
-custom_units:
- # - "人次"
- # - "笔订单"
- # - "亿元RMB"
-
-# ── 自动列检测规则 ──
-column_rules:
- # 白名单:不管是否数值,强制脱敏这些列(正则匹配列名)
- force_mask_patterns:
- # - "金额"
- # - ".*收入"
- # - "工资$"
-
- # 黑名单:数值列也跳过脱敏(正则匹配列名)
- skip_patterns:
- - "^id$"
- - "^ID$"
- - "^序号$"
- - "^编号$"
- - "^年份$"
- - "^年度$"
- - "^月$"
- - "^季度$"
- - "^排名$"
- - "^排序$"
- - "^百分比$"
- - "^占比$"
-
-# ── 脱敏排除规则 ──
-skip_rules:
- # 是否跳过电话号码格式 (11位手机号)
- skip_phone: false
-
- # 排除特定格式的正则列表
- skip_patterns:
- # - "^\\d{18}$" # 18位身份证号
- # - "^\\d{6}$" # 邮编
-
-# ── 默认参数 ──
-defaults:
- strategy: hash # 默认脱敏策略: hash | placeholder | blur | range
- seed: 42 # 默认随机种子
- auto_detect_numeric: true # DataFrame 脱敏时是否自动检测数值列
-
-# ── PDF 输出设置 ──
-pdf:
- # 默认输出格式: pdf (生成新PDF) 或 txt (纯文本)
- output_format: pdf
- # PDF 中文字体路径(留空则自动查找系统字体)
- cjk_font_path: ""
- # 字体大小
- font_size: 10
-
-# ── 图片 OCR 设置 ──
-ocr:
- # 默认识别语言 (Tesseract 语言代码)
- # chi_sim+eng = 简体中文 + 英文(推荐)
- # eng = 纯英文
- # chi_sim = 纯简体中文
- # jpn+eng = 日文 + 英文
- default_lang: chi_sim+eng
- # Windows 上 Tesseract 安装路径的候选列表(留空则用默认)
- tesseract_cmd_candidates:
- - "C:\\Program Files\\Tesseract-OCR\\tesseract.exe"
- - "C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe"
-
-# ── 未知格式处理 ──
-fallback:
- # 遇到未识别格式时,是否自动尝试文本读取并脱敏
- auto_attempt_text: true
- # 文本读取尝试的编码顺序
- encodings:
- - utf-8
- - gbk
- - gb2312
- - latin-1
- # 回退输出格式(始终为 txt)
- output_format: txt
+# 数据脱敏工具配置文件
+# 支持自定义:单位后缀、白名单列、黑名单列、脱敏排除规则
+
+# ── 自定义单位后缀 ──
+# 将自动追加到内置单位列表(万元、亿美元...)之后
+custom_units:
+ # - "人次"
+ # - "笔订单"
+ # - "亿元RMB"
+
+# ── 自动列检测规则 ──
+column_rules:
+ # 白名单:不管是否数值,强制脱敏这些列(正则匹配列名)
+ force_mask_patterns:
+ # - "金额"
+ # - ".*收入"
+ # - "工资$"
+
+ # 黑名单:数值列也跳过脱敏(正则匹配列名)
+ skip_patterns:
+ - "^id$"
+ - "^ID$"
+ - "^序号$"
+ - "^编号$"
+ - "^年份$"
+ - "^年度$"
+ - "^月$"
+ - "^季度$"
+ - "^排名$"
+ - "^排序$"
+ - "^百分比$"
+ - "^占比$"
+
+# ── 脱敏排除规则 ──
+skip_rules:
+ # 是否跳过电话号码格式 (11位手机号)
+ skip_phone: false
+
+ # 排除特定格式的正则列表
+ skip_patterns:
+ # - "^\\d{18}$" # 18位身份证号
+ # - "^\\d{6}$" # 邮编
+
+# ── 默认参数 ──
+defaults:
+ strategy: hash # 默认脱敏策略: hash | placeholder | blur | range
+ seed: 42 # 默认随机种子
+ auto_detect_numeric: true # DataFrame 脱敏时是否自动检测数值列
+
+# ── PDF 输出设置 ──
+pdf:
+ # 默认输出格式: pdf (生成新PDF) 或 txt (纯文本)
+ output_format: pdf
+ # PDF 中文字体路径(留空则自动查找系统字体)
+ cjk_font_path: ""
+ # 字体大小
+ font_size: 10
+
+# ── 图片 OCR 设置 ──
+ocr:
+ # 默认识别语言 (Tesseract 语言代码)
+ # chi_sim+eng = 简体中文 + 英文(推荐)
+ # eng = 纯英文
+ # chi_sim = 纯简体中文
+ # jpn+eng = 日文 + 英文
+ default_lang: chi_sim+eng
+ # Windows 上 Tesseract 安装路径的候选列表(留空则用默认)
+ tesseract_cmd_candidates:
+ - "C:\\Program Files\\Tesseract-OCR\\tesseract.exe"
+ - "C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe"
+
+# ── 未知格式处理 ──
+fallback:
+ # 遇到未识别格式时,是否自动尝试文本读取并脱敏
+ auto_attempt_text: true
+ # 文本读取尝试的编码顺序
+ encodings:
+ - utf-8
+ - gbk
+ - gb2312
+ - latin-1
+ # 回退输出格式(始终为 txt)
+ output_format: txt
diff --git a/src/backend/features/__init__.py b/src/backend/features/__init__.py
new file mode 100644
index 00000000..233a7460
--- /dev/null
+++ b/src/backend/features/__init__.py
@@ -0,0 +1,157 @@
+"""Host composition for product tools until their owning extensions move in P2/P3."""
+
+
+def register_product_tools():
+ from core.agent_runtime.host import register as register_chat_execution
+ register_chat_execution()
+ from core.ontology.host import register as register_governance
+ register_governance()
+ from core.harness.registry.system_configuration import register_system_configuration_store
+ from core.db.system_configuration import SystemConfigurationStore
+ register_system_configuration_store(SystemConfigurationStore())
+ from core.harness.registry.cache_invalidation import cache_invalidators
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from features.desktop.capabilities.local_projection import sync_local_skills
+ from prompts.prompt_runtime import invalidate_prompt_cache
+ cache_invalidators().register("before_skill_refresh", "desktop_skills",
+ lambda: sync_local_skills() if has_run_capabilities() else None)
+ cache_invalidators().register("after_skill_refresh", "prompt_cache", invalidate_prompt_cache)
+
+ from core.evolution.settlement.memory_service import register as register_settlement
+ register_settlement()
+ from core.runs.host import register as register_runs
+ register_runs()
+ from core.runs.tool_effects.host import register as register_tool_effects
+ register_tool_effects()
+ from core.llm.host_services import register
+ register()
+ register_marketplace_catalog()
+ register_retrieval_context()
+ from importlib import import_module
+ from features.installed import INSTALLED_FEATURES
+ for package in INSTALLED_FEATURES:
+ register = getattr(import_module(f"{package}.extension"), "register_tools", None)
+ if register:
+ register()
+ gateways = getattr(import_module(f"{package}.extension"), "register_gateway_contributions", None)
+ if gateways:
+ gateways()
+
+
+def register_package_sources():
+ # CLI installs packages before API composition, so bind import policy here.
+ from core.ontology.host import register as register_governance
+ register_governance()
+ register_marketplace_catalog()
+ from core.harness.registry.package_views import register_package_view
+ from features.workbench.ui import contract
+ from features.workbench.ui import workspace_enabled
+ register_package_view(
+ contract.public_contributions, normalize=contract.normalize_ui, find=contract.find_module,
+ enabled=workspace_enabled.enabled, workspace_only=workspace_enabled.workspace_only,
+ set_enabled=workspace_enabled.set_enabled,
+ )
+ from core.harness.registry.package_sources import register_package_source, register_package_upgrade_policy
+ from features.marketplace.package_source import MarketplacePackageSource
+ from features.workbench.package_source import WorkbenchPackagePolicy
+ register_package_source("marketplace", MarketplacePackageSource())
+ register_package_upgrade_policy(WorkbenchPackagePolicy())
+
+
+def register_resource_owners():
+ from core.harness.registry.resource_owners import ResourceOwner, register_resource_owner
+ from features.workbench import background, diagnostics, installation_binding, revocation_sweep
+ from features.workbench import resource_owner, resource_deadlines, revocation
+ from features.hosted_apps.resources import admission, cleanup
+ register_resource_owner(ResourceOwner(
+ binding=lambda *args, **kwargs: resource_owner.application_binding(*args, **kwargs),
+ resolve_installation=lambda *args, **kwargs: resource_owner.resolve_installation(*args, **kwargs),
+ exact_installation=lambda *args, **kwargs: installation_binding.exact(*args, **kwargs),
+ authorize=lambda *args, **kwargs: background.authorized_resource(*args, **kwargs),
+ observe=lambda *args, **kwargs: diagnostics.observe_resource(*args, **kwargs),
+ record=lambda *args, **kwargs: diagnostics.record(*args, **kwargs),
+ renew=lambda *args, **kwargs: background.renew(*args, **kwargs),
+ arm=lambda *args, **kwargs: resource_deadlines.arm(*args, **kwargs),
+ invalidate=lambda *args, **kwargs: resource_deadlines.invalidate(*args, **kwargs),
+ sweep=lambda *args, **kwargs: revocation_sweep.sweep(*args, **kwargs),
+ collect=lambda *args, **kwargs: resource_owner.collect(*args, **kwargs),
+ create=admission.create,
+ owned=admission.owned,
+ lease=admission.renew,
+ reserve=admission.reserve,
+ close=cleanup.close,
+ revoke_session=revocation.revoke_user_workbench,
+ ))
+
+
+def register_run_capabilities():
+ from core.harness.registry.capability_revisions import register_capability_revisions
+ from features.desktop.capability.revision_provider import CloudCapabilityRevisions
+ register_capability_revisions(CloudCapabilityRevisions())
+ from core.harness.registry.capability_management import register_manager_executor
+ from features.desktop.local.manager_execution import execute
+ register_manager_executor(execute)
+ from core.harness.registry.device_catalog import register_device_catalog
+ from features.desktop.capabilities import device_catalog
+ register_device_catalog(device_catalog)
+ from core.harness.registry.local_files import register_local_file_services
+ from features.desktop.local import grants, snapshots
+ register_local_file_services(grants, snapshots)
+ from core.harness.registry.capability_runtime import register_capability_runtime
+ from features.desktop.capabilities import runtime as capability_runtime
+ register_capability_runtime(capability_runtime)
+ from core.harness.registry.local_publication import register_local_publication
+ from features.desktop.local.publication_provider import DesktopLocalPublication
+ register_local_publication(DesktopLocalPublication())
+ from core.harness.registry.execution_identity import register_execution_identity
+ from features.desktop.bridge import identity
+ register_execution_identity(identity)
+ from core.harness.registry.execution_catalog import register_execution_catalog
+ from core.harness.registry.execution_account import register_execution_account
+ from features.desktop.bridge import gateway
+ from features.desktop.capabilities import skill_account
+ register_execution_catalog(gateway)
+ register_execution_account(skill_account)
+ from core.harness.registry.host_runtime import register_remote_execution_policy
+ from features.desktop.auth import bridge as desktop_bridge
+ from core.harness.registry.authentication import register_host_authentication
+ register_host_authentication(desktop_bridge.claims_request, desktop_bridge.resolve_bridge_user)
+ register_remote_execution_policy(lambda: desktop_bridge.bridge_enabled())
+ from core.harness.registry.run_capabilities import register_run_capability_provider
+ from features.desktop.capabilities import paths
+ from features.desktop.run_capabilities.provider import DesktopRunCapabilityProvider
+ register_run_capability_provider("desktop", DesktopRunCapabilityProvider(), enabled=lambda: paths.capabilities_enabled())
+
+
+def register_model_credentials():
+ from core.harness.registry.model_credentials import register_request_header_resolver
+ from features.desktop.bridge import model_credentials as desktop_model_credentials
+ from core.harness.registry.model_credentials import register_model_credential_provider
+ register_model_credential_provider(desktop_model_credentials)
+ register_request_header_resolver(
+ lambda *args: desktop_model_credentials.prepare_request_headers(*args)
+ )
+
+
+def register_catalog_options():
+ from importlib import import_module
+ from features.installed import INSTALLED_FEATURES
+ for package in INSTALLED_FEATURES:
+ register = getattr(import_module(f"{package}.extension"), "register_catalog_options", None)
+ if register:
+ register()
+
+
+def register_retrieval_context():
+ from importlib import import_module
+ from features.installed import INSTALLED_FEATURES
+ for package in INSTALLED_FEATURES:
+ register = getattr(import_module(f"{package}.extension"), "register_retrieval_context", None)
+ if register:
+ register()
+
+
+def register_marketplace_catalog():
+ from core.harness.registry.marketplace_catalog import register_marketplace_catalog as register_catalog
+ from features.marketplace.core_catalog import MarketplaceCatalogProvider
+ register_catalog(MarketplaceCatalogProvider())
diff --git a/src/backend/features/automation/README.md b/src/backend/features/automation/README.md
new file mode 100644
index 00000000..f5553e21
--- /dev/null
+++ b/src/backend/features/automation/README.md
@@ -0,0 +1,15 @@
+# 自动化与任务循环
+
+管理计划任务、执行记录、展示摘要、执行位置、通知投递与任务循环。`service.AutomationService` 保留 CRUD 和原子状态变更;`scheduling.compute_next_run` 保留时区感知的 Cron 计算;其余模块按执行绑定、投递目标、通知与远端副作用拆分。
+
+对外接口包括原 `/v1/automations` 路由、MCP 任务服务调用的公开服务方法,以及 orchestration 调度器/恢复器使用的任务接口。路由顺序与权限键不变,调度器仍留在 orchestration。模型 `ScheduledTask`、`ScheduledTaskRun` 在 FULL 和 CE 均通过安装清单注册,表名与关系不变;执行状态新增 pending,升级脚本与本机 schema hook 同步更新状态约束。
+
+依赖内核数据库、运行账本、项目权限、沙箱和临时状态接口。显式功能依赖为 **automation → desktop**:`tool_routing` 核验当前设备与账号授权,再选择本机任务执行;cloud 路径不使用此能力。当前 core.auth.desktop_bridge/core.capabilities 的物理路径待 P5 按清单迁移。
+
+P5 边界待办:桌面工具执行器当前反向调用本模块的路由与副作用服务,属于原始导入基线中的循环债务;不能作为最终允许的反向依赖保留,应通过已注册的工具适配接口接入。本批不改变既有授权和幂等语义。
+
+模块安装不自动启动调度器;现有 orchestration 启停和环境配置保持原行为。浏览器自动化属于另一套能力,不归入本目录。
+
+执行记录的条件写入由 `runs.AutomationRunRepository` 管理。完成只接受 running 且 lease_token 匹配的记录。恢复先认领,再重读 chat_id。子执行通过事务内 on_accept 钩子关联并保存恢复参数,内核不导入功能模型。调度器取消只转交观察权,用户取消仍通过会话执行器处理。
+
+接纳由 admission.admit 管理:条件更新消费触发信号并插入 pending 记录,同事务提交;禁止恢复独立清空触发信号或提前推进排期的旧路径。
diff --git a/src/backend/features/automation/__init__.py b/src/backend/features/automation/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/automation/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/features/automation/admission.py b/src/backend/features/automation/admission.py
new file mode 100644
index 00000000..0630c400
--- /dev/null
+++ b/src/backend/features/automation/admission.py
@@ -0,0 +1,80 @@
+"""Durable admission: consume a trigger and insert its run in one transaction."""
+import uuid
+
+from sqlalchemy import and_, or_
+
+from core.infra.time import utc_now
+from features.automation.models import ScheduledTask, ScheduledTaskRun
+from features.automation import scheduling
+
+LIVE = ("pending", "running")
+
+
+def admit(db, limit):
+ if limit <= 0:
+ return []
+ now = utc_now()
+ outstanding = db.query(ScheduledTaskRun.run_id).filter(
+ ScheduledTaskRun.task_id == ScheduledTask.task_id,
+ ScheduledTaskRun.status.in_(LIVE),
+ ).exists()
+ tasks = (
+ db.query(ScheduledTask)
+ .filter(
+ ScheduledTask.status.in_(("active", "paused", "completed")),
+ ~outstanding,
+ or_(
+ ScheduledTask.manual_trigger_at.isnot(None),
+ and_(ScheduledTask.status == "active",
+ ScheduledTask.schedule_type != "manual",
+ ScheduledTask.next_run_at <= now),
+ ),
+ )
+ .order_by(ScheduledTask.updated_at, ScheduledTask.task_id)
+ .with_for_update(skip_locked=True)
+ .limit(limit).all()
+ )
+ ids = []
+ for task in tasks:
+ manual = task.manual_trigger_at is not None
+ count = (task.run_count or 0) + 1
+ values = {"run_count": count, "updated_at": now}
+ if manual:
+ values["manual_trigger_at"] = None
+ elif (not task.recurring or task.schedule_type == "once"
+ or (task.max_runs and count >= task.max_runs)):
+ values["next_run_at"] = None
+ else:
+ values["next_run_at"] = scheduling.compute_next_run(task.cron_expression, task.timezone)
+ # Row locks are an optimization; the conditional write also fences
+ # SQLite and a contender whose initial SELECT observed an older row.
+ changed = db.query(ScheduledTask).filter(
+ ScheduledTask.task_id == task.task_id,
+ ScheduledTask.run_count == task.run_count,
+ ScheduledTask.manual_trigger_at == task.manual_trigger_at,
+ ScheduledTask.next_run_at == task.next_run_at,
+ ScheduledTask.updated_at == task.updated_at,
+ ~outstanding,
+ ).update(values, synchronize_session=False)
+ if changed != 1:
+ continue
+ run = ScheduledTaskRun(
+ run_id="run_" + uuid.uuid4().hex[:16],
+ task_id=task.task_id, status="pending", started_at=now,
+ )
+ db.add(run)
+ ids.append(run.run_id)
+ db.commit()
+ db.expire_all()
+ return ids
+
+
+def candidates(db, limit):
+ return [
+ task_id
+ for (task_id,) in db.query(ScheduledTask.task_id)
+ .join(ScheduledTaskRun)
+ .filter(ScheduledTaskRun.status.in_(LIVE))
+ .order_by(ScheduledTaskRun.started_at)
+ .limit(limit).all()
+ ]
diff --git a/src/backend/features/automation/api/__init__.py b/src/backend/features/automation/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/automation/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/automations.py b/src/backend/features/automation/api/automations.py
similarity index 94%
rename from src/backend/api/routes/v1/automations.py
rename to src/backend/features/automation/api/automations.py
index 00566c8a..1e70b1da 100644
--- a/src/backend/api/routes/v1/automations.py
+++ b/src/backend/features/automation/api/automations.py
@@ -10,8 +10,8 @@
from core.auth.backend import get_current_user, UserContext
from core.db.engine import get_db
from core.infra.responses import success_response, created_response
-from core.services import automation_notifications as notifications
-from core.services.automation_service import AutomationService, SUMMARY_LIMIT_LIST
+from features.automation import notifications as notifications
+from features.automation.service import AutomationService, SUMMARY_LIMIT_LIST
from core.infra.logging import get_logger, quiet_access_log
logger = get_logger(__name__)
@@ -83,7 +83,7 @@ def create_automation(
if req.task_type == "plan":
if not req.plan_id:
raise HTTPException(status_code=400, detail="计划类型任务必须提供 plan_id")
- from core.services.plan_service import PlanService
+ from core.chat.plans import PlanService
plan_svc = PlanService(db)
plan = plan_svc.get_plan(req.plan_id, user.user_id)
if not plan:
@@ -92,7 +92,7 @@ def create_automation(
if req.task_type == "loop":
if not req.loop_id:
raise HTTPException(status_code=400, detail="循环类型任务必须提供 loop_id")
- from core.services.loop_service import LoopService
+ from features.automation.loops import LoopService
if not LoopService(db).get_loop(req.loop_id, user_id=user.user_id):
raise HTTPException(status_code=404, detail="循环不存在或无权访问")
@@ -103,14 +103,15 @@ def create_automation(
if req.task_type == "loop":
task_metadata = {"loop_id": req.loop_id}
if req.channel_id and req.conversation_id:
- from core.db.repository.channel import ChannelConnectionRepository
+ from integrations.channels.repository import ChannelConnectionRepository
conn = ChannelConnectionRepository(db).get_by_id(req.channel_id)
if conn is None or conn.owner_user_id != str(user.user_id):
raise HTTPException(status_code=403, detail="无权投递到该渠道机器人")
task_metadata = {**(task_metadata or {}),
- "channel_id": req.channel_id, "conversation_id": req.conversation_id}
+ "delivery_targets": [{"type": "inapp"}, {"type": "channel",
+ "channel_id": req.channel_id, "conversation_id": req.conversation_id}]}
- from core.services.automation_execution import execution_metadata
+ from features.automation.execution import execution_metadata
import pytz
try:
pytz.timezone(req.timezone)
@@ -201,9 +202,7 @@ def update_automation(
if not task0:
raise HTTPException(status_code=404, detail="任务不存在")
new_ed = dict(task0.extra_data or {})
- # First clear old delivery markers (flat ones + channel targets inside the list)
- new_ed.pop("channel_id", None)
- new_ed.pop("conversation_id", None)
+ # Replace channel targets while retaining the other delivery destinations.
if isinstance(new_ed.get("delivery_targets"), list):
new_ed["delivery_targets"] = [
t for t in new_ed["delivery_targets"]
@@ -213,12 +212,14 @@ def update_automation(
new_ed.pop("delivery_targets")
# If a channel conversation is given -> verify bot ownership then write; otherwise keep it cleared (= in-site only)
if req.channel_id and req.conversation_id:
- from core.db.repository.channel import ChannelConnectionRepository
+ from integrations.channels.repository import ChannelConnectionRepository
conn = ChannelConnectionRepository(db).get_by_id(req.channel_id)
if conn is None or conn.owner_user_id != str(user.user_id):
raise HTTPException(status_code=403, detail="无权投递到该渠道机器人")
- new_ed["channel_id"] = req.channel_id
- new_ed["conversation_id"] = req.conversation_id
+ new_ed.setdefault("delivery_targets", [{"type": "inapp"}]).append({
+ "type": "channel", "channel_id": req.channel_id,
+ "conversation_id": req.conversation_id,
+ })
updates["extra_data"] = new_ed
try:
diff --git a/src/backend/core/services/delivery_targets.py b/src/backend/features/automation/delivery_targets.py
similarity index 75%
rename from src/backend/core/services/delivery_targets.py
rename to src/backend/features/automation/delivery_targets.py
index 187b3a5a..5eaee4fc 100644
--- a/src/backend/core/services/delivery_targets.py
+++ b/src/backend/features/automation/delivery_targets.py
@@ -7,10 +7,6 @@
{"type": "channel", "channel_id": "chan_…", "conversation_id": "oc_…"} # IM conversation (Feishu, etc.)
{"type": "email", "to": "a@b.com"} # reserved, delivery not yet implemented
-Backward compatibility: legacy tasks put flat ``channel_id`` / ``conversation_id`` only
-at the top level of extra_data (old #7 style); ``resolve_delivery_targets`` interprets
-this as ``[{inapp}, {channel…}]``. No markers at all → in-app only.
-
See internal design docs.
"""
@@ -22,12 +18,7 @@
def resolve_delivery_targets(extra_data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
- """Resolve the final delivery-target list from a task's extra_data (with backward
- compatibility).
-
- Priority: explicit ``delivery_targets`` > legacy flat ``channel_id/conversation_id``
- > in-app only.
- """
+ """Read the canonical delivery-target list; omitted targets mean in-app."""
meta = extra_data if isinstance(extra_data, dict) else {}
targets = meta.get("delivery_targets")
if isinstance(targets, list) and targets:
@@ -38,13 +29,7 @@ def resolve_delivery_targets(extra_data: Optional[Dict[str, Any]]) -> List[Dict[
if out:
return out
- # Backward compatibility: legacy #7 channel tasks only put flat channel_id/conversation_id at the top level
- ch = meta.get("channel_id")
- conv = meta.get("conversation_id")
- legacy: List[Dict[str, Any]] = [{"type": "inapp"}]
- if ch and conv:
- legacy.append({"type": "channel", "channel_id": ch, "conversation_id": conv})
- return legacy
+ return [{"type": "inapp"}]
def build_delivery_targets(
diff --git a/src/backend/core/services/automation_execution.py b/src/backend/features/automation/execution.py
similarity index 96%
rename from src/backend/core/services/automation_execution.py
rename to src/backend/features/automation/execution.py
index faca455a..ba8f5d3f 100644
--- a/src/backend/core/services/automation_execution.py
+++ b/src/backend/features/automation/execution.py
@@ -26,7 +26,7 @@ def references_host_path(prompt):
def execution_metadata(db, user_id, *, location=None, project_id=None, prompt=""):
from core.db.models import Project
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
actual = instance_location()
location = location or actual
@@ -66,7 +66,7 @@ def execution_metadata(db, user_id, *, location=None, project_id=None, prompt=""
def task_execution_context(db, task):
- from core.services.project_scope import build_project_ctx
+ from core.projects.edition import build_project_ctx
saved = dict(task.extra_data or {})
# Legacy tasks belong to the backend where they were stored.
diff --git a/src/backend/features/automation/extension.py b/src/backend/features/automation/extension.py
new file mode 100644
index 00000000..ecc708de
--- /dev/null
+++ b/src/backend/features/automation/extension.py
@@ -0,0 +1,28 @@
+"""Automation installation; existing scheduler enablement remains authoritative."""
+NAME = "automation"
+
+
+def register_schema():
+ from core.harness.registry.schema_lifecycle import register_schema_lifecycle
+ from features.automation.schema import initialize_schema
+ register_schema_lifecycle(NAME, initialize_schema)
+
+
+def setup():
+ register_schema()
+ return True
+
+
+def register_gateway_contributions():
+ from core.harness.registry.tool_operations import ToolOperations, register_tool_operations
+ from features.automation import remote_effect, tool_routing
+ register_tool_operations("automation", ToolOperations(
+ receipt_tools=remote_effect.RECEIPT_TOOLS,
+ prepare_arguments=lambda name, args, user, chat: (tool_routing.creation_arguments(args, user, chat)
+ if name == "create_scheduled_task" else args),
+ local_tools=frozenset(tool_routing.TOOLS),
+ bind=lambda *args: remote_effect.bind_remote_effect(*args),
+ lookup=lambda *args: remote_effect.lookup_remote_receipt(*args),
+ remember=lambda *args: remote_effect.remember_remote_call(*args),
+ dispatch_local=lambda *args, **kwargs: tool_routing.local_automation_result(*args, **kwargs),
+ ))
diff --git a/src/backend/core/services/loop_service.py b/src/backend/features/automation/loops.py
similarity index 99%
rename from src/backend/core/services/loop_service.py
rename to src/backend/features/automation/loops.py
index 01ff5843..9ac49b41 100644
--- a/src/backend/core/services/loop_service.py
+++ b/src/backend/features/automation/loops.py
@@ -8,10 +8,9 @@
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
-from sqlalchemy.orm import Session
-
from core.db.models import AgentLoop, LoopIteration
from core.infra.logging import get_logger
+from sqlalchemy.orm import Session
logger = get_logger(__name__)
@@ -196,7 +195,7 @@ def load_ledger(self, loop_id: str) -> Optional[Dict[str, Any]]:
def persist_result(self, loop_id: str, result: Any, *, commit: bool = True) -> None:
"""Persist the result of one run_autonomous_loop: loop terminal state + each round's LoopIteration.
- ``result`` is an orchestration.autonomous_loop.LoopResult.
+ ``result`` is an orchestration.loops.driver.LoopResult.
"""
loop = self.get_loop(loop_id)
if not loop:
diff --git a/src/backend/core/db/models/automation.py b/src/backend/features/automation/models.py
similarity index 64%
rename from src/backend/core/db/models/automation.py
rename to src/backend/features/automation/models.py
index 6ea34374..6a75b100 100644
--- a/src/backend/core/db/models/automation.py
+++ b/src/backend/features/automation/models.py
@@ -1,10 +1,10 @@
-"""SQLAlchemy ORM models — automation / distillation."""
+"""Scheduled task and run models; loaded independently of scheduler enablement."""
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
from datetime import datetime, timezone
-from core.db.engine import Base
+from core.db.models import Base, Plan, UserShadow
from sqlalchemy import (
JSON,
BigInteger,
@@ -79,8 +79,8 @@ class ScheduledTask(Base):
updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
# Relationships
- user = relationship("UserShadow")
- plan = relationship("Plan")
+ user = relationship(UserShadow)
+ plan = relationship(Plan)
run_history = relationship(
"ScheduledTaskRun",
back_populates="task",
@@ -121,6 +121,7 @@ class ScheduledTaskRun(Base):
String(64), ForeignKey("scheduled_tasks.task_id", ondelete="CASCADE"), nullable=False
)
status = Column(String(20), nullable=False, default="running")
+ lease_token = Column(String(64), nullable=True)
chat_id = Column(String(64))
result_summary = Column(Text)
error_message = Column(Text)
@@ -133,57 +134,9 @@ class ScheduledTaskRun(Base):
__table_args__ = (
CheckConstraint(
- "status IN ('running', 'success', 'failed')",
+ "status IN ('pending', 'running', 'success', 'failed')",
name="scheduled_task_runs_status_check",
),
Index("idx_scheduled_task_runs_task_id", "task_id"),
Index("idx_scheduled_task_runs_started_at", "started_at"),
)
-
-
-class PersonaDistillJob(Base):
- """Persona-level distillation job (colleague skill / personal skill).
-
- Unlike DistillationRun (single conversation, cron-driven): one job
- aggregates a user's multiple conversations + memories, executed in two
- stages — map (per-conversation summaries) → reduce (synthesize SKILL.md).
- In colleague mode the output is mirrored into admin_skill_drafts for
- review; in personal mode it lands directly as AdminSkill(owner=the user)
- after the user's own confirmation.
- """
-
- __tablename__ = "persona_distill_jobs"
-
- job_id = Column(String(64), primary_key=True) # pdj_<16hex>
- kind = Column(String(16), nullable=False) # colleague | personal
- target_user_id = Column(String(64), nullable=False) # the user being distilled
- requested_by = Column(String(64), nullable=False) # initiator (config_admin or user_id)
- scope = Column(JSONType, default=dict) # chat_ids / date range / memory switch / hint
- status = Column(String(16), nullable=False, default="queued")
- progress_done = Column(
- Integer, nullable=False, default=0
- ) # conversations completed in the map stage
- progress_total = Column(Integer, nullable=False, default=0)
- intermediate = Column(JSONType, default=list) # list of conversation summaries (map output)
- result_skill_content = Column(Text) # full SKILL.md text (reduce output)
- result_meta = Column(JSONType, default=dict) # proposed_skill_id/display_name/...
- result_draft_id = Column(String(64)) # mirrored draft in colleague mode
- saved_skill_id = Column(String(100)) # AdminSkill.skill_id after confirmed persistence
- cost_usd = Column(Numeric(8, 4), default=0)
- error = Column(Text)
- created_at = Column(UTCDateTime(timezone=True), default=utc_now)
- started_at = Column(UTCDateTime(timezone=True))
- finished_at = Column(UTCDateTime(timezone=True))
-
- __table_args__ = (
- CheckConstraint(
- "kind IN ('colleague', 'personal')",
- name="persona_distill_jobs_kind_check",
- ),
- CheckConstraint(
- "status IN ('queued','running','completed','failed','cancelled')",
- name="persona_distill_jobs_status_check",
- ),
- Index("idx_persona_distill_jobs_target", "target_user_id", "created_at"),
- Index("idx_persona_distill_jobs_status", "status"),
- )
diff --git a/src/backend/core/services/automation_notifications.py b/src/backend/features/automation/notifications.py
similarity index 100%
rename from src/backend/core/services/automation_notifications.py
rename to src/backend/features/automation/notifications.py
diff --git a/src/backend/core/services/automation_remote_effect.py b/src/backend/features/automation/remote_effect.py
similarity index 97%
rename from src/backend/core/services/automation_remote_effect.py
rename to src/backend/features/automation/remote_effect.py
index 7568a377..6a3aee83 100644
--- a/src/backend/core/services/automation_remote_effect.py
+++ b/src/backend/features/automation/remote_effect.py
@@ -3,8 +3,7 @@
import json
from sqlalchemy.exc import IntegrityError
-class RemoteOperationConflict(ValueError):
- """The authenticated operation key is already bound to another request."""
+from core.harness.registry.tool_operations import RemoteOperationConflict
RECEIPT_TOOLS = frozenset({
diff --git a/src/backend/features/automation/runs.py b/src/backend/features/automation/runs.py
new file mode 100644
index 00000000..0b22d120
--- /dev/null
+++ b/src/backend/features/automation/runs.py
@@ -0,0 +1,66 @@
+"""Conditional ownership writes for scheduled execution records."""
+
+from core.infra.time import utc_now
+from features.automation.models import ScheduledTaskRun
+
+
+class AutomationRunRepository:
+ def __init__(self, db):
+ self.db = db
+
+ def claim(self, run_id, lease_token, *, expected_token):
+ """Only a scheduler holding Redis ownership may transfer a running row."""
+ if not lease_token:
+ raise ValueError("Execution ownership token is required")
+ count = (
+ self.db.query(ScheduledTaskRun)
+ .filter(
+ ScheduledTaskRun.run_id == run_id,
+ ScheduledTaskRun.status.in_(("pending", "running")),
+ ScheduledTaskRun.lease_token == expected_token,
+ )
+ .update({ScheduledTaskRun.lease_token: lease_token, ScheduledTaskRun.status: "running"}, synchronize_session=False)
+ )
+ self.db.commit()
+ return count == 1
+
+ def complete(self, run_id, *, lease_token, commit=True, **values):
+ if not lease_token:
+ raise ValueError("Execution ownership token is required")
+ count = (
+ self.db.query(ScheduledTaskRun)
+ .filter(
+ ScheduledTaskRun.run_id == run_id,
+ ScheduledTaskRun.status == "running",
+ ScheduledTaskRun.lease_token == lease_token,
+ )
+ .update({**values, "completed_at": utc_now()}, synchronize_session=False)
+ )
+ if commit:
+ self.db.commit()
+ else:
+ self.db.flush()
+ if not count:
+ return None
+ self.db.expire_all()
+ return self.db.get(ScheduledTaskRun, run_id)
+
+
+def owned_run_guard(run_id, token):
+ """Lock the automation owner in the journal's cancellation transaction."""
+ if not token:
+ raise ValueError("Execution ownership token is required")
+
+ def guard(db):
+ row = (
+ db.query(ScheduledTaskRun)
+ .filter(
+ ScheduledTaskRun.run_id == run_id,
+ ScheduledTaskRun.lease_token == token,
+ )
+ .with_for_update()
+ .one_or_none()
+ )
+ return row is not None
+
+ return guard
diff --git a/src/backend/features/automation/scheduling.py b/src/backend/features/automation/scheduling.py
new file mode 100644
index 00000000..619ef9fa
--- /dev/null
+++ b/src/backend/features/automation/scheduling.py
@@ -0,0 +1,23 @@
+"""Timezone-aware cron scheduling shared by automation and personal-agent schedules."""
+from datetime import datetime
+from typing import Optional
+from croniter import croniter
+import pytz
+
+
+def compute_next_run(
+ cron_expression: str,
+ timezone: str = "Asia/Shanghai",
+ base_time: Optional[datetime] = None,
+) -> datetime:
+ """Compute the next fire time from a cron expression in the given timezone.
+
+ Returns a timezone-aware UTC datetime.
+ """
+ tz = pytz.timezone(timezone)
+ base = base_time or datetime.now(tz)
+ if base.tzinfo is None:
+ base = tz.localize(base)
+ cron = croniter(cron_expression, base)
+ next_local = cron.get_next(datetime)
+ return next_local.astimezone(pytz.utc)
diff --git a/src/backend/features/automation/schema.py b/src/backend/features/automation/schema.py
new file mode 100644
index 00000000..9ac2ca52
--- /dev/null
+++ b/src/backend/features/automation/schema.py
@@ -0,0 +1,59 @@
+"""Schema upgrades for automation admission and canonical delivery targets."""
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import inspect
+
+RUN_STATUSES = "status IN ('pending', 'running', 'success', 'failed')"
+
+
+def upgrade_queue(connection):
+ inspector = inspect(connection)
+ if not inspector.has_table("scheduled_task_runs"):
+ return
+ current = next((item for item in inspector.get_check_constraints("scheduled_task_runs")
+ if item.get("name") == "scheduled_task_runs_status_check"), None)
+ if current and "'pending'" in current["sqltext"]:
+ return
+ operations = Operations(MigrationContext.configure(connection))
+ with operations.batch_alter_table("scheduled_task_runs") as batch:
+ if current:
+ batch.drop_constraint("scheduled_task_runs_status_check", type_="check")
+ batch.create_check_constraint("scheduled_task_runs_status_check", RUN_STATUSES)
+
+
+def initialize_schema():
+ from core.db.engine import engine
+ with engine.begin() as connection:
+ upgrade_queue(connection)
+ upgrade_delivery_targets(connection)
+
+
+def upgrade_delivery_targets(connection):
+ """Migrate stored flat destinations once; request handling reads only the new form."""
+ from sqlalchemy import Column, JSON, MetaData, String, Table, select, text
+ if not inspect(connection).has_table("scheduled_tasks"):
+ return
+ tasks = Table("scheduled_tasks", MetaData(),
+ Column("task_id", String), Column("metadata", JSON))
+ predicate = (
+ "metadata::jsonb ?| array['channel_id', 'conversation_id']"
+ if connection.dialect.name == "postgresql" else
+ "json_type(metadata, '$.channel_id') IS NOT NULL OR "
+ "json_type(metadata, '$.conversation_id') IS NOT NULL"
+ )
+ query = select(tasks.c.task_id, tasks.c.metadata).where(text(predicate))
+ for task_id, data in connection.execute(query):
+ if not isinstance(data, dict) or not ({"channel_id", "conversation_id"} & data.keys()):
+ continue
+ data = dict(data)
+ channel = data.pop("channel_id", None)
+ conversation = data.pop("conversation_id", None)
+ targets = data.get("delivery_targets")
+ valid = [dict(item) for item in targets
+ if isinstance(item, dict) and item.get("type") in ("inapp", "channel", "email")] if isinstance(targets, list) else []
+ if not valid:
+ valid = [{"type": "inapp"}]
+ if channel and conversation:
+ valid.append({"type": "channel", "channel_id": channel, "conversation_id": conversation})
+ data["delivery_targets"] = valid
+ connection.execute(tasks.update().where(tasks.c.task_id == task_id).values(metadata=data))
diff --git a/src/backend/core/services/automation_service.py b/src/backend/features/automation/service.py
similarity index 73%
rename from src/backend/core/services/automation_service.py
rename to src/backend/features/automation/service.py
index 8cf34ed1..b1117403 100644
--- a/src/backend/core/services/automation_service.py
+++ b/src/backend/features/automation/service.py
@@ -4,12 +4,11 @@
from datetime import datetime, timezone
import uuid
-from croniter import croniter
-import pytz
+from features.automation import scheduling
from sqlalchemy import func, update
from sqlalchemy.orm import Session
-from core.db.models import ScheduledTask, ScheduledTaskRun
+from features.automation.models import ScheduledTask, ScheduledTaskRun
from core.infra.logging import get_logger
logger = get_logger(__name__)
@@ -33,12 +32,11 @@ def _channel_target_fields(extra_data: Optional[Dict[str, Any]]) -> Dict[str, An
"""Parse the first channel delivery target from a task's extra_data, filling in
{channel_id, conversation_id}.
- Handles both the delivery_targets list and the old flat form (resolve_delivery_targets
- unifies them). No channel target → both are None (= site-internal only). Exposed via
- task_to_dict to backfill the frontend edit form.
+ Reads delivery_targets for the frontend edit form. Without a channel target,
+ both fields are None.
"""
try:
- from core.services.delivery_targets import resolve_delivery_targets
+ from features.automation.delivery_targets import resolve_delivery_targets
tgt = next(
(t for t in resolve_delivery_targets(extra_data or {}) if t.get("type") == "channel"),
@@ -52,22 +50,6 @@ def _channel_target_fields(extra_data: Optional[Dict[str, Any]]) -> Dict[str, An
}
-def compute_next_run(
- cron_expression: str,
- timezone: str = "Asia/Shanghai",
- base_time: Optional[datetime] = None,
-) -> datetime:
- """Compute the next fire time from a cron expression in the given timezone.
-
- Returns a timezone-aware UTC datetime.
- """
- tz = pytz.timezone(timezone)
- base = base_time or datetime.now(tz)
- if base.tzinfo is None:
- base = tz.localize(base)
- cron = croniter(cron_expression, base)
- next_local = cron.get_next(datetime)
- return next_local.astimezone(pytz.utc)
class AutomationService:
@@ -100,7 +82,7 @@ def create_task(
execution_location: Optional[str] = None,
project_id: Optional[str] = None,
) -> ScheduledTask:
- from core.services.automation_execution import execution_metadata
+ from features.automation.execution import execution_metadata
binding = execution_metadata(self.db, user_id, location=execution_location,
project_id=project_id, prompt=prompt)
task_id = f"auto_{uuid.uuid4().hex[:16]}"
@@ -112,7 +94,7 @@ def create_task(
if schedule_type == "manual":
next_run = None
else:
- next_run = compute_next_run(cron_expression, timezone)
+ next_run = scheduling.compute_next_run(cron_expression, timezone)
if not recurring:
max_runs = 1
@@ -195,7 +177,7 @@ def update_task(
if not task:
return None
if "prompt" in kwargs:
- from core.services.automation_execution import execution_metadata
+ from features.automation.execution import execution_metadata
saved = task.extra_data or {}
execution_metadata(self.db, user_id, location=saved.get("execution_location"),
project_id=saved.get("project_id"), prompt=kwargs["prompt"])
@@ -210,7 +192,7 @@ def update_task(
if task.schedule_type == "manual":
task.next_run_at = None
elif "cron_expression" in kwargs or "schedule_type" in kwargs:
- task.next_run_at = compute_next_run(task.cron_expression, task.timezone)
+ task.next_run_at = scheduling.compute_next_run(task.cron_expression, task.timezone)
task.updated_at = datetime.now(timezone.utc)
if commit:
self.db.commit()
@@ -265,7 +247,7 @@ def resume_task(self, task_id: str, user_id: str) -> Optional[ScheduledTask]:
if not task or task.status != "paused":
return None
task.status = "active"
- task.next_run_at = compute_next_run(task.cron_expression, task.timezone)
+ task.next_run_at = scheduling.compute_next_run(task.cron_expression, task.timezone)
task.updated_at = datetime.now(timezone.utc)
self.db.commit()
self.db.refresh(task)
@@ -273,18 +255,6 @@ def resume_task(self, task_id: str, user_id: str) -> Optional[ScheduledTask]:
# ── Due tasks (for scheduler) ──────────────────────────────────
- def get_due_tasks(self, now: datetime) -> List[ScheduledTask]:
- # Manual-type tasks are never auto-fired; they can only be triggered via /trigger.
- return (
- self.db.query(ScheduledTask)
- .filter(
- ScheduledTask.status == "active",
- ScheduledTask.schedule_type != "manual",
- ScheduledTask.next_run_at <= now,
- )
- .all()
- )
-
@staticmethod
def _is_oneshot_done(task: ScheduledTask) -> bool:
"""True when no more runs are scheduled after the current one
@@ -298,30 +268,6 @@ def _is_oneshot_done(task: ScheduledTask) -> bool:
or bool(task.max_runs and (task.run_count or 0) >= task.max_runs)
)
- def advance_next_run(self, task_id: str) -> None:
- task = self.get_task_by_id(task_id)
- if not task:
- return
- task.run_count = (task.run_count or 0) + 1
-
- # Manual tasks: stay active, no next_run_at, run_count just grows.
- if task.schedule_type == "manual":
- task.next_run_at = None
- # One-shot / max_runs reached: clear next_run_at so the schedule stops
- # re-firing. Do NOT flip status to "completed" here — this runs as a
- # PRE-advance (before execute_task), and execute_task's guard refuses any
- # task whose status isn't active/paused. Marking it completed now would
- # make the executor skip the very run it just fired (never records a run,
- # never delivers to the channel). Terminal status is set AFTER the run by
- # finalize_after_run().
- elif self._is_oneshot_done(task):
- task.next_run_at = None
- else:
- task.next_run_at = compute_next_run(task.cron_expression, task.timezone)
-
- task.updated_at = datetime.now(timezone.utc)
- self.db.commit()
-
def request_manual_trigger(self, task: ScheduledTask) -> None:
"""登记一次「立即执行」,留给调度器取走。
@@ -335,44 +281,9 @@ def request_manual_trigger(self, task: ScheduledTask) -> None:
task.updated_at = task.manual_trigger_at
self.db.commit()
- def take_manual_triggers(self) -> List[Tuple[str, str]]:
- """取走所有待执行的手动触发,返回 (task_id, user_id)。
-
- 一条语句取走并清空,取到即归本次调用所有,不必再加行锁。
- """
- taken = self.db.execute(
- update(ScheduledTask)
- .where(ScheduledTask.manual_trigger_at.isnot(None))
- .values(manual_trigger_at=None)
- .returning(ScheduledTask.task_id, ScheduledTask.user_id)
- ).all()
- self.db.commit()
- return [(row.task_id, row.user_id) for row in taken]
-
- def bump_run_count(self, task_id: str) -> None:
- """手动触发路径专用的计数递增。
-
- run_count 原本只在 advance_next_run 里 +1,而那是**调度器轮询**的前置步骤。
- 手动「立即执行」和 schedule_type='manual' 的任务根本不走调度器(get_due_tasks
- 把 manual 排除在外),于是这些执行只落了一条 run 记录、累计次数却纹丝不动——
- 详情页出现「有 5 条执行记录、累计执行 0 次」的自相矛盾。
-
- 这里不动调度器那条路径的语义(那边 +1 与 next_run_at 推进必须原子发生),
- 只补上手动触发这一路的计数。
- """
- self.db.execute(
- update(ScheduledTask)
- .where(ScheduledTask.task_id == task_id)
- .values(
- run_count=func.coalesce(ScheduledTask.run_count, 0) + 1,
- updated_at=datetime.now(timezone.utc),
- )
- )
- self.db.commit()
-
def finalize_after_run(self, task_id: str) -> None:
"""Mark one-shot / exhausted tasks 'completed' once their run has
- finished. Recurring tasks stay active (advance_next_run already moved
+ finished. Recurring tasks stay active (admission already moved
their next_run_at forward). Never overrides an auto-disabled task."""
task = self.get_task_by_id(task_id)
if not task or task.status == "disabled":
@@ -385,42 +296,32 @@ def finalize_after_run(self, task_id: str) -> None:
# ── Run history ───────────────────────────────────────────────
- def record_run_start(self, task_id: str) -> ScheduledTaskRun:
- run = ScheduledTaskRun(
- run_id=f"run_{uuid.uuid4().hex[:16]}",
- task_id=task_id,
- status="running",
- started_at=datetime.now(timezone.utc),
- )
- self.db.add(run)
- self.db.commit()
- self.db.refresh(run)
- return run
-
def record_run_complete(
self,
run_id: str,
*,
+ lease_token: str,
status: str,
chat_id: Optional[str] = None,
result_summary: Optional[str] = None,
error_message: Optional[str] = None,
duration_ms: int = 0,
usage: Optional[Dict] = None,
+ commit: bool = True,
) -> Optional[ScheduledTaskRun]:
- run = self.db.query(ScheduledTaskRun).filter(ScheduledTaskRun.run_id == run_id).first()
- if not run:
- return None
- run.status = status
- run.chat_id = chat_id
- run.result_summary = result_summary
- run.error_message = error_message
- run.completed_at = datetime.now(timezone.utc)
- run.duration_ms = duration_ms
- run.usage = usage or {}
- self.db.commit()
- self.db.refresh(run)
- return run
+ from .runs import AutomationRunRepository
+
+ return AutomationRunRepository(self.db).complete(
+ run_id,
+ lease_token=lease_token,
+ status=status,
+ chat_id=chat_id,
+ result_summary=result_summary,
+ error_message=error_message,
+ duration_ms=duration_ms,
+ usage=usage or {},
+ commit=commit,
+ )
def get_task_runs(self, task_id: str, user_id: str, limit: int = 10) -> List[ScheduledTaskRun]:
task = self.get_task(task_id, user_id)
@@ -438,7 +339,7 @@ def get_task_runs(self, task_id: str, user_id: str, limit: int = 10) -> List[Sch
@staticmethod
def task_to_dict(task: ScheduledTask) -> Dict[str, Any]:
- from core.services.automation_execution import instance_location, device_identity
+ from features.automation.execution import instance_location, device_identity
location = (task.extra_data or {}).get("execution_location") or instance_location()
plan_title = None
if task.plan_id and task.plan:
@@ -475,7 +376,7 @@ def task_to_dict(task: ScheduledTask) -> Dict[str, Any]:
"description": task.description,
"sidebar_activated": bool(task.sidebar_activated),
# Channel delivery target (for frontend edit backfill): parse the first channel
- # target from delivery_targets / the old flat channel_id. None if absent (= site-internal only).
+ # target from delivery_targets. None if absent (= site-internal only).
**_channel_target_fields(task.extra_data),
"created_at": task.created_at.isoformat() if task.created_at else None,
"updated_at": task.updated_at.isoformat() if task.updated_at else None,
diff --git a/src/backend/features/automation/tool_routing.py b/src/backend/features/automation/tool_routing.py
new file mode 100644
index 00000000..5c62d479
--- /dev/null
+++ b/src/backend/features/automation/tool_routing.py
@@ -0,0 +1,116 @@
+"""Automation component execution across a desktop's two authorized backends."""
+from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
+import asyncio
+import json
+
+TOOLS = {
+ "create_scheduled_task": "create_task", "list_scheduled_tasks": "list_tasks",
+ "get_scheduled_task": "get_task", "update_scheduled_task": "update_task",
+ "pause_scheduled_task": "pause_task", "resume_scheduled_task": "resume_task",
+ "delete_scheduled_task": "delete_task",
+}
+
+
+def creation_arguments(arguments, user_id, chat_id):
+ from core.db.engine import SessionLocal
+ from core.db.models import ChatSession, Project
+ from features.desktop.auth.bridge import bridge_enabled
+ from features.automation.execution import instance_location
+
+ args = dict(arguments)
+ with SessionLocal() as db:
+ chat = db.get(ChatSession, chat_id) if chat_id else None
+ if not args.get("project_id") and chat and chat.user_id == user_id and chat.deleted_at is None:
+ if chat.project_id:
+ args["project_id"] = chat.project_id
+ project = db.get(Project, args["project_id"]) if args.get("project_id") else None
+ if not args.get("execution_location"):
+ args["execution_location"] = (
+ "local" if project and project.kind == "local"
+ else "cloud" if bridge_enabled() else instance_location()
+ )
+ return args
+
+
+def _local_target(db, tool_name, args, headers, effect):
+ """Resolve the local owner and arguments, or None when the call runs in the cloud."""
+ from core.db.models import ChatRun, ChatSession
+ from features.automation.models import ScheduledTask
+
+ run = db.get(ChatRun, effect.run_id) if effect else None
+ chat_id = next((v for k, v in headers.items() if k.lower() == "x-chat-id"), "")
+ chat = db.get(ChatSession, chat_id) if chat_id else None
+ user_id = run.user_id if run else chat.user_id if chat else None
+ if not user_id:
+ raise ValueError("无法确定本机任务的当前用户")
+ from features.desktop.capabilities.skill_account import current_local_user_id
+ # The same cloud account has a device-local shadow ID. Resolve its
+ # cloud-address-scoped mapping before comparing local resource owners.
+ if current_local_user_id() != user_id:
+ raise ValueError("本机任务所属账号与当前云端账号不匹配")
+ location = args.get("execution_location") or ""
+ if location not in {"", "local", "cloud"}:
+ raise ValueError("执行位置必须为本机或云端")
+ if tool_name == "create_scheduled_task":
+ args = creation_arguments(args, user_id, run.chat_id if run else chat_id)
+ location = args["execution_location"]
+ if location == "cloud" and args.get("project_id"):
+ from core.db.models import Project
+ project = db.get(Project, args["project_id"])
+ if project is not None and project.kind == "local":
+ raise ValueError("本机项目不能在云端执行,请选择本机")
+ elif not location and args.get("task_ref"):
+ local = db.get(ScheduledTask, args["task_ref"])
+ if local and local.user_id == user_id:
+ location = "local"
+ elif db.query(ScheduledTask).filter(
+ ScheduledTask.user_id == user_id,
+ ScheduledTask.name.ilike(f"%{args['task_ref']}%"),
+ ).first() is not None:
+ raise ValueError("本机存在同名任务,请明确 execution_location 或使用任务 ID")
+ merge_cloud = tool_name == "list_scheduled_tasks" and not location
+ if merge_cloud:
+ location = "local"
+ if location != "local":
+ return None
+ return user_id, args, merge_cloud
+
+
+async def local_automation_result(source_plugin, tool_name, arguments, headers, *, authorize=None):
+ """Return None for cloud execution; local dispatch requires the automation component."""
+ if source_plugin != "automation" or tool_name not in TOOLS:
+ return None
+ from core.db.threads import run_in_session
+ from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT
+ from features.automation.execution import instance_location
+
+ if instance_location() != "local":
+ return None
+ identity_state = bridge_identity.get_state()
+ bridge_identity.ensure_current_authorization(identity_state)
+ # User identity belongs to the durable local run, not model arguments.
+ target = await run_in_session(
+ _local_target, tool_name, dict(arguments), headers, CURRENT_TOOL_EFFECT.get()
+ )
+ if target is None:
+ return None
+ user_id, args, merge_cloud = target
+ if authorize is None:
+ raise ValueError("本机工具缺少当前授权校验")
+ await authorize()
+ bridge_identity.ensure_current_authorization(identity_state)
+ if tool_name != "create_scheduled_task":
+ args.pop("execution_location", None)
+ from extensions.mcp_servers.automation_task_mcp import impl
+ def dispatch():
+ # Account changes must not race the queued local database operation.
+ with bridge_identity.account_scope(identity_state):
+ return getattr(impl, TOOLS[tool_name])(user_id=user_id, **args)
+
+ result = await asyncio.to_thread(dispatch)
+ from agentscope.message import TextBlock, ToolResultState
+ from agentscope.tool._response import ToolChunk
+ return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))],
+ state=ToolResultState.SUCCESS if result.get("ok") else ToolResultState.ERROR,
+ metadata={"origin": "local", "execution_location": "local", "merge_cloud": merge_cloud})
diff --git a/src/backend/features/desktop/README.md b/src/backend/features/desktop/README.md
new file mode 100644
index 00000000..af2dce1e
--- /dev/null
+++ b/src/backend/features/desktop/README.md
@@ -0,0 +1,11 @@
+# Desktop
+
+Owns the device capability store, candidate resolution, runtime views, cloud capability authorization, workcopies and desktop observation records. Desktop schemas are installed on cloud and local hosts independently of whether local file-store execution is enabled.
+
+The host registers `run_capabilities.provider.DesktopRunCapabilityProvider` through the core run-capability interface. `capabilities.paths` controls whether the current runtime has a local capability root. Selection and account authorization are evaluated at use time.
+
+Installation records and their SQLAlchemy commit/rollback notifications live in `capabilities.registry`; immutable row projections live in `registry_types`. Skill candidates, hashing, identity, views and publication use one shared cache/lock state. Cloud capability API groups preserve the original route order and observation route class; model and MCP streaming share one HTTP client and close each upstream response.
+
+Core dependencies include database/session primitives, extension ports, sandbox runtime and capability package parsing. Direct feature dependencies are marketplace archives and knowledge access permissions. Workbench presentation uses a host-injected package view. Sites register upload handlers and receipt localization; automation registers local dispatch and durable operation callbacks. Desktop does not import those component implementations, so feature dependencies remain acyclic. Remaining core-to-feature layering is handled in P6.
+
+Verification covers selection/account revocation, projection persistence, token tampering and expiry, gateway credential/header isolation, streaming response cleanup, workcopy compare-and-swap, original route/schema snapshots, and CE empty-database upgrade/release checks.
diff --git a/src/backend/features/desktop/__init__.py b/src/backend/features/desktop/__init__.py
new file mode 100644
index 00000000..2e93078f
--- /dev/null
+++ b/src/backend/features/desktop/__init__.py
@@ -0,0 +1 @@
+"""desktop package."""
diff --git a/src/backend/features/desktop/api/__init__.py b/src/backend/features/desktop/api/__init__.py
new file mode 100644
index 00000000..27806988
--- /dev/null
+++ b/src/backend/features/desktop/api/__init__.py
@@ -0,0 +1 @@
+"""api package."""
diff --git a/src/backend/features/desktop/api/capability_auth.py b/src/backend/features/desktop/api/capability_auth.py
new file mode 100644
index 00000000..21eacab9
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_auth.py
@@ -0,0 +1,48 @@
+"""capability auth responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+from core.infra.responses import success_response
+from fastapi import HTTPException, Request, Response
+from pydantic import BaseModel, Field
+from features.desktop.api import desktop_capability_state
+
+class CapabilityTokenBody(BaseModel):
+ device_id: str = Field(
+ ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
+ )
+
+
+@router.post("/token", summary="签发桌面能力令牌")
+async def issue_token(body: CapabilityTokenBody, request: Request, response: Response):
+ """Only an actual live cloud cookie session can mint a desktop credential."""
+ import hashlib
+
+ from core.auth.session import validate_session
+ from core.config.settings import settings
+ from features.desktop.capability.service import issue_capability_token, session_authorization_epoch
+
+ cookie = request.cookies.get(settings.session.cookie_name, "")
+ try:
+ current = await validate_session(cookie) if cookie else None
+ except Exception:
+ current = None
+ if (
+ not current
+ or not current.get("user_id")
+ or not current.get("user_center_id")
+ or not session_authorization_epoch(current)
+ ):
+ raise HTTPException(status_code=401, detail="valid cloud session required")
+ data = issue_capability_token(
+ str(current["user_id"]),
+ device_id=body.device_id,
+ issuer=str(request.base_url),
+ session_hash=hashlib.sha256(cookie.encode()).hexdigest(),
+ authorization_epoch=session_authorization_epoch(current),
+ user_center_id=str(current["user_center_id"]),
+ )
+ response.headers["Cache-Control"] = "no-store"
+ response.headers["Pragma"] = "no-cache"
+ return success_response(data=data)
diff --git a/src/backend/features/desktop/api/capability_bridge.py b/src/backend/features/desktop/api/capability_bridge.py
new file mode 100644
index 00000000..a1b372d8
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_bridge.py
@@ -0,0 +1,92 @@
+"""capability bridge responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+import asyncio
+from core.auth.backend import UserContext, get_current_user
+from core.infra.logging import quiet_access_log
+from core.infra.responses import success_response
+from fastapi import Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.api import desktop_capability_state
+
+class CloudBridgeBody(BaseModel):
+ cloud_base: str = Field(..., min_length=1, description="云端后端根地址(含协议)")
+ token: str = Field(..., min_length=8, description="capability token")
+ expires_in: int = Field(default=600, ge=60, le=600)
+ device_id: str = Field(
+ ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
+ )
+
+
+def require_desktop_bridge_process() -> None:
+ from features.desktop.auth.bridge import bridge_enabled
+
+ if not bridge_enabled():
+ raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用")
+
+
+def require_desktop_shell_control(request: Request) -> None:
+ from features.desktop.capability.service import is_desktop_shell_control
+
+ require_desktop_bridge_process()
+ if not is_desktop_shell_control(
+ request.headers.get("authorization", ""), request.headers.get("origin")
+ ):
+ raise HTTPException(status_code=401, detail="desktop shell authorization required")
+
+
+@router.post("/cloud-bridge", summary="推送云端能力桥配置(桌面壳 → 本机后端)")
+async def set_cloud_bridge(
+ body: CloudBridgeBody,
+ _: None = Depends(require_desktop_shell_control),
+):
+ require_desktop_bridge_process()
+ from features.desktop.bridge.client import set_state
+
+ base = body.cloud_base.strip().rstrip("/")
+ from urllib.parse import urlsplit
+
+ try:
+ parsed = urlsplit(base)
+ except ValueError:
+ raise HTTPException(status_code=400, detail="cloud_base 地址无效")
+ if (
+ parsed.scheme not in ("http", "https")
+ or not parsed.hostname
+ or parsed.username
+ or parsed.password
+ or parsed.query
+ or parsed.fragment
+ ):
+ raise HTTPException(status_code=400, detail="cloud_base 必须是不含凭据的 http(s) 地址")
+ set_state(base, body.token, body.expires_in, device_id=body.device_id)
+ logger.info("[cloud-bridge] 桥配置已更新(cloud_base=%s)", base)
+ return success_response(data={"ok": True})
+
+
+@router.delete("/cloud-bridge", summary="清除云端能力桥(桌面退出登录)")
+async def clear_cloud_bridge(_: None = Depends(require_desktop_shell_control)):
+ require_desktop_bridge_process()
+ from features.desktop.bridge.identity import clear_state
+
+ clear_state()
+ return success_response(data={"ok": True})
+
+
+@router.get("/cloud-bridge/status", summary="云端能力桥状态")
+async def cloud_bridge_status(_user: UserContext = Depends(get_current_user)):
+ from features.desktop.bridge.status import bridge_status
+
+ return success_response(data=bridge_status())
+
+
+@router.get("/cloud-bridge/readiness", summary="桌面壳首次能力同步进度")
+@quiet_access_log
+async def cloud_bridge_readiness(_: None = Depends(require_desktop_shell_control)):
+ from features.desktop.bridge.status import initial_sync_status
+
+ return success_response(data=await asyncio.to_thread(initial_sync_status))
diff --git a/src/backend/features/desktop/api/capability_dependencies.py b/src/backend/features/desktop/api/capability_dependencies.py
new file mode 100644
index 00000000..1664ee4c
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_dependencies.py
@@ -0,0 +1,107 @@
+"""capability dependencies responsibilities."""
+from __future__ import annotations
+from typing import Optional
+import httpx
+from core.infra.responses import success_response
+from fastapi import Depends, HTTPException, Request, Response
+from fastapi.security import HTTPAuthorizationCredentials
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.api import desktop_capability_state
+
+def client() -> httpx.AsyncClient:
+ pass # State is owned by its responsibility module.
+ if desktop_capability_state.gateway_client is None:
+ desktop_capability_state.gateway_client = httpx.AsyncClient(
+ timeout=httpx.Timeout(600.0),
+ # 桌面端两条消息之间常常隔着几十秒;默认 5s 的 keepalive 意味着几乎
+ # 每次模型调用都要重新 TCP + TLS 握手。
+ limits=httpx.Limits(max_keepalive_connections=64, keepalive_expiry=300.0),
+ )
+ return desktop_capability_state.gateway_client
+
+
+async def require_capability_user(
+ request: Request,
+ credentials: Optional[HTTPAuthorizationCredentials] = Depends(desktop_capability_state.bearer),
+) -> str:
+ from features.desktop.capability.service import CAPABILITY_DEVICE_HEADER, verify_capability_token
+
+ token = credentials.credentials if credentials else ""
+ user_id = await verify_capability_token(
+ token,
+ device_id=request.headers.get(CAPABILITY_DEVICE_HEADER, ""),
+ issuer=str(request.base_url),
+ )
+ if not user_id:
+ raise HTTPException(status_code=401, detail="invalid capability token")
+ request.state.desktop_observation_user = user_id
+ return user_id
+
+
+def public_content(user_id: str, value, *, bundle: bool = False):
+ from features.desktop.capability.service import CapabilityContentRejected, guard_capability_bundle, guard_capability_content
+
+ try:
+ if callable(value):
+ value = value()
+ return (
+ guard_capability_bundle(user_id, value)
+ if bundle
+ else guard_capability_content(user_id, value)
+ )
+ except CapabilityContentRejected:
+ raise HTTPException(
+ status_code=422,
+ detail={
+ "code": "integrity_failed",
+ "message": "capability content blocked by credential policy",
+ },
+ ) from None
+
+
+def stream_secrets(user_id: str, target: dict) -> set[str]:
+ from features.desktop.capability.service import CapabilityContentRejected, gateway_stream_secrets
+
+ try:
+ return gateway_stream_secrets(user_id, target)
+ except CapabilityContentRejected:
+ raise HTTPException(
+ status_code=422,
+ detail={"code": "integrity_failed", "message": "credential policy unavailable"},
+ ) from None
+
+
+async def checked_upstream_bytes(upstream: httpx.Response, secrets: set[str]):
+ from features.desktop.capability.service import CapabilityContentRejected, guard_capability_stream
+
+ try:
+ async for chunk in guard_capability_stream(upstream.aiter_raw(), secrets):
+ yield chunk
+ except CapabilityContentRejected:
+ logger.warning("[desktop-capability] upstream content blocked code=integrity_failed")
+ # Streaming headers may already be sent. Return only a fixed diagnostic;
+ # the buffered bytes containing the credential are never released.
+ yield b'data: {"error":{"code":"integrity_failed","message":"upstream content blocked"}}\n\n'
+ finally:
+ await upstream.aclose()
+
+
+def entity_manifest_response(request: Request, response: Response, manifest: dict):
+ etag = f'"{manifest["revision"]}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ response.headers.update(headers)
+ return success_response(data=manifest)
+
+
+def bundle_response(request: Request, resolved):
+ if resolved is None:
+ raise HTTPException(status_code=404, detail="not available")
+ data, content_hash = resolved
+ etag = f'"{content_hash}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ return Response(content=data, media_type="application/zip", headers=headers)
diff --git a/src/backend/features/desktop/api/capability_manifests.py b/src/backend/features/desktop/api/capability_manifests.py
new file mode 100644
index 00000000..720a0ac1
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_manifests.py
@@ -0,0 +1,145 @@
+"""capability manifests responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+from core.infra.responses import success_response
+from fastapi import Depends, HTTPException, Request, Response
+from starlette.concurrency import run_in_threadpool
+from features.desktop.api import capability_dependencies
+from features.desktop.api import desktop_capability_state
+
+@router.get("/manifest", summary="当前用户的云端能力清单")
+async def get_manifest(
+ request: Request,
+ response: Response,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import build_user_capability_manifest
+
+ use_cache = request.headers.get("cache-control") != "no-cache"
+ manifest = await run_in_threadpool(
+ capability_dependencies.public_content, user_id,
+ lambda: build_user_capability_manifest(user_id, use_cache=use_cache),
+ )
+ etag = f'"{manifest["revision"]}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ response.headers.update(headers)
+ return success_response(data=manifest)
+
+
+@router.get("/models", summary="桌面本机执行面的无密钥模型清单")
+async def get_model_manifest(
+ request: Request,
+ response: Response,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import build_user_model_manifest
+
+ manifest = await run_in_threadpool(
+ capability_dependencies.public_content, user_id, lambda: build_user_model_manifest(user_id)
+ )
+ etag = f'"{manifest["revision"]}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ response.headers.update(headers)
+ return success_response(data=manifest)
+
+
+@router.get("/skills/manifest", summary="当前用户的云端技能清单")
+async def get_skill_manifest(
+ request: Request,
+ response: Response,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import build_user_skill_manifest
+
+ use_cache = request.headers.get("cache-control") != "no-cache"
+ manifest = await run_in_threadpool(
+ capability_dependencies.public_content, user_id,
+ lambda: build_user_skill_manifest(user_id, use_cache=use_cache),
+ )
+ etag = f'"{manifest["revision"]}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ response.headers.update(headers)
+ return success_response(data=manifest)
+
+
+@router.get("/skills/{skill_id}/bundle", summary="下载一个当前授权技能的完整 zip 包")
+async def get_skill_bundle(
+ skill_id: str,
+ request: Request,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import resolve_skill_bundle
+
+ resolved = capability_dependencies.public_content(
+ user_id, lambda: resolve_skill_bundle(user_id, skill_id), bundle=True
+ )
+ if resolved is None:
+ raise HTTPException(status_code=404, detail="skill not available")
+ data, content_hash = resolved
+ etag = f'"{content_hash}"'
+ headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
+ if request.headers.get("if-none-match") == etag:
+ return Response(status_code=304, headers=headers)
+ return Response(content=data, media_type="application/zip", headers=headers)
+
+
+@router.get("/agents/manifest", summary="当前用户可见的智能体定义清单")
+async def get_agent_manifest(
+ request: Request, response: Response, user_id: str = Depends(capability_dependencies.require_capability_user)
+):
+ from features.desktop.capability.service import build_user_agent_manifest
+
+ use_cache = request.headers.get("cache-control") != "no-cache"
+ manifest = await run_in_threadpool(
+ capability_dependencies.public_content, user_id,
+ lambda: build_user_agent_manifest(user_id, use_cache=use_cache),
+ )
+ return capability_dependencies.entity_manifest_response(request, response, manifest)
+
+
+@router.get(
+ "/agents/{agent_id}/bundle", summary="下载一个智能体定义包(agent.json + instructions.md)"
+)
+async def get_agent_bundle(
+ agent_id: str, request: Request, user_id: str = Depends(capability_dependencies.require_capability_user)
+):
+ from features.desktop.capability.service import resolve_agent_bundle
+
+ return capability_dependencies.bundle_response(
+ request,
+ capability_dependencies.public_content(user_id, lambda: resolve_agent_bundle(user_id, agent_id), bundle=True),
+ )
+
+
+@router.get("/plugins/manifest", summary="当前用户已安装插件清单")
+async def get_plugin_manifest(
+ request: Request, response: Response, user_id: str = Depends(capability_dependencies.require_capability_user)
+):
+ from features.desktop.capability.service import build_user_plugin_manifest
+
+ use_cache = request.headers.get("cache-control") != "no-cache"
+ manifest = await run_in_threadpool(
+ capability_dependencies.public_content, user_id,
+ lambda: build_user_plugin_manifest(user_id, use_cache=use_cache),
+ )
+ return capability_dependencies.entity_manifest_response(request, response, manifest)
+
+
+@router.get("/plugins/{install_id}/bundle", summary="下载一个插件定义包(plugin.json)")
+async def get_plugin_bundle(
+ install_id: str, request: Request, user_id: str = Depends(capability_dependencies.require_capability_user)
+):
+ from features.desktop.capability.service import resolve_plugin_bundle
+
+ return capability_dependencies.bundle_response(
+ request,
+ capability_dependencies.public_content(user_id, lambda: resolve_plugin_bundle(user_id, install_id), bundle=True),
+ )
diff --git a/src/backend/features/desktop/api/capability_mcp_gateway.py b/src/backend/features/desktop/api/capability_mcp_gateway.py
new file mode 100644
index 00000000..27eddbb4
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_mcp_gateway.py
@@ -0,0 +1,79 @@
+"""capability mcp gateway responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+import httpx
+from fastapi import Depends, HTTPException, Request
+from starlette.background import BackgroundTask
+from starlette.concurrency import run_in_threadpool
+from starlette.responses import StreamingResponse
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.api import capability_dependencies
+from features.desktop.api import desktop_capability_state
+
+@router.api_route(
+ "/gateway/{server_id}/mcp",
+ methods=["POST", "GET", "DELETE"],
+ summary="桌面能力网关(MCP streamable-http 反代)",
+)
+async def gateway_mcp(
+ server_id: str,
+ request: Request,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ # 授权解析走 30s per-user 缓存、内部短会话——不经 Depends(get_db),
+ # 避免流式转发全程占住连接池里的一条 DB 连接。
+ from features.desktop.capability.service import resolve_gateway_target
+
+ target = await run_in_threadpool(resolve_gateway_target, user_id, server_id)
+ if target is None:
+ raise HTTPException(status_code=404, detail="server not available")
+
+ headers: dict[str, str] = {
+ k: v for k, v in request.headers.items() if k.lower() not in desktop_capability_state.DROP_REQUEST_HEADERS
+ }
+ # 云端侧连接凭据(远程第三方 MCP 的 headers 等)只在此处物化,不出云端。
+ for k, v in (target.get("headers") or {}).items():
+ if isinstance(k, str) and isinstance(v, str):
+ headers[k] = v
+ # 身份以 capability token 为准,客户端注入的同名头已在 deny-list 拦下。
+ headers["X-Current-User-Id"] = user_id
+ # StreamingResponse 使用 aiter_raw 原样转发,因此不允许上游压缩后再丢失
+ # Content-Encoding;这是旧客户端透明 MCP 端点的传输完整性要求。
+ headers["accept-encoding"] = "identity"
+
+ body = await request.body()
+ response_secrets = await run_in_threadpool(capability_dependencies.stream_secrets, user_id, target)
+ client = capability_dependencies.client()
+ upstream_req = client.build_request(
+ request.method, target["url"], headers=headers, content=body
+ )
+ try:
+ upstream = await client.send(upstream_req, stream=True)
+ except httpx.HTTPError as exc:
+ logger.warning(
+ "[desktop-capability] gateway upstream error user=%s server=%s error_type=%s",
+ user_id,
+ server_id,
+ type(exc).__name__,
+ )
+ raise HTTPException(status_code=502, detail="upstream mcp unreachable")
+
+ logger.info(
+ "[desktop-capability] gateway %s user=%s server=%s status=%s",
+ request.method,
+ user_id,
+ server_id,
+ upstream.status_code,
+ )
+ resp_headers = {
+ name: upstream.headers[name] for name in desktop_capability_state.FWD_RESPONSE_HEADERS if name in upstream.headers
+ }
+ return StreamingResponse(
+ capability_dependencies.checked_upstream_bytes(upstream, response_secrets),
+ status_code=upstream.status_code,
+ headers=resp_headers,
+ background=BackgroundTask(upstream.aclose),
+ )
diff --git a/src/backend/features/desktop/api/capability_model_gateway.py b/src/backend/features/desktop/api/capability_model_gateway.py
new file mode 100644
index 00000000..d980500b
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_model_gateway.py
@@ -0,0 +1,89 @@
+"""capability model gateway responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+import json
+import httpx
+from fastapi import Depends, HTTPException, Request
+from starlette.background import BackgroundTask
+from starlette.concurrency import run_in_threadpool
+from starlette.responses import StreamingResponse
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.api import capability_dependencies
+from features.desktop.api import desktop_capability_state
+
+@router.post(
+ "/gateway/models/{provider_id}/{model_path:path}",
+ summary="桌面模型网关(OpenAI-compatible 流式反代)",
+)
+async def gateway_model(
+ provider_id: str,
+ model_path: str,
+ request: Request,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ """把本机 Agent 的对话/向量/重排请求转发到云端内网模型。
+
+ 路径、模型名和上游凭据全由云端 DB 重算;客户端提供的
+ Authorization / x-api-key / model 字段都不可信。
+ """
+ from features.desktop.capability.service import resolve_model_gateway_target
+
+ target = await run_in_threadpool(resolve_model_gateway_target, user_id, provider_id)
+ if target is None or model_path.strip("/") != target["path"]:
+ raise HTTPException(status_code=404, detail="model not available")
+
+ try:
+ payload = await request.json()
+ except (json.JSONDecodeError, UnicodeDecodeError, ValueError):
+ raise HTTPException(status_code=400, detail="model request must be JSON")
+ if not isinstance(payload, dict):
+ raise HTTPException(status_code=400, detail="model request must be a JSON object")
+ payload["model"] = target["model_name"]
+ request.state.desktop_observation_model = target["model_name"]
+
+ headers = {
+ "authorization": f"Bearer {target['api_key']}",
+ "content-type": "application/json",
+ "accept": request.headers.get("accept", "application/json"),
+ # aiter_raw 保留原始字节,因此明确要求上游不压缩,避免在没有
+ # Content-Encoding 响应头的情况下把 gzip 字节直接送给 OpenAI SDK。
+ "accept-encoding": "identity",
+ }
+ response_secrets = await run_in_threadpool(capability_dependencies.stream_secrets, user_id, target)
+ client = capability_dependencies.client()
+ upstream_req = client.build_request(
+ "POST",
+ target["url"],
+ headers=headers,
+ content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
+ )
+ try:
+ upstream = await client.send(upstream_req, stream=True)
+ except httpx.HTTPError as exc:
+ logger.warning(
+ "[desktop-capability] model gateway upstream error user=%s provider=%s error_type=%s",
+ user_id,
+ provider_id,
+ type(exc).__name__,
+ )
+ raise HTTPException(status_code=502, detail="upstream model unreachable")
+
+ logger.info(
+ "[desktop-capability] model gateway user=%s provider=%s type=%s status=%s",
+ user_id,
+ provider_id,
+ target["provider_type"],
+ upstream.status_code,
+ )
+ response_headers = {}
+ if "content-type" in upstream.headers:
+ response_headers["content-type"] = upstream.headers["content-type"]
+ return StreamingResponse(
+ capability_dependencies.checked_upstream_bytes(upstream, response_secrets),
+ status_code=upstream.status_code,
+ headers=response_headers,
+ background=BackgroundTask(upstream.aclose),
+ )
diff --git a/src/backend/features/desktop/api/capability_tools.py b/src/backend/features/desktop/api/capability_tools.py
new file mode 100644
index 00000000..6ca1922a
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_tools.py
@@ -0,0 +1,192 @@
+"""capability tools responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+import asyncio
+import time
+from core.infra.responses import success_response
+from fastapi import Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+from starlette.concurrency import run_in_threadpool
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.api import capability_dependencies
+from features.desktop.api import desktop_capability_state
+
+class GatewayToolCallBody(BaseModel):
+ tool_name: str = Field(..., min_length=1, max_length=200)
+ arguments: dict = Field(default_factory=dict)
+ schema_hash: str = Field(..., min_length=64, max_length=64)
+
+
+@router.post("/gateway/{server_id}/site-publish", summary="上传桌面构建产物并云端托管")
+async def gateway_site_publish(
+ server_id: str,
+ request: Request,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ import tarfile
+
+ from core.infra.exceptions import AppException
+ from features.desktop.capability.service import resolve_gateway_tool
+ from core.capabilities.manifest_protocol import CapabilityManifestStaleError
+ from core.harness.registry.transfer_channels import MAX_UPLOAD_OPTIONS_CHARS, UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER, endpoint_plugin
+ from core.harness.registry.transfer_channels import upload_receiver
+ receiver = upload_receiver("site-publish")
+ from pydantic import ValidationError
+
+ try:
+ resolved = await run_in_threadpool(
+ resolve_gateway_tool,
+ user_id,
+ server_id,
+ "publish_site",
+ schema_hash=request.headers.get(UPLOAD_SCHEMA_HEADER, ""),
+ )
+ except CapabilityManifestStaleError:
+ raise HTTPException(status_code=409, detail="capability manifest changed") from None
+ if resolved is None or str(
+ resolved["target"].get("source_plugin") or ""
+ ) != endpoint_plugin("site-publish"):
+ raise HTTPException(status_code=403, detail="site publishing is not authorized")
+ raw_options = request.headers.get(UPLOAD_OPTIONS_HEADER, "{}")
+ if len(raw_options) > MAX_UPLOAD_OPTIONS_CHARS:
+ raise HTTPException(status_code=400, detail="site options too large")
+ try:
+ options = receiver.options_model.model_validate_json(raw_options)
+ except ValidationError:
+ raise HTTPException(status_code=400, detail="invalid site options") from None
+ archive = bytearray()
+ async for chunk in request.stream():
+ if len(archive) + len(chunk) > receiver.max_bytes:
+ raise HTTPException(status_code=413, detail="site archive exceeds 40 MB")
+ archive.extend(chunk)
+ try:
+ result = await asyncio.to_thread(receiver.publish, user_id, bytes(archive), options)
+ except (ValueError, tarfile.TarError):
+ raise HTTPException(status_code=400, detail="invalid site build archive") from None
+ except AppException as exc:
+ raise HTTPException(status_code=400, detail=exc.message) from None
+ return success_response(data=capability_dependencies.public_content(user_id, result))
+
+
+@router.post(
+ "/gateway/{server_id}/call",
+ summary="桌面云端工具调用网关(动态 manifest schema + JSON invocation)",
+)
+async def gateway_mcp_call(
+ server_id: str,
+ body: GatewayToolCallBody,
+ request: Request,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ """Execute a currently-authorized MCP tool from the cloud network."""
+ from features.desktop.capability.service import CapabilityContentRejected, invoke_gateway_tool, resolve_gateway_tool
+ from core.capabilities.manifest_protocol import CapabilityManifestStaleError
+
+ try:
+ resolved = await run_in_threadpool(
+ resolve_gateway_tool,
+ user_id,
+ server_id,
+ body.tool_name,
+ schema_hash=body.schema_hash,
+ )
+ except CapabilityManifestStaleError as exc:
+ raise HTTPException(status_code=409, detail="capability manifest changed") from exc
+ if resolved is None:
+ raise HTTPException(status_code=404, detail="tool not available")
+
+ runtime_headers = {
+ k: v for k, v in request.headers.items() if k.lower() in desktop_capability_state.RUNTIME_CONTEXT_HEADERS
+ }
+ from core.harness.registry.tool_operations import RemoteOperationConflict
+ started = time.monotonic()
+ try:
+ result = await invoke_gateway_tool(resolved, body.arguments, runtime_headers)
+ except HTTPException:
+ from extensions.mcp_servers.registry import gateway_server
+ if gateway_server(resolved["server_id"]) is not None:
+ raise
+ raise HTTPException(502, "Cloud tool call failed") from None
+ except RemoteOperationConflict:
+ raise HTTPException(status_code=400, detail="operation key already bound to different arguments") from None
+ except CapabilityContentRejected:
+ raise HTTPException(
+ status_code=422,
+ detail={"code": "integrity_failed", "message": "upstream content blocked"},
+ ) from None
+ except asyncio.TimeoutError as exc:
+ logger.warning(
+ "[desktop-capability] tool call timeout user=%s server=%s tool=%s",
+ user_id,
+ server_id,
+ body.tool_name,
+ )
+ raise HTTPException(status_code=504, detail="upstream mcp tool timed out") from exc
+ except Exception as exc: # noqa: BLE001 - never expose cloud credentials/errors
+ logger.warning(
+ "[desktop-capability] tool call failed user=%s server=%s tool=%s error_type=%s",
+ user_id,
+ server_id,
+ body.tool_name,
+ type(exc).__name__,
+ )
+ raise HTTPException(status_code=502, detail="upstream mcp tool failed") from exc
+
+ logger.info(
+ "[desktop-capability] tool call user=%s server=%s tool=%s duration_ms=%.0f",
+ user_id,
+ server_id,
+ body.tool_name,
+ (time.monotonic() - started) * 1000,
+ )
+ return success_response(data=capability_dependencies.public_content(user_id, result))
+
+
+class GatewayReceiptBody(BaseModel):
+ tool_name: str = Field(..., min_length=1, max_length=128)
+ schema_hash: str = Field(..., min_length=64, max_length=64)
+ operation_id: str = Field(..., min_length=1, max_length=128)
+
+
+@router.post("/gateway/{server_id}/authorize", summary="校验本机任务工具的当前授权")
+async def gateway_authorize_local(
+ server_id: str, body: GatewayReceiptBody,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import resolve_gateway_tool
+ from core.capabilities.manifest_protocol import CapabilityManifestStaleError
+ from core.harness.registry.tool_operations import local_tool_allowed
+ try:
+ resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id,
+ body.tool_name, schema_hash=body.schema_hash)
+ except CapabilityManifestStaleError:
+ raise HTTPException(status_code=409, detail="capability manifest changed") from None
+ if (resolved is None or not local_tool_allowed(
+ resolved["target"].get("source_plugin", ""), body.tool_name)):
+ raise HTTPException(status_code=403, detail="local automation is not authorized")
+ return success_response(data={"authorized": True})
+
+
+@router.post("/gateway/{server_id}/receipt", summary="查询云端操作回执")
+async def gateway_effect_receipt(
+ server_id: str, body: GatewayReceiptBody,
+ user_id: str = Depends(capability_dependencies.require_capability_user),
+):
+ from features.desktop.capability.service import resolve_gateway_tool
+ from core.capabilities.manifest_protocol import CapabilityManifestStaleError
+ from core.harness.registry.tool_operations import receipt_operations
+
+ try:
+ resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id,
+ body.tool_name, schema_hash=body.schema_hash)
+ except CapabilityManifestStaleError:
+ raise HTTPException(status_code=409, detail="capability manifest changed") from None
+ operations = receipt_operations(body.tool_name)
+ if resolved is None or operations is None:
+ raise HTTPException(status_code=404, detail="tool not available")
+ result = await run_in_threadpool(operations.lookup, user_id, server_id,
+ body.tool_name, body.operation_id)
+ return success_response(data=capability_dependencies.public_content(user_id, result))
diff --git a/src/backend/features/desktop/api/capability_workcopies.py b/src/backend/features/desktop/api/capability_workcopies.py
new file mode 100644
index 00000000..9152c456
--- /dev/null
+++ b/src/backend/features/desktop/api/capability_workcopies.py
@@ -0,0 +1,38 @@
+"""capability workcopies responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+from features.desktop.gateway.observability import ObservedGatewayRoute
+router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute)
+from core.infra.responses import success_response
+from fastapi import Depends, Response
+from pydantic import BaseModel, Field
+from starlette.concurrency import run_in_threadpool
+from features.desktop.api import capability_dependencies
+from features.desktop.api import desktop_capability_state
+
+class WorkcopyCommitBody(BaseModel):
+ files: dict[str, str]
+ expected_revision: str | None = None
+ expected_model_version: str | None = None
+ request_id: str = Field(pattern=r"^[a-f0-9]{32}$")
+ acknowledge_sensitive: bool = False
+ create_only: bool = False
+
+
+@router.get("/workcopies/{kind}/{key}", summary="比较当前账号的能力文件版本")
+async def workcopy_snapshot(kind: str, key: str, response: Response,
+ user_id: str = Depends(capability_dependencies.require_capability_user)):
+ from features.desktop.capability.workcopies import snapshot
+ response.headers["Cache-Control"] = "no-store"
+ result = await run_in_threadpool(snapshot, user_id, kind, key)
+ return success_response(data=await run_in_threadpool(capability_dependencies.public_content, user_id, result))
+
+
+@router.post("/workcopies/{kind}/{key}", summary="显式提交能力文件版本")
+async def workcopy_commit(kind: str, key: str, body: WorkcopyCommitBody, response: Response,
+ user_id: str = Depends(capability_dependencies.require_capability_user)):
+ from features.desktop.capability.workcopies import commit
+ response.headers["Cache-Control"] = "no-store"
+ await run_in_threadpool(capability_dependencies.public_content, user_id, body.files)
+ result = await run_in_threadpool(commit, user_id, kind, key, body.model_dump())
+ return success_response(data=await run_in_threadpool(capability_dependencies.public_content, user_id, result))
diff --git a/src/backend/api/routes/v1/desktop.py b/src/backend/features/desktop/api/desktop.py
similarity index 99%
rename from src/backend/api/routes/v1/desktop.py
rename to src/backend/features/desktop/api/desktop.py
index 25ef16bc..8640e054 100644
--- a/src/backend/api/routes/v1/desktop.py
+++ b/src/backend/features/desktop/api/desktop.py
@@ -47,7 +47,7 @@
from fastapi import APIRouter, Request
from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse
-from core.services.desktop_release_events import ReleaseEvents
+from features.desktop.bridge.release_events import ReleaseEvents
from core.infra.logging import get_logger
logger = get_logger(__name__)
diff --git a/src/backend/api/routes/v1/desktop_capabilities.py b/src/backend/features/desktop/api/desktop_capabilities.py
similarity index 80%
rename from src/backend/api/routes/v1/desktop_capabilities.py
rename to src/backend/features/desktop/api/desktop_capabilities.py
index 7c3fd7ca..b2317818 100644
--- a/src/backend/api/routes/v1/desktop_capabilities.py
+++ b/src/backend/features/desktop/api/desktop_capabilities.py
@@ -12,6 +12,10 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_candidates
+from features.desktop.capabilities import skill_views
+from features.desktop.capabilities import skills
import asyncio
import logging
@@ -37,9 +41,9 @@ async def preview_changes(body: ChangePreviewBody, user: UserContext = Depends(g
_require_desktop_store()
uid = str(user.user_id)
state = _bridge_state(uid)
- from core.capabilities.change_sync import preview
- from core.capabilities.errors import CapabilityError
- from core.capabilities.mcp_json import McpJsonError
+ from features.desktop.capabilities.change_sync import preview
+ from core.harness.capability_errors import CapabilityError
+ from features.desktop.capabilities.mcp_json import McpJsonError
try:
return success_response(data=await asyncio.to_thread(preview, uid, body.install_id, state))
except (CapabilityError, McpJsonError, OSError, ValueError) as exc:
@@ -58,9 +62,9 @@ async def commit_changes(body: ChangeCommitBody, user: UserContext = Depends(get
_require_desktop_store()
uid = str(user.user_id)
state = _bridge_state(uid)
- from core.capabilities.change_sync import commit
- from core.capabilities.errors import CapabilityError
- from core.capabilities.mcp_json import McpJsonError
+ from features.desktop.capabilities.change_sync import commit
+ from core.harness.capability_errors import CapabilityError
+ from features.desktop.capabilities.mcp_json import McpJsonError
try:
return success_response(data=await asyncio.to_thread(
commit, uid, body.preview_id, body.choices, state,
@@ -70,23 +74,23 @@ async def commit_changes(body: ChangeCommitBody, user: UserContext = Depends(get
def _require_desktop_store() -> None:
- from core.auth.desktop_bridge import bridge_enabled
- from core.capabilities.paths import capabilities_enabled
+ from features.desktop.auth.bridge import bridge_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
if not bridge_enabled():
raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用")
- if not capabilities_enabled():
+ if not has_run_capabilities():
raise HTTPException(status_code=403, detail="本机未配置能力文件存储(HUGAGENT_CAPS_ROOT)")
def _authorized_profile(user_id: str) -> Optional[str]:
- from core.capabilities import skills
+ from features.desktop.capabilities import skills
- return skills.current_account_profile() if skills.account_authorized_for(user_id) else None
+ return skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None
def _bridge_state(user_id: str) -> Dict[str, Any]:
- from core.services.desktop_cloud_bridge import get_state
+ from features.desktop.bridge.identity import get_state
st = get_state()
if not st:
@@ -110,17 +114,15 @@ def _device_item(inst, *, runtime_name: str, usable: bool) -> Dict[str, Any]:
return item
-from core.capabilities.management_context import (
- management_context as _readiness_context, connectors_view as _connectors_view,
-)
+from features.desktop.capabilities.management_context import management_context as _readiness_context, connectors_view as _connectors_view
def _agents_view(user_id: str) -> Dict[str, Any]:
- from core.capabilities import agents as caps_agents
- from core.capabilities import registry, skills
- from core.capabilities.paths import KIND_AGENT
+ from features.desktop.capabilities import agents as caps_agents
+ from features.desktop.capabilities import registry, skills
+ from core.capabilities.identity_rules import KIND_AGENT
from core.db.engine import SessionLocal
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
with SessionLocal() as db:
svc = UserAgentService(db)
@@ -132,7 +134,7 @@ def _agents_view(user_id: str) -> Dict[str, Any]:
items: List[Dict[str, Any]] = []
profile = _authorized_profile(user_id)
context = _readiness_context(user_id)
- from core.capabilities.readiness import apply_to_item, file_readiness, files_ready
+ from features.desktop.capabilities.readiness import apply_to_item, file_readiness, files_ready
for inst in registry.list_installations(
kind=KIND_AGENT, profiles=[p for p in ("local", profile) if p]
@@ -162,12 +164,12 @@ def _agents_view(user_id: str) -> Dict[str, Any]:
def _plugins_view(user_id: str) -> Dict[str, Any]:
- from core.capabilities import registry
- from core.capabilities.paths import KIND_PLUGIN
+ from features.desktop.capabilities import registry
+ from core.capabilities.identity_rules import KIND_PLUGIN
profile = _authorized_profile(user_id)
context = _readiness_context(user_id)
- from core.capabilities.readiness import file_readiness, files_ready
+ from features.desktop.capabilities.readiness import file_readiness, files_ready
items: List[Dict[str, Any]] = []
for inst in registry.list_installations(
@@ -195,8 +197,8 @@ def _plugins_view(user_id: str) -> Dict[str, Any]:
def _installations_view(user_id: str, kind: str) -> Dict[str, Any]:
- from core.capabilities import skills
- from core.capabilities.paths import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL
+ from features.desktop.capabilities import skills
+ from core.capabilities.identity_rules import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL
if kind == KIND_MCP:
return _connectors_view(user_id)
@@ -206,7 +208,7 @@ def _installations_view(user_id: str, kind: str) -> Dict[str, Any]:
return _plugins_view(user_id)
if kind != KIND_SKILL:
raise HTTPException(status_code=400, detail=f"unknown kind {kind!r}")
- from core.capabilities.discovery import scan
+ from features.desktop.capabilities.discovery import scan
discovery_errors = scan(user_id)
res = skills.resolve_for_user(user_id)
chosen = {c.install_id: name for name, c in res.chosen.items()}
@@ -226,17 +228,12 @@ def _resolution_of(install_id: str) -> Dict[str, Any]:
return {"outcome": "absent", "reason": None}
context = _readiness_context(user_id)
- from core.capabilities.readiness import (
- apply_to_item,
- builtin_readiness,
- file_readiness,
- files_ready,
- )
+ from features.desktop.capabilities.readiness import apply_to_item, builtin_readiness, file_readiness, files_ready
items: List[Dict[str, Any]] = []
- for cand in skills.candidates(user_id):
+ for cand in skill_candidates.candidates(user_id):
entry = cand.to_dict()
- from core.capabilities import registry
+ from features.desktop.capabilities import registry
inst = registry.get(cand.install_id)
entry["registered"] = inst is not None
@@ -260,7 +257,7 @@ def _resolution_of(install_id: str) -> Dict[str, Any]:
def _preferences(kind: str, user_id: str) -> Dict[str, str]:
- from core.capabilities import registry
+ from features.desktop.capabilities import registry
return registry.preferences(kind, user_id=user_id)
@@ -269,7 +266,7 @@ def _preferences(kind: str, user_id: str) -> Dict[str, str]:
@quiet_access_log
async def list_installations(kind: str = "skill", user: UserContext = Depends(get_current_user)):
_require_desktop_store()
- from core.capabilities.change_sync import annotate
+ from features.desktop.capabilities.change_sync import annotate
def listing():
return annotate(str(user.user_id), _installations_view(str(user.user_id), kind))
return success_response(
@@ -281,8 +278,8 @@ def listing():
async def check_sync(user: UserContext = Depends(get_current_user)):
_require_desktop_store()
st = _bridge_state(str(user.user_id))
- from core.capabilities.errors import CloudUnavailable
- from core.services.desktop_capability_sync_check import check
+ from core.harness.capability_errors import CloudUnavailable
+ from features.desktop.capability.sync_check import check
try:
return success_response(data=await asyncio.to_thread(check, st))
@@ -294,10 +291,8 @@ async def check_sync(user: UserContext = Depends(get_current_user)):
async def sync_now(user: UserContext = Depends(get_current_user)):
_require_desktop_store()
st = _bridge_state(str(user.user_id))
- from core.services import desktop_cloud_bundles, desktop_cloud_skills
- from core.services.desktop_cloud_bridge import (
- sync_capabilities_blocking, sync_tool_manifest_blocking,
- )
+ from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills
+ from features.desktop.bridge.client import sync_capabilities_blocking, sync_tool_manifest_blocking
await asyncio.to_thread(sync_tool_manifest_blocking, st)
await asyncio.to_thread(sync_capabilities_blocking, st)
@@ -310,18 +305,18 @@ async def sync_now(user: UserContext = Depends(get_current_user)):
status_code=502,
detail={"code": "cloud_unavailable", "message": "; ".join(errors), "retryable": True},
)
- from core.capabilities import skills
+ from features.desktop.capabilities import skills
- await asyncio.to_thread(skills.rebuild_views, str(user.user_id))
+ await asyncio.to_thread(skill_views.rebuild_views, str(user.user_id))
return success_response(data=status)
@router.post("/views/rebuild", summary="重建运行视图")
async def rebuild_views(user: UserContext = Depends(get_current_user)):
_require_desktop_store()
- from core.capabilities import skills
+ from features.desktop.capabilities import skills
- reports = await asyncio.to_thread(skills.rebuild_views, str(user.user_id))
+ reports = await asyncio.to_thread(skill_views.rebuild_views, str(user.user_id))
return success_response(data={k: v.to_dict() for k, v in reports.items()})
@@ -329,7 +324,7 @@ async def rebuild_views(user: UserContext = Depends(get_current_user)):
def _mcp_json_doc(user_id: str) -> Dict[str, Any]:
- from core.capabilities import mcp_json, skills
+ from features.desktop.capabilities import mcp_json, skills
try:
doc = mcp_json.load()
@@ -363,7 +358,7 @@ async def get_mcp_json(_user: UserContext = Depends(get_current_user)):
async def initial_sync_status_route(user: UserContext = Depends(get_current_user)):
_require_desktop_store()
_bridge_state(str(user.user_id))
- from core.services.desktop_cloud_bridge import initial_sync_status
+ from features.desktop.bridge.status import initial_sync_status
return success_response(data=await asyncio.to_thread(initial_sync_status))
@@ -371,16 +366,17 @@ async def initial_sync_status_route(user: UserContext = Depends(get_current_user
async def retry_initial_sync_route(user: UserContext = Depends(get_current_user)):
_require_desktop_store()
st = _bridge_state(str(user.user_id))
- from core.services.desktop_cloud_bridge import retry_initial_sync
+ from features.desktop.bridge.client import retry_initial_sync
await asyncio.to_thread(retry_initial_sync, st)
return success_response(data={"ok": True})
def _accept_synced_capabilities(st, user_id):
- from core.services.desktop_cloud_bridge import account_scope, initial_sync_status
- from core.capabilities import registry, session_availability, skills
- from core.capabilities.readiness import file_readiness
- from core.capabilities.paths import revision_for_hash
+ from features.desktop.bridge.identity import account_scope
+ from features.desktop.bridge.status import initial_sync_status
+ from features.desktop.capabilities import registry, session_availability, skills
+ from features.desktop.capabilities.readiness import file_readiness
+ from features.desktop.capabilities.paths import revision_for_hash
with account_scope(st):
status = initial_sync_status()
@@ -399,7 +395,7 @@ def _accept_synced_capabilities(st, user_id):
session_availability.activate(profile, revisions)
registry._bump()
skills.bump_view_generation()
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache(user_id)
return {"ok": True, "partial": True, "available_count": len(revisions)}
diff --git a/src/backend/features/desktop/api/desktop_capability.py b/src/backend/features/desktop/api/desktop_capability.py
new file mode 100644
index 00000000..62ddd6f6
--- /dev/null
+++ b/src/backend/features/desktop/api/desktop_capability.py
@@ -0,0 +1,12 @@
+"""Compose desktop capability routes in their original security and API order."""
+from fastapi import APIRouter
+from features.desktop.api import capability_auth, capability_workcopies, capability_manifests, capability_tools, capability_model_gateway, capability_mcp_gateway, capability_bridge
+
+router = APIRouter()
+router.include_router(capability_auth.router)
+router.include_router(capability_workcopies.router)
+router.include_router(capability_manifests.router)
+router.include_router(capability_tools.router)
+router.include_router(capability_model_gateway.router)
+router.include_router(capability_mcp_gateway.router)
+router.include_router(capability_bridge.router)
diff --git a/src/backend/features/desktop/api/desktop_capability_state.py b/src/backend/features/desktop/api/desktop_capability_state.py
new file mode 100644
index 00000000..2189ecf4
--- /dev/null
+++ b/src/backend/features/desktop/api/desktop_capability_state.py
@@ -0,0 +1,52 @@
+"""desktop capability state responsibilities."""
+from __future__ import annotations
+from typing import Optional
+import httpx
+from fastapi import APIRouter
+from fastapi.security import HTTPBearer
+from features.desktop.gateway.observability import ObservedGatewayRoute
+
+
+
+
+gateway_client: Optional[httpx.AsyncClient] = None
+
+
+bearer = HTTPBearer(auto_error=False)
+
+
+DROP_REQUEST_HEADERS = {
+ # 身份与凭据:capability token 不上传上游;身份头由网关重写
+ "authorization",
+ "cookie",
+ "x-current-user-id",
+ # 知识库授权按云端用户重算(头缺失时 KB MCP 自动解析可用知识库)
+ "x-allowed-kb-ids",
+ # 桌面桥内部头
+ "x-desktop-bridge",
+ "x-desktop-bridge-user",
+ "x-desktop-device-id",
+ "x-hugagent-target",
+ # 逐跳 / 传输层头,由 httpx 按新连接重建
+ "host",
+ "content-length",
+ "connection",
+ "accept-encoding",
+ "transfer-encoding",
+ "keep-alive",
+ "proxy-authorization",
+ "te",
+ "upgrade",
+}
+
+
+FWD_RESPONSE_HEADERS = ("content-type", "mcp-session-id", "mcp-protocol-version")
+
+
+RUNTIME_CONTEXT_HEADERS = {
+ "x-chat-id",
+ "x-channel-id",
+ "x-conversation-id",
+ "x-allowed-dataset-ids",
+ "x-reranker-enabled",
+}
diff --git a/src/backend/api/routes/v1/desktop_login.py b/src/backend/features/desktop/api/desktop_login.py
similarity index 99%
rename from src/backend/api/routes/v1/desktop_login.py
rename to src/backend/features/desktop/api/desktop_login.py
index b788782d..0deca1c8 100644
--- a/src/backend/api/routes/v1/desktop_login.py
+++ b/src/backend/features/desktop/api/desktop_login.py
@@ -7,7 +7,7 @@
from fastapi import APIRouter, HTTPException, Request, Response
from pydantic import BaseModel, Field
-from core.auth import desktop_login_store as store
+from features.desktop.auth import login_store as store
from core.auth.session import validate_session
from core.config.settings import settings
from core.infra.responses import success_response
diff --git a/src/backend/api/routes/v1/desktop_observability.py b/src/backend/features/desktop/api/desktop_observability.py
similarity index 90%
rename from src/backend/api/routes/v1/desktop_observability.py
rename to src/backend/features/desktop/api/desktop_observability.py
index 5baae543..83d57783 100644
--- a/src/backend/api/routes/v1/desktop_observability.py
+++ b/src/backend/features/desktop/api/desktop_observability.py
@@ -4,12 +4,12 @@
from typing import Literal
from api.deps import require_config
-from api.routes.v1.desktop_capability import _require_capability_user
+from features.desktop.api.capability_dependencies import require_capability_user as _require_capability_user
from core.db.engine import SessionLocal, get_db
from core.db.models import UserShadow
-from core.db.models.observability import DesktopSyncDevice
+from features.desktop.models import DesktopSyncDevice
from core.infra.responses import paginated_response, success_response
-from core.services.desktop_observability import MAX_BATCH_BYTES, ingest, list_records
+from features.desktop.observability.projection import MAX_BATCH_BYTES, ingest, list_records
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy import select
from starlette.concurrency import run_in_threadpool
@@ -101,6 +101,6 @@ def devices(db=Depends(get_db)):
@admin_router.get("/health")
def health():
- from core.services.desktop_gateway_observability import health_snapshot
+ from features.desktop.gateway.observability import health_snapshot
return success_response(data=health_snapshot())
diff --git a/src/backend/api/routes/v1/device_link.py b/src/backend/features/desktop/api/device_link.py
similarity index 95%
rename from src/backend/api/routes/v1/device_link.py
rename to src/backend/features/desktop/api/device_link.py
index 1b64995a..b7b6bbbe 100644
--- a/src/backend/api/routes/v1/device_link.py
+++ b/src/backend/features/desktop/api/device_link.py
@@ -10,9 +10,9 @@
from typing import Optional
from core.auth.backend import UserContext, get_current_user
-from core.auth.desktop_bridge import bridge_enabled
-from core.device_link import ACTIONS, SESSIONS, store
-from core.device_link.runner import current_account, current_link, run_driver, runner
+from features.desktop.auth.bridge import bridge_enabled
+from personal_agent.device_link import ACTIONS, SESSIONS, store
+from personal_agent.device_link.runner import current_account, current_link, run_driver, runner
from core.infra.responses import success_response
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
diff --git a/src/backend/api/routes/v1/local.py b/src/backend/features/desktop/api/local.py
similarity index 82%
rename from src/backend/api/routes/v1/local.py
rename to src/backend/features/desktop/api/local.py
index 61b91b5c..87959709 100644
--- a/src/backend/api/routes/v1/local.py
+++ b/src/backend/features/desktop/api/local.py
@@ -12,8 +12,8 @@
from core.auth.backend import UserContext, get_current_user
from core.config.local_mode import local_mode_enabled
from core.infra.responses import success_response
-from core.services import local_grant_service as grants
-from core.services import local_snapshot_service as snapshots
+from features.desktop.local import grants as grants
+from features.desktop.local import snapshots as snapshots
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
@@ -77,20 +77,8 @@ async def set_policy(body: PolicyBody, _user: UserContext = Depends(get_current_
return success_response(data=saved, message="策略已更新")
-class ApprovalBody(BaseModel):
- mode: Literal["strict", "standard", "full"]
-
-
-@router.get("/approval-mode", summary="获取当前审批档(对话框权限栏)")
-async def get_approval_mode(_user: UserContext = Depends(get_current_user)):
- _require_local()
- return success_response(data={"mode": grants.get_approval_mode()})
-
-
-@router.put("/approval-mode", summary="设置审批档")
-async def set_approval_mode(body: ApprovalBody, _user: UserContext = Depends(get_current_user)):
- _require_local()
- return success_response(data={"mode": grants.set_approval_mode(body.mode)}, message="已切换权限档")
+from features.desktop.edition import register_local_routes
+register_local_routes(router, _require_local)
class RollbackBody(BaseModel):
diff --git a/src/backend/api/routes/local_file_actions.py b/src/backend/features/desktop/api/local_file_actions.py
similarity index 93%
rename from src/backend/api/routes/local_file_actions.py
rename to src/backend/features/desktop/api/local_file_actions.py
index 925339b8..3301aae2 100644
--- a/src/backend/api/routes/local_file_actions.py
+++ b/src/backend/features/desktop/api/local_file_actions.py
@@ -27,7 +27,7 @@ def applications(
include_icons: bool = True,
):
from api.routes.files import local_file_location
- from core.services.local_file_actions import file_applications
+ from features.desktop.local.file_actions import file_applications
location = local_file_location(file_id, user, db)
try:
@@ -42,7 +42,7 @@ def open_local_file(
user: UserContext = Depends(require_auth()), db: Session = Depends(get_db),
):
from api.routes.files import local_file_location
- from core.services.local_file_actions import open_file
+ from features.desktop.local.file_actions import open_file
location = local_file_location(file_id, user, db)
try:
diff --git a/src/backend/features/desktop/auth/__init__.py b/src/backend/features/desktop/auth/__init__.py
new file mode 100644
index 00000000..95d9f8f1
--- /dev/null
+++ b/src/backend/features/desktop/auth/__init__.py
@@ -0,0 +1 @@
+"""auth package."""
diff --git a/src/backend/core/auth/desktop_bridge.py b/src/backend/features/desktop/auth/bridge.py
similarity index 92%
rename from src/backend/core/auth/desktop_bridge.py
rename to src/backend/features/desktop/auth/bridge.py
index 3a462777..2abd09eb 100644
--- a/src/backend/core/auth/desktop_bridge.py
+++ b/src/backend/features/desktop/auth/bridge.py
@@ -85,7 +85,7 @@ def _ensure_bridge_capabilities(db: Session, shadow) -> None:
meta = dict(shadow.extra_data or {})
if meta.get(_BRIDGE_META_FLAG):
return
- from core.services.local_user_service import CE_DEFAULT_ADMIN_CAPABILITIES
+ from core.auth.local_users_constants import CE_DEFAULT_ADMIN_CAPABILITIES
meta.update(CE_DEFAULT_ADMIN_CAPABILITIES)
meta["auth_source"] = "desktop_bridge"
@@ -103,6 +103,11 @@ def _ensure_bridge_capabilities(db: Session, shadow) -> None:
db.commit()
+def claims_request(request: Request) -> bool:
+ """Whether this request presents a bridge credential (header check, no I/O)."""
+ return bridge_enabled() and bool(request.headers.get(BRIDGE_SECRET_HEADER, ""))
+
+
def resolve_bridge_user(request: Request, db: Session):
"""尝试按桌面桥接头解析用户;未启用/未命中/校验失败返回 None(回落常规认证)。
@@ -124,16 +129,16 @@ def resolve_bridge_user(request: Request, db: Session):
# The shell header must belong to the current signed cloud identity. A
# header captured before account switch/logout must never create a local user.
- from core.capabilities.paths import capabilities_enabled
+ from features.desktop.capabilities.paths import capabilities_enabled
if capabilities_enabled():
- from core.services.desktop_cloud_bridge import get_identity_state
+ from features.desktop.bridge.identity import get_identity_state
identity = get_identity_state()
# 壳送来的是按云端地址加了命名空间的标识;本机侧从凭据推出同样的标识再比对。
if not identity or identity.get("shell_user_center_id") != info["user_center_id"]:
return None
from core.auth.backend import UserContext
- from core.services import UserService
+ from core.auth.users import UserService
shadow = UserService(db).get_or_create_user_shadow(
user_center_id=info["user_center_id"],
diff --git a/src/backend/core/auth/desktop_login_store.py b/src/backend/features/desktop/auth/login_store.py
similarity index 100%
rename from src/backend/core/auth/desktop_login_store.py
rename to src/backend/features/desktop/auth/login_store.py
diff --git a/src/backend/core/auth/desktop_ticket_store.py b/src/backend/features/desktop/auth/ticket_store.py
similarity index 100%
rename from src/backend/core/auth/desktop_ticket_store.py
rename to src/backend/features/desktop/auth/ticket_store.py
diff --git a/src/backend/features/desktop/bridge/__init__.py b/src/backend/features/desktop/bridge/__init__.py
new file mode 100644
index 00000000..e7e03d85
--- /dev/null
+++ b/src/backend/features/desktop/bridge/__init__.py
@@ -0,0 +1 @@
+"""bridge package."""
diff --git a/src/backend/core/services/desktop_cloud_bundles.py b/src/backend/features/desktop/bridge/bundles.py
similarity index 88%
rename from src/backend/core/services/desktop_cloud_bundles.py
rename to src/backend/features/desktop/bridge/bundles.py
index 1f1c8365..ab6578de 100644
--- a/src/backend/core/services/desktop_cloud_bundles.py
+++ b/src/backend/features/desktop/bridge/bundles.py
@@ -8,16 +8,18 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import registry_types
import logging
import threading
from typing import Any, Dict, List, Optional
-from core.capabilities import plugins, registry, store, manifest_order
-from core.capabilities.errors import CloudUnavailable, IntegrityFailed
-from core.capabilities.paths import KIND_AGENT, KIND_PLUGIN, capabilities_enabled, revision_for_hash
-from core.capabilities.ref import cloud_ref, profile_id
-from core.services.desktop_capability_protocol import entity_content_hash, entity_id_key
+from features.desktop.capabilities import plugins, registry, store, manifest_order
+from core.harness.capability_errors import CloudUnavailable, IntegrityFailed
+from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN
+from features.desktop.capabilities.paths import capabilities_enabled, revision_for_hash
+from core.capabilities.reference import cloud_ref, profile_id
+from core.capabilities.manifest_protocol import entity_content_hash, entity_id_key
logger = logging.getLogger(__name__)
@@ -29,7 +31,7 @@
def _profile(state: Dict[str, Any]) -> str:
- from core.services.desktop_capability_protocol import token_subject
+ from core.capabilities.manifest_protocol import token_subject
subject = token_subject(str(state.get("token") or ""))
if not subject:
@@ -38,14 +40,14 @@ def _profile(state: Dict[str, Any]) -> str:
def _headers(state: Dict[str, Any]) -> Dict[str, str]:
- from core.services.desktop_cloud_bridge import cloud_headers
+ from features.desktop.bridge.identity import cloud_headers
return cloud_headers(state)
def _fetch(kind: str, state: Dict[str, Any]) -> Dict[str, Any]:
import httpx
- from core.services.desktop_capability_protocol import validate_entity_manifest
+ from core.capabilities.manifest_protocol import validate_entity_manifest
ticket = manifest_order.begin(kind, _profile(state))
with _lock:
@@ -89,7 +91,7 @@ def _dir_hash(comp: store.StoredComponent) -> str:
def _text_files(comp: store.StoredComponent):
- from core.capabilities import archive
+ from core.capabilities import archive as archive
for rel, path in archive.iter_files(comp.path):
if rel == ".inventory.json":
@@ -97,8 +99,8 @@ def _text_files(comp: store.StoredComponent):
yield rel, path
-def _prepare(kind: str, state: Dict[str, Any], inst: registry.Installation) -> None:
- from core.capabilities.preparation import prepare_component
+def _prepare(kind: str, state: Dict[str, Any], inst: registry_types.Installation) -> None:
+ from features.desktop.capabilities.preparation import prepare_component
remote_id = str(inst.payload.get("cloud_install_id") or inst.key)
@@ -110,7 +112,7 @@ def after_publish(comp):
)
# 云端同步下来的插件同样要在本机铺它自己的资产:站点插件的建站流程要在
# 本机跑 init-react-site.sh,插件从云端来不代表这一步可以省。
- from core.plugins.local.assets import provision_for
+ from features.desktop.local.plugins.assets import provision_for
provision_for(inst.key)
@@ -125,7 +127,7 @@ def after_publish(comp):
def prepare(state: Dict[str, Any], install_ids: List[str]) -> List[Dict[str, Any]]:
"""Prepare selected agent/plugin definitions using the same verified publication."""
- from core.capabilities.errors import PackageMissing
+ from core.harness.capability_errors import PackageMissing
results = []
for iid in install_ids:
@@ -151,7 +153,7 @@ def _reconcile(
kind: str, manifest: Dict[str, Any], state: Dict[str, Any], *, prepare: bool = True
) -> int:
"""Publish all current intentions before any optional package download."""
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
profile = _profile(state)
with account_scope(state):
@@ -193,7 +195,7 @@ def prepare_item(inst):
def _apply_intent(
kind: str, manifest: Dict[str, Any], state: Dict[str, Any]
-) -> List[registry.Installation]:
+) -> List[registry_types.Installation]:
profile = _profile(state)
cloud_base = str(state["cloud_base"])
id_key = entity_id_key(kind)
@@ -242,7 +244,7 @@ def sync_kind(kind: str, state: Dict[str, Any]) -> bool:
_errors[kind] = "capability store disabled (HUGAGENT_CAPS_ROOT unset)"
return False
try:
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
manifest = _fetch(kind, state)
with _lock:
@@ -253,9 +255,7 @@ def sync_kind(kind: str, state: Dict[str, Any]) -> bool:
except manifest_order.StaleManifest:
return False
except Exception as exc: # noqa: BLE001
- from core.services.desktop_cloud_bridge import (
- _state_fingerprint, get_state, note_rejected_credential,
- )
+ from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, get_state, note_rejected_credential
note_rejected_credential(state, exc)
if _state_fingerprint(get_state()) != _state_fingerprint(state):
@@ -280,7 +280,7 @@ def sync_blocking(state: Dict[str, Any]) -> None:
for kind in (KIND_AGENT, KIND_PLUGIN):
changed = sync_kind(kind, state) or changed
if changed:
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
invalidate_capability_cache()
@@ -301,7 +301,7 @@ def synced_manifests() -> Dict[str, Optional[Dict[str, Any]]]:
def status() -> Dict[str, Any]:
- from core.services.desktop_cloud_bridge import get_state
+ from features.desktop.bridge.identity import get_state
st = get_state()
out: Dict[str, Any] = {}
diff --git a/src/backend/features/desktop/bridge/client.py b/src/backend/features/desktop/bridge/client.py
new file mode 100644
index 00000000..863d1f10
--- /dev/null
+++ b/src/backend/features/desktop/bridge/client.py
@@ -0,0 +1,214 @@
+"""client responsibilities."""
+from __future__ import annotations
+from features.desktop.capabilities import skills
+import threading
+import time
+from typing import Any, Dict, Optional
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.bridge import client_state as bridge_client_state
+from features.desktop.bridge import identity as bridge_identity
+from features.desktop.bridge import managed_connectors as bridge_managed_connectors
+
+def notify_cloud_changed() -> None:
+ """云端能力可能已变(前端写操作 / 网关告知):后台同步一次清单与本机文件。"""
+ refresh_manifest_async(force=True)
+
+
+def set_state(
+ cloud_base: str,
+ token: str,
+ expires_in: int,
+ *,
+ device_id: Optional[str] = None,
+ authorization_epoch: Optional[int] = None,
+) -> None:
+ """壳侧推送桥配置(幂等)。立即触发一次后台 manifest 刷新。"""
+ pass # State is owned by its responsibility module.
+ payload = {
+ "cloud_base": cloud_base.strip().rstrip("/"),
+ "token": token.strip(),
+ "expires_at": time.time() + max(1, int(expires_in or 0)),
+ "device_id": device_id,
+ "authorization_epoch": authorization_epoch,
+ }
+ from core.capabilities.manifest_protocol import token_claims
+
+ claims = token_claims(token)
+ if device_id and claims.get("d") and device_id != claims["d"]:
+ raise ValueError("desktop device identity does not match token")
+ if claims.get("e"):
+ payload["expires_at"] = min(payload["expires_at"], float(claims["e"]))
+ with bridge_client_state.state_lock:
+ changed = bridge_identity.state_fingerprint(bridge_client_state.state) != bridge_identity.state_fingerprint(payload)
+ renewed_after_rejection = not changed and bridge_client_state.credential_rejected == bridge_identity.state_fingerprint(payload)
+ bridge_client_state.credential_rejected = ""
+ bridge_client_state.state = payload
+ bridge_client_state.state_loaded = True
+ if changed:
+ bridge_identity.clear_partial_selection()
+ # Never expose a previous cloud account's tools while the replacement
+ # dynamic manifest is still in flight. Skill files stay in the previous
+ # account's isolated profile directory; only the runtime view changes.
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest = None
+ bridge_client_state.manifest_ts = 0.0
+ bridge_client_state.manifest_error = None
+ from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills
+
+ desktop_cloud_skills.on_account_switch()
+ desktop_cloud_bundles.on_account_switch()
+ bridge_identity.purge_persisted_state()
+ if changed:
+ bridge_identity.rebuild_identity_views()
+ refresh_manifest_async(force=True)
+ elif renewed_after_rejection:
+ refresh_manifest_async(force=True)
+
+
+def synced_manifest() -> Optional[Dict[str, Any]]:
+ """Read the last accepted connector snapshot without starting a refresh."""
+ import copy
+
+ with bridge_client_state.manifest_lock:
+ return copy.deepcopy(bridge_client_state.manifest)
+
+
+def sync_tool_manifest_blocking(st: Dict[str, Any]) -> None:
+ """Refresh connectors for both login and explicit synchronization."""
+ with bridge_client_state.tool_sync_lock:
+ sync_tool_manifest(st)
+
+
+def sync_tool_manifest(st: Dict[str, Any]) -> None:
+ pass # State is owned by its responsibility module.
+ from core.harness.capability_errors import CloudUnavailable
+
+ bridge_identity.require_current_account(st)
+ url = f"{st['cloud_base']}/api/v1/desktop/capability/manifest"
+ import httpx
+ from core.capabilities.manifest_protocol import validate_manifest
+
+ headers = {**bridge_identity.cloud_headers(st), "Cache-Control": "no-cache"}
+ with bridge_client_state.manifest_lock:
+ current_revision = str((bridge_client_state.manifest or {}).get("revision") or "")
+ if current_revision:
+ headers["If-None-Match"] = f'"{current_revision}"'
+
+ resp = httpx.get(
+ url,
+ headers=headers,
+ timeout=httpx.Timeout(10.0, connect=5.0),
+ )
+ bridge_identity.require_current_account(st)
+ if resp.status_code in (401, 403):
+ with bridge_identity.account_scope(st):
+ bridge_identity.clear_state()
+ raise CloudUnavailable("cloud authorization unavailable")
+ if resp.status_code == 304:
+ with bridge_identity.account_scope(st), bridge_client_state.manifest_lock:
+ bridge_client_state.manifest_ts = time.monotonic()
+ bridge_client_state.manifest_error = None
+ else:
+ resp.raise_for_status()
+ body = resp.json()
+ data = body.get("data") if isinstance(body, dict) else None
+ manifest = validate_manifest(data)
+ with bridge_identity.account_scope(st):
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest = manifest
+ bridge_client_state.manifest_ts = time.monotonic()
+ bridge_client_state.manifest_error = None
+ bridge_managed_connectors.project_managed_profile(st, manifest)
+ logger.info(
+ "[cloud-bridge] manifest refreshed revision=%s servers=%d",
+ manifest["revision"][:12],
+ len(manifest["servers"]),
+ )
+
+
+def fetch_manifest_blocking(st: Dict[str, Any]) -> None:
+ pass # State is owned by its responsibility module.
+ refresh_replacement = False
+ try:
+ sync_tool_manifest_blocking(st)
+ # Skills, agents and plugins ride the same poll and token: their intent
+ # is reconciled right after the tool manifest is known to be current.
+ sync_capabilities_blocking(st)
+ except Exception as exc: # noqa: BLE001
+ with bridge_client_state.state_lock:
+ if bridge_identity.state_fingerprint(bridge_identity.get_state()) != bridge_identity.state_fingerprint(st):
+ refresh_replacement = True
+ return
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest_ts = time.monotonic()
+ bridge_client_state.manifest_error = str(exc)
+ logger.warning("[cloud-bridge] manifest 拉取失败: %s", exc)
+ finally:
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest_fetching = False
+ rerun = bridge_client_state.refresh_pending
+ bridge_client_state.refresh_pending = False
+ if refresh_replacement or rerun:
+ refresh_manifest_async(force=True)
+
+
+def sync_capabilities_blocking(st: Dict[str, Any]) -> None:
+ """One pass over every cloud-synced kind (skills, agents, plugins). Blocking; call off-loop.
+
+ Both the background manifest poll and the user-triggered sync endpoint go
+ through here so no kind can be forgotten by one of the two paths.
+ """
+ from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills
+
+ desktop_cloud_skills.sync_blocking(st)
+ desktop_cloud_bundles.sync_blocking(st)
+ try:
+ from features.desktop.capabilities.warmup import warmup_current_account
+
+ warmup_current_account(st)
+ except Exception as exc: # Best effort: ordinary execution retains all checks.
+ logger.warning("[caps] startup reads deferred: %s", type(exc).__name__)
+
+
+def refresh_manifest_async(force: bool = False) -> None:
+ """后台同步一次云端清单。没有定时器:``force`` 由登录 / 变更通知触发,非
+ ``force`` 只是「还没有可用清单」时的有界重试(离线登录后的自愈),一旦拿到
+ 清单就不再有任何后台请求。"""
+ pass # State is owned by its responsibility module.
+ st = bridge_identity.get_state()
+ if not st:
+ return
+ with bridge_client_state.manifest_lock:
+ if bridge_client_state.manifest_fetching:
+ bridge_client_state.refresh_pending = bridge_client_state.refresh_pending or force
+ return
+ if not force:
+ if bridge_client_state.manifest is not None and not bridge_client_state.manifest_error:
+ return
+ if bridge_client_state.manifest_ts and time.monotonic() - bridge_client_state.manifest_ts < bridge_client_state.MANIFEST_NEG_TTL_S:
+ return
+ bridge_client_state.manifest_fetching = True
+ threading.Thread(
+ target=fetch_manifest_blocking, args=(st,), name="cloud-bridge-manifest", daemon=True
+ ).start()
+
+
+def get_cached_manifest() -> Optional[Dict[str, Any]]:
+ """返回缓存的 manifest(可能为 None)。调用方只读,不得修改。
+
+ 还没有清单时(离线登录等)触发一次有界重试;已有清单则不发任何请求,更新
+ 靠登录与变更通知。不做激活判定——守门统一在公共入口(``_bridge_context``)。
+ 清单只在 ``_fetch_manifest_blocking`` 里整体替换,从不原地修改,所以可以安全共享。
+ """
+ refresh_manifest_async()
+ with bridge_client_state.manifest_lock:
+ return bridge_client_state.manifest
+
+
+def retry_initial_sync(st):
+ from features.desktop.capabilities import skills
+ with bridge_identity.account_scope(st):
+ bridge_identity.clear_partial_selection()
+ skills.bump_view_generation()
+ refresh_manifest_async(force=True)
diff --git a/src/backend/features/desktop/bridge/client_state.py b/src/backend/features/desktop/bridge/client_state.py
new file mode 100644
index 00000000..d2065f81
--- /dev/null
+++ b/src/backend/features/desktop/bridge/client_state.py
@@ -0,0 +1,48 @@
+"""client state responsibilities."""
+from __future__ import annotations
+import threading
+from typing import Any, Dict, Optional, Tuple
+
+BRIDGE_BLOCK_ID = "desktop_cloud_bridge"
+
+
+MANIFEST_NEG_TTL_S = 30.0
+
+
+state_lock = threading.RLock()
+
+
+state: Optional[Dict[str, Any]] = None # {"cloud_base", "token", "expires_at"}
+
+
+credential_rejected = "" # account fingerprint whose current token the cloud refused
+
+
+state_loaded = False
+
+
+manifest_lock = threading.Lock()
+
+
+tool_sync_lock = threading.Lock()
+
+
+manifest: Optional[Dict[str, Any]] = None
+
+
+manifest_ts: float = 0.0
+
+
+manifest_error: Optional[str] = None
+
+
+manifest_fetching = False
+
+
+refresh_pending = False # a forced refresh arrived while a pass was running
+
+
+context_lock = threading.Lock()
+
+
+context_cache: Optional[Tuple[tuple, Dict[str, Any]]] = None
diff --git a/src/backend/features/desktop/bridge/gateway.py b/src/backend/features/desktop/bridge/gateway.py
new file mode 100644
index 00000000..565f0cf6
--- /dev/null
+++ b/src/backend/features/desktop/bridge/gateway.py
@@ -0,0 +1,225 @@
+"""gateway responsibilities."""
+from __future__ import annotations
+from features.desktop.bridge import client as bridge_client
+from typing import Any, Dict, List, Optional, Set
+from features.desktop.auth import bridge as bridge_auth
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.bridge import client as bridge_client
+from features.desktop.bridge import client_state as bridge_client_state
+from features.desktop.bridge import identity as bridge_identity
+from features.desktop.bridge import managed_connectors as bridge_managed_connectors
+
+def bridge_context() -> Optional[Dict[str, Any]]:
+ """唯一守门点:桥激活且 manifest 就绪时返回上下文,否则 None。
+
+ 返回 {"state": st, "servers": [含 component 的云端 server,已剔除 mcp.json 里
+ 用户停用的项], "profile": 账号 profile}。同名裁决交给解析器。
+ server 列表按 (账号, 清单 revision, 停用集合) 记忆化;``state`` 每次现取,
+ 因为它携带轮换中的 token。
+ """
+ pass # State is owned by its responsibility module.
+ if not bridge_auth.bridge_enabled():
+ return None
+ st = bridge_identity.get_state()
+ if not st:
+ return None
+ manifest = bridge_client.get_cached_manifest()
+ if not manifest:
+ return None
+ enabled = bridge_managed_connectors.managed_enabled(st)
+ key = (bridge_identity.state_fingerprint(st), str(manifest["revision"]), tuple(sorted(enabled.items())))
+ with bridge_client_state.context_lock:
+ hit = bridge_client_state.context_cache
+ if hit is None or hit[0] != key:
+ servers: List[Dict[str, Any]] = []
+ for s in manifest.get("servers") or []:
+ if not isinstance(s, dict):
+ continue
+ # A server with no current tool contracts cannot replace a local
+ # implementation. The manifest validator already guarantees every
+ # non-empty entry contains complete, cloud-supplied schemas.
+ if not s.get("tools"):
+ continue
+ sid = str(s.get("server_id") or "").strip()
+ if not sid or not enabled.get(sid, True):
+ continue
+ entry = dict(s)
+ entry["server_id"] = sid
+ entry["component"] = str(s.get("component") or sid).strip()
+ servers.append(entry)
+ hit = (
+ key,
+ {
+ "servers": servers,
+ "manifest_revision": str(manifest["revision"]),
+ "profile": bridge_managed_connectors.account_profile(st),
+ },
+ )
+ with bridge_client_state.context_lock:
+ bridge_client_state.context_cache = hit
+ if not hit[1]["servers"]:
+ return None
+ return {"state": st, **hit[1]}
+
+
+def mcp_json_local_configs() -> Dict[str, dict]:
+ from features.desktop.capabilities import mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+
+ if not capabilities_enabled():
+ return {}
+ try:
+ return mcp_json.local_server_configs()
+ except mcp_json.McpJsonError as exc:
+ logger.warning("[cloud-bridge] mcp.json local servers unavailable: %s", exc)
+ return {}
+
+
+def cloud_gateway_mcp_configs(
+ enabled_mcp_ids: Optional[List[str]] = None, *, resolution_out=None, unavailable_out=None
+) -> Dict[str, dict]:
+ """Desktop-only MCP config sources: cloud gateway bindings + mcp.json local servers.
+
+ Only the components the resolver chose for the cloud binding get a gateway
+ config; the run's enabled-id allowlist still collapses the final set.
+ """
+ from features.desktop.capabilities import connectors
+
+ configs: Dict[str, dict] = dict(mcp_json_local_configs())
+ ctx = bridge_context()
+ res = resolve_mcp_bindings(enabled_mcp_ids, ctx)
+ if resolution_out is not None:
+ resolution_out.append(res)
+ requested = set(enabled_mcp_ids or [])
+ for name, group in {**res.conflicts, **res.unusable}.items():
+ if name in requested or any(connectors.server_id_of(c) in requested for c in group):
+ from core.harness.capability_errors import NameConflict, PackageMissing
+
+ error = NameConflict if name in res.conflicts else PackageMissing
+ if unavailable_out is not None:
+ unavailable_out.update({connectors.server_id_of(c): error.code for c in group})
+ continue
+ raise error(
+ "selected connector binding is unavailable; choose its source", runtime_name=name
+ )
+ chosen = res.chosen_ids()
+ configs = {
+ sid: cfg
+ for sid, cfg in configs.items()
+ if f"mcp:{connectors.MCP_JSON_PROFILE}:{sid}" in chosen
+ }
+ if not ctx:
+ return configs
+ st = ctx["state"]
+ manifest_revision = ctx["manifest_revision"]
+ for s in ctx["servers"]:
+ # Source selection is complete; only the exact authorized cloud binding
+ # chosen by the resolver may produce a gateway config.
+ sid = s["server_id"]
+ if f"mcp:{ctx['profile']}:{sid}" not in chosen:
+ continue
+ from features.desktop.bridge.resource_plugins import local_config
+ from features.desktop.capabilities.skill_account import current_local_user_id
+ native = local_config(s, current_local_user_id(), ctx["profile"])
+ if native is not None:
+ configs[sid] = native
+ continue
+ invoke_url = f"{st['cloud_base']}/api/v1/desktop/capability/gateway/{sid}/call"
+ configs[sid] = {
+ "transport": "streamable_http",
+ # This endpoint invokes the authorized manifest tool through JSON.
+ "url": invoke_url,
+ "headers": bridge_identity.cloud_headers(st),
+ "execution_timeout": 180,
+ "schema_source": "cloud_manifest",
+ "manifest_revision": manifest_revision,
+ "manifest_tools": s["tools"],
+ "schema_hash": str(s["schema_hash"]),
+ "gateway_invoke_url": invoke_url,
+ "gateway_plugin": str(s.get("source_plugin") or ""),
+ }
+ return configs
+
+
+def local_server_ids() -> Set[str]:
+ """本机 DB 内全部 MCP 的 server_id。
+
+ 复用 ``McpServerConfigService`` 自带的 30s 缓存与失效链路,不另建缓存。
+ """
+ try:
+ from core.capabilities.mcp.service import McpServerConfigService
+
+ return set(McpServerConfigService.get_instance().get_all_servers(enabled_only=False))
+ except Exception as exc: # noqa: BLE001
+ logger.debug("[cloud-bridge] local server ids failed: %s", exc)
+ return set()
+
+
+def apply_to_enabled_mcp_ids(mcp_ids: Optional[List[str]]) -> Optional[List[str]]:
+ """把本轮 enabled_mcp_ids 交给解析器,按 server_id 裁决每个连接器的绑定。
+
+ 候选:本机目录行(device)、云端 manifest(account)、mcp.json 本机声明。
+ 同一 server_id 多个候选时,云端账号级候选默认胜出,
+ 用户可用同名偏好显式改选;落选绑定记为 shadowed,可在界面
+ 切换,而不是被静默顶掉。云端 manifest 尚未就绪 / 桥未激活:原样返回。
+ 顺序稳定(已选项按原顺序,新增云端与 mcp.json 项追加),幂等。
+ """
+ if mcp_ids is None:
+ return None
+ ctx = bridge_context()
+ from features.desktop.capabilities import connectors
+
+ local_ids = local_server_ids()
+ json_local = mcp_json_local_declarations()
+ res = resolve_mcp_bindings(mcp_ids, ctx)
+
+ chosen_ids = {connectors.server_id_of(c) for c in res.chosen.values()}
+ known = local_ids | {s["server_id"] for s in (ctx or {}).get("servers", [])} | set(json_local)
+ kept = [mid for mid in mcp_ids if mid in chosen_ids or mid not in known]
+ existing = set(kept)
+ for name in sorted(res.chosen):
+ sid = connectors.server_id_of(res.chosen[name])
+ if sid not in existing:
+ kept.append(sid)
+ existing.add(sid)
+ return kept
+
+
+def resolve_mcp_bindings(mcp_ids: Optional[List[str]], ctx):
+ from features.desktop.capabilities import connectors
+
+ local_ids = local_server_ids()
+ enabled = set(local_ids) if mcp_ids is None else set(mcp_ids)
+ candidates = connectors.db_candidates(local_ids, enabled) + connectors.json_candidates(
+ mcp_json_local_declarations()
+ )
+ if ctx:
+ candidates += connectors.cloud_candidates(ctx["profile"], ctx["servers"], {})
+ return connectors.resolve_bindings(candidates)
+
+
+def mcp_json_local_declarations() -> Dict[str, Dict[str, Any]]:
+ from features.desktop.capabilities import mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+
+ if not capabilities_enabled():
+ return {}
+ try:
+ return mcp_json.load().local
+ except mcp_json.McpJsonError:
+ return {}
+
+
+def apply_to_enabled_skill_ids(skill_ids: Optional[List[str]]) -> Optional[List[str]]:
+ """把云端技能合并进本轮 enabled_skill_ids(云端为真源,见 desktop_cloud_skills)。"""
+ if skill_ids is None or not bridge_identity.bridge_active():
+ return skill_ids
+ from features.desktop.bridge import skills as desktop_cloud_skills
+
+ return desktop_cloud_skills.apply_to_enabled_skill_ids(skill_ids)
+
+
+def local_plugin_configs(user_id, enabled_only=True):
+ from features.desktop.capabilities.local_plugin_runtime import configs
+ return configs(user_id, enabled_only=enabled_only)
diff --git a/src/backend/features/desktop/bridge/identity.py b/src/backend/features/desktop/bridge/identity.py
new file mode 100644
index 00000000..d94458e6
--- /dev/null
+++ b/src/backend/features/desktop/bridge/identity.py
@@ -0,0 +1,254 @@
+"""identity responsibilities."""
+from __future__ import annotations
+from features.desktop.capabilities import skill_paths
+from features.desktop.capabilities import skill_views
+from contextlib import contextmanager
+import hashlib
+import time
+from typing import Any, Dict, Optional
+from features.desktop.auth import bridge as bridge_auth
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.bridge import client_state as bridge_client_state
+
+def state_fingerprint(state: Optional[Dict[str, Any]]) -> str:
+ """Bind a manifest snapshot to exactly one cloud/token identity."""
+ if not state:
+ return ""
+ from core.capabilities.manifest_protocol import token_subject, token_claims
+
+ token = str(state.get("token") or "")
+ # A rotated token for the same account keeps the snapshot; only a real
+ # account switch (or an opaque token) invalidates it.
+ claims = token_claims(token)
+ payload = f"{state.get('cloud_base') or ''}\0{token_subject(token) or token}\0{claims.get('a', '')}\0{claims.get('h', '')}\0{claims.get('d', state.get('device_id') or '')}"
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def require_current_account(state: Dict[str, Any]) -> None:
+ """Reject work captured before logout/account switch, including late HTTP replies."""
+ from core.harness.capability_errors import CloudUnavailable
+
+ if state_fingerprint(get_state()) != state_fingerprint(state):
+ raise CloudUnavailable("cloud account changed; retry with the current account")
+
+
+@contextmanager
+def account_scope(state: Dict[str, Any]):
+ """Serialize a short publication with identity changes; never hold across network IO."""
+ with bridge_client_state.state_lock:
+ require_current_account(state)
+ yield
+
+
+def ensure_current_authorization(state: Optional[Dict[str, Any]] = None) -> None:
+ """账号仍然有效才允许使用私有字节;这条判定不发网络请求。
+
+ 本机不做「事前探测」——云端清单只在登录、云端能力被改动(前端写操作后调
+ ``POST /v1/desktop/capabilities/sync``)以及云端在网关调用里明确告知能力已变
+ 这三种事件下同步,没有任何定时轮询。真正的撤权由云端在网关调用时裁决:返回
+ 401/403 会立刻清掉本机的桥状态,被撤权的能力下一次调用即失败。
+ """
+ from core.harness.capability_errors import CloudUnavailable
+
+ st = state or get_state()
+ if not st:
+ raise CloudUnavailable("cloud authorization expired; sign in again")
+ require_current_account(st)
+
+
+def purge_persisted_state() -> None:
+ """Remove credentials left by an earlier release; bridge tokens are memory-only."""
+ try:
+ from features.desktop.bridge.model_credentials import scrub_legacy_rows
+
+ scrub_legacy_rows()
+ except Exception as exc:
+ logger.warning(
+ "[cloud-bridge] legacy model credential cleanup unavailable: %s", type(exc).__name__
+ )
+ try:
+ from core.db.engine import SessionLocal
+ from core.db.models import ContentBlock
+
+ with SessionLocal() as db:
+ row = db.get(ContentBlock, bridge_client_state.BRIDGE_BLOCK_ID)
+ if row is not None:
+ db.delete(row)
+ db.commit()
+ except Exception as exc:
+ logger.warning(
+ "[cloud-bridge] legacy credential cleanup unavailable: %s", type(exc).__name__
+ )
+
+
+def load_state_from_db() -> None:
+ purge_persisted_state()
+ return None
+
+
+def current_credentials(state: Dict[str, Any]) -> Dict[str, Any]:
+ """The credential belongs to the account, not to the snapshot a pass started with.
+
+ A sync pass can outlive the token it started with while the shell has already
+ renewed it. Same account → newest token; another account → untouched, so the
+ surrounding identity checks reject it.
+ """
+ with bridge_client_state.state_lock:
+ live = bridge_client_state.state
+ if live and state_fingerprint(live) == state_fingerprint(state):
+ return live
+ return state
+
+
+def note_rejected_credential(state: Dict[str, Any], exc: BaseException) -> None:
+ """Only a refusal of the live token schedules re-sync on its next renewal."""
+ pass # State is owned by its responsibility module.
+ import httpx
+
+ if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401:
+ authorization = exc.request.headers.get("Authorization")
+ with bridge_client_state.state_lock:
+ if (
+ bridge_client_state.state
+ and authorization == f"Bearer {bridge_client_state.state.get('token')}"
+ and state_fingerprint(bridge_client_state.state) == state_fingerprint(state)
+ ):
+ bridge_client_state.credential_rejected = state_fingerprint(state)
+
+
+def cloud_headers(state: Dict[str, Any]) -> Dict[str, str]:
+ from core.capabilities.manifest_protocol import token_claims
+
+ state = current_credentials(state)
+ device_id = str(
+ state.get("device_id") or token_claims(str(state.get("token") or "")).get("d") or ""
+ )
+ headers = {"Authorization": f"Bearer {state['token']}"}
+ if device_id:
+ headers["X-Desktop-Device-Id"] = device_id
+ return headers
+
+
+def clear_state() -> None:
+ """Logout invalidates the account immediately, including in-flight responses."""
+ pass # State is owned by its responsibility module.
+ with bridge_client_state.state_lock:
+ bridge_client_state.state, bridge_client_state.state_loaded = None, True
+ bridge_client_state.credential_rejected = ""
+ clear_partial_selection()
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest, bridge_client_state.manifest_error, bridge_client_state.manifest_ts = None, None, 0.0
+ from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills
+
+ desktop_cloud_skills.on_account_switch()
+ desktop_cloud_bundles.on_account_switch()
+ purge_persisted_state()
+ rebuild_identity_views()
+
+
+def rebuild_identity_views() -> None:
+ from features.desktop.capabilities import skills
+ from features.desktop.capabilities.paths import capabilities_enabled
+
+ if not capabilities_enabled():
+ return
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
+
+ refresh_skill_caches()
+ shared = skill_paths.device_view_dir()
+ root = shared.parent / (shared.name + "_u")
+ if root.is_dir():
+ for child in root.iterdir():
+ if child.is_dir():
+ skill_views.rebuild_user_view(child.name)
+
+
+def get_state() -> Optional[Dict[str, Any]]:
+ pass # State is owned by its responsibility module.
+ with bridge_client_state.state_lock:
+ if not bridge_client_state.state_loaded:
+ bridge_client_state.state = load_state_from_db()
+ bridge_client_state.state_loaded = True
+ st = bridge_client_state.state
+ if not st:
+ return None
+ if float(st.get("expires_at") or 0) < time.time():
+ return None
+ return dict(st)
+
+
+def shell_user_center_id(cloud_base: str, user_center_id: str) -> str:
+ """壳送来的桥接用户标识:``cloud:::<云端 user_center_id>``。
+
+ 壳按云端地址给用户加命名空间,同一台机器连不同云端时本机身份天然隔离。本机侧
+ 用完全相同的规则从凭据里的云端身份推出壳会送来的标识,两者必须逐字相同。
+ 端口缺省与壳一致:https 443、其它 80。
+ """
+ from urllib.parse import urlsplit
+
+ parts = urlsplit(cloud_base.strip())
+ host = parts.hostname or "cloud"
+ port = parts.port or (443 if parts.scheme == "https" else 80)
+ return f"cloud:{host}:{port}:{user_center_id}"
+
+
+def get_identity_state() -> Optional[Dict[str, Any]]:
+ """Last shell-authenticated identity for offline local resources; never a cloud grant."""
+ from core.capabilities.manifest_protocol import token_claims
+
+ with bridge_client_state.state_lock:
+ st = dict(bridge_client_state.state) if bridge_client_state.state else None
+ if not st:
+ return None
+ claims = token_claims(str(st.get("token") or ""))
+ if not claims.get("c"):
+ return None
+ return {
+ "cloud_base": st["cloud_base"],
+ "user_center_id": str(claims["c"]),
+ "shell_user_center_id": shell_user_center_id(str(st["cloud_base"]), str(claims["c"])),
+ "subject": str(claims.get("u") or ""),
+ "device_id": str(claims.get("d") or ""),
+ "authorization_epoch": claims.get("a"),
+ }
+
+
+def bridge_active() -> bool:
+ """本进程是否应启用云端能力桥(仅桌面壳孵化的双端本机后端为 True)。"""
+ return bridge_auth.bridge_enabled() and get_state() is not None
+
+
+def clear_partial_selection():
+ from features.desktop.capabilities import session_availability, registry
+ if session_availability.active():
+ session_availability.clear()
+ registry._bump() # Runtime selection changed even though cloud intent did not.
+
+
+def reset_for_tests() -> None: # pragma: no cover - 仅测试用
+ pass # State is owned by its responsibility module.
+ pass # State is owned by its responsibility module.
+ from features.desktop.bridge import skills as desktop_cloud_skills
+
+ with bridge_client_state.state_lock:
+ bridge_client_state.state = None
+ bridge_client_state.state_loaded = False
+ bridge_client_state.credential_rejected = ""
+ with bridge_client_state.manifest_lock:
+ bridge_client_state.manifest = None
+ bridge_client_state.manifest_ts = 0.0
+ bridge_client_state.manifest_error = None
+ bridge_client_state.manifest_fetching = False
+ bridge_client_state.refresh_pending = False
+ desktop_cloud_skills.reset_for_tests()
+
+
+def context_headers(*, model=False):
+ from features.desktop.observability.context import context_headers as correlate
+ return correlate(model=model)
+
+
+def notify_cloud_changed():
+ from features.desktop.bridge.client import notify_cloud_changed as notify
+ return notify()
diff --git a/src/backend/features/desktop/bridge/managed_connectors.py b/src/backend/features/desktop/bridge/managed_connectors.py
new file mode 100644
index 00000000..d5a0ee70
--- /dev/null
+++ b/src/backend/features/desktop/bridge/managed_connectors.py
@@ -0,0 +1,144 @@
+"""managed connectors responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict, List
+from features.desktop.auth import bridge as bridge_auth
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.bridge import client_state as bridge_client_state
+from features.desktop.bridge import identity as bridge_identity
+
+def account_profile(st: Dict[str, Any]) -> str:
+ """Profile label of the bridged account (opaque tokens get a deterministic label)."""
+ from core.capabilities.reference import profile_id
+ from core.capabilities.manifest_protocol import token_subject
+
+ return profile_id(
+ str(st["cloud_base"]), token_subject(str(st.get("token") or "")) or "opaque-subject"
+ )
+
+
+def project_managed_profile(st: Dict[str, Any], manifest: Dict[str, Any]) -> None:
+ """Write the account's cloud connector references into mcp.json (managedProfiles)."""
+ from features.desktop.capabilities import mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+ from core.capabilities.reference import cloud_issuer
+
+ if not capabilities_enabled():
+ return
+ try:
+ mcp_json.project_managed_profile(
+ account_profile(st),
+ cloud_instance_id=cloud_issuer(str(st["cloud_base"])),
+ catalog_revision=str(manifest["revision"]),
+ servers=list(manifest.get("servers") or []),
+ initial_enabled={
+ str(s["server_id"]): bool(s.get("enabled", True))
+ for s in manifest.get("servers") or []
+ if isinstance(s, dict) and s.get("server_id")
+ },
+ )
+ except mcp_json.McpJsonCorrupt as exc:
+ logger.error("[cloud-bridge] mcp.json unreadable, projection skipped: %s", exc)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("[cloud-bridge] mcp.json projection failed: %s", exc)
+
+
+def managed_enabled(st: Dict[str, Any]) -> Dict[str, bool]:
+ from features.desktop.capabilities import mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+
+ if not capabilities_enabled():
+ return {}
+ try:
+ return mcp_json.managed_enabled(account_profile(st))
+ except mcp_json.McpJsonError as exc:
+ logger.warning("[cloud-bridge] mcp.json managed flags unavailable: %s", exc)
+ return {}
+
+
+def managed_connectors() -> List[Dict[str, Any]]:
+ """云端下发的连接器全集(含本机停用的),供本机能力中心展示。
+
+ ``_bridge_context`` 是装配用的,它按本机停用标志把关掉的过滤掉了——界面要的是
+ 「有哪些」,关掉的也得列出来,开关才有地方可点。启停仍以本机 mcp.json 为准。
+
+ 只读已缓存的清单,不触发任何刷新:这是界面读取路径,同步有它自己的时机。
+ """
+ st = bridge_identity.get_state()
+ if not st or not bridge_auth.bridge_enabled():
+ return []
+ with bridge_client_state.manifest_lock:
+ manifest = bridge_client_state.manifest
+ if not manifest:
+ return []
+ enabled = managed_enabled(st)
+ out: List[Dict[str, Any]] = []
+ for server in manifest.get("servers") or []:
+ if not isinstance(server, dict):
+ continue
+ sid = str(server.get("server_id") or "").strip()
+ if not sid:
+ continue
+ out.append(
+ {
+ "server_id": sid,
+ "display_name": str(server.get("display_name") or sid),
+ "description": str(server.get("description") or ""),
+ "created_at": server.get("created_at"),
+ "enabled": bool(enabled.get(sid, True)),
+ "source_plugin": str(server.get("source_plugin") or ""),
+ "icon": str(server.get("icon") or ""),
+ "tools": [
+ {
+ "name": str(tool.get("name") or ""),
+ "description": str(tool.get("description") or ""),
+ }
+ for tool in server.get("tools") or []
+ if isinstance(tool, dict)
+ ],
+ }
+ )
+ return out
+
+
+def set_managed_connector_enabled(server_id: str, enabled: bool) -> bool:
+ """记下用户对某个云端下发连接器的启停,落本机 mcp.json。
+
+ 本机是这个开关的唯一记账处:装配走 ``_bridge_context()``,它就按这里的标志
+ 过滤,云端不参与也不需要回写。返回 False 表示该 server_id 不是本机托管的
+ 云端连接器,调用方应继续按普通目录项处理。
+
+ 「数据库查询」是个伞形条目(见 ``device_catalog._db_umbrella_item``),本身没有
+ 对应的 server,开关要落到它收拢的那几个成员上——否则界面上的开关点了没有任何
+ 东西真的被打开。
+ """
+ from features.desktop.capabilities import mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+ from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID
+
+ st = bridge_identity.get_state()
+ if not capabilities_enabled() or not bridge_auth.bridge_enabled() or not st:
+ return False
+ profile = account_profile(st)
+ if not profile:
+ return False
+ sid = str(server_id)
+ if sid == DB_UMBRELLA_ID:
+ members = [
+ str(s["server_id"])
+ for s in managed_connectors()
+ if str(s.get("server_id")) in DB_HIDDEN_SERVERS
+ ]
+ if not members:
+ return False
+ for member in members:
+ try:
+ mcp_json.set_managed_enabled(profile, member, bool(enabled))
+ except mcp_json.McpJsonError:
+ return False
+ return True
+ try:
+ mcp_json.set_managed_enabled(profile, sid, bool(enabled))
+ except mcp_json.McpJsonError:
+ return False
+ return True
diff --git a/src/backend/core/services/desktop_model_credentials.py b/src/backend/features/desktop/bridge/model_credentials.py
similarity index 86%
rename from src/backend/core/services/desktop_model_credentials.py
rename to src/backend/features/desktop/bridge/model_credentials.py
index 3d5bc3a2..1f330714 100644
--- a/src/backend/core/services/desktop_model_credentials.py
+++ b/src/backend/features/desktop/bridge/model_credentials.py
@@ -1,10 +1,11 @@
"""Account-bound model gateway references; bearer credentials never enter model rows."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
import copy
from urllib.parse import urlsplit
-from core.capabilities.errors import CloudUnavailable
+from core.harness.capability_errors import CloudUnavailable
PREFIX = "desktop-capability:"
_GATEWAY = "/api/v1/desktop/capability/gateway/models/"
@@ -31,10 +32,10 @@ def gateway_cloud_base(base_url):
def sanitize_import(data):
"""Accept the legacy shell payload at the boundary, persist only a reference."""
- from core.auth.desktop_bridge import bridge_enabled
- from core.capabilities.paths import capabilities_enabled
- from core.capabilities.ref import profile_id
- from core.services.desktop_capability_protocol import token_subject
+ from features.desktop.auth.bridge import bridge_enabled
+ from features.desktop.capabilities.paths import capabilities_enabled
+ from core.capabilities.reference import profile_id
+ from core.capabilities.manifest_protocol import token_subject
out = copy.deepcopy(data)
for provider in out.get("providers") or []:
@@ -54,11 +55,11 @@ def sanitize_import(data):
def bind(reference, base_url):
"""Capture identity once per model instance; token rotation stays in memory."""
- from core.services import desktop_cloud_bridge as bridge
- from core.capabilities.ref import profile_id
- from core.services.desktop_capability_protocol import token_subject
+ from features.desktop.bridge import client as bridge
+ from core.capabilities.reference import profile_id
+ from core.capabilities.manifest_protocol import token_subject
- state = bridge.get_state()
+ state = bridge_identity.get_state()
if not state or not is_reference(reference):
raise CloudUnavailable("sign in again to use the cloud model")
expected_base = gateway_cloud_base(base_url)
@@ -71,10 +72,10 @@ def bind(reference, base_url):
def headers(reference, base_url, captured_state=None):
- from core.services import desktop_cloud_bridge as bridge
+ from features.desktop.bridge import client as bridge
captured_state = captured_state or bind(reference, base_url)
- bridge.require_current_account(captured_state)
+ bridge_identity.require_current_account(captured_state)
state = bind(reference, base_url)
token = str(state.get("token") or "")
# The signed device claim is decoded only for routing; the cloud validates it.
@@ -86,7 +87,7 @@ def headers(reference, base_url, captured_state=None):
device = str(state.get("device_id") or claims.get("device_id") or claims.get("d") or "")
except (ValueError, IndexError, TypeError):
device = ""
- from core.services.desktop_observation_context import context_headers
+ from features.desktop.observability.context import context_headers
return {"Authorization": "Bearer " + token, "X-Desktop-Device-Id": device, **context_headers(model=True)}
diff --git a/src/backend/core/services/desktop_release_events.py b/src/backend/features/desktop/bridge/release_events.py
similarity index 100%
rename from src/backend/core/services/desktop_release_events.py
rename to src/backend/features/desktop/bridge/release_events.py
diff --git a/src/backend/core/services/desktop_resource_plugins.py b/src/backend/features/desktop/bridge/resource_plugins.py
similarity index 86%
rename from src/backend/core/services/desktop_resource_plugins.py
rename to src/backend/features/desktop/bridge/resource_plugins.py
index 45040494..c913f803 100644
--- a/src/backend/core/services/desktop_resource_plugins.py
+++ b/src/backend/features/desktop/bridge/resource_plugins.py
@@ -8,13 +8,13 @@ def package_root(installed):
revision = (installed.get("import_report") or {}).get("package_revision")
if revision:
from core.db.engine import SessionLocal
- from core.plugins.packaging.runtime_assets import materialize
+ from core.capabilities.plugins.packaging.runtime_assets import materialize
with SessionLocal() as db:
return materialize(db, revision)
- from core.plugins.packaging.sources import _resolve_plugin_dir
+ from core.capabilities.plugins.packaging.sources import resolve_plugin_dir
- return _resolve_plugin_dir(installed["slug"])
+ return resolve_plugin_dir(installed["slug"])
def package_files(installed):
@@ -62,14 +62,14 @@ def local_config(server, user_id, profile):
if not settings.deploy.is_local:
return None
- from core.capabilities import device_catalog, store
- from core.capabilities.plugins import load_manifest
- from core.capabilities.dependency import component_hash
- from core.capabilities.local_plugin_runtime import enabled_for
- from core.config.mcp_config import _mcp_http_url
+ from features.desktop.capabilities import device_catalog, store
+ from features.desktop.capabilities.plugins import load_manifest
+ from features.desktop.capabilities.dependency import component_hash
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
+ from core.config.mcp_config import mcp_http_url as _mcp_http_url
from extensions.mcp_servers._ports import PORTS
- for inst in device_catalog._installations("plugin", user_id=user_id):
+ for inst in device_catalog.installations("plugin", user_id=user_id):
if inst.profile_id != profile or server["server_id"] not in inst.payload.get(
"components", {}
).get("mcp", []):
@@ -103,7 +103,6 @@ def local_config(server, user_id, profile):
"url": _mcp_http_url(audience),
"origin": "local_plugin",
"execution_scope": "local",
- "is_stable": False,
"owner_user_id": user_id,
"source_plugin": inst.key,
"plugin_install_id": inst.install_id,
diff --git a/src/backend/core/services/desktop_cloud_skills.py b/src/backend/features/desktop/bridge/skills.py
similarity index 86%
rename from src/backend/core/services/desktop_cloud_skills.py
rename to src/backend/features/desktop/bridge/skills.py
index 60582aaf..e983ec26 100644
--- a/src/backend/core/services/desktop_cloud_skills.py
+++ b/src/backend/features/desktop/bridge/skills.py
@@ -18,16 +18,21 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_hashes
+from features.desktop.capabilities import skill_views
+from features.desktop.capabilities import skills
import copy
import logging
import threading
from typing import Any, Dict, List, Optional
-from core.capabilities import registry, skills, store, manifest_order
-from core.capabilities.errors import CloudUnavailable, IntegrityFailed, PackageMissing
-from core.capabilities.paths import KIND_SKILL, capabilities_enabled, revision_for_hash
-from core.capabilities.ref import cloud_ref, profile_id
+from features.desktop.capabilities import registry, skills, store, manifest_order
+from core.harness.capability_errors import CloudUnavailable, IntegrityFailed, PackageMissing
+from core.capabilities.identity_rules import KIND_SKILL
+from features.desktop.capabilities.paths import capabilities_enabled, revision_for_hash
+from core.capabilities.reference import cloud_ref, profile_id
logger = logging.getLogger(__name__)
@@ -38,7 +43,7 @@
def _profile(state: Dict[str, Any]) -> str:
- from core.services.desktop_capability_protocol import token_subject
+ from core.capabilities.manifest_protocol import token_subject
subject = token_subject(str(state.get("token") or ""))
if not subject:
@@ -47,13 +52,13 @@ def _profile(state: Dict[str, Any]) -> str:
def _headers(state: Dict[str, Any]) -> Dict[str, str]:
- from core.services.desktop_cloud_bridge import cloud_headers
+ from features.desktop.bridge.identity import cloud_headers
return cloud_headers(state)
def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]:
import httpx
- from core.services.desktop_capability_protocol import validate_skill_manifest
+ from core.capabilities.manifest_protocol import validate_skill_manifest
ticket = manifest_order.begin(KIND_SKILL, _profile(state))
with _lock:
@@ -76,7 +81,7 @@ def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]:
def _reconcile_intent(manifest: Dict[str, Any], state: Dict[str, Any]) -> List[str]:
"""Atomically gate intent/cache publication against newer in-flight requests."""
global _manifest, _error
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
with account_scope(state):
with manifest_order.apply(KIND_SKILL, _profile(state), manifest):
@@ -132,7 +137,7 @@ def sync_blocking(state: Dict[str, Any]) -> None:
return
with _sync_lock:
try:
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
manifest = _fetch_manifest(state)
with account_scope(state):
@@ -143,9 +148,7 @@ def sync_blocking(state: Dict[str, Any]) -> None:
except manifest_order.StaleManifest:
return
except Exception as exc: # noqa: BLE001
- from core.services.desktop_cloud_bridge import (
- _state_fingerprint, get_state, note_rejected_credential,
- )
+ from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, get_state, note_rejected_credential
note_rejected_credential(state, exc)
if _state_fingerprint(get_state()) != _state_fingerprint(state):
@@ -157,8 +160,8 @@ def sync_blocking(state: Dict[str, Any]) -> None:
prepared = prepare(state, pending) if pending else []
if changed and not any(row.get("ok") for row in prepared):
skills.bump_view_generation()
- skills.rebuild_views(None)
- from core.agent_skills.cache_refresh import refresh_skill_caches
+ skill_views.rebuild_views(None)
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
refresh_skill_caches()
if changed or prepared:
@@ -190,7 +193,7 @@ def _download(state: Dict[str, Any], skill_id: str) -> bytes:
def prepare_one(state: Dict[str, Any], install_id: str) -> Dict[str, Any]:
"""Publish verified bytes for this account; failed updates keep the old revision."""
- from core.capabilities.preparation import prepare_component
+ from features.desktop.capabilities.preparation import prepare_component
inst = registry.get(install_id)
if inst is None or inst.kind != KIND_SKILL:
@@ -198,7 +201,7 @@ def prepare_one(state: Dict[str, Any], install_id: str) -> Dict[str, Any]:
return prepare_component(
state, inst,
download=lambda: _download(state, inst.key),
- content_hash=lambda comp: skills.skill_dir_hash(comp.path, fresh=True),
+ content_hash=lambda comp: skill_hashes.skill_dir_hash(comp.path, fresh=True),
).to_dict()
@@ -230,8 +233,8 @@ def prepare_item(iid):
(time.perf_counter_ns() - started) / 1_000_000)
if any(r["ok"] for r in results):
skills.bump_view_generation()
- skills.rebuild_views(None)
- from core.agent_skills.cache_refresh import refresh_skill_caches
+ skill_views.rebuild_views(None)
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
refresh_skill_caches()
return results
@@ -243,10 +246,10 @@ def apply_to_enabled_skill_ids(skill_ids: List[str]) -> List[str]:
manifest = _manifest
if not manifest:
return skill_ids
- resolution = skills.resolve_for_user(skills.current_local_user_id())
- from core.capabilities.plugins import enabled_cloud_skill_intents
+ resolution = skills.resolve_for_user(skill_account.current_local_user_id())
+ from features.desktop.capabilities.plugins import enabled_cloud_skill_intents
- plugin_intents = enabled_cloud_skill_intents(skills.current_local_user_id())
+ plugin_intents = enabled_cloud_skill_intents(skill_account.current_local_user_id())
hidden = set(resolution.conflicts) | (set(resolution.unusable) - plugin_intents)
ready = [c.runtime_name for c in resolution.chosen.values() if c.source == "cloud"]
kept = [sid for sid in skill_ids if sid not in hidden]
@@ -277,7 +280,7 @@ def status() -> Dict[str, Any]:
with _lock:
manifest = _manifest
err = _error
- from core.services.desktop_cloud_bridge import get_state
+ from features.desktop.bridge.identity import get_state
st = get_state()
profile = None
diff --git a/src/backend/features/desktop/bridge/status.py b/src/backend/features/desktop/bridge/status.py
new file mode 100644
index 00000000..495e4ac3
--- /dev/null
+++ b/src/backend/features/desktop/bridge/status.py
@@ -0,0 +1,113 @@
+"""status responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict
+from features.desktop.bridge import client_state as bridge_client_state
+from features.desktop.bridge import identity as bridge_identity
+
+def initial_sync_status() -> Dict[str, Any]:
+ """Report actual package counts without disk hashing or network probes."""
+ from features.desktop.bridge import skills as desktop_cloud_skills, bundles as desktop_cloud_bundles
+ from features.desktop.capabilities import session_availability
+ from features.desktop.capabilities.paths import revision_for_hash
+
+ st = bridge_identity.get_state()
+ if not st:
+ return {"ready": False, "syncing": False, "partial": False, "groups": [],
+ "pending": 0, "error": "云端身份尚未就绪", "can_continue": False}
+ with bridge_client_state.manifest_lock:
+ manifest_ready = bridge_client_state.manifest is not None
+ fetching = bridge_client_state.manifest_fetching
+ error = bridge_client_state.manifest_error
+ groups = {"skill": desktop_cloud_skills.status(), **desktop_cloud_bundles.status()}
+ pending, completed, total = 0, 0, 0
+ progress = []
+ for kind, group in groups.items():
+ error = error or group.get("last_error")
+ rows = [row for row in group.get("installations", []) if row.get("state") != "removed"]
+ done = sum(bool(row.get("content_hash")) and row.get("state") == "ready"
+ and row.get("resolved_revision") == revision_for_hash(row["content_hash"])
+ for row in rows)
+ failed = sum(row.get("state") == "failed" or bool(row.get("last_error")) for row in rows)
+ completed += done
+ total += len(rows)
+ pending += len(rows) - done
+ progress.append({"kind": kind, "total": len(rows), "completed": done,
+ "failed": failed, "manifest_ready": bool(group.get("revision"))})
+ complete_manifests = all(group.get("revision") for group in groups.values())
+ partial = session_availability.active(desktop_cloud_skills._profile(st))
+ # Progress is read without the identity lock: package rows may wait on the
+ # database, and the shell's token push must never queue behind that wait.
+ bridge_identity.require_current_account(st)
+ complete = (manifest_ready and not fetching and not error and pending == 0 and complete_manifests)
+ if not fetching and not complete and not partial:
+ error = error or "部分能力未能同步完成,请选择重试或使用已同步能力继续"
+ return {"ready": bool(complete or partial), "syncing": fetching, "partial": partial,
+ "groups": progress, "completed": completed, "total": total,
+ "totals_known": bool(complete_manifests), "pending": pending,
+ "error": error, "can_continue": bool(manifest_ready and not fetching and not complete)}
+
+
+def bridge_status() -> Dict[str, Any]:
+ """诊断视图(/v1/desktop/capability/cloud-bridge/status)。"""
+ from features.desktop.bridge import skills as desktop_cloud_skills
+
+ st = bridge_identity.get_state()
+ with bridge_client_state.manifest_lock:
+ servers = (bridge_client_state.manifest or {}).get("servers") or []
+ err = bridge_client_state.manifest_error
+ manifest_revision = str((bridge_client_state.manifest or {}).get("revision") or "")
+ schema_ready_count = sum(
+ 1 for server in servers if isinstance(server, dict) and bool(server.get("tools"))
+ )
+ return {
+ "configured": st is not None,
+ "cloud_base": (st or {}).get("cloud_base"),
+ "active": bridge_identity.bridge_active(),
+ "cloud_server_count": len(servers),
+ "manifest_revision": manifest_revision,
+ "schema_mode": "dynamic_manifest" if manifest_revision else "none",
+ "schema_ready_server_count": schema_ready_count,
+ "cloud_servers": [
+ {
+ "server_id": s.get("server_id"),
+ "component": s.get("component"),
+ "display_name": s.get("display_name"),
+ "origin": "cloud",
+ }
+ for s in servers
+ if isinstance(s, dict)
+ ],
+ "last_error": err,
+ "skills": desktop_cloud_skills.status(),
+ "bundles": bundles_status(),
+ "mcp_json": mcp_json_status(),
+ }
+
+
+def bundles_status() -> Dict[str, Any]:
+ from features.desktop.bridge import bundles as desktop_cloud_bundles
+
+ return desktop_cloud_bundles.status()
+
+
+def mcp_json_status() -> Dict[str, Any]:
+ from features.desktop.capabilities import connectors, mcp_json
+ from features.desktop.capabilities.paths import capabilities_enabled
+
+ if not capabilities_enabled():
+ return {"enabled": False}
+ try:
+ doc = mcp_json.load()
+ except mcp_json.McpJsonCorrupt as exc:
+ return {"enabled": True, "corrupt": True, "error": str(exc)}
+ res = connectors.last_resolution()
+ return {
+ "enabled": True,
+ "generation": doc.generation,
+ "local_server_count": len(doc.local),
+ "managed_profiles": {
+ p: len((v or {}).get("servers") or {}) for p, v in doc.managed.items()
+ },
+ "conflicts": sorted(res.conflicts) if res else [],
+ "shadowed": sorted(res.shadowed) if res else [],
+ }
diff --git a/src/backend/features/desktop/capabilities/__init__.py b/src/backend/features/desktop/capabilities/__init__.py
new file mode 100644
index 00000000..b98922f8
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/__init__.py
@@ -0,0 +1,16 @@
+"""Desktop capability runtime: partitioned storage, name resolution, flat runtime views.
+
+Three layers, deliberately separated (capability plan v2.0, ch. 04):
+
+- **store** — ``/s////``: immutable versions, one
+ sub-tree per source profile, same-named keys from different profiles coexist.
+- **resolver** — the single place that decides which candidate a runtime name points
+ to for this device/user. Explicit user choice → explicit request → the current
+ account's usable candidate → the only usable candidate → otherwise a conflict.
+- **view** — a flat directory of directory links (``/``) the
+ execution plane and the model see. The model-facing path contract
+ ``/workspace/skills/`` never changes.
+
+The package is inert unless :func:`paths.capability_root` is set — only the desktop
+local runtime keeps a file store; cloud deployments keep their database truth.
+"""
diff --git a/src/backend/core/capabilities/agents.py b/src/backend/features/desktop/capabilities/agents.py
similarity index 93%
rename from src/backend/core/capabilities/agents.py
rename to src/backend/features/desktop/capabilities/agents.py
index e25f2b26..1d8d13da 100644
--- a/src/backend/core/capabilities/agents.py
+++ b/src/backend/features/desktop/capabilities/agents.py
@@ -17,10 +17,11 @@
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Set
-from . import registry, store
-from .paths import KIND_AGENT, LOCAL_PROFILE, revision_for_hash
-from .ref import local_ref
-from .resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve
+from features.desktop.capabilities import registry, store
+from core.capabilities.identity_rules import KIND_AGENT, LOCAL_PROFILE
+from features.desktop.capabilities.paths import revision_for_hash
+from core.capabilities.reference import local_ref
+from features.desktop.capabilities.resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve
logger = logging.getLogger(__name__)
@@ -136,7 +137,7 @@ def to_serialized(self) -> Dict[str, Any]:
return data
def content_hash(self) -> str:
- from core.services.desktop_capability_protocol import entity_content_hash
+ from core.capabilities.manifest_protocol import entity_content_hash
return entity_content_hash(self.to_files())
@@ -152,7 +153,7 @@ def load_definition(comp: store.StoredComponent, *, origin: str) -> AgentDefinit
def account_definitions(profile: Optional[str] = None) -> List[AgentDefinition]:
"""Ready + enabled agents of the bridged account, loaded from the store."""
- from .skills import current_account_profile
+ from features.desktop.capabilities.skill_account import current_account_profile
profile = profile or current_account_profile()
if not profile:
@@ -200,7 +201,7 @@ def publish_local_agent(serialized: Dict[str, Any]) -> store.StoredComponent:
def remove_local_agent(agent_id: str) -> bool:
- from core.capabilities.runtime import references
+ from features.desktop.capabilities.runtime import references
if references("agent", LOCAL_PROFILE, agent_id):
# Source deletion revokes use; history still needs its exact bytes.
@@ -242,7 +243,7 @@ def resolve_visible(user_id: str, local_rows: Iterable[Dict[str, Any]]) -> Resol
install_id=registry.install_id(KIND_AGENT, LOCAL_PROFILE, defn.agent_id),
)
)
- from .skills import account_authorized_for
+ from features.desktop.capabilities.skill_account import account_authorized_for
for defn in account_definitions() if account_authorized_for(user_id) else []:
cands.append(
diff --git a/src/backend/core/capabilities/change_merge.py b/src/backend/features/desktop/capabilities/change_merge.py
similarity index 94%
rename from src/backend/core/capabilities/change_merge.py
rename to src/backend/features/desktop/capabilities/change_merge.py
index b675e747..e105c0f2 100644
--- a/src/backend/core/capabilities/change_merge.py
+++ b/src/backend/features/desktop/capabilities/change_merge.py
@@ -3,14 +3,8 @@
import hashlib
import json
import re
-from .archive import (
- _normalize_member,
- _validate_names,
- MAX_MEMBERS,
- MAX_TOTAL_BYTES,
- MAX_MEMBER_BYTES,
-)
-from core.agent_skills.binary_files import decode_binary, is_binary_value
+from core.capabilities.archive import normalize_member as _normalize_member, validate_names as _validate_names, MAX_MEMBERS, MAX_TOTAL_BYTES, MAX_MEMBER_BYTES
+from core.capabilities.skills.binary_files import decode_binary, is_binary_value
def validate_files(files):
diff --git a/src/backend/core/capabilities/change_signal.py b/src/backend/features/desktop/capabilities/change_signal.py
similarity index 100%
rename from src/backend/core/capabilities/change_signal.py
rename to src/backend/features/desktop/capabilities/change_signal.py
diff --git a/src/backend/core/capabilities/change_sync.py b/src/backend/features/desktop/capabilities/change_sync.py
similarity index 87%
rename from src/backend/core/capabilities/change_sync.py
rename to src/backend/features/desktop/capabilities/change_sync.py
index 1bf527bf..3e63c8da 100644
--- a/src/backend/core/capabilities/change_sync.py
+++ b/src/backend/features/desktop/capabilities/change_sync.py
@@ -1,4 +1,6 @@
"""Device-side explicit capability changes, isolated from conversation execution."""
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skills
import hashlib
import json
@@ -9,12 +11,16 @@
import httpx
from fastapi import HTTPException
from core.db.models import ContentBlock
-from core.agent_skills.binary_files import encode_upload
-from . import archive, registry, skills, store
-from .paths import assert_managed_path, LOCAL_PROFILE, BUILTIN_PROFILE
-from .change_merge import plan, revision, validate_files, sensitive_paths
-from .errors import CapabilityError
-from .mcp_json import McpJsonError
+from core.capabilities.skills.binary_files import encode_upload
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import skills
+from features.desktop.capabilities import store
+from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE
+from features.desktop.capabilities.paths import assert_managed_path
+from features.desktop.capabilities.change_merge import plan, revision, validate_files, sensitive_paths
+from core.harness.capability_errors import CapabilityError
+from features.desktop.capabilities.mcp_json import McpJsonError
from sqlalchemy.exc import SQLAlchemyError
from core.infra.crypto import encrypt_secret, decrypt_secret
@@ -43,7 +49,7 @@ def _save(key, payload):
def cloud_request(state, method, kind, key, body=None):
- from core.services.desktop_cloud_bridge import cloud_headers, require_current_account
+ from features.desktop.bridge.identity import cloud_headers, require_current_account
require_current_account(state)
url = (
@@ -86,16 +92,16 @@ def source(user_id, iid):
raise HTTPException(400, detail="能力标识无效") from exc
if kind not in ("skill", "plugin", "mcp"):
raise HTTPException(400, detail="此类型暂不支持文件提交")
- current = skills.current_account_profile()
+ current = skill_account.current_account_profile()
if profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, "local-json", current):
raise HTTPException(403, detail="该能力不属于当前账号")
- if profile == current and not skills.account_authorized_for(user_id):
+ if profile == current and not skill_account.account_authorized_for(user_id):
raise HTTPException(403, detail="账号不匹配")
inst = registry.get(iid)
if kind == "mcp":
if profile == LOCAL_PROFILE:
from core.db.models import AdminMcpServer
- from core.services.mcp_management_service import decrypt_mcp_headers
+ from core.capabilities.mcp.management import decrypt_mcp_headers
with registry._session() as db:
row = db.get(AdminMcpServer, key)
@@ -116,8 +122,8 @@ def source(user_id, iid):
}
spec["headers"] = decrypt_mcp_headers(row.headers or {})
elif profile == "local-json":
- from .mcp_json import load
- from .paths import mcp_json_path
+ from features.desktop.capabilities.mcp_json import load
+ from features.desktop.capabilities.paths import mcp_json_path
doc = load()
raw = assert_managed_path(mcp_json_path()).read_bytes()
@@ -157,7 +163,7 @@ def source(user_id, iid):
def preview(user_id, iid, state):
kind, key, files, _ = source(user_id, iid)
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
base = _read(_key("cap-change-base:", user_id, profile, iid))
key = base.get("cloud_key", key) if base else key
try:
@@ -173,7 +179,7 @@ def preview(user_id, iid, state):
base_files = base["files"] if base else ({} if not remote.get("exists") else None)
result = plan(base_files, files, remote["files"])
token = uuid.uuid4().hex
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
with account_scope(state):
_save(
@@ -204,16 +210,16 @@ def preview(user_id, iid, state):
def _apply_local(user_id, iid, files, path, expected_revision):
import os
- from .paths import require_root, revision_for_hash
- from core.agent_skills.binary_files import decode_binary, is_binary_value
- from core.services.desktop_capability_protocol import skill_content_hash, entity_content_hash
+ from features.desktop.capabilities.paths import require_root, revision_for_hash
+ from core.capabilities.skills.binary_files import decode_binary, is_binary_value
+ from core.capabilities.manifest_protocol import skill_content_hash, entity_content_hash
kind, profile, key = iid.split(":", 2)
if kind == "mcp":
if profile != "local-json":
raise HTTPException(409, detail="云端已保存,请在本机连接器编辑器确认合并配置")
- from .mcp_json import load, write, _validate_local_server
- from .paths import mcp_json_path
+ from features.desktop.capabilities.mcp_json import load, write, _validate_local_server
+ from features.desktop.capabilities.paths import mcp_json_path
spec = json.loads(files["connector.json"])
_validate_local_server(key, spec)
@@ -260,7 +266,7 @@ def _apply_local(user_id, iid, files, path, expected_revision):
except OSError:
os.replace(backup, path)
raise
- from .discovery import scan
+ from features.desktop.capabilities.discovery import scan
scan(user_id)
else:
@@ -277,7 +283,7 @@ def _apply_local(user_id, iid, files, path, expected_revision):
def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_key=None):
- from .change_merge import resolve
+ from features.desktop.capabilities.change_merge import resolve
record_key = "cap-change-preview:" + token
record = _read(record_key)
@@ -293,7 +299,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_
)
}
)
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
if not record or record["user_id"] != str(user_id) or record["profile"] != profile:
raise HTTPException(404, detail="比较记录不存在")
if record.get("result"):
@@ -345,7 +351,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_
}
# A remote success is never retried as a new write merely because local
# reconciliation failed. Keep its receipt and leave newer local edits alone.
- from core.services.desktop_cloud_bridge import account_scope
+ from features.desktop.bridge.identity import account_scope
with account_scope(state):
record["result"] = result
@@ -376,7 +382,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_
def annotate(user_id, listing):
"""Local-only change hints. Network comparison happens only on explicit preview."""
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
for item in listing["items"]:
iid = item["install_id"]
item["change_state"] = "unavailable"
@@ -392,7 +398,7 @@ def annotate(user_id, listing):
else:
inst = registry.get(iid)
if kind == "skill" and inst:
- from core.services.desktop_capability_protocol import skill_content_hash
+ from core.capabilities.manifest_protocol import skill_content_hash
digest = skill_content_hash(
files.get("SKILL.md", ""),
@@ -400,7 +406,7 @@ def annotate(user_id, listing):
)
item["change_state"] = "modified" if digest != inst.content_hash else "compare"
elif kind == "plugin" and inst:
- from core.services.desktop_capability_protocol import entity_content_hash
+ from core.capabilities.manifest_protocol import entity_content_hash
item["change_state"] = (
"modified" if entity_content_hash(files) != inst.content_hash else "compare"
@@ -425,7 +431,7 @@ def record_downloaded(inst):
import logging
try:
- user_id = skills.current_local_user_id()
+ user_id = skill_account.current_local_user_id()
if not user_id:
return
_, key, files, _ = source(user_id, inst.install_id)
diff --git a/src/backend/core/capabilities/connectors.py b/src/backend/features/desktop/capabilities/connectors.py
similarity index 90%
rename from src/backend/core/capabilities/connectors.py
rename to src/backend/features/desktop/capabilities/connectors.py
index 9d8dd679..c4f8f28e 100644
--- a/src/backend/core/capabilities/connectors.py
+++ b/src/backend/features/desktop/capabilities/connectors.py
@@ -16,9 +16,9 @@
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Set
-from . import registry
-from .paths import KIND_MCP, LOCAL_PROFILE
-from .resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve
+from features.desktop.capabilities import registry
+from core.capabilities.identity_rules import KIND_MCP, LOCAL_PROFILE
+from features.desktop.capabilities.resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve
MCP_JSON_PROFILE = "local-json"
_DB_PATH = Path("")
@@ -36,8 +36,8 @@ def db_candidates(
Include those authorized declarations in every device binding resolution;
the caller's enabled-ID allowlist still controls their ambient availability.
"""
- from .local_plugin_runtime import configs
- from .skills import current_local_user_id
+ from features.desktop.capabilities.local_plugin_runtime import configs
+ from features.desktop.capabilities.skill_account import current_local_user_id
selected_user = user_id if user_id is not None else current_local_user_id()
ids = set(server_ids) | set(configs(selected_user))
@@ -115,7 +115,7 @@ def server_id_of(candidate: Candidate) -> str:
def resolve_bindings(candidates: List[Candidate], *, user_id: Optional[str] = None) -> Resolution:
"""Resolve exact source bindings without deployment-specific tool filters."""
- from .skills import current_local_user_id
+ from features.desktop.capabilities.skill_account import current_local_user_id
selected_user_id = user_id if user_id is not None else current_local_user_id()
res = resolve(
diff --git a/src/backend/core/capabilities/credentials.py b/src/backend/features/desktop/capabilities/credentials.py
similarity index 100%
rename from src/backend/core/capabilities/credentials.py
rename to src/backend/features/desktop/capabilities/credentials.py
diff --git a/src/backend/core/capabilities/dependency.py b/src/backend/features/desktop/capabilities/dependency.py
similarity index 94%
rename from src/backend/core/capabilities/dependency.py
rename to src/backend/features/desktop/capabilities/dependency.py
index c715eca4..13e29467 100644
--- a/src/backend/core/capabilities/dependency.py
+++ b/src/backend/features/desktop/capabilities/dependency.py
@@ -7,6 +7,8 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_hashes
import copy
import importlib.metadata
@@ -25,9 +27,9 @@
from packaging.specifiers import SpecifierSet
from packaging.version import Version
-from . import registry, skills, store
-from .errors import CapabilityError
-from .paths import BUILTIN_PROFILE, LOCAL_PROFILE
+from features.desktop.capabilities import registry, skills, store
+from core.harness.capability_errors import CapabilityError
+from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE
logger = logging.getLogger(__name__)
@@ -37,7 +39,7 @@ class DependencyMissing(CapabilityError):
recovery_action = "inspect_dependencies"
-from .dependency_declarations import _entries, _identifier
+from features.desktop.capabilities.dependency_declarations import _entries, _identifier
def _parse_skill_definition(text):
@@ -68,11 +70,11 @@ def skill_definition(path: Path):
def component_hash(comp, *, fresh: bool = False):
"""A component's content digest. ``fresh`` reads every byte; see ``dir_digest``."""
if comp.kind == "skill":
- return skills.skill_dir_hash(comp.path, fresh=fresh)
- from core.agent_skills.binary_files import encode_upload
- from core.services.desktop_capability_protocol import entity_content_hash
+ return skill_hashes.skill_dir_hash(comp.path, fresh=fresh)
+ from core.capabilities.skills.binary_files import encode_upload
+ from core.capabilities.manifest_protocol import entity_content_hash
- from .archive import iter_files
+ from core.capabilities.archive import iter_files
def read_all():
return entity_content_hash(
@@ -83,7 +85,7 @@ def read_all():
}
)
- return skills.dir_digest(comp.path, read_all, fresh=fresh)
+ return skill_hashes.dir_digest(comp.path, read_all, fresh=fresh)
@dataclass
@@ -228,8 +230,8 @@ def resolve(self, kind, profile, key):
_, selected_profile, key = parts
if selected_profile not in (LOCAL_PROFILE, BUILTIN_PROFILE):
if (
- selected_profile != skills.current_account_profile()
- or not skills.account_authorized_for(self.context.user_id)
+ selected_profile != skill_account.current_account_profile()
+ or not skill_account.account_authorized_for(self.context.user_id)
):
return None, None, None
profile = selected_profile
@@ -316,7 +318,7 @@ def visit(self, entry, profile, chain=(), required=True):
):
self.issue("component_not_ready", path, required)
return
- if comp.profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, skills.current_account_profile()):
+ if comp.profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, skill_account.current_account_profile()):
self.issue("not_authorized_or_missing", path, required)
return
if (
@@ -471,7 +473,7 @@ def require_report(report):
def allowed_model_ids(user_id):
"""Reuse the live model gateway's role/user-switch authorization policy."""
from core.db.models import ModelProvider
- from core.services.desktop_capability_models import _model_provider_allowed
+ from features.desktop.capability.models import _model_provider_allowed
with registry._session() as db:
return {
diff --git a/src/backend/core/capabilities/dependency_declarations.py b/src/backend/features/desktop/capabilities/dependency_declarations.py
similarity index 100%
rename from src/backend/core/capabilities/dependency_declarations.py
rename to src/backend/features/desktop/capabilities/dependency_declarations.py
diff --git a/src/backend/core/capabilities/device_catalog.py b/src/backend/features/desktop/capabilities/device_catalog.py
similarity index 81%
rename from src/backend/core/capabilities/device_catalog.py
rename to src/backend/features/desktop/capabilities/device_catalog.py
index bc7c2335..6f39f4a3 100644
--- a/src/backend/core/capabilities/device_catalog.py
+++ b/src/backend/features/desktop/capabilities/device_catalog.py
@@ -11,14 +11,16 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
import logging
from functools import lru_cache
from typing import Any, Dict, List, Optional, Tuple
-from core.config.catalog_runtime import merge_items_by_id
+from core.capabilities.catalog.runtime import merge_items_by_id
-from .paths import KIND_AGENT, KIND_PLUGIN, KIND_SKILL, capabilities_enabled, capability_root
+from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL
+from features.desktop.capabilities.paths import capabilities_enabled, capability_root
logger = logging.getLogger(__name__)
@@ -33,38 +35,38 @@ def active() -> bool:
自带的内置技能与内置连接器整体不参与——既不进能力中心的清单,也不进装配,用户
才不会看到云端账号里根本没有、点了也没反应的能力。
"""
- from core.auth.desktop_bridge import bridge_enabled
+ from features.desktop.auth.bridge import bridge_enabled
return bool(bridge_enabled() and capabilities_enabled())
def _profile() -> Optional[str]:
- from core.capabilities import skills
+ from features.desktop.capabilities import skills
try:
- return skills.current_account_profile()
+ return skill_account.current_account_profile()
except Exception as exc: # noqa: BLE001 - 身份未就绪时清单为空,不是错误
logger.debug("[device-catalog] account profile unavailable: %s", exc)
return None
-def _installations(kind: str, user_id=None) -> List[Any]:
- from . import registry, skills
- from .paths import LOCAL_PROFILE
+def installations(kind: str, user_id=None) -> List[Any]:
+ from features.desktop.capabilities import registry, skills
+ from core.capabilities.identity_rules import LOCAL_PROFILE
if not capabilities_enabled():
return []
- user_id = user_id or skills.current_local_user_id()
+ user_id = user_id or skill_account.current_local_user_id()
rows = [row for row in registry.list_installations(kind=kind, profile_id=LOCAL_PROFILE)
if user_id and (row.payload.get("owner_user_id") == user_id or (row.payload.get("shared_installation") and not active()))]
profile = _profile()
- if profile and (user_id is None or skills.account_authorized_for(user_id)):
+ if profile and (user_id is None or skill_account.account_authorized_for(user_id)):
rows += registry.list_installations(kind=kind, profile_id=profile)
return rows
def _managed_connectors() -> List[Dict[str, Any]]:
- from core.services.desktop_cloud_bridge import managed_connectors
+ from features.desktop.bridge.managed_connectors import managed_connectors
return managed_connectors()
@@ -82,7 +84,7 @@ def _read_component(
的失效点。否则每打开一次能力中心,就把每个技能的目录重读一遍。``root`` 只为
进键——换一个能力仓就是另一套文件。
"""
- from . import store
+ from features.desktop.capabilities import store
component = store.get(KIND_SKILL, profile, key, revision)
if component is None:
@@ -114,7 +116,7 @@ def _skill_body(inst) -> Optional[Tuple[str, Tuple[str, ...]]]:
def _skill_item(inst) -> Dict[str, Any]:
- from core.config.catalog_details import skill_body_from_raw
+ from core.capabilities.catalog.details import skill_body_from_raw
description = inst.description or ""
item = {
@@ -169,12 +171,7 @@ def _db_umbrella_item(members: List[Dict[str, Any]]) -> Dict[str, Any]:
启停取成员的并集:任一成员开着,伞形就是开着的。
"""
- from core.config.catalog_loader import (
- DB_UMBRELLA_DESC,
- DB_UMBRELLA_ICON,
- DB_UMBRELLA_ID,
- DB_UMBRELLA_NAME,
- )
+ from core.capabilities.catalog.loader import DB_UMBRELLA_DESC, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, DB_UMBRELLA_NAME
return {
"id": DB_UMBRELLA_ID,
@@ -203,15 +200,15 @@ def catalog_overlay(user_id=None) -> Dict[str, Any]:
"""
if not capabilities_enabled():
return {"skills": [], "mcp": [], "hidden_skills": set(), "hidden_mcp": set()}
- skills = _installations(KIND_SKILL, user_id=user_id)
+ skills = installations(KIND_SKILL, user_id=user_id)
connectors = _managed_connectors()
hidden_skills = {inst.key for inst in skills if inst.source_plugin}
hidden_mcp = {str(s["server_id"]) for s in connectors if s.get("source_plugin")}
- for inst in _installations(KIND_PLUGIN, user_id=user_id):
+ for inst in installations(KIND_PLUGIN, user_id=user_id):
components = dict(inst.payload.get("components") or {})
hidden_skills.update(str(sid) for sid in components.get("skills") or [])
hidden_mcp.update(str(mid) for mid in components.get("mcp") or [])
- from core.config.catalog_loader import DB_HIDDEN_SERVERS
+ from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS
standalone = [s for s in connectors if not s.get("source_plugin")]
db_members = [s for s in standalone if str(s.get("server_id")) in DB_HIDDEN_SERVERS]
@@ -237,7 +234,7 @@ def _account_definitions() -> Dict[str, Any]:
逐个 ``account_definition(id)`` 会把整份清单连同文件重查一遍,条目一多就是
N+1 次查询加 N×M 次文件解析。
"""
- from . import agents as caps_agents
+ from features.desktop.capabilities import agents as caps_agents
try:
return {defn.agent_id: defn for defn in caps_agents.account_definitions()}
@@ -286,11 +283,11 @@ def agent_entries() -> List[Dict[str, Any]]:
"""云端账号的智能体投影成 ``/v1/agents`` 的条目结构。"""
if not active():
return []
- installations = _installations(KIND_AGENT)
- if not installations:
+ installed_agents = installations(KIND_AGENT)
+ if not installed_agents:
return []
definitions = _account_definitions()
- return [_agent_entry(inst, definitions.get(inst.key)) for inst in installations]
+ return [_agent_entry(inst, definitions.get(inst.key)) for inst in installed_agents]
def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
@@ -299,7 +296,7 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict
定义(提示词、绑定的技能与工具)是云端那一份,改它要去云端改完再同步;这台
机器上能决定的只有开不开。
"""
- from . import registry
+ from features.desktop.capabilities import registry
if "is_enabled" not in data:
return None
@@ -312,9 +309,7 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict
return _agent_entry(inst, _account_definitions().get(agent_id))
-from .device_plugin_catalog import (
- plugin_entries, plugin_detail, plugin_ui_contributions, _projected_plugin,
-)
+from features.desktop.capabilities.device_plugin_catalog import plugin_entries, plugin_detail, plugin_ui_contributions, _projected_plugin
# ── 启停 ──────────────────────────────────────────────────────────────
@@ -322,8 +317,8 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict
def set_enabled(kind: str, item_id: str, enabled: bool, user_id=None) -> bool:
"""把启停写进登记表;返回 False 表示这条不是云端投影项,调用方照常处理。"""
- from . import registry
- matches = [row for row in _installations(kind, user_id=user_id)
+ from features.desktop.capabilities import registry
+ matches = [row for row in installations(kind, user_id=user_id)
if item_id in (row.key, row.install_id)]
if len(matches) != 1:
return False
@@ -337,13 +332,13 @@ def set_plugin_enabled(install_id_or_slug: str, enabled: bool, user_id=None) ->
"""插件按云端 install_id 或 slug 定位后写登记表。"""
if not capabilities_enabled():
return False
- from . import registry
+ from features.desktop.capabilities import registry
inst = _projected_plugin(install_id_or_slug, user_id)
if inst is None:
return False
if inst.profile_id == "local":
- from core.plugins.local.service import set_enabled_for_user
+ from features.desktop.local.plugins.service import set_enabled_for_user
set_enabled_for_user(user_id, inst.install_id, enabled)
else:
registry.set_enabled(inst.install_id, bool(enabled))
@@ -354,8 +349,8 @@ def set_plugin_enabled(install_id_or_slug: str, enabled: bool, user_id=None) ->
def _invalidate() -> None:
"""启停变了,运行时解析与目录缓存都要重算。"""
try:
- from core.agent_skills.cache_refresh import refresh_skill_caches
- from core.config.catalog_runtime import invalidate_runtime_catalog_cache
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
+ from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache
invalidate_runtime_catalog_cache()
refresh_skill_caches()
@@ -374,3 +369,26 @@ def _invalidate() -> None:
"set_plugin_enabled",
"toggle_projected_agent",
]
+
+
+def resolve_plugin(slug, user_id, *, install_id=None):
+ from features.desktop.capabilities.device_plugin_catalog import resolve_resource_plugin
+ return resolve_resource_plugin(slug, user_id, install_id=install_id)
+
+
+def account_plugin_installations(user_id):
+ from features.desktop.capabilities import registry
+ profile = skill_account.current_account_profile()
+ if not profile or not skill_account.account_authorized_for(user_id):
+ return []
+ return registry.list_installations(kind="plugin", profile_id=profile)
+
+
+def owned_identities(user_id):
+ from features.desktop.capabilities import registry
+ from core.capabilities.identity_rules import LOCAL_PROFILE
+ profiles = [LOCAL_PROFILE]
+ if skill_account.account_authorized_for(user_id):
+ profiles.append(skill_account.current_account_profile())
+ return {(row.kind, row.key) for row in registry.list_installations(profiles=profiles)
+ if row.payload.get("owner_user_id") in (None, "", user_id)}
diff --git a/src/backend/core/capabilities/device_plugin_catalog.py b/src/backend/features/desktop/capabilities/device_plugin_catalog.py
similarity index 69%
rename from src/backend/core/capabilities/device_plugin_catalog.py
rename to src/backend/features/desktop/capabilities/device_plugin_catalog.py
index 5c6d2605..d5b383ac 100644
--- a/src/backend/core/capabilities/device_plugin_catalog.py
+++ b/src/backend/features/desktop/capabilities/device_plugin_catalog.py
@@ -1,8 +1,8 @@
"""Plugin catalog views over authorized device installations."""
from typing import Any, Dict, List, Optional
-from . import device_catalog as catalog
-from .local_plugin_runtime import enabled_for
-from .paths import KIND_PLUGIN, KIND_SKILL, capabilities_enabled
+from features.desktop.capabilities.local_plugin_runtime import enabled_for
+from core.capabilities.identity_rules import KIND_PLUGIN, KIND_SKILL
+from features.desktop.capabilities.paths import capabilities_enabled
# ── 插件 ──────────────────────────────────────────────────────────────
@@ -11,11 +11,11 @@ def plugin_entries(user_id=None) -> List[Dict[str, Any]]:
"""云端账号的插件投影成 ``/v1/plugins/installed`` 的条目结构。"""
if not capabilities_enabled():
return []
- from .management_context import management_context
- from .readiness import file_readiness
+ from features.desktop.capabilities.management_context import management_context
+ from features.desktop.capabilities.readiness import file_readiness
context = management_context(user_id)
entries: List[Dict[str, Any]] = []
- for inst in catalog._installations(KIND_PLUGIN, user_id=user_id):
+ for inst in catalog.installations(KIND_PLUGIN, user_id=user_id):
components = dict(inst.payload.get("components") or {})
readiness = file_readiness(inst, context)
entries.append(
@@ -51,12 +51,12 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]:
"""
if not capabilities_enabled():
return []
- from core.plugins.ui.contract import public_contributions
+ from core.harness.registry.package_views import public_package_view
- from . import plugins as caps_plugins, store
+ from features.desktop.capabilities import plugins as caps_plugins, store
out: List[Dict[str, Any]] = []
- for inst in catalog._installations(KIND_PLUGIN, user_id=user_id):
+ for inst in catalog.installations(KIND_PLUGIN, user_id=user_id):
if not (enabled_for(inst, user_id) and inst.ready):
continue
comp = store.get(KIND_PLUGIN, inst.profile_id, inst.key, inst.resolved_revision)
@@ -67,7 +67,7 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]:
except (OSError, ValueError):
# 清单读不出来只影响这一个插件的界面,不该让整页拿不到贡献。
continue
- public = public_contributions(manifest.get("ui_contributions"), slug=inst.key)
+ public = public_package_view(manifest.get("ui_contributions"), slug=inst.key)
if public.get("contributes"):
out.append(public)
return out
@@ -75,7 +75,7 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]:
def _projected_plugin(install_id_or_slug: str, user_id=None):
"""按云端 install_id、slug 或本机 install_id 定位一条投影插件。"""
- for inst in catalog._installations(KIND_PLUGIN, user_id=user_id):
+ for inst in catalog.installations(KIND_PLUGIN, user_id=user_id):
cloud_id = str(inst.payload.get("cloud_install_id") or "")
if install_id_or_slug in (inst.key, cloud_id, inst.install_id, *inst.payload.get("legacy_ids", [])):
return inst
@@ -83,9 +83,9 @@ def _projected_plugin(install_id_or_slug: str, user_id=None):
def _component_skill(profile: str, skill_id: str) -> Optional[Dict[str, Any]]:
- from core.config.catalog_details import skill_body_from_raw
+ from core.capabilities.catalog.details import skill_body_from_raw
- from . import registry
+ from features.desktop.capabilities import registry
inst = registry.get(registry.install_id(KIND_SKILL, profile, skill_id))
if inst is None or inst.state == "removed":
@@ -126,8 +126,8 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A
inst = _projected_plugin(install_id_or_slug, user_id)
if inst is None:
return None
- from core.plugins.connections import manifest_connection
- from . import plugins, store
+ from core.capabilities.plugins.connections import manifest_connection
+ from features.desktop.capabilities import plugins, store
comp = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision)
manifest = plugins.load_manifest(comp) if comp else {}
@@ -140,7 +140,7 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A
]
known = {c["server_id"]: c for c in catalog._managed_connectors()}
if profile == "local":
- from .local_plugin_runtime import configs
+ from features.desktop.capabilities.local_plugin_runtime import configs
known.update(configs(user_id or inst.payload.get("owner_user_id"), enabled_only=False))
return {
"install_id": str(inst.payload.get("cloud_install_id") or inst.install_id),
@@ -158,3 +158,40 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A
"admin_config": None,
"connection": manifest_connection(manifest),
}
+
+
+def resolve_resource_plugin(slug, user_id, *, install_id=None):
+ """Resolve the exact device installation for a resource launch."""
+ from core.capabilities.plugins.resources.installation import Installation
+ from fastapi import HTTPException
+ from features.desktop.capabilities import store
+ from features.desktop.capabilities.device_plugin_catalog import _projected_plugin
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
+ from features.desktop.capabilities.plugins import load_manifest
+
+ candidate = _projected_plugin(install_id or slug, user_id=user_id)
+ if candidate is not None:
+ if candidate.key != slug or not candidate.ready or not enabled_for(candidate, user_id):
+ raise HTTPException(403, "plugin_not_ready")
+ component = store.get(
+ "plugin", candidate.profile_id, candidate.key, candidate.resolved_revision
+ )
+ if component is None:
+ raise HTTPException(409, "plugin_package_missing")
+ manifest = load_manifest(component)
+ package = component.path / "package"
+ if not package.is_dir():
+ raise HTTPException(409, "plugin_package_missing")
+ return Installation(
+ candidate.install_id,
+ candidate.resolved_revision,
+ slug,
+ manifest.get("ui_contributions") or {},
+ package,
+ )
+ return None
+
+
+# device_catalog re-exports this module's functions at import time, so it is
+# bound last: by then those names exist whichever module was imported first.
+from features.desktop.capabilities import device_catalog as catalog # noqa: E402
diff --git a/src/backend/core/capabilities/discovery.py b/src/backend/features/desktop/capabilities/discovery.py
similarity index 85%
rename from src/backend/core/capabilities/discovery.py
rename to src/backend/features/desktop/capabilities/discovery.py
index 5e366e7c..8a894a92 100644
--- a/src/backend/core/capabilities/discovery.py
+++ b/src/backend/features/desktop/capabilities/discovery.py
@@ -1,8 +1,8 @@
"""Read-only inventory of unregistered skill folders. Installation is explicit."""
import re
-from . import archive
-from .errors import CapabilityError
-from .paths import capabilities_enabled, kind_root, assert_managed_path
+from core.capabilities import archive as archive
+from core.harness.capability_errors import CapabilityError
+from features.desktop.capabilities.paths import capabilities_enabled, kind_root, assert_managed_path
def scan(user_id):
diff --git a/src/backend/core/capabilities/invocation.py b/src/backend/features/desktop/capabilities/invocation.py
similarity index 74%
rename from src/backend/core/capabilities/invocation.py
rename to src/backend/features/desktop/capabilities/invocation.py
index c38bfe0f..c8cf5356 100644
--- a/src/backend/core/capabilities/invocation.py
+++ b/src/backend/features/desktop/capabilities/invocation.py
@@ -1,12 +1,16 @@
"""Explicit desktop selections use the same authorized sources as runtime views."""
+from features.desktop.bridge import gateway as bridge_gateway
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_candidates as skill_catalog
+from features.desktop.capabilities import skills
-from . import connectors, plugins, registry, skills
-from .errors import NameConflict
+from features.desktop.capabilities import connectors, plugins, registry, skills
+from core.harness.capability_errors import NameConflict
def cloud_plugin_selection(ident, *, user_id):
- from .local_plugin_runtime import configs, enabled_for
- from .device_catalog import active
+ from features.desktop.capabilities.local_plugin_runtime import configs, enabled_for
+ from features.desktop.capabilities.device_catalog import active
local = [row for row in registry.list_installations(kind="plugin", profile_id="local")
if (row.payload.get("owner_user_id") == user_id or (row.payload.get("shared_installation") and not active()))
and ident in (row.key, row.install_id, *row.payload.get("legacy_ids", []))]
@@ -18,9 +22,9 @@ def cloud_plugin_selection(ident, *, user_id):
"skills": [sid for sid in components.get("skills", [])
if (child := registry.get(registry.install_id("skill", "local", sid))) and child.enabled and child.ready],
"mcp": [sid for sid in components.get("mcp", []) if sid in available]}
- if not skills.account_authorized_for(user_id):
+ if not skill_account.account_authorized_for(user_id):
return None
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
matches = [
row
for row in registry.list_installations(kind="plugin", profile_id=profile)
@@ -33,9 +37,9 @@ def cloud_plugin_selection(ident, *, user_id):
row = matches[0]
if row.enabled and not row.ready:
# 用户在对话里直接选了一个尚未下载的云端插件:按需准备定义与组件。
- from .preparation import ensure_cloud_ready
+ from features.desktop.capabilities.preparation import ensure_cloud_ready
- from .errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
try:
ensure_cloud_ready(user_id, install_ids=[row.install_id])
@@ -51,9 +55,9 @@ def cloud_plugin_selection(ident, *, user_id):
"skills": [],
"mcp": [],
}
- from .preparation import ensure_cloud_ready as _ensure_components
+ from features.desktop.capabilities.preparation import ensure_cloud_ready as _ensure_components
- from .errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
try:
_ensure_components(user_id, install_ids=[row.install_id])
@@ -75,17 +79,17 @@ def resolve_explicit_ids(user_id, allowed_skill_ids, allowed_mcp_ids):
For names present in the desktop store, its current resolver is authoritative.
This function never installs a component or imports a cloud credential.
"""
- from core.services import desktop_cloud_bridge as bridge
+ from features.desktop.bridge import client as bridge
- skill_candidates = skills.candidates(user_id)
+ skill_candidates = skill_catalog.candidates(user_id)
skill_resolution = skills.resolve_for_user(user_id)
known_skills = {candidate.runtime_name for candidate in skill_candidates}
allowed_skills = set(allowed_skill_ids) - known_skills
allowed_skills.update(skill_resolution.chosen)
- context = bridge._bridge_context() if skills.account_authorized_for(user_id) else None
- local_ids = bridge._local_server_ids()
- local_json = bridge._mcp_json_local_declarations()
+ context = bridge_gateway.bridge_context() if skill_account.account_authorized_for(user_id) else None
+ local_ids = bridge_gateway.local_server_ids()
+ local_json = bridge_gateway.mcp_json_local_declarations()
candidates = connectors.db_candidates(local_ids, set(allowed_mcp_ids), user_id=user_id)
candidates.extend(connectors.json_candidates(local_json))
if context:
diff --git a/src/backend/core/capabilities/local_plugin_runtime.py b/src/backend/features/desktop/capabilities/local_plugin_runtime.py
similarity index 72%
rename from src/backend/core/capabilities/local_plugin_runtime.py
rename to src/backend/features/desktop/capabilities/local_plugin_runtime.py
index 47dd1004..ec7b5e80 100644
--- a/src/backend/core/capabilities/local_plugin_runtime.py
+++ b/src/backend/features/desktop/capabilities/local_plugin_runtime.py
@@ -1,7 +1,9 @@
"""Resolve plugin-owned local connectors directly from retained, authorized packages."""
import json
-from . import registry, store
-from .paths import LOCAL_PROFILE, capabilities_enabled
+from features.desktop.capabilities import registry, store
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from features.desktop.capabilities.paths import capabilities_enabled
+from core.capabilities.mcp.service import rewrite_builtin_mcp_host
def enabled_for(inst, user_id):
@@ -11,10 +13,10 @@ def enabled_for(inst, user_id):
def configs(user_id, enabled_only=True):
if not user_id or not capabilities_enabled():
return {}
- from .dependency import component_hash
+ from features.desktop.capabilities.dependency import component_hash
out = {}
for inst in registry.list_installations(kind="plugin", profile_id=LOCAL_PROFILE):
- from .device_catalog import active
+ from features.desktop.capabilities.device_catalog import active
owned = inst.payload.get("owner_user_id") == user_id or (inst.payload.get("shared_installation") and not active())
if not owned or not inst.ready or (enabled_only and not enabled_for(inst, user_id)):
continue
@@ -26,15 +28,18 @@ def configs(user_id, enabled_only=True):
if enabled_only and not entry.get("enabled", True):
continue
config = json.loads(json.dumps(entry).replace("{plugin_root}", str(comp.path / "package").replace("\\", "\\\\")))
- config.update(source_plugin=inst.key, owner_user_id=user_id, origin="local_plugin", execution_scope="local", is_stable=False)
+ if config.get("url"):
+ # Packaged manifests name the compose host; local planes listen elsewhere.
+ config["url"] = rewrite_builtin_mcp_host(config["url"])
+ config.update(source_plugin=inst.key, owner_user_id=user_id, origin="local_plugin", execution_scope="local")
executor = definition.get("manager_executor")
if executor:
- from core.services.management_contract import tools, VERSION
- from core.services.desktop_capability_protocol import canonical_hash
+ from core.capabilities.plugins.management.contract import tools, VERSION
+ from core.capabilities.manifest_protocol import canonical_hash
manager = executor.get("id")
if executor.get("version") != VERSION:
raise ValueError("plugin executor upgrade required")
- from .skills import account_authorized_for
+ from features.desktop.capabilities.skill_account import account_authorized_for
cloud_available = account_authorized_for(user_id)
schemas = tools(manager, "local")
if not cloud_available:
diff --git a/src/backend/core/capabilities/local_projection.py b/src/backend/features/desktop/capabilities/local_projection.py
similarity index 79%
rename from src/backend/core/capabilities/local_projection.py
rename to src/backend/features/desktop/capabilities/local_projection.py
index dd27018d..07cb6ee4 100644
--- a/src/backend/core/capabilities/local_projection.py
+++ b/src/backend/features/desktop/capabilities/local_projection.py
@@ -1,10 +1,13 @@
"""Publish committed device DB skill mutations into the capability store/index."""
from __future__ import annotations
+from sqlalchemy import inspect as inspect_model
+from features.desktop.capabilities import skill_publication
+from features.desktop.capabilities import skill_views
import threading
-from . import registry, skills
-from .paths import capabilities_enabled
+from features.desktop.capabilities import registry, skills
+from features.desktop.capabilities.paths import capabilities_enabled
_lock = threading.RLock()
@@ -13,23 +16,23 @@ def sync_local_skills() -> None:
if not capabilities_enabled():
return
from core.db.models import AdminSkill
- from core.agent_skills.binary_files import decode_binary, is_binary_value
- from core.services.desktop_capability_protocol import skill_content_hash
+ from core.capabilities.skills.binary_files import decode_binary, is_binary_value
+ from core.capabilities.manifest_protocol import skill_content_hash
- from core.services.local_management_migration import migrate
+ from features.desktop.local.management_migration import migrate
migrate()
owners = set()
with _lock:
with registry._session() as db:
from sqlalchemy import inspect
- if not inspect(db.get_bind()).has_table(AdminSkill.__tablename__):
+ if not inspect(db.get_bind()).has_table(inspect_model(AdminSkill).local_table.name):
return
rows = db.query(AdminSkill).order_by(AdminSkill.skill_id).all()
# 混合模式:本机业务库里的插件整体不参与(见 plugin_service.list_installed),
# 它们的技能同样不投影——否则会和云端同步下来的同名技能抢运行名。用户自建的
# 技能没有归属插件,照常投影:那是他自己的东西,只有这台机器上有。
- from . import device_catalog
+ from features.desktop.capabilities import device_catalog
hybrid = device_catalog.active()
definitions = [
@@ -53,7 +56,7 @@ def sync_local_skills() -> None:
sid, owner = definition["skill_id"], definition["owner"]
existing = registry.get(registry.install_id("skill", "local", sid))
if existing is not None and not existing.payload.get("from_db"):
- from .errors import NameConflict
+ from core.harness.capability_errors import NameConflict
raise NameConflict(
"local upload key collides with an independent installation", runtime_name=sid
@@ -69,7 +72,7 @@ def sync_local_skills() -> None:
for name, value in extras.items()
},
}
- skills.publish_local_skill(
+ skill_publication.publish_local_skill(
sid,
files=files,
content_hash=skill_content_hash(definition["content"], extras),
@@ -92,8 +95,8 @@ def sync_local_skills() -> None:
if inst.payload.get("from_db") and inst.key not in live:
if inst.payload.get("owner_user_id"):
owners.add(inst.payload["owner_user_id"])
- skills.prune_local(live)
+ skill_publication.prune_local(live)
# Views depend on identity; do not hold the projection lock while resolving it.
- skills.rebuild_views(None)
+ skill_views.rebuild_views(None)
for owner in owners:
- skills.rebuild_user_view(owner)
+ skill_views.rebuild_user_view(owner)
diff --git a/src/backend/core/capabilities/management_context.py b/src/backend/features/desktop/capabilities/management_context.py
similarity index 82%
rename from src/backend/core/capabilities/management_context.py
rename to src/backend/features/desktop/capabilities/management_context.py
index 3665ad95..ed3ad599 100644
--- a/src/backend/core/capabilities/management_context.py
+++ b/src/backend/features/desktop/capabilities/management_context.py
@@ -1,14 +1,15 @@
"""Shared connector resolution and dependency context for management surfaces."""
+from features.desktop.capabilities import skill_account
from typing import Any, Dict
-from . import registry, skills
+from features.desktop.capabilities import registry, skills
def _authorized_profile(user_id):
- return skills.current_account_profile() if skills.account_authorized_for(user_id) else None
+ return skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None
def management_context(user_id: str):
- from core.capabilities.readiness import context_for_user
+ from features.desktop.capabilities.readiness import context_for_user
connectors = connectors_view(user_id)
available = {
@@ -21,8 +22,8 @@ def management_context(user_id: str):
def connectors_view(user_id: str) -> Dict[str, Any]:
"""Connector bindings as the resolver last decided them for this device."""
- from core.capabilities import connectors, skills, mcp_json
- from core.services.mcp_service import McpServerConfigService
+ from features.desktop.capabilities import connectors, skills, mcp_json
+ from core.capabilities.mcp.service import McpServerConfigService
svc = McpServerConfigService.get_instance()
all_cfgs = {
@@ -32,14 +33,14 @@ def connectors_view(user_id: str) -> Dict[str, Any]:
}
all_cfgs.update(svc.get_owned_servers(user_id, enabled_only=False))
enabled_ids = set(svc.get_all_servers(enabled_only=True)) | set(svc.get_owned_servers(user_id))
- from core.services.desktop_cloud_bridge import _mcp_json_local_declarations
+ from features.desktop.bridge.gateway import mcp_json_local_declarations as _mcp_json_local_declarations
candidates = connectors.db_candidates(all_cfgs, enabled_ids, user_id=user_id) + connectors.json_candidates(
_mcp_json_local_declarations()
)
# The runtime context excludes disabled bindings; the management view must
# keep them visible so users can turn them back on.
- from core.services.desktop_cloud_bridge import get_cached_manifest
+ from features.desktop.bridge.client import get_cached_manifest
profile = _authorized_profile(user_id)
manifest = get_cached_manifest() if profile else None
@@ -77,7 +78,7 @@ def connectors_view(user_id: str) -> Dict[str, Any]:
)
)
entry["resolution"] = {"outcome": outcome, "reason": res.reasons.get(c.runtime_name)}
- from core.capabilities.readiness import apply_to_item, connector_readiness
+ from features.desktop.capabilities.readiness import apply_to_item, connector_readiness
config = (
_mcp_json_local_declarations().get(entry["server_id"], {})
diff --git a/src/backend/core/capabilities/manifest_order.py b/src/backend/features/desktop/capabilities/manifest_order.py
similarity index 96%
rename from src/backend/core/capabilities/manifest_order.py
rename to src/backend/features/desktop/capabilities/manifest_order.py
index 53d680bc..4cc3b08c 100644
--- a/src/backend/core/capabilities/manifest_order.py
+++ b/src/backend/features/desktop/capabilities/manifest_order.py
@@ -8,7 +8,7 @@
from contextlib import contextmanager
from dataclasses import dataclass
import threading
-from .errors import CloudUnavailable
+from core.harness.capability_errors import CloudUnavailable
class StaleManifest(CloudUnavailable):
diff --git a/src/backend/core/capabilities/mcp_json.py b/src/backend/features/desktop/capabilities/mcp_json.py
similarity index 98%
rename from src/backend/core/capabilities/mcp_json.py
rename to src/backend/features/desktop/capabilities/mcp_json.py
index 37d7f298..721af3f2 100644
--- a/src/backend/core/capabilities/mcp_json.py
+++ b/src/backend/features/desktop/capabilities/mcp_json.py
@@ -28,9 +28,10 @@
from pathlib import Path
from typing import Any, Dict, List, Optional
-from . import credentials
-from .lockfile import locked
-from .paths import mcp_json_path, meta_root, safe_segment
+from features.desktop.capabilities import credentials
+from core.infra.file_lock import locked
+from core.capabilities.identity_rules import safe_segment
+from features.desktop.capabilities.paths import mcp_json_path, meta_root
SCHEMA_VERSION = 2
LOCAL_TRANSPORTS = ("stdio", "streamable_http", "sse")
@@ -342,7 +343,6 @@ def local_server_configs(doc: Optional[McpJson] = None) -> Dict[str, dict]:
continue
cfg: Dict[str, Any] = {
"transport": entry["transport"],
- "is_stable": False,
"source_plugin": None,
"owner_user_id": None,
"origin": "mcp_json_local",
diff --git a/src/backend/core/capabilities/migration.py b/src/backend/features/desktop/capabilities/migration.py
similarity index 92%
rename from src/backend/core/capabilities/migration.py
rename to src/backend/features/desktop/capabilities/migration.py
index a26bc26d..c9e48581 100644
--- a/src/backend/core/capabilities/migration.py
+++ b/src/backend/features/desktop/capabilities/migration.py
@@ -17,6 +17,8 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_paths
+from features.desktop.capabilities import skill_publication
import hashlib
import json
@@ -27,8 +29,11 @@
from pathlib import Path
from typing import Dict, List, Optional
-from . import junction, skills, store
-from .paths import KIND_SKILL, capabilities_enabled, migrations_root, safe_segment
+from core.infra import directory_links as junction
+from features.desktop.capabilities import skills
+from features.desktop.capabilities import store
+from core.capabilities.identity_rules import KIND_SKILL, safe_segment
+from features.desktop.capabilities.paths import capabilities_enabled, migrations_root
logger = logging.getLogger(__name__)
@@ -84,8 +89,8 @@ def _sweep_view_dir(report: MigrationReport, view: Path) -> None:
def _import_flat_skills(report: MigrationReport) -> None:
- from core.agent_skills.binary_files import decode_binary, is_binary_value, pack_directory
- from core.services.desktop_capability_protocol import skill_content_hash
+ from core.capabilities.skills.binary_files import decode_binary, is_binary_value, pack_directory
+ from core.capabilities.manifest_protocol import skill_content_hash
for flat in store.flat_import_candidates(KIND_SKILL):
try:
@@ -94,7 +99,7 @@ def _import_flat_skills(report: MigrationReport) -> None:
files = {"SKILL.md": content}
for rel, body in packed.items():
files[rel] = decode_binary(body) if is_binary_value(body) else body
- skills.publish_local_skill(
+ skill_publication.publish_local_skill(
safe_segment(flat.name),
files=files,
content_hash=skill_content_hash(content, packed),
@@ -138,7 +143,7 @@ def _tree_inventory(root: Path) -> Optional[Dict[str, str]]:
def _session_copy_sources(report: MigrationReport, workspace: Path) -> Dict[str, List[Path]]:
"""Preserved original views and immutable components from which copies can be rebuilt."""
- roots = [workspace / "skills", skills.builtin_dir()]
+ roots = [workspace / "skills", skill_paths.builtin_dir()]
users = workspace / "skills_u"
if users.is_dir():
roots.extend(
@@ -236,9 +241,9 @@ def _drop_run_copies(report: MigrationReport) -> None:
"""
from core.db.models import ContentBlock
- from . import registry
- from .paths import require_root
- from core.capabilities.runtime.state import _PREFIX, PreparedRun
+ from features.desktop.capabilities import registry
+ from features.desktop.capabilities.paths import require_root
+ from features.desktop.capabilities.runtime.state import _PREFIX, PreparedRun
known = set(PreparedRun.__dataclass_fields__)
with registry._session() as db:
@@ -259,7 +264,7 @@ def migrate_legacy_layout() -> Optional[MigrationReport]:
"""Run the migration once per process start when the store is enabled."""
if not capabilities_enabled():
return None
- from core.agent_skills.config import get_sandbox_skills_dir, get_user_skills_root
+ from core.capabilities.skills.config import get_sandbox_skills_dir, get_user_skills_root
report = MigrationReport()
shared = get_sandbox_skills_dir()
diff --git a/src/backend/core/capabilities/paths.py b/src/backend/features/desktop/capabilities/paths.py
similarity index 80%
rename from src/backend/core/capabilities/paths.py
rename to src/backend/features/desktop/capabilities/paths.py
index 37d0920a..5b3e29e0 100644
--- a/src/backend/core/capabilities/paths.py
+++ b/src/backend/features/desktop/capabilities/paths.py
@@ -12,33 +12,18 @@
import os
import re
from pathlib import Path
+from core.capabilities.identity_rules import KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP, KINDS, LOCAL_PROFILE, BUILTIN_PROFILE, safe_segment
from typing import Optional
-CAPS_ROOT_ENV = "HUGAGENT_CAPS_ROOT"
+from core.config.runtime_env import CAPS_ROOT_ENV
-KIND_SKILL = "skill"
-KIND_PLUGIN = "plugin"
-KIND_AGENT = "agent"
-KIND_MCP = "mcp"
-KINDS = (KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP)
_KIND_DIRS = {KIND_SKILL: "skills", KIND_PLUGIN: "plugins", KIND_AGENT: "agents"}
-LOCAL_PROFILE = "local"
-BUILTIN_PROFILE = "builtin"
META_DIR = ".capabilities"
MCP_JSON_NAME = "mcp.json"
-_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,120}$")
-_WINDOWS_RESERVED = {
- "CON",
- "PRN",
- "AUX",
- "NUL",
- *(f"COM{i}" for i in range(1, 10)),
- *(f"LPT{i}" for i in range(1, 10)),
-}
class CapabilityStoreDisabled(RuntimeError):
@@ -73,8 +58,8 @@ def assert_managed_path(path: Path) -> Path:
def _assert_managed_path(path: Path, root: Path) -> Path:
- from .errors import IntegrityFailed
- from .junction import is_directory_link
+ from core.harness.capability_errors import IntegrityFailed
+ from core.infra.directory_links import is_directory_link
root = root.absolute()
path = path.absolute()
@@ -92,14 +77,6 @@ def _assert_managed_path(path: Path, root: Path) -> Path:
return path
-def safe_segment(value: str) -> str:
- """A single path segment safe on every platform, or ``ValueError``."""
- seg = (value or "").strip()
- if seg != value or seg.endswith((".", " ")) or not _SEGMENT_RE.match(seg) or seg in (".", ".."):
- raise ValueError(f"unsafe path segment: {value!r}")
- if seg.split(".")[0].upper() in _WINDOWS_RESERVED:
- raise ValueError(f"reserved name: {value!r}")
- return seg
def _kind_path(kind: str, *segments: str) -> Path:
diff --git a/src/backend/core/capabilities/plugins.py b/src/backend/features/desktop/capabilities/plugins.py
similarity index 87%
rename from src/backend/core/capabilities/plugins.py
rename to src/backend/features/desktop/capabilities/plugins.py
index f7516228..501979cf 100644
--- a/src/backend/core/capabilities/plugins.py
+++ b/src/backend/features/desktop/capabilities/plugins.py
@@ -7,23 +7,18 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import registry_types
+from features.desktop.capabilities import skill_account
import json
import logging
from typing import Any, Dict, Iterable, List, Optional
-from . import registry, store
-from .paths import (
- KIND_AGENT,
- KIND_MCP,
- KIND_PLUGIN,
- KIND_SKILL,
- LOCAL_PROFILE,
- revision_for_hash,
- safe_segment,
-)
-from .ref import local_ref
-from .resolver import SOURCE_LOCAL
+from features.desktop.capabilities import registry, store
+from core.capabilities.identity_rules import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL, LOCAL_PROFILE, safe_segment
+from features.desktop.capabilities.paths import revision_for_hash
+from core.capabilities.reference import local_ref
+from features.desktop.capabilities.resolver import SOURCE_LOCAL
logger = logging.getLogger(__name__)
@@ -40,13 +35,13 @@ def cloud_binding_ids(
plugin_ids: Iterable[str], *, user_id: Optional[str]
) -> tuple[list[str], list[str]]:
"""Expand selected, authorized cloud definitions without preparing their files."""
- from . import skills
- from .dependency import Context, Inspector, _identifier, component_hash
- from .errors import IntegrityFailed, NameConflict
+ from features.desktop.capabilities import skills
+ from features.desktop.capabilities.dependency import Context, Inspector, _identifier, component_hash
+ from core.harness.capability_errors import IntegrityFailed, NameConflict
- if not skills.account_authorized_for(user_id):
+ if not skill_account.account_authorized_for(user_id):
return [], []
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
account_profile = profile
installed = registry.list_installations(kind=KIND_PLUGIN, profile_id=profile)
selected_skills, selected_mcp = set(), set()
@@ -126,7 +121,7 @@ def publish_local_plugin(
definition: Dict[str, Any], *, owner_user_id: Optional[str]
) -> store.StoredComponent:
"""Project a device-installed plugin row into the store and record its components."""
- from core.services.desktop_capability_protocol import entity_content_hash
+ from core.capabilities.manifest_protocol import entity_content_hash
slug = safe_segment(str(definition["slug"]))
files = plugin_manifest_files(definition)
@@ -159,7 +154,7 @@ def publish_local_plugin(
def remove_local_plugin(slug: str) -> bool:
- from core.capabilities.runtime import references
+ from features.desktop.capabilities.runtime import references
if references("plugin", LOCAL_PROFILE, slug):
# Source deletion revokes use; history still needs its exact bytes.
@@ -169,14 +164,14 @@ def remove_local_plugin(slug: str) -> bool:
def readiness(
- inst: registry.Installation,
+ inst: registry_types.Installation,
*,
cloud_server_ids: Iterable[str],
managed_enabled: Optional[Dict[str, bool]] = None,
user_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Component-by-component readiness of one plugin installation on this device."""
- from core.services.mcp_service import McpServerConfigService
+ from core.capabilities.mcp.service import McpServerConfigService
service = McpServerConfigService.get_instance()
selected_user = user_id or inst.payload.get("owner_user_id")
@@ -219,7 +214,7 @@ def readiness(
)
if required and not ok:
missing.append(cid)
- from .dependency import check_installation
+ from features.desktop.capabilities.dependency import check_installation
report = check_installation(
inst,
@@ -240,12 +235,12 @@ def readiness(
def enabled_cloud_skill_intents(user_id):
"""Skill intentions of enabled plugins in the current authorized account."""
- from . import skills
- from .dependency import _identifier
+ from features.desktop.capabilities import skills
+ from features.desktop.capabilities.dependency import _identifier
- if not skills.account_authorized_for(user_id):
+ if not skill_account.account_authorized_for(user_id):
return set()
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
selected = set()
for row in registry.list_installations(kind=KIND_PLUGIN, profile_id=profile):
if not row.enabled:
diff --git a/src/backend/core/capabilities/preparation.py b/src/backend/features/desktop/capabilities/preparation.py
similarity index 89%
rename from src/backend/core/capabilities/preparation.py
rename to src/backend/features/desktop/capabilities/preparation.py
index 22fbda79..56f51811 100644
--- a/src/backend/core/capabilities/preparation.py
+++ b/src/backend/features/desktop/capabilities/preparation.py
@@ -1,14 +1,16 @@
"""Account-bound publication and recovery shared by the three file package kinds."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
+from features.desktop.capabilities import skill_account
import threading
import logging
import time
-from . import registry, store
-from .errors import CloudUnavailable, IntegrityFailed, PackageMissing
-from .paths import revision_for_hash
+from features.desktop.capabilities import registry, store
+from core.harness.capability_errors import CloudUnavailable, IntegrityFailed, PackageMissing
+from features.desktop.capabilities.paths import revision_for_hash
_prepare_lock = threading.RLock()
# (registry generation, profile) → ids still awaiting preparation + ownership edges.
@@ -36,8 +38,8 @@ def _pending_closure(profile):
def prepare_component(state, inst, *, download, content_hash, after_publish=None):
- from core.services.desktop_cloud_bridge import account_scope, require_current_account
- from core.services.desktop_cloud_skills import _profile
+ from features.desktop.bridge.identity import account_scope, require_current_account
+ from features.desktop.bridge.skills import _profile
if inst.profile_id != _profile(state):
raise PackageMissing("installation belongs to another account", ref=inst.install_id)
@@ -105,7 +107,7 @@ def prepare_component(state, inst, *, download, content_hash, after_publish=None
payload_update={"resolved_content_hash": inst.content_hash},
)
if inst.kind in ("skill", "plugin"):
- from .change_sync import record_downloaded
+ from features.desktop.capabilities.change_sync import record_downloaded
record_downloaded(done)
if tx:
registry.advance_transaction(tx, "committed")
@@ -145,13 +147,13 @@ def ensure_cloud_ready(user_id, *, skill_keys=(), plugin_keys=(), install_ids=()
时,这里就是"按需拉取"的落点。桥未激活、没有对应的云端安装记录、或组件已就绪时不做
任何事。返回准备失败的 install_id 列表(下载/校验失败),由调用方决定如何报错。
"""
- from . import registry, skills
- from core.services import desktop_cloud_bridge as bridge
+ from features.desktop.capabilities import registry, skills
+ from features.desktop.bridge import client as bridge
- if not skills.account_authorized_for(user_id):
+ if not skill_account.account_authorized_for(user_id):
return []
- state = bridge.get_state()
- profile = skills.current_account_profile()
+ state = bridge_identity.get_state()
+ profile = skill_account.current_account_profile()
if not state or not profile:
return []
@@ -184,14 +186,14 @@ def pending(kind_filter):
failures = []
definitions = pending(("plugin", "agent"))
if definitions:
- from core.services import desktop_cloud_bundles
+ from features.desktop.bridge import bundles as desktop_cloud_bundles
failures += [r["install_id"] for r in desktop_cloud_bundles.prepare(state, definitions) if not r["ok"]]
for owner in definitions:
wanted.update(registry.components_of(owner))
skill_ids = pending(("skill",))
if skill_ids:
- from core.services import desktop_cloud_skills
+ from features.desktop.bridge import skills as desktop_cloud_skills
failures += [r["install_id"] for r in desktop_cloud_skills.prepare(state, skill_ids) if not r["ok"]]
return failures
diff --git a/src/backend/core/capabilities/readiness.py b/src/backend/features/desktop/capabilities/readiness.py
similarity index 92%
rename from src/backend/core/capabilities/readiness.py
rename to src/backend/features/desktop/capabilities/readiness.py
index 1192069d..188892d0 100644
--- a/src/backend/core/capabilities/readiness.py
+++ b/src/backend/features/desktop/capabilities/readiness.py
@@ -1,15 +1,17 @@
"""Read-only management readiness, separate from retained package publication state."""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skills
import shutil
import os
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
-from . import registry, store
-from .dependency import Context, Inspector, allowed_model_ids, skill_definition
-from .errors import CapabilityError
-from .paths import LOCAL_PROFILE
+from features.desktop.capabilities import registry, store
+from features.desktop.capabilities.dependency import Context, Inspector, allowed_model_ids, skill_definition
+from core.harness.capability_errors import CapabilityError
+from core.capabilities.identity_rules import LOCAL_PROFILE
def context_for_user(user_id, *, available_mcp=()):
@@ -18,13 +20,13 @@ def context_for_user(user_id, *, available_mcp=()):
except Exception:
models = None
try:
- from core.auth.kb_permissions import get_accessible_local_kb_ids
+ from features.knowledge.edition import get_accessible_local_kb_ids
with registry._session() as db:
kbs = get_accessible_local_kb_ids(db, user_id)
except Exception:
kbs = None
- from . import skills
+ from features.desktop.capabilities import skills
choices = skills.resolve_for_user(user_id).chosen
return Context(
@@ -208,12 +210,12 @@ def resolve(self, kind, profile, key):
parts = key.split(":", 2)
if len(parts) != 3 or parts[0] != kind:
return None, None, None
- from . import skills
- from .paths import BUILTIN_PROFILE
+ from features.desktop.capabilities import skills
+ from core.capabilities.identity_rules import BUILTIN_PROFILE
if parts[1] not in (LOCAL_PROFILE, BUILTIN_PROFILE) and (
- parts[1] != skills.current_account_profile()
- or not skills.account_authorized_for(self.context.user_id)
+ parts[1] != skill_account.current_account_profile()
+ or not skill_account.account_authorized_for(self.context.user_id)
):
return None, None, None
key = parts[2]
@@ -243,7 +245,7 @@ def eligible_skill_candidates(candidates, user_id, *, preferences=None, requeste
conservatively pruned until no selected node depends on an excluded node.
"""
from dataclasses import replace
- from .resolver import resolve
+ from features.desktop.capabilities.resolver import resolve
candidates = list(candidates)
preferences = (
diff --git a/src/backend/core/capabilities/registry.py b/src/backend/features/desktop/capabilities/registry.py
similarity index 84%
rename from src/backend/core/capabilities/registry.py
rename to src/backend/features/desktop/capabilities/registry.py
index f1caa892..31f13c84 100644
--- a/src/backend/core/capabilities/registry.py
+++ b/src/backend/features/desktop/capabilities/registry.py
@@ -1,27 +1,21 @@
"""Device installation index: intent + state, kept in the device's business database."""
from __future__ import annotations
+from features.desktop.capabilities import registry_types
import hashlib
import itertools
import threading
import uuid
from contextlib import contextmanager
-from dataclasses import dataclass
-from datetime import datetime
from typing import Any, Dict, Iterator, List, Optional
from core.db.engine import SessionLocal
-from core.db.models import (
- DeviceCapabilityComponent,
- DeviceCapabilityInstallation,
- DeviceCapabilityNamePreference,
- DeviceCapabilityTransaction,
-)
+from features.desktop.models import DeviceCapabilityComponent, DeviceCapabilityInstallation, DeviceCapabilityNamePreference, DeviceCapabilityTransaction
from sqlalchemy import event, func
from sqlalchemy.orm import Session
-from .ref import ResourceRef
+from core.capabilities.reference import ResourceRef
_generation_lock = threading.Lock()
@@ -87,77 +81,10 @@ def preference_id(kind: str, runtime_name: str, user_id: Optional[str] = None) -
return f"{kind}:user:{key}"
-@dataclass
-class Installation:
- install_id: str
- profile_id: str
- kind: str
- key: str
- ref: ResourceRef
- display_name: str
- description: str
- version: str
- content_hash: Optional[str]
- resolved_revision: Optional[str]
- state: str
- enabled: bool
- source: str
- source_plugin: Optional[str]
- generation: int
- last_error: Optional[str]
- payload: Dict[str, Any]
- created_at: Optional[datetime]
-
- @property
- def ready(self) -> bool:
- return self.state == "ready" and bool(self.resolved_revision)
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "install_id": self.install_id,
- "profile_id": self.profile_id,
- "kind": self.kind,
- "key": self.key,
- "resource_ref": self.ref.to_dict(),
- "display_name": self.display_name,
- "description": self.description,
- "version": self.version,
- "content_hash": self.content_hash,
- "resolved_revision": self.resolved_revision,
- "state": self.state,
- "enabled": self.enabled,
- "source": self.source,
- "source_plugin": self.source_plugin,
- "generation": self.generation,
- "last_error": self.last_error,
- "payload": dict(self.payload or {}),
- "created_at": self.created_at.isoformat() if self.created_at else None,
- }
-
-
-def _to_installation(row: DeviceCapabilityInstallation) -> Installation:
- from .session_availability import permits
-
- return Installation(
- install_id=row.install_id,
- profile_id=row.profile_id,
- kind=row.kind,
- key=row.key,
- ref=ResourceRef(row.ref_issuer, row.ref_namespace, row.kind, row.ref_id),
- display_name=row.display_name or "",
- description=row.description or "",
- version=row.version or "",
- content_hash=row.content_hash,
- resolved_revision=row.resolved_revision,
- state=row.state,
- enabled=bool(row.enabled) and permits(row.profile_id, row.install_id, row.resolved_revision),
- source=row.source,
- source_plugin=row.source_plugin,
- generation=int(row.generation or 0),
- last_error=row.last_error,
- payload=dict(row.payload or {}),
- created_at=row.created_at,
- )
+
+
+
+
@contextmanager
@@ -190,7 +117,7 @@ def upsert(
enabled: Optional[bool] = None,
initial_enabled: Optional[bool] = None,
db: Optional[Session] = None,
-) -> Installation:
+) -> registry_types.Installation:
"""Create or refresh the intent row. State transitions are done by the setters below.
``enabled`` 是本机意图,每次都写;``initial_enabled`` 只在这台机器第一次见到这个
@@ -239,16 +166,16 @@ def upsert(
if row.state == "removed":
row.state = "pending"
s.flush()
- return _to_installation(row)
+ return registry_types.to_installation(row)
-def get(install_id_: str, db: Optional[Session] = None) -> Optional[Installation]:
+def get(install_id_: str, db: Optional[Session] = None) -> Optional[registry_types.Installation]:
with _session(db) as s:
row = s.get(DeviceCapabilityInstallation, install_id_)
- return _to_installation(row) if row else None
+ return registry_types.to_installation(row) if row else None
-def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installation]:
+def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, registry_types.Installation]:
"""Whole-closure lookup in one query.
``validate`` re-checks every frozen node and binding on every message —
@@ -258,7 +185,7 @@ def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installatio
ids = [str(i) for i in dict.fromkeys(install_ids)]
if not ids:
return {}
- found: Dict[str, Installation] = {}
+ found: Dict[str, registry_types.Installation] = {}
with _session(db) as s:
for start in range(0, len(ids), 400):
rows = (
@@ -266,7 +193,7 @@ def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installatio
.filter(DeviceCapabilityInstallation.install_id.in_(ids[start : start + 400]))
.all()
)
- found.update({row.install_id: _to_installation(row) for row in rows})
+ found.update({row.install_id: registry_types.to_installation(row) for row in rows})
return found
@@ -277,7 +204,7 @@ def list_installations(
profiles: Optional[List[str]] = None,
include_removed: bool = False,
db: Optional[Session] = None,
-) -> List[Installation]:
+) -> List[registry_types.Installation]:
with _session(db) as s:
q = s.query(DeviceCapabilityInstallation)
if kind:
@@ -289,7 +216,7 @@ def list_installations(
if not include_removed:
q = q.filter(DeviceCapabilityInstallation.state != "removed")
rows = q.order_by(DeviceCapabilityInstallation.kind, DeviceCapabilityInstallation.key).all()
- return [_to_installation(r) for r in rows]
+ return [registry_types.to_installation(r) for r in rows]
def source_token(*, kind: str, profile_id: str, db: Optional[Session] = None) -> tuple:
@@ -334,7 +261,7 @@ def set_state(
last_error: Optional[str] = None,
payload_update: Optional[Dict[str, Any]] = None,
db: Optional[Session] = None,
-) -> Optional[Installation]:
+) -> Optional[registry_types.Installation]:
with _session(db) as s:
row = s.get(DeviceCapabilityInstallation, install_id_)
if row is None:
@@ -347,7 +274,7 @@ def set_state(
row.generation = int(row.generation or 0) + 1
payload = dict(row.payload or {})
payload.pop("update_available", None)
- from .paths import revision_for_hash
+ from features.desktop.capabilities.paths import revision_for_hash
if row.content_hash and row.resolved_revision == revision_for_hash(row.content_hash):
payload["resolved_content_hash"] = row.content_hash
@@ -355,19 +282,19 @@ def set_state(
if payload_update:
row.payload = {**dict(row.payload or {}), **payload_update}
s.flush()
- return _to_installation(row)
+ return registry_types.to_installation(row)
def set_enabled(
install_id_: str, enabled: bool, db: Optional[Session] = None
-) -> Optional[Installation]:
+) -> Optional[registry_types.Installation]:
with _session(db) as s:
row = s.get(DeviceCapabilityInstallation, install_id_)
if row is None:
return None
row.enabled = enabled
s.flush()
- return _to_installation(row)
+ return registry_types.to_installation(row)
def mark_removed(install_id_: str, db: Optional[Session] = None) -> bool:
diff --git a/src/backend/features/desktop/capabilities/registry_types.py b/src/backend/features/desktop/capabilities/registry_types.py
new file mode 100644
index 00000000..b71feec3
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/registry_types.py
@@ -0,0 +1,79 @@
+"""Typed installation projections of device capability records."""
+from __future__ import annotations
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, Optional
+from features.desktop.models import DeviceCapabilityInstallation
+from core.capabilities.reference import ResourceRef
+
+@dataclass
+class Installation:
+ install_id: str
+ profile_id: str
+ kind: str
+ key: str
+ ref: ResourceRef
+ display_name: str
+ description: str
+ version: str
+ content_hash: Optional[str]
+ resolved_revision: Optional[str]
+ state: str
+ enabled: bool
+ source: str
+ source_plugin: Optional[str]
+ generation: int
+ last_error: Optional[str]
+ payload: Dict[str, Any]
+ created_at: Optional[datetime]
+
+ @property
+ def ready(self) -> bool:
+ return self.state == "ready" and bool(self.resolved_revision)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "install_id": self.install_id,
+ "profile_id": self.profile_id,
+ "kind": self.kind,
+ "key": self.key,
+ "resource_ref": self.ref.to_dict(),
+ "display_name": self.display_name,
+ "description": self.description,
+ "version": self.version,
+ "content_hash": self.content_hash,
+ "resolved_revision": self.resolved_revision,
+ "state": self.state,
+ "enabled": self.enabled,
+ "source": self.source,
+ "source_plugin": self.source_plugin,
+ "generation": self.generation,
+ "last_error": self.last_error,
+ "payload": dict(self.payload or {}),
+ "created_at": self.created_at.isoformat() if self.created_at else None,
+ }
+
+
+def to_installation(row: DeviceCapabilityInstallation) -> Installation:
+ from features.desktop.capabilities.session_availability import permits
+
+ return Installation(
+ install_id=row.install_id,
+ profile_id=row.profile_id,
+ kind=row.kind,
+ key=row.key,
+ ref=ResourceRef(row.ref_issuer, row.ref_namespace, row.kind, row.ref_id),
+ display_name=row.display_name or "",
+ description=row.description or "",
+ version=row.version or "",
+ content_hash=row.content_hash,
+ resolved_revision=row.resolved_revision,
+ state=row.state,
+ enabled=bool(row.enabled) and permits(row.profile_id, row.install_id, row.resolved_revision),
+ source=row.source,
+ source_plugin=row.source_plugin,
+ generation=int(row.generation or 0),
+ last_error=row.last_error,
+ payload=dict(row.payload or {}),
+ created_at=row.created_at,
+ )
diff --git a/src/backend/core/capabilities/resolver.py b/src/backend/features/desktop/capabilities/resolver.py
similarity index 98%
rename from src/backend/core/capabilities/resolver.py
rename to src/backend/features/desktop/capabilities/resolver.py
index 487fb434..6ceb3a33 100644
--- a/src/backend/core/capabilities/resolver.py
+++ b/src/backend/features/desktop/capabilities/resolver.py
@@ -19,8 +19,8 @@
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Set
-from .paths import BUILTIN_PROFILE, LOCAL_PROFILE
-from .ref import ResourceRef
+from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE
+from core.capabilities.reference import ResourceRef
SOURCE_CLOUD = "cloud"
SOURCE_LOCAL = "local"
diff --git a/src/backend/features/desktop/capabilities/runtime/__init__.py b/src/backend/features/desktop/capabilities/runtime/__init__.py
new file mode 100644
index 00000000..677c0c1f
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/runtime/__init__.py
@@ -0,0 +1,37 @@
+"""Public capability runtime API; state and operations have one-way dependencies."""
+
+from features.desktop.capabilities.runtime.connectors import bind_mcp as bind_mcp
+from features.desktop.capabilities.runtime.connectors import pin_agent_definition as pin_agent_definition
+from features.desktop.capabilities.runtime.connectors import references as references
+from features.desktop.capabilities.runtime.dependencies import preflight as preflight
+from features.desktop.capabilities.runtime.dependencies import validate_activation as validate_activation
+from features.desktop.capabilities.runtime.preparation import prepare as prepare
+from features.desktop.capabilities.runtime.state import PreparedRun as PreparedRun
+from features.desktop.capabilities.runtime.state import child_scope as child_scope
+from features.desktop.capabilities.runtime.state import frozen_loader as frozen_loader
+from features.desktop.capabilities.runtime.state import get as get
+from features.desktop.capabilities.runtime.state import rebuild as rebuild
+from features.desktop.capabilities.runtime.state import record_tool_scope as record_tool_scope
+from features.desktop.capabilities.runtime.state import require_root_tool_scope as require_root_tool_scope
+from features.desktop.capabilities.runtime.state import save as save
+from features.desktop.capabilities.runtime.state import validate as validate
+from features.desktop.capabilities.runtime.state import view_for_execution as view_for_execution
+
+__all__ = [
+ "child_scope",
+ "PreparedRun",
+ "get",
+ "validate",
+ "rebuild",
+ "save",
+ "frozen_loader",
+ "view_for_execution",
+ "record_tool_scope",
+ "require_root_tool_scope",
+ "prepare",
+ "bind_mcp",
+ "pin_agent_definition",
+ "references",
+ "preflight",
+ "validate_activation",
+]
diff --git a/src/backend/core/capabilities/runtime/connectors.py b/src/backend/features/desktop/capabilities/runtime/connectors.py
similarity index 92%
rename from src/backend/core/capabilities/runtime/connectors.py
rename to src/backend/features/desktop/capabilities/runtime/connectors.py
index 74451c72..8cb45c6f 100644
--- a/src/backend/core/capabilities/runtime/connectors.py
+++ b/src/backend/features/desktop/capabilities/runtime/connectors.py
@@ -1,6 +1,7 @@
"""Pin connector and agent identities for durable replay."""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
import copy
import hashlib
@@ -8,10 +9,10 @@
from dataclasses import replace
from typing import Optional
-from core.capabilities import registry, skills
-from core.capabilities.errors import PermissionDenied
-from core.capabilities.paths import LOCAL_PROFILE
-from core.capabilities.runtime import state as runtime
+from features.desktop.capabilities import registry, skills
+from core.harness.capability_errors import PermissionDenied
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from features.desktop.capabilities.runtime import state as runtime
from core.db.models import ContentBlock
@@ -47,7 +48,7 @@ def public_values(values):
def bind_mcp(run: runtime.PreparedRun, configs, choices):
"""Freeze MCP source identities/contracts; refresh credentials independently."""
- from core.capabilities.connectors import server_id_of
+ from features.desktop.capabilities.connectors import server_id_of
selected = {server_id_of(c): c.install_id for c in choices.chosen.values()} if choices else {}
current = {
@@ -112,24 +113,20 @@ def bind_mcp(run: runtime.PreparedRun, configs, choices):
def pin_agent_definition(run_id, user_id, definition, *, scope_id: str = ""):
"""Keep the selected agent's instructions and dependency IDs stable on replay."""
- from core.capabilities.agents import _JSON_FIELDS, AgentDefinition
+ from features.desktop.capabilities.agents import _JSON_FIELDS, AgentDefinition
ident = str(definition.agent_id)
key = "desktop_capability_agent:" + runtime._snapshot_key(str(run_id) + ":" + ident, scope_id)
profile = (
- skills.current_account_profile()
+ skill_account.current_account_profile()
if getattr(definition, "origin", "local") == "cloud"
else None
)
- from core.services.desktop_cloud_bridge import (
- _state_fingerprint,
- ensure_current_authorization,
- get_state,
- )
+ from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state
fingerprint = _state_fingerprint(get_state()) if profile else None
if getattr(definition, "origin", "local") == "cloud":
- if not skills.account_authorized_for(user_id):
+ if not skill_account.account_authorized_for(user_id):
raise PermissionDenied("cloud agent belongs to another user")
ensure_current_authorization()
current = registry.get(
diff --git a/src/backend/core/capabilities/runtime/dependencies.py b/src/backend/features/desktop/capabilities/runtime/dependencies.py
similarity index 93%
rename from src/backend/core/capabilities/runtime/dependencies.py
rename to src/backend/features/desktop/capabilities/runtime/dependencies.py
index 061c7767..cdd4b35a 100644
--- a/src/backend/core/capabilities/runtime/dependencies.py
+++ b/src/backend/features/desktop/capabilities/runtime/dependencies.py
@@ -4,10 +4,10 @@
from dataclasses import replace
-from core.capabilities import registry, store
-from core.capabilities.errors import CapabilityError, IntegrityFailed, PermissionDenied
-from core.capabilities.paths import BUILTIN_PROFILE, LOCAL_PROFILE
-from core.capabilities.runtime import state as runtime
+from features.desktop.capabilities import registry, store
+from core.harness.capability_errors import CapabilityError, IntegrityFailed, PermissionDenied
+from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE
+from features.desktop.capabilities.runtime import state as runtime
def validate_activation(run: runtime.PreparedRun, nodes, *, available_mcp=()) -> None:
@@ -18,13 +18,7 @@ def validate_activation(run: runtime.PreparedRun, nodes, *, available_mcp=()) ->
技能版本去对,而不是拿当前安装的版本。``nodes`` 是推迟那一刻记下的身份。
只查这一个插件及其定义闭包,不牵连整轮。
"""
- from core.capabilities.dependency import (
- Context,
- Inspector,
- _identifier,
- component_hash,
- require_report,
- )
+ from features.desktop.capabilities.dependency import Context, Inspector, _identifier, component_hash, require_report
nodes = list(nodes or [])
if not nodes:
@@ -88,7 +82,7 @@ def preflight(
绝不因此让整个助手不可用。连接器与插件的从属关系记进 ``connector_parents``,
供「加载插件」一类的工具反查。
"""
- from core.capabilities.dependency import Context, Inspector
+ from features.desktop.capabilities.dependency import Context, Inspector
runtime.validate(run)
run = runtime.get(run.run_id, scope_id=run.scope_id) or run
diff --git a/src/backend/core/capabilities/runtime/preparation.py b/src/backend/features/desktop/capabilities/runtime/preparation.py
similarity index 86%
rename from src/backend/core/capabilities/runtime/preparation.py
rename to src/backend/features/desktop/capabilities/runtime/preparation.py
index 001f466e..b7ee6d64 100644
--- a/src/backend/core/capabilities/runtime/preparation.py
+++ b/src/backend/features/desktop/capabilities/runtime/preparation.py
@@ -1,19 +1,26 @@
"""Resolve and freeze user-selected capability revisions."""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_hashes
+from features.desktop.capabilities import skills
import copy
-from core.capabilities import archive, registry, skills, store
-from core.capabilities.errors import CapabilityError, IntegrityFailed, PackageMissing
-from core.capabilities.paths import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE, revision_for_hash
-from core.capabilities.runtime import state as runtime
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import skills
+from features.desktop.capabilities import store
+from core.harness.capability_errors import CapabilityError, IntegrityFailed, PackageMissing
+from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE
+from features.desktop.capabilities.paths import revision_for_hash
+from features.desktop.capabilities.runtime import state as runtime
def _freeze_candidate(name, candidate):
# 摘要走内容指纹缓存:候选路径是不可变存储里的版本目录,字节一改元数据签名
# 就变、缓存随之失效,所以复用安全,而重复冻结同一版本不必再读一遍磁盘。
- actual = skills.skill_dir_hash(candidate.path)
+ actual = skill_hashes.skill_dir_hash(candidate.path)
revision = candidate.revision or revision_for_hash(actual)
profile = candidate.profile
if profile == BUILTIN_PROFILE:
@@ -75,7 +82,7 @@ def prepare(
永不原地修改,所以编辑安装目录只会产生新版本,已冻结的这一轮看到的字节自始
至终不变。准备不了的单个组件记进 ``unavailable``,不影响其余组件。
"""
- from core.capabilities.preparation import ensure_cloud_ready
+ from features.desktop.capabilities.preparation import ensure_cloud_ready
if not run_id:
raise ValueError("a run id is required for a durable capability snapshot")
@@ -99,7 +106,7 @@ def prepare(
)
# 这一轮的根:选中的技能,加上选中的插件。先各自按需下载,再从它们出发
# 走一遍依赖,把闭包里牵连到的技能也收进来。一个根准备不了只记下它自己。
- account = skills.current_account_profile() or LOCAL_PROFILE
+ account = skill_account.current_account_profile() or LOCAL_PROFILE
roots = [("skill", name, LOCAL_PROFILE, name) for name in requested]
roots += [("plugin", key, account, "plugin:" + key) for key in plugin_ids]
unavailable = {}
@@ -110,8 +117,8 @@ def prepare(
except (CapabilityError, OSError, ValueError) as exc:
unavailable[label] = getattr(exc, "code", "package_missing")
resolution = skills.resolve_for_user(str(user_id))
- from core.capabilities.dependency import Context
- from core.capabilities.readiness import _BindingInspector, _choice_bindings
+ from features.desktop.capabilities.dependency import Context
+ from features.desktop.capabilities.readiness import _BindingInspector, _choice_bindings
discovery = _BindingInspector(
Context(
diff --git a/src/backend/core/capabilities/runtime/state.py b/src/backend/features/desktop/capabilities/runtime/state.py
similarity index 89%
rename from src/backend/core/capabilities/runtime/state.py
rename to src/backend/features/desktop/capabilities/runtime/state.py
index 02b6b727..2fad7cd9 100644
--- a/src/backend/core/capabilities/runtime/state.py
+++ b/src/backend/features/desktop/capabilities/runtime/state.py
@@ -6,6 +6,10 @@
"""
from __future__ import annotations
+from core.harness.registry.capability_runtime import child_scope
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_hashes
+from features.desktop.capabilities import skills
import copy
import hashlib
@@ -15,15 +19,11 @@
from pathlib import Path
from typing import Any, Dict, Optional
-from core.capabilities import registry, skills, store, view
-from core.capabilities.errors import (
- CapabilityError,
- IntegrityFailed,
- PackageMissing,
- PermissionDenied,
- ViewUnavailable,
-)
-from core.capabilities.paths import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE, require_root
+from features.desktop.capabilities import registry, skills, store, view
+from features.desktop.run_capabilities.provider import PreparedCapabilityOperations
+from core.harness.capability_errors import CapabilityError, IntegrityFailed, PackageMissing, PermissionDenied, ViewUnavailable
+from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE
+from features.desktop.capabilities.paths import require_root
from core.db.models import ContentBlock
_lock = threading.RLock()
@@ -34,14 +34,6 @@ def _key(run_id):
return hashlib.sha256(str(run_id).encode()).hexdigest()
-def child_scope(parent_scope: str, kind: str, *identities: str) -> str:
- """Bounded, deterministic scope from durable orchestration identities."""
- if not kind or not identities or any(not str(value or "").strip() for value in identities):
- raise IntegrityFailed("capability scope requires durable invocation identities")
- material = json.dumps(
- [str(parent_scope or ""), str(kind), *map(str, identities)], separators=(",", ":")
- )
- return "s_" + _key(material)
def _snapshot_key(run_id, scope_id=""):
@@ -54,7 +46,7 @@ def _snapshot_key(run_id, scope_id=""):
@dataclass(frozen=True)
-class PreparedRun:
+class PreparedRun(PreparedCapabilityOperations):
run_id: str
user_id: str
profile: Optional[str]
@@ -128,7 +120,7 @@ def _validate_skill_binding(name: str, binding: dict, run: PreparedRun) -> None:
owner = dependency.payload.get("owner_user_id")
if owner and str(owner) != run.user_id:
raise PermissionDenied("skill dependency belongs to another user", runtime_name=name)
- if comp is None or skills.skill_dir_hash(comp.path) != binding["content_hash"]:
+ if comp is None or skill_hashes.skill_dir_hash(comp.path) != binding["content_hash"]:
raise IntegrityFailed("prepared revision is missing or changed", runtime_name=name)
@@ -153,14 +145,10 @@ def validate(
details={"recovery_action": "switch_execution_plane"},
)
if run.profile is not None:
- from core.services.desktop_cloud_bridge import (
- _state_fingerprint,
- ensure_current_authorization,
- get_state,
- )
+ from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state
if (
- run.profile != skills.current_account_profile()
+ run.profile != skill_account.current_account_profile()
or run.authorization_fingerprint != _state_fingerprint(get_state())
):
raise PermissionDenied("cloud session changed; prepare a new run")
@@ -225,15 +213,11 @@ def rebuild(run: PreparedRun) -> Path:
def _bind_cloud_identity(run):
- from core.services.desktop_cloud_bridge import (
- _state_fingerprint,
- ensure_current_authorization,
- get_state,
- )
+ from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state
- if not skills.account_authorized_for(run.user_id):
+ if not skill_account.account_authorized_for(run.user_id):
raise PermissionDenied("cloud capabilities belong to another user")
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
fingerprint = _state_fingerprint(get_state())
if not profile:
raise PermissionDenied("cloud session is unavailable")
@@ -284,8 +268,8 @@ def _ensure_view(run: PreparedRun) -> None:
def frozen_loader(run: PreparedRun):
- from core.agent_skills.backends import CompositeBackend, FilesystemBackend
- from core.agent_skills.loader import MultiSourceSkillLoader
+ from core.capabilities.skills.backends import CompositeBackend, FilesystemBackend
+ from core.capabilities.skills.loader import MultiSourceSkillLoader
_ensure_view(run)
# 视图重建可能刚摘掉某个组件,读取方必须拿到落库后的最新 ``unavailable``。
diff --git a/src/backend/core/capabilities/session_availability.py b/src/backend/features/desktop/capabilities/session_availability.py
similarity index 100%
rename from src/backend/core/capabilities/session_availability.py
rename to src/backend/features/desktop/capabilities/session_availability.py
diff --git a/src/backend/features/desktop/capabilities/skill_account.py b/src/backend/features/desktop/capabilities/skill_account.py
new file mode 100644
index 00000000..4c759199
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_account.py
@@ -0,0 +1,54 @@
+"""skill account responsibilities."""
+from __future__ import annotations
+from typing import Optional
+from features.desktop.capabilities import registry
+from core.capabilities.reference import profile_id
+from features.desktop.capabilities import skills_state
+
+def current_account_profile() -> Optional[str]:
+ """Profile id of the cloud account this device is currently bridged to."""
+ from core.capabilities.manifest_protocol import token_subject
+ from features.desktop.bridge.identity import get_state
+
+ st = get_state()
+ if not st:
+ return None
+ subject = token_subject(str(st.get("token") or ""))
+ if not subject:
+ return None
+ return profile_id(str(st["cloud_base"]), subject)
+
+
+def current_local_user_id() -> Optional[str]:
+ """Map the signed stable cloud subject to this device's shadow-user ID."""
+ from features.desktop.bridge.identity import get_identity_state
+
+ identity = get_identity_state()
+ # 本机影子用户按壳送来的命名空间标识建档(cloud:::),
+ # 查找必须用同一标识,而不是凭据里的原始云端 id。
+ center = str((identity or {}).get("shell_user_center_id") or "")
+ if not center:
+ return None
+ # user_center_id → user_id is a primary-key mapping fixed at creation; only
+ # hits are remembered so a shadow created later is still found.
+ hit = skills_state.shadow_user_ids.get(center)
+ if hit is not None:
+ return hit
+ from core.db.models import UserShadow
+
+ with registry._session() as db:
+ row = db.query(UserShadow).filter(UserShadow.user_center_id == center).first()
+ if row is None:
+ return None
+ skills_state.shadow_user_ids[center] = str(row.user_id)
+ return skills_state.shadow_user_ids[center]
+
+
+def account_authorized_for(user_id: Optional[str]) -> bool:
+ from features.desktop.bridge.identity import get_state
+
+ st = get_state()
+ if not st or not str(st.get("token") or "").startswith("dcap2."):
+ return False
+ current = current_local_user_id()
+ return bool(user_id and current and str(user_id) == current)
diff --git a/src/backend/features/desktop/capabilities/skill_candidates.py b/src/backend/features/desktop/capabilities/skill_candidates.py
new file mode 100644
index 00000000..5315132a
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_candidates.py
@@ -0,0 +1,140 @@
+"""skill candidates responsibilities."""
+from __future__ import annotations
+from features.desktop.capabilities import registry_types
+import os
+from concurrent.futures import ThreadPoolExecutor
+from contextvars import copy_context
+from typing import Dict, List, Optional, Tuple
+from features.desktop.capabilities import registry, store
+from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE
+from core.capabilities.reference import builtin_ref
+from features.desktop.capabilities.resolver import SOURCE_BUILTIN, Candidate
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_hashes
+from features.desktop.capabilities import skill_paths
+
+def builtin_candidates() -> List[Candidate]:
+ """安装包自带的技能。
+
+ 混合模式下它们不进能力中心、也不会被启用(目录层在 ``get_runtime_catalog`` 里就清空
+ 了),但**候选必须留着**:云端技能会声明对自带技能的依赖,自带的那份还是云端副本在
+ 本平台跑不了时的兜底,候选一空这两条链路就断了。
+ """
+ root = skill_paths.builtin_dir()
+ out: List[Candidate] = []
+ if not root.is_dir():
+ return out
+ for d in sorted(p for p in root.iterdir() if p.is_dir()):
+ if not (d / "SKILL.md").is_file():
+ continue
+ out.append(
+ Candidate(
+ install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, d.name),
+ runtime_name=d.name,
+ kind=KIND_SKILL,
+ profile=BUILTIN_PROFILE,
+ source=SOURCE_BUILTIN,
+ path=d,
+ content_hash=None, # computed lazily only on a name collision
+ ref=builtin_ref(KIND_SKILL, d.name),
+ usable=True,
+ account_level=False,
+ display_name=d.name,
+ )
+ )
+ return out
+
+
+def installation_candidate(inst: registry_types.Installation, *, account_level: bool) -> Candidate:
+ path = None
+ if inst.ready:
+ from core.harness.capability_errors import CapabilityError
+
+ try:
+ comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "")
+ except (CapabilityError, OSError):
+ comp = None
+ path = comp.path if comp else None
+ usable = inst.enabled and path is not None
+ return Candidate(
+ install_id=inst.install_id,
+ runtime_name=str(inst.payload.get("runtime_name") or inst.key),
+ kind=KIND_SKILL,
+ profile=inst.profile_id,
+ source=inst.source,
+ path=path,
+ content_hash=inst.payload.get("resolved_content_hash") or inst.content_hash,
+ revision=inst.resolved_revision,
+ ref=inst.ref,
+ usable=usable,
+ account_level=account_level and inst.enabled,
+ state=inst.state if path is not None or not inst.ready else "files_missing",
+ display_name=inst.display_name,
+ description=inst.description,
+ version=inst.version,
+ )
+
+
+def installation_candidates(rows):
+ rows = list(rows)
+ if os.name == "nt" and len(rows) >= 8:
+ with ThreadPoolExecutor(max_workers=4, thread_name_prefix="cap-candidates") as pool:
+ futures = [
+ pool.submit(copy_context().run, installation_candidate, row, account_level=True)
+ for row in rows
+ ]
+ return [future.result() for future in futures]
+ return [installation_candidate(row, account_level=True) for row in rows]
+
+
+def local_candidates(user_id: Optional[str], *, shared_only: bool = False) -> List[Candidate]:
+ rows = []
+ for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE):
+ owner = inst.payload.get("owner_user_id")
+ if owner and (shared_only or owner != user_id):
+ continue
+ rows.append(inst)
+ return installation_candidates(rows)
+
+
+def account_candidates(profile: Optional[str] = None) -> List[Candidate]:
+ profile = profile or skill_account.current_account_profile()
+ if not profile:
+ return []
+ return installation_candidates(
+ registry.list_installations(kind=KIND_SKILL, profile_id=profile)
+ )
+
+
+def candidates(user_id: Optional[str]) -> List[Candidate]:
+ return (
+ builtin_candidates()
+ + local_candidates(user_id)
+ + (account_candidates() if skill_account.account_authorized_for(user_id) else [])
+ )
+
+
+def fill_hashes(cands: List[Candidate]) -> List[Candidate]:
+ """Hash only the members of multi-candidate names (identical-content detection)."""
+ by_name: Dict[str, List[Candidate]] = {}
+ for c in cands:
+ by_name.setdefault(c.runtime_name, []).append(c)
+ out: List[Candidate] = []
+ for group in by_name.values():
+ if len(group) < 2:
+ out.extend(group)
+ continue
+ for c in group:
+ if c.content_hash is None and c.path is not None:
+ c = Candidate(**{**c.__dict__, "content_hash": skill_hashes.skill_dir_hash(c.path)})
+ out.append(c)
+ return out
+
+
+def builtin_signature() -> Tuple:
+ """Shipped built-ins live on disk, outside every write signal — list them."""
+ root = skill_paths.builtin_dir()
+ try:
+ return (str(root), tuple(sorted(p.name for p in root.iterdir() if p.is_dir())))
+ except OSError:
+ return (str(root), ())
diff --git a/src/backend/features/desktop/capabilities/skill_hashes.py b/src/backend/features/desktop/capabilities/skill_hashes.py
new file mode 100644
index 00000000..08604cab
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_hashes.py
@@ -0,0 +1,101 @@
+"""skill hashes responsibilities."""
+from __future__ import annotations
+import hashlib
+from pathlib import Path
+from typing import Callable, Tuple
+from features.desktop.capabilities import skills_state
+
+def dir_signature(path: Path) -> Tuple:
+ from core.capabilities.archive import iter_file_stats
+
+ # A maximum mtime misses edits to any file older than the newest file.
+ return tuple(
+ (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino)
+ for rel, _, metadata in iter_file_stats(path)
+ if rel != skills_state.INVENTORY_NAME
+ )
+
+
+def dir_digest(path: Path, compute: Callable[[], str], *, fresh: bool = False) -> str:
+ """A package directory's content digest, cached on its file signature.
+
+ Identity is *established* with ``fresh=True``: every byte is read, so the
+ digest a binding freezes is always a true content hash. Later passes only
+ *verify* that digest, and a run re-verifies its whole closure on every
+ message — re-reading a hundred unchanged packages each time is pure cost.
+ So verification recomputes exactly when the signature moved (a file was
+ added, removed, or written), and reuses the frozen digest otherwise.
+ """
+ key = str(path.resolve())
+ sig = dir_signature(path)
+ with skills_state.hash_lock:
+ hit = skills_state.hash_cache.get(key)
+ if not fresh and hit and hit[0] == sig:
+ return hit[1]
+ digest = compute()
+ with skills_state.hash_lock:
+ skills_state.hash_cache[key] = (sig, digest)
+ return digest
+
+
+def skill_dir_hash(path: Path, *, fresh: bool = False) -> str:
+ """Hash every package file; reject links and excess sizes rather than omit bytes."""
+ from core.capabilities.archive import iter_file_stats
+
+ key = str(path.resolve())
+ entries = [item for item in iter_file_stats(path) if item[0] != skills_state.INVENTORY_NAME]
+ sig = tuple(
+ (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino)
+ for rel, _, metadata in entries
+ )
+ with skills_state.hash_lock:
+ hit = skills_state.hash_cache.get(key)
+ if not fresh and hit and hit[0] == sig:
+ return hit[1]
+ # Reuse this traversal's metadata; fresh verification still reads every byte.
+ digest = hash_skill_package(path, entries=entries)
+ with skills_state.hash_lock:
+ skills_state.hash_cache[key] = (sig, digest)
+ return digest
+
+
+def hash_skill_package(path: Path, *, entries=None) -> str:
+ from core.capabilities.skills.binary_files import encode_upload
+ from core.capabilities.manifest_protocol import skill_content_hash
+
+ from core.capabilities import archive as archive
+ from core.harness.capability_errors import IntegrityFailed
+ from core.infra.directory_links import native_path as _native
+
+ raw_files = {}
+ total = 0
+ for rel, file, metadata in archive.iter_file_stats(path) if entries is None else entries:
+ if rel == skills_state.INVENTORY_NAME:
+ continue
+ size = metadata.st_size
+ total += size
+ if (
+ size > archive.MAX_MEMBER_BYTES
+ or total > archive.MAX_TOTAL_BYTES
+ or len(raw_files) >= archive.MAX_MEMBERS
+ ):
+ raise IntegrityFailed("skill package exceeds content verification limits")
+ # Scanning already supports extended Windows paths; byte reads must too.
+ raw = _native(file).read_bytes()
+ if len(raw) != size:
+ raise IntegrityFailed("skill package changed during verification")
+ raw_files[rel] = raw
+ content_key = tuple((rel, hashlib.sha256(raw).digest()) for rel, raw in raw_files.items())
+ with skills_state.hash_lock:
+ cached = skills_state.content_hash_cache.get(content_key)
+ if cached is not None:
+ skills_state.content_hash_cache.move_to_end(content_key)
+ return cached
+ files = {rel: encode_upload(rel, raw) for rel, raw in raw_files.items()}
+ digest = skill_content_hash(files.pop("SKILL.md", ""), files)
+ with skills_state.hash_lock:
+ skills_state.content_hash_cache[content_key] = digest
+ skills_state.content_hash_cache.move_to_end(content_key)
+ while len(skills_state.content_hash_cache) > 256:
+ skills_state.content_hash_cache.popitem(last=False)
+ return digest
diff --git a/src/backend/features/desktop/capabilities/skill_paths.py b/src/backend/features/desktop/capabilities/skill_paths.py
new file mode 100644
index 00000000..d05d5584
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_paths.py
@@ -0,0 +1,29 @@
+"""skill paths responsibilities."""
+from __future__ import annotations
+from pathlib import Path
+from typing import List, Optional
+from extensions.paths import SKILLS_DIR
+from features.desktop.capabilities.paths import capability_root
+
+def builtin_dir() -> Path:
+ return SKILLS_DIR / "default"
+
+
+def allowed_link_roots() -> List[Path]:
+ roots = [builtin_dir()]
+ root = capability_root()
+ if root is not None:
+ roots.insert(0, root)
+ return roots
+
+
+def device_view_dir() -> Path:
+ from core.capabilities.skills.config import get_sandbox_skills_dir
+
+ return get_sandbox_skills_dir()
+
+
+def user_view_dir(user_id: Optional[str]) -> Optional[Path]:
+ from core.capabilities.skills.config import get_user_skills_dir
+
+ return get_user_skills_dir(user_id)
diff --git a/src/backend/features/desktop/capabilities/skill_publication.py b/src/backend/features/desktop/capabilities/skill_publication.py
new file mode 100644
index 00000000..91b4ca3e
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_publication.py
@@ -0,0 +1,88 @@
+"""skill publication responsibilities."""
+from __future__ import annotations
+from features.desktop.capabilities import skills
+from typing import Dict, Optional
+from features.desktop.capabilities import registry, store
+from core.capabilities.identity_rules import KIND_SKILL, LOCAL_PROFILE
+from features.desktop.capabilities.paths import revision_for_hash
+from core.capabilities.reference import local_ref
+from features.desktop.capabilities.resolver import SOURCE_LOCAL, SOURCE_PLUGIN
+from features.desktop.capabilities import skills
+
+def publish_local_skill(
+ skill_id: str,
+ *,
+ files: Dict[str, bytes | str],
+ content_hash: str,
+ owner_user_id: Optional[str] = None,
+ source: str = SOURCE_LOCAL,
+ source_plugin: Optional[str] = None,
+ display_name: str = "",
+ description: str = "",
+ version: str = "",
+ enabled: bool = True,
+ from_db: bool = False,
+) -> store.StoredComponent:
+ """Write one revision of a device-local skill and make it the ready revision."""
+ revision = revision_for_hash(content_hash)
+ before_generation = registry.generation()
+ published = False
+ ref = local_ref(KIND_SKILL, skill_id)
+ inst = registry.upsert(
+ profile_id=LOCAL_PROFILE,
+ ref=ref,
+ display_name=display_name or skill_id,
+ description=description,
+ version=version,
+ content_hash=content_hash,
+ source=source,
+ source_plugin=source_plugin,
+ payload={"owner_user_id": owner_user_id, "from_db": from_db},
+ enabled=enabled,
+ )
+ comp = store.get(KIND_SKILL, LOCAL_PROFILE, ref.key, revision)
+ if comp is None:
+ tx = registry.begin_transaction(inst.install_id, inst.generation + 1)
+ try:
+ comp = store.write_from_files(KIND_SKILL, LOCAL_PROFILE, ref.key, revision, files)
+ published = True
+ registry.advance_transaction(tx, "published", inventory=store.inventory(comp))
+ except Exception as exc:
+ registry.advance_transaction(tx, "failed", error=str(exc))
+ registry.set_state(inst.install_id, "failed", last_error=str(exc))
+ raise
+ registry.set_state(inst.install_id, "ready", resolved_revision=revision)
+ registry.advance_transaction(tx, "committed")
+ elif inst.resolved_revision != revision or inst.state != "ready":
+ registry.set_state(inst.install_id, "ready", resolved_revision=revision)
+ # A DB-backed skill is materialized again after restart/cache refresh. An
+ # identical publication must not invalidate every user's resolved closure.
+ if published or registry.generation() != before_generation:
+ skills.bump_view_generation()
+ return comp
+
+
+def remove_local_skill(skill_id: str) -> bool:
+ from features.desktop.capabilities.runtime import references
+
+ if references("skill", LOCAL_PROFILE, skill_id):
+ # Source deletion revokes use; history still needs its exact bytes.
+ return registry.mark_removed(registry.install_id("skill", LOCAL_PROFILE, skill_id))
+ iid = registry.install_id(KIND_SKILL, LOCAL_PROFILE, skill_id)
+ removed_files = store.remove_key(KIND_SKILL, LOCAL_PROFILE, skill_id) > 0
+ removed_row = registry.delete(iid)
+ registry.clear_preference(KIND_SKILL, skill_id)
+ skills.bump_view_generation()
+ return removed_files or removed_row
+
+
+def prune_local(live_skill_owners: Dict[str, Optional[str]]) -> int:
+ """Drop local-profile entries whose database row no longer exists."""
+ removed = 0
+ for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE):
+ if inst.source in (SOURCE_LOCAL, SOURCE_PLUGIN) and inst.payload.get("from_db"):
+ if inst.key not in live_skill_owners:
+ registry.mark_removed(inst.install_id)
+ removed += 1
+ # Unindexed and superseded bytes may still be retained by history snapshots.
+ return removed
diff --git a/src/backend/features/desktop/capabilities/skill_views.py b/src/backend/features/desktop/capabilities/skill_views.py
new file mode 100644
index 00000000..96344992
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skill_views.py
@@ -0,0 +1,62 @@
+"""skill views responsibilities."""
+from __future__ import annotations
+from features.desktop.capabilities import skills
+from pathlib import Path
+from typing import Dict, Optional
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import view as view_mod
+from core.capabilities.identity_rules import KIND_SKILL
+from features.desktop.capabilities.resolver import Resolution, resolve
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.capabilities import skill_candidates
+from features.desktop.capabilities import skill_paths
+from features.desktop.capabilities import skills
+
+def targets(res: Resolution) -> Dict[str, Path]:
+ return {name: c.path for name, c in res.chosen.items() if c.path is not None}
+
+
+def rebuild_device_view() -> view_mod.ViewReport:
+ res = resolve(
+ KIND_SKILL,
+ skill_candidates.fill_hashes(skill_candidates.builtin_candidates() + skill_candidates.local_candidates(None, shared_only=True)),
+ preferences=registry.preferences(KIND_SKILL),
+ )
+ report = view_mod.build_view(
+ skill_paths.device_view_dir(), targets(res), allowed_roots=skill_paths.allowed_link_roots()
+ )
+ log_report("device", report)
+ return report
+
+
+def rebuild_user_view(user_id: str) -> Optional[view_mod.ViewReport]:
+ vdir = skill_paths.user_view_dir(user_id)
+ if vdir is None:
+ return None
+ res = skills.resolve_for_user(user_id)
+ report = view_mod.build_view(vdir, targets(res), allowed_roots=skill_paths.allowed_link_roots())
+ log_report(f"user:{user_id}", report)
+ return report
+
+
+def rebuild_views(user_id: Optional[str]) -> Dict[str, view_mod.ViewReport]:
+ reports = {"device": rebuild_device_view()}
+ if user_id:
+ user_report = rebuild_user_view(user_id)
+ if user_report is not None:
+ reports["user"] = user_report
+ return reports
+
+
+def log_report(label: str, report: view_mod.ViewReport) -> None:
+ if report.changed:
+ logger.info(
+ "[caps-view] %s linked=%d relinked=%d removed=%d",
+ label,
+ len(report.linked),
+ len(report.relinked),
+ len(report.removed),
+ )
+ for name, why in report.blocked.items():
+ logger.warning("[caps-view] %s name '%s' unavailable: %s", label, name, why)
diff --git a/src/backend/features/desktop/capabilities/skills.py b/src/backend/features/desktop/capabilities/skills.py
new file mode 100644
index 00000000..29a866cb
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skills.py
@@ -0,0 +1,161 @@
+"""skills responsibilities."""
+from __future__ import annotations
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+from features.desktop.capabilities import registry
+from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE
+from features.desktop.capabilities.resolver import SOURCE_BUILTIN, SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve
+import logging
+logger = logging.getLogger(__name__)
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skill_candidates
+from features.desktop.capabilities import skills_state
+
+def view_generation() -> int:
+ """Bumped whenever store contents change; view consumers refresh on a new value."""
+ return skills_state.view_generation
+
+
+def bump_view_generation() -> int:
+ pass # State is owned by its responsibility module.
+ with skills_state.resolution_lock:
+ skills_state.view_generation += 1
+ return skills_state.view_generation
+
+
+def resolve_for_user(user_id: Optional[str], *, requested: Optional[Set[str]] = None) -> Resolution:
+ """Which candidate wins each runtime name, recomputed only when it can differ.
+
+ The answer is a pure function of the installations, the store contents, the
+ user's name preferences and the bound cloud account — every one of which
+ moves ``registry.generation()``, ``view_generation()`` or the account
+ profile. Agent assembly asks for this several times per message and the
+ eligibility fixpoint walks every candidate's package each time, so without
+ this the same answer was recomputed from disk two or three times a turn.
+ """
+ from features.desktop.capabilities.discovery import scan
+ from features.desktop.capabilities.readiness import eligible_skill_candidates
+ scan(user_id)
+ key = user_id or ""
+ signal = (
+ registry.generation(),
+ view_generation(),
+ skill_account.current_account_profile(),
+ skill_candidates.builtin_signature(),
+ frozenset(requested) if requested is not None else None,
+ )
+ with skills_state.resolution_lock:
+ hit = skills_state.resolution_cache.get(key)
+ if hit is not None and hit[0] == signal:
+ skills_state.last_resolution[key] = hit[1]
+ return hit[1]
+
+ preferences = registry.preferences(KIND_SKILL, user_id=user_id)
+ eligible = eligible_skill_candidates(
+ skill_candidates.fill_hashes(skill_candidates.candidates(user_id)), user_id, preferences=preferences, requested=requested
+ )
+ res = resolve(KIND_SKILL, eligible, preferences=preferences, requested=requested)
+ with skills_state.resolution_lock:
+ skills_state.last_resolution[key] = res
+ # A cloud deployment shares this process across users; keep the memo
+ # bounded rather than letting it grow with the user table.
+ if len(skills_state.resolution_cache) >= 64 and key not in skills_state.resolution_cache:
+ skills_state.resolution_cache.pop(next(iter(skills_state.resolution_cache)), None)
+ skills_state.resolution_cache[key] = (signal, res)
+ return res
+
+
+def filter_available_names(names, *, user_id=None):
+ """Implicit defaults omit unusable names; explicit bindings still fail in preflight."""
+ chosen = resolve_for_user(user_id).chosen
+ return [name for name in dict.fromkeys(names) if name in chosen]
+
+
+def last_resolution(user_id: Optional[str]) -> Optional[Resolution]:
+ with skills_state.resolution_lock:
+ return skills_state.last_resolution.get(user_id or "")
+
+
+def info_candidate(info: Any) -> Candidate:
+ """Map a backend ``SkillFileInfo`` onto a resolver candidate."""
+ src = info.source_name
+ origin = dict(getattr(info, "origin", None) or {})
+ if src == "built-in":
+ return Candidate(
+ install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, info.skill_id),
+ runtime_name=info.skill_id,
+ kind=KIND_SKILL,
+ profile=BUILTIN_PROFILE,
+ source=SOURCE_BUILTIN,
+ path=Path(info.file_path).parent,
+ account_level=False,
+ )
+ if src in ("cloud", "device-store"):
+ return Candidate(
+ install_id=str(origin.get("install_id") or ""),
+ runtime_name=info.skill_id,
+ kind=KIND_SKILL,
+ profile=str(origin.get("profile") or ""),
+ source=SOURCE_CLOUD if src == "cloud" else SOURCE_LOCAL,
+ path=Path(info.file_path).parent,
+ content_hash=origin.get("content_hash"),
+ revision=origin.get("revision"),
+ account_level=True,
+ )
+ if src == "admin":
+ # Device catalog row (created here, installed from a plugin or the market).
+ return Candidate(
+ install_id=registry.install_id(KIND_SKILL, LOCAL_PROFILE, info.skill_id),
+ runtime_name=info.skill_id,
+ kind=KIND_SKILL,
+ profile=LOCAL_PROFILE,
+ source=SOURCE_LOCAL,
+ path=Path(""),
+ account_level=True,
+ )
+ return Candidate(
+ install_id=registry.install_id(KIND_SKILL, src, info.skill_id),
+ runtime_name=info.skill_id,
+ kind=KIND_SKILL,
+ profile=src,
+ source=SOURCE_LOCAL,
+ path=Path(info.file_path).parent,
+ account_level=False,
+ )
+
+
+def merge_skill_infos(groups: Dict[str, List[Any]], *, backend_hash) -> Dict[str, Any]:
+ """Composite-backend merge hook: one winner per id, conflicts load nothing.
+
+ ``backend_hash(info) -> str`` computes a content hash for a backend entry; it
+ is only called for ids with more than one candidate.
+ """
+ cands: List[Candidate] = []
+ lookup: Dict[str, Any] = {}
+ for skill_id, infos in groups.items():
+ for info in infos:
+ cand = info_candidate(info)
+ if len(infos) > 1 and cand.content_hash is None:
+ cand = Candidate(**{**cand.__dict__, "content_hash": backend_hash(info)})
+ cands.append(cand)
+ lookup[cand.install_id] = info
+ res = resolve(
+ KIND_SKILL,
+ cands,
+ preferences=registry.preferences(KIND_SKILL, user_id=skill_account.current_local_user_id()),
+ )
+ with skills_state.resolution_lock:
+ skills_state.last_resolution["__loader__"] = res
+ for name in res.conflicts:
+ logger.warning("[caps] skill name conflict, nothing loaded for '%s'", name)
+ return {name: lookup[c.install_id] for name, c in res.chosen.items()}
+
+
+def loader_resolution() -> Optional[Resolution]:
+ with skills_state.resolution_lock:
+ return skills_state.last_resolution.get("__loader__")
+
+
+def conflicted_names() -> Set[str]:
+ res = loader_resolution()
+ return set(res.conflicts) if res else set()
diff --git a/src/backend/features/desktop/capabilities/skills_state.py b/src/backend/features/desktop/capabilities/skills_state.py
new file mode 100644
index 00000000..05ee6099
--- /dev/null
+++ b/src/backend/features/desktop/capabilities/skills_state.py
@@ -0,0 +1,35 @@
+"""skills state responsibilities."""
+from __future__ import annotations
+import threading
+from collections import OrderedDict
+from typing import Dict, Tuple
+from features.desktop.capabilities.resolver import Resolution
+
+SKIP_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"}
+
+
+INVENTORY_NAME = ".inventory.json"
+
+
+hash_lock = threading.Lock()
+
+
+hash_cache: Dict[str, Tuple[Tuple, str]] = {}
+
+
+content_hash_cache: OrderedDict[tuple, str] = OrderedDict()
+
+
+resolution_lock = threading.Lock()
+
+
+last_resolution: Dict[str, Resolution] = {}
+
+
+resolution_cache: Dict[str, Tuple[Tuple, Resolution]] = {}
+
+
+view_generation = 0
+
+
+shadow_user_ids: Dict[str, str] = {}
diff --git a/src/backend/core/capabilities/store.py b/src/backend/features/desktop/capabilities/store.py
similarity index 94%
rename from src/backend/core/capabilities/store.py
rename to src/backend/features/desktop/capabilities/store.py
index e003fe9f..fc18d55e 100644
--- a/src/backend/core/capabilities/store.py
+++ b/src/backend/features/desktop/capabilities/store.py
@@ -16,19 +16,10 @@
from pathlib import Path
from typing import Dict, Iterator, List, Optional
-from . import archive
-from .errors import IntegrityFailed
-from .paths import (
- KIND_AGENT,
- KIND_PLUGIN,
- KIND_SKILL,
- component_dir,
- assert_managed_path,
- kind_root,
- profile_dir,
- safe_segment,
- staging_root,
-)
+from core.capabilities import archive as archive
+from core.harness.capability_errors import IntegrityFailed
+from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL, safe_segment
+from features.desktop.capabilities.paths import component_dir, assert_managed_path, kind_root, profile_dir, staging_root
_ENTRY_FILE = {KIND_SKILL: "SKILL.md", KIND_PLUGIN: "plugin.json", KIND_AGENT: "agent.json"}
_INVENTORY = ".inventory.json"
diff --git a/src/backend/core/capabilities/view.py b/src/backend/features/desktop/capabilities/view.py
similarity index 97%
rename from src/backend/core/capabilities/view.py
rename to src/backend/features/desktop/capabilities/view.py
index ed572680..6a888f67 100644
--- a/src/backend/core/capabilities/view.py
+++ b/src/backend/features/desktop/capabilities/view.py
@@ -14,8 +14,8 @@
from pathlib import Path
from typing import Dict, Iterable, List
-from . import junction
-from .errors import ViewUnavailable
+from core.infra import directory_links as junction
+from core.harness.capability_errors import ViewUnavailable
@dataclass
diff --git a/src/backend/core/capabilities/warmup.py b/src/backend/features/desktop/capabilities/warmup.py
similarity index 72%
rename from src/backend/core/capabilities/warmup.py
rename to src/backend/features/desktop/capabilities/warmup.py
index ec7092f3..51daf1c7 100644
--- a/src/backend/core/capabilities/warmup.py
+++ b/src/backend/features/desktop/capabilities/warmup.py
@@ -5,6 +5,8 @@
"""
from __future__ import annotations
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skills
import logging
import os
@@ -16,10 +18,11 @@
def warmup_current_account(state):
- from . import skills, registry, store
- from .dependency import component_hash, skill_definition
- from .paths import capabilities_enabled, LOCAL_PROFILE, BUILTIN_PROFILE
- from core.services.desktop_cloud_bridge import get_state, _state_fingerprint
+ from features.desktop.capabilities import skills, registry, store
+ from features.desktop.capabilities.dependency import component_hash, skill_definition
+ from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE
+ from features.desktop.capabilities.paths import capabilities_enabled
+ from features.desktop.bridge.identity import get_state, state_fingerprint as _state_fingerprint
if os.name != "nt" or not capabilities_enabled():
return
@@ -31,9 +34,9 @@ def current():
if not current():
return
- user_id = skills.current_local_user_id()
- profile = skills.current_account_profile()
- if not user_id or not profile or not skills.account_authorized_for(user_id):
+ user_id = skill_account.current_local_user_id()
+ profile = skill_account.current_account_profile()
+ if not user_id or not profile or not skill_account.account_authorized_for(user_id):
return
rows = [
row
diff --git a/src/backend/features/desktop/capability/__init__.py b/src/backend/features/desktop/capability/__init__.py
new file mode 100644
index 00000000..78a2d786
--- /dev/null
+++ b/src/backend/features/desktop/capability/__init__.py
@@ -0,0 +1 @@
+"""capability package."""
diff --git a/src/backend/core/services/desktop_capability_configs.py b/src/backend/features/desktop/capability/configs.py
similarity index 91%
rename from src/backend/core/services/desktop_capability_configs.py
rename to src/backend/features/desktop/capability/configs.py
index 2768c41f..2590d5aa 100644
--- a/src/backend/core/services/desktop_capability_configs.py
+++ b/src/backend/features/desktop/capability/configs.py
@@ -42,9 +42,9 @@ def _user_capability_configs(
if hit and (now - hit[0]) < _EFFECTIVE_TTL_S:
return list(hit[1]), list(hit[2]), dict(hit[3])
- from core.config.catalog_resolver import resolve_all_runtime_enabled
- from core.llm.factory.tools.mcp_config import _effective_mcp_server_keys
- from core.services.mcp_service import McpServerConfigService
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+ from core.llm.factory.tools.mcp_config import effective_mcp_server_keys as _effective_mcp_server_keys
+ from core.capabilities.mcp.service import McpServerConfigService
svc = McpServerConfigService.get_instance()
owned = svc.get_owned_servers(uid, enabled_only=False, strict=not use_cache)
diff --git a/src/backend/core/services/desktop_capability_credentials.py b/src/backend/features/desktop/capability/credentials.py
similarity index 98%
rename from src/backend/core/services/desktop_capability_credentials.py
rename to src/backend/features/desktop/capability/credentials.py
index 961f4744..eb44f488 100644
--- a/src/backend/core/services/desktop_capability_credentials.py
+++ b/src/backend/features/desktop/capability/credentials.py
@@ -6,7 +6,7 @@
from typing import Any, Dict
from urllib.parse import unquote, unquote_plus, urlsplit
-from core.llm.reasoning_effort import EFFORT_KEYS
+from core.models.reasoning_effort import EFFORT_KEYS
class CapabilityContentRejected(ValueError):
diff --git a/src/backend/core/services/desktop_capability_entities.py b/src/backend/features/desktop/capability/entities.py
similarity index 91%
rename from src/backend/core/services/desktop_capability_entities.py
rename to src/backend/features/desktop/capability/entities.py
index 2c7f7289..12f4efea 100644
--- a/src/backend/core/services/desktop_capability_entities.py
+++ b/src/backend/features/desktop/capability/entities.py
@@ -8,8 +8,8 @@
from typing import Any, Dict, List, Optional, Tuple
from core.db.engine import SessionLocal
-from core.services.desktop_capability_configs import _EFFECTIVE_TTL_S, _effective_lock
-from core.services.desktop_capability_credentials import CapabilityContentRejected
+from features.desktop.capability.configs import _EFFECTIVE_TTL_S, _effective_lock
+from features.desktop.capability.credentials import CapabilityContentRejected
# ── 智能体 / 插件清单与定义包(云端侧) ──────────────────────────────────
#
@@ -184,7 +184,7 @@ def _agent_files(serialized: Dict[str, Any]) -> Dict[str, str]:
def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]:
- from core.plugins.connections import connection_type
+ from core.capabilities.plugins.connections import connection_type
uploaded = installed.get("_uploaded_files")
definition = {
@@ -206,7 +206,7 @@ def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]:
definition.update(_public_declarations(installed))
if uploaded is not None:
definition = json.loads(uploaded["plugin.json"])
- from core.services.desktop_resource_plugins import package_files
+ from features.desktop.bridge.resource_plugins import package_files
assets, native = package_files(installed)
if uploaded is not None and not native:
return dict(uploaded)
@@ -220,7 +220,7 @@ def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]:
def _user_agents(user_id: str) -> List[Dict[str, Any]]:
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
with SessionLocal() as db:
return list(UserAgentService(db).list_for_user(user_id) or [])
@@ -228,10 +228,10 @@ def _user_agents(user_id: str) -> List[Dict[str, Any]]:
def _user_plugins(user_id: str) -> List[Dict[str, Any]]:
from core.db.models import InstalledPlugin
- from core.plugins import management as plugin_service
+ from core.capabilities.plugins import management as plugin_service
with SessionLocal() as db:
- from core.services.capability_workcopies import active_plugin_files
+ from features.desktop.capability.workcopies import active_plugin_files
rows = plugin_service.list_installed(db, user_id, include_global=True)
metadata = {
@@ -244,7 +244,7 @@ def _user_plugins(user_id: str) -> List[Dict[str, Any]]:
InstalledPlugin.install_id.in_([r["install_id"] for r in rows] or [""])
)
}
- from core.plugins.management.admin_config import _connection_for_slug
+ from core.capabilities.plugins.management.admin_config import connection_for_slug as _connection_for_slug
for r in rows:
r.update(metadata.get(r["install_id"], {}))
@@ -253,7 +253,7 @@ def _user_plugins(user_id: str) -> List[Dict[str, Any]]:
def build_user_agent_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]:
- from core.services.desktop_capability_protocol import build_entity_manifest, entity_content_hash
+ from core.capabilities.manifest_protocol import build_entity_manifest, entity_content_hash
uid = str(user_id)
now = time.monotonic()
@@ -280,7 +280,7 @@ def build_user_agent_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s
def resolve_agent_bundle(user_id: str, agent_id: str) -> Optional[Tuple[bytes, str]]:
- from core.services.desktop_capability_protocol import entity_content_hash
+ from core.capabilities.manifest_protocol import entity_content_hash
for a in _user_agents(str(user_id)):
if a["agent_id"] == agent_id:
@@ -290,7 +290,7 @@ def resolve_agent_bundle(user_id: str, agent_id: str) -> Optional[Tuple[bytes, s
def build_user_plugin_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]:
- from core.services.desktop_capability_protocol import build_entity_manifest, entity_content_hash
+ from core.capabilities.manifest_protocol import build_entity_manifest, entity_content_hash
uid = str(user_id)
now = time.monotonic()
@@ -321,7 +321,7 @@ def build_user_plugin_manifest(user_id: str, *, use_cache: bool = True) -> Dict[
def resolve_plugin_bundle(user_id: str, install_id: str) -> Optional[Tuple[bytes, str]]:
- from core.services.desktop_capability_protocol import entity_content_hash
+ from core.capabilities.manifest_protocol import entity_content_hash
for p in _user_plugins(str(user_id)):
if p["install_id"] == install_id:
diff --git a/src/backend/core/services/desktop_capability_mcp.py b/src/backend/features/desktop/capability/mcp.py
similarity index 85%
rename from src/backend/core/services/desktop_capability_mcp.py
rename to src/backend/features/desktop/capability/mcp.py
index 4111f551..27ba9cbc 100644
--- a/src/backend/core/services/desktop_capability_mcp.py
+++ b/src/backend/features/desktop/capability/mcp.py
@@ -7,15 +7,10 @@
from core.db.engine import SessionLocal
from core.db.models import AdminMcpServer
-from core.services.desktop_capability_configs import _user_capability_configs
-from core.services.desktop_capability_credentials import _secrets_from_config
-from core.services.desktop_capability_protocol import (
- CapabilityManifestStaleError,
- build_manifest,
- canonical_hash,
- public_tool_schemas,
-)
-from core.services.desktop_capability_security import guard_capability_content
+from features.desktop.capability.configs import _user_capability_configs
+from features.desktop.capability.credentials import _secrets_from_config
+from core.capabilities.manifest_protocol import CapabilityManifestStaleError, build_manifest, canonical_hash, public_tool_schemas
+from features.desktop.capability.security import guard_capability_content
def build_user_capability_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]:
@@ -26,7 +21,7 @@ def build_user_capability_manifest(user_id: str, *, use_cache: bool = True) -> D
MCP streamable-http 协议透明反代;stdio / sse 传输的(本就极少)不进桌面清单。
凭据(URL 内嵌密钥、headers、OAuth)一律留在云端连接层,manifest 不携带任何密钥。
"""
- from core.config.catalog_runtime import _DEFAULT_MCP_ICONS
+ from core.capabilities.catalog.runtime import DEFAULT_MCP_ICONS as _DEFAULT_MCP_ICONS
keys, enabled_keys, all_cfgs = _user_capability_configs(user_id, use_cache=use_cache)
enabled = set(enabled_keys)
@@ -152,16 +147,17 @@ async def invoke_gateway_tool(
embedded.interface().execute, resolved["user_id"], resolved["tool"]["name"], arguments,
)
import mcp.types
- from core.llm.mcp_pool import make_client
+ from core.llm.mcp.client import make_client
arguments = dict(arguments or {})
- from core.services.automation_remote_effect import RECEIPT_TOOLS, bind_remote_effect
+ from core.harness.registry.tool_operations import receipt_operations
tool_name = str(resolved["tool"]["name"])
- if tool_name in RECEIPT_TOOLS and arguments.get("tool_effect_id"):
+ operations = receipt_operations(tool_name)
+ if operations is not None and arguments.get("tool_effect_id"):
operation_id = arguments.pop("tool_effect_id")
arguments["tool_effect_id"] = await asyncio.to_thread(
- bind_remote_effect,
+ operations.bind,
str(resolved["user_id"]),
str(resolved["server_id"]),
tool_name,
@@ -184,7 +180,7 @@ async def invoke_gateway_tool(
upstream_headers["accept-encoding"] = "identity"
# The authenticated cloud gateway is a new invocation hop. Never forward
# a device-issued proof to an internal MCP that trusts the cloud key.
- from core.llm.mcp_invocation import HEADER, PLUGIN_HEADER, for_url
+ from core.llm.mcp.invocation import HEADER, PLUGIN_HEADER, for_url
upstream_headers.pop(HEADER.lower(), None)
upstream_headers.pop(PLUGIN_HEADER.lower(), None)
@@ -193,15 +189,13 @@ async def invoke_gateway_tool(
))
target["headers"] = upstream_headers
- client = make_client(str(resolved["server_id"]), target, is_stateful=False)
+ from core.llm.mcp.results import tool_result
+ from core.llm.mcp.client import _execution_timeout
raw_tool = mcp.types.Tool.model_validate(resolved["tool"])
- # Skip a second tools/list call in the cloud: the allowlisted schema was read
- # from the same DB row immediately above. get_tool then performs only the
- # real initialize + tools/call lifecycle against the private MCP target.
- client._cached_tools = [raw_tool] # noqa: SLF001 - AgentScope has no public preload API
- tool = await client.get_tool(raw_tool.name)
- timeout = max(1.0, float(client.execution_timeout or 120.0)) + 10.0
- chunk = await asyncio.wait_for(tool(**dict(arguments or {})), timeout=timeout)
+ async with make_client(str(resolved["server_id"]), target) as client:
+ result = await asyncio.wait_for(client.call_tool(raw_tool.name, dict(arguments or {})),
+ timeout=_execution_timeout(str(resolved["server_id"]), target))
+ chunk = tool_result(result, raw_tool)
chunk.metadata.setdefault("origin", "cloud")
chunk.metadata.setdefault("mcp_server_id", str(resolved["server_id"]))
return guard_capability_content(
diff --git a/src/backend/core/services/desktop_capability_models.py b/src/backend/features/desktop/capability/models.py
similarity index 93%
rename from src/backend/core/services/desktop_capability_models.py
rename to src/backend/features/desktop/capability/models.py
index 820e786d..d5083f7c 100644
--- a/src/backend/core/services/desktop_capability_models.py
+++ b/src/backend/features/desktop/capability/models.py
@@ -7,9 +7,9 @@
from core.db.engine import SessionLocal
from core.db.models import ModelProvider, ModelRoleAssignment
-from core.services.desktop_capability_credentials import CapabilityContentRejected
-from core.services.desktop_capability_protocol import canonical_hash
-from core.services.desktop_capability_security import _guard_value, _known_cloud_secrets
+from features.desktop.capability.credentials import CapabilityContentRejected
+from core.capabilities.manifest_protocol import canonical_hash
+from features.desktop.capability.security import _guard_value, _known_cloud_secrets
logger = logging.getLogger(__name__)
@@ -33,9 +33,9 @@ def _upstream_model_path(provider: ModelProvider) -> Optional[str]:
provider_type = str(provider.provider_type or "")
if provider_type != "chat":
return _MODEL_PATHS.get(provider_type)
- from core.llm.chat_models import wants_responses
- from core.llm.providers.protocol_probe import PROTOCOL_RESPONSES
- from core.llm.providers.registry import get_spec
+ from core.models.chat_models import wants_responses
+ from core.models.providers.protocol_probe import PROTOCOL_RESPONSES
+ from core.models.providers.registry import get_spec
spec = get_spec(getattr(provider, "provider", None) or "openai_compatible")
protocol = (provider.extra_config or {}).get("api_protocol")
@@ -62,7 +62,7 @@ def _model_is_gateway_compatible(provider: ModelProvider) -> bool:
Gemini / Bedrock 也不是同一线上协议;在专用适配器完成前不把它们
伪装成可用,更不会为了兼容而下发真实凭据。
"""
- from core.llm.providers.registry import get_spec
+ from core.models.providers.registry import get_spec
provider_id = getattr(provider, "provider", None) or "openai_compatible"
spec = get_spec(provider_id)
@@ -160,7 +160,7 @@ def _model_provider_allowed(db, user_id: str, provider: ModelProvider) -> bool:
return True
if provider.provider_type != "chat":
return False
- from core.services.user_model_selection import user_can_switch_model
+ from core.auth.model_selection import user_can_switch_model
return user_can_switch_model(db, str(user_id))
diff --git a/src/backend/features/desktop/capability/revision_provider.py b/src/backend/features/desktop/capability/revision_provider.py
new file mode 100644
index 00000000..090ae82d
--- /dev/null
+++ b/src/backend/features/desktop/capability/revision_provider.py
@@ -0,0 +1,22 @@
+"""Expose existing account revision authorization and compare-and-swap semantics."""
+from features.desktop.capability import workcopies
+
+
+class CloudCapabilityRevisions:
+ def model(self, db, kind, key):
+ return workcopies.get_model(db, kind, key)
+
+ def snapshot_in_session(self, db, user, kind, key):
+ return workcopies.snapshot_in_session(db, user, kind, key)
+
+ def snapshot(self, user, kind, key):
+ return workcopies.snapshot(user, kind, key)
+
+ def commit(self, user, kind, key, body):
+ return workcopies.commit(user, kind, key, body)
+
+ def revision(self, files):
+ return workcopies.revision(files)
+
+ def record_id(self, prefix, user, kind, key):
+ return workcopies.record_id(prefix, user, kind, key)
diff --git a/src/backend/core/services/desktop_capability_security.py b/src/backend/features/desktop/capability/security.py
similarity index 96%
rename from src/backend/core/services/desktop_capability_security.py
rename to src/backend/features/desktop/capability/security.py
index bee6e847..264065b1 100644
--- a/src/backend/core/services/desktop_capability_security.py
+++ b/src/backend/features/desktop/capability/security.py
@@ -7,11 +7,8 @@
from core.db.engine import SessionLocal
from core.db.models import ModelProvider
-from core.services.desktop_capability_configs import _effective_lock, _user_capability_configs
-from core.services.desktop_capability_credentials import (
- CapabilityContentRejected,
- _secrets_from_config,
-)
+from features.desktop.capability.configs import _effective_lock, _user_capability_configs
+from features.desktop.capability.credentials import CapabilityContentRejected, _secrets_from_config
def _without_public_identifiers(secrets: set[str], public_identifiers: set[str]) -> set[str]:
@@ -27,7 +24,7 @@ def _without_public_identifiers(secrets: set[str], public_identifiers: set[str])
def _model_credentials_and_public_identifiers(*, fresh: bool = True) -> Tuple[set[str], set[str]]:
global _model_secret_cache
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
version = ModelConfigService.get_instance().version
if not fresh:
diff --git a/src/backend/core/services/desktop_capability.py b/src/backend/features/desktop/capability/service.py
similarity index 90%
rename from src/backend/core/services/desktop_capability.py
rename to src/backend/features/desktop/capability/service.py
index 9f4d7c32..99eb1cf2 100644
--- a/src/backend/core/services/desktop_capability.py
+++ b/src/backend/features/desktop/capability/service.py
@@ -35,42 +35,15 @@
from core.db.engine import SessionLocal
from core.db.models import ContentBlock
-from core.services.desktop_capability_credentials import CapabilityContentRejected
-from core.services.desktop_capability_entities import (
- build_user_agent_manifest,
- build_user_plugin_manifest,
- resolve_agent_bundle,
- resolve_plugin_bundle,
-)
+from features.desktop.capability.credentials import CapabilityContentRejected
+from features.desktop.capability.entities import build_user_agent_manifest, build_user_plugin_manifest, resolve_agent_bundle, resolve_plugin_bundle
# Public entry points remain stable for route consumers; implementations own their state.
-from core.services.desktop_capability_mcp import (
- build_user_capability_manifest,
- invoke_gateway_tool,
- resolve_gateway_target,
- resolve_gateway_tool,
-)
-from core.services.desktop_capability_models import (
- build_user_model_manifest,
- resolve_model_gateway_target,
-)
-from core.services.desktop_capability_protocol import (
- CapabilityManifestStaleError,
- build_manifest,
- build_skill_manifest,
- canonical_hash,
- public_tool_schema,
- public_tool_schemas,
- skill_content_hash,
-)
-from core.services.desktop_capability_security import (
- gateway_stream_secrets,
- guard_capability_bundle,
- guard_capability_content,
- guard_capability_stream,
- invalidate_model_gateway_cache,
-)
-from core.services.desktop_capability_skills import build_user_skill_manifest, resolve_skill_bundle
+from features.desktop.capability.mcp import build_user_capability_manifest, invoke_gateway_tool, resolve_gateway_target, resolve_gateway_tool
+from features.desktop.capability.models import build_user_model_manifest, resolve_model_gateway_target
+from core.capabilities.manifest_protocol import CapabilityManifestStaleError, build_manifest, build_skill_manifest, canonical_hash, public_tool_schema, public_tool_schemas, skill_content_hash
+from features.desktop.capability.security import gateway_stream_secrets, guard_capability_bundle, guard_capability_content, guard_capability_stream, invalidate_model_gateway_cache
+from features.desktop.capability.skills import build_user_skill_manifest, resolve_skill_bundle
logger = logging.getLogger(__name__)
@@ -137,7 +110,7 @@ def _sign(data: bytes) -> str:
def is_desktop_shell_control(authorization: str, origin: Optional[str]) -> bool:
"""Only the shell process secret authorizes local capability management."""
- from core.auth.desktop_bridge import BRIDGE_SECRET_ENV
+ from features.desktop.auth.bridge import BRIDGE_SECRET_ENV
secret = os.getenv(BRIDGE_SECRET_ENV, "").strip()
if not secret or origin is not None:
diff --git a/src/backend/core/services/desktop_capability_skills.py b/src/backend/features/desktop/capability/skills.py
similarity index 86%
rename from src/backend/core/services/desktop_capability_skills.py
rename to src/backend/features/desktop/capability/skills.py
index 68075f5f..45079045 100644
--- a/src/backend/core/services/desktop_capability_skills.py
+++ b/src/backend/features/desktop/capability/skills.py
@@ -8,8 +8,8 @@
from typing import Any, Dict, List, Optional, Tuple
from core.db.engine import SessionLocal
-from core.services.desktop_capability_configs import _EFFECTIVE_TTL_S, _effective_lock
-from core.services.desktop_capability_protocol import build_skill_manifest, skill_content_hash
+from features.desktop.capability.configs import _EFFECTIVE_TTL_S, _effective_lock
+from core.capabilities.manifest_protocol import build_skill_manifest, skill_content_hash
# ── 技能清单 / 技能包(云端为真源,本机只缓存文件快照) ───────────────────
@@ -18,7 +18,7 @@
def _skill_info(skill_id: str):
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
return get_skill_loader()._backend.get_skill_info(skill_id)
@@ -30,8 +30,8 @@ def _skill_snapshot(
``info`` 给已经取过的调用方复用,省一次后端查询。
"""
- from core.agent_skills.binary_files import pack_directory
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.binary_files import pack_directory
+ from core.capabilities.skills.loader_factory import get_skill_loader
loader = get_skill_loader()
info = _skill_info(skill_id) if info is None else info
@@ -67,9 +67,9 @@ def build_user_skill_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s
if hit and (now - hit[0]) < _EFFECTIVE_TTL_S:
return copy.deepcopy(hit[1])
- from core.agent_skills.loader import get_skill_loader
- from core.config.catalog_resolver import resolve_all_runtime_enabled
- from core.llm.factory.selection.capabilities import _filter_skill_ids_for_user
+ from core.capabilities.skills.loader_factory import get_skill_loader
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
+ from core.llm.factory.selection.capabilities import filter_skill_ids_for_user as _filter_skill_ids_for_user
with SessionLocal() as db:
enabled, _agents, _mcps = resolve_all_runtime_enabled(db, uid)
@@ -111,7 +111,7 @@ def build_user_skill_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s
def resolve_skill_bundle(user_id: str, skill_id: str) -> Optional[Tuple[bytes, str]]:
"""打包一个当前授权技能为 zip,返回 (bytes, content_hash);未授权返回 None。"""
- from core.services.marketplace_service import build_skill_zip, build_skill_zip_from_dir
+ from features.marketplace.skill_archives import build_skill_zip, build_skill_zip_from_dir
manifest = build_user_skill_manifest(user_id, use_cache=False)
if not any(s["skill_id"] == skill_id for s in manifest["skills"]):
diff --git a/src/backend/core/services/desktop_capability_sync_check.py b/src/backend/features/desktop/capability/sync_check.py
similarity index 79%
rename from src/backend/core/services/desktop_capability_sync_check.py
rename to src/backend/features/desktop/capability/sync_check.py
index e3fd4396..86fc3926 100644
--- a/src/backend/core/services/desktop_capability_sync_check.py
+++ b/src/backend/features/desktop/capability/sync_check.py
@@ -5,6 +5,8 @@
"""
from __future__ import annotations
+from features.desktop.bridge import client as bridge_client
+from features.desktop.bridge import identity as bridge_identity
import threading
import time
@@ -12,15 +14,10 @@
import httpx
-from core.services import desktop_cloud_bridge as bridge
-from core.services import desktop_cloud_bundles as bundles
-from core.services import desktop_cloud_skills as skills
-from core.services.desktop_capability_protocol import (
- canonical_hash,
- validate_manifest,
- validate_skill_manifest,
- validate_entity_manifest,
-)
+from features.desktop.bridge import client as bridge
+from features.desktop.bridge import bundles as bundles
+from features.desktop.bridge import skills as skills
+from core.capabilities.manifest_protocol import canonical_hash, validate_manifest, validate_skill_manifest, validate_entity_manifest
_lock = threading.Lock()
_cache_key = ""
@@ -31,7 +28,7 @@
def _snapshots():
return {
- "mcp": bridge.synced_manifest(),
+ "mcp": bridge_client.synced_manifest(),
"skill": skills.synced_manifest(),
**bundles.synced_manifests(),
}
@@ -57,13 +54,13 @@ def _fetch(st, kind, known):
response = httpx.get(
f"{st['cloud_base']}/api/v1/desktop/capability/{path}",
headers={
- **bridge.cloud_headers(st),
+ **bridge_identity.cloud_headers(st),
"Cache-Control": "no-cache",
"If-None-Match": f'"{known["revision"]}"',
},
timeout=httpx.Timeout(10.0, connect=5.0),
)
- bridge.require_current_account(st)
+ bridge_identity.require_current_account(st)
if response.status_code == 304:
return known
response.raise_for_status()
@@ -79,11 +76,11 @@ def check(st: dict[str, Any]) -> dict[str, Any]:
"""Return changed=None when comparison is unavailable, never a false update."""
global _cache_key, _cache_until, _cache_result
with _lock:
- with bridge.account_scope(st):
+ with bridge_identity.account_scope(st):
local = _snapshots()
if any(snapshot is None for snapshot in local.values()):
return {"changed": None}
- key = canonical_hash([bridge._state_fingerprint(st), local])
+ key = canonical_hash([bridge_identity.state_fingerprint(st), local])
if key == _cache_key and time.monotonic() < _cache_until:
return dict(_cache_result)
try:
@@ -95,14 +92,14 @@ def check(st: dict[str, Any]) -> dict[str, Any]:
)
# Automatic recovery or another window may have synchronized while
# the comparison was in flight. Do not publish against an old base.
- with bridge.account_scope(st):
+ with bridge_identity.account_scope(st):
if local != _snapshots():
return {"changed": None}
result = {"changed": changed}
except Exception:
# Network/protocol failures are unknown, not an update. No credentials
# or raw upstream error bodies are exposed by this advisory endpoint.
- bridge.require_current_account(st)
+ bridge_identity.require_current_account(st)
result = {"changed": None}
_cache_key, _cache_result = key, result
_cache_until = time.monotonic() + CHECK_INTERVAL_SECONDS
diff --git a/src/backend/core/services/capability_workcopies.py b/src/backend/features/desktop/capability/workcopies.py
similarity index 89%
rename from src/backend/core/services/capability_workcopies.py
rename to src/backend/features/desktop/capability/workcopies.py
index 7f3c60dc..ce621fea 100644
--- a/src/backend/core/services/capability_workcopies.py
+++ b/src/backend/features/desktop/capability/workcopies.py
@@ -6,9 +6,9 @@
import threading
from datetime import datetime, timezone
-from core.capabilities.change_merge import revision, sensitive_paths, validate_files
-from core.capabilities.errors import CapabilityError
-from core.capabilities.paths import safe_segment
+from features.desktop.capabilities.change_merge import revision, sensitive_paths, validate_files
+from core.harness.capability_errors import CapabilityError
+from core.capabilities.identity_rules import safe_segment
from core.db.engine import SessionLocal
from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin
from core.infra.crypto import decrypt_secret, encrypt_secret
@@ -19,11 +19,11 @@
_lock = threading.RLock()
-def _id(prefix, user, kind, key):
+def record_id(prefix, user, kind, key):
return prefix + hashlib.sha256((str(user) + "\0" + kind + "\0" + key).encode()).hexdigest()
-def _model(db, kind, key):
+def get_model(db, kind, key):
models = {
"skill": (AdminSkill, AdminSkill.skill_id),
"plugin": (InstalledPlugin, InstalledPlugin.install_id),
@@ -48,7 +48,7 @@ def _files(row, kind):
data["components"] = row.component_ids or {}
data["ui_contributions"] = row.ui_contributions
data["import_report"] = row.import_report or {}
- from core.services.desktop_capability_entities import _plugin_files
+ from features.desktop.capability.entities import _plugin_files
return _plugin_files(data)
# Existing cloud credentials are never exported to a device.
@@ -57,7 +57,7 @@ def _files(row, kind):
def _stored(db, user, kind, key):
- record = db.get(ContentBlock, _id("cap-cloud-copy:", user, kind, key))
+ record = db.get(ContentBlock, record_id("cap-cloud-copy:", user, kind, key))
if record is None:
return None
text = decrypt_secret(record.payload.get("encrypted"))
@@ -66,12 +66,12 @@ def _stored(db, user, kind, key):
return json.loads(text)
-def _snapshot(db, user, kind, key):
- row = _model(db, kind, key)
+def snapshot_in_session(db, user, kind, key):
+ row = get_model(db, kind, key)
if row is not None and row.owner_user_id not in (None, str(user)):
raise HTTPException(404, detail="capability not available")
if row is not None and row.owner_user_id is None:
- from core.services import desktop_capability as capabilities
+ from features.desktop.capability import service as capabilities
if kind == "skill":
allowed = {
@@ -121,7 +121,7 @@ def _snapshot(db, user, kind, key):
def snapshot(user, kind, key):
with SessionLocal() as db:
- return _snapshot(db, str(user), kind, key)[0]
+ return snapshot_in_session(db, str(user), kind, key)[0]
def _activate(db, user, kind, key, files, row):
@@ -135,10 +135,10 @@ def _activate(db, user, kind, key, files, row):
}
_require_flag(str(user), db, *flags[kind])
if kind == "skill":
- from core.agent_skills.registry import _load_skill_metadata_from_str
+ from core.capabilities.skills.registry import load_skill_metadata_from_str
content = files.get("SKILL.md", "")
- metadata = _load_skill_metadata_from_str(content, key)
+ metadata = load_skill_metadata_from_str(content, key)
from api.routes.v1.me_capabilities import extract_mcp_server_ids, resolve_mcp_bindings
from core.ontology.build_validator import ensure_ontology_build_valid
@@ -168,8 +168,8 @@ def _activate(db, user, kind, key, files, row):
metadata.allowed_tools,
)
row.extra_files = {name: value for name, value in files.items() if name != "SKILL.md"}
- from core.agent_skills.binary_files import is_binary_value
- from core.agent_skills.deps_detector import detect_dependencies
+ from core.capabilities.skills.binary_files import is_binary_value
+ from core.capabilities.skills.deps.detector import detect_dependencies
row.dependencies = detect_dependencies(
{name: value for name, value in row.extra_files.items() if not is_binary_value(value)}
@@ -208,7 +208,7 @@ def _activate(db, user, kind, key, files, row):
import asyncio
from core.config.settings import settings
- from core.services.mcp_management_service import encrypt_mcp_headers, validate_remote_mcp_url
+ from core.capabilities.mcp.management import encrypt_mcp_headers, validate_remote_mcp_url
asyncio.run(
validate_remote_mcp_url(
@@ -223,7 +223,6 @@ def _activate(db, user, kind, key, files, row):
owner_user_id=str(user),
created_by=str(user),
display_name=spec.get("display_name") or key,
- is_stable=False,
)
db.add(row)
for field in ("display_name", "description", "transport", "url"):
@@ -244,7 +243,7 @@ def commit(user, kind, key, body):
if sensitive_paths(files) and not body.get("acknowledge_sensitive"):
raise HTTPException(409, detail="sensitive upload needs confirmation")
request_id = body["request_id"]
- receipt_key = _id("cap-cloud-receipt:", user, kind, request_id)
+ receipt_key = record_id("cap-cloud-receipt:", user, kind, request_id)
request_hash = revision({"request.json": json.dumps({"key": key, **body}, sort_keys=True)})
with _lock, SessionLocal() as db:
try:
@@ -263,7 +262,7 @@ def commit(user, kind, key, body):
409, detail="request identity reused with different content"
)
return json.loads(decrypt_secret(receipt.payload["encrypted"]))
- before, row = _snapshot(db, user, kind, key)
+ before, row = snapshot_in_session(db, user, kind, key)
if not before["can_edit"]:
raise HTTPException(403, detail="create a private copy instead")
if body.get("create_only"):
@@ -294,7 +293,7 @@ def commit(user, kind, key, body):
if applied
else previous_active
)
- storage_key = _id("cap-cloud-copy:", user, kind, key)
+ storage_key = record_id("cap-cloud-copy:", user, kind, key)
record = db.get(ContentBlock, storage_key)
if record is None:
record = ContentBlock(id=storage_key)
@@ -304,7 +303,7 @@ def commit(user, kind, key, body):
"updated_at": datetime.now(timezone.utc).isoformat(),
}
for version in (before["files"], files):
- history_key = _id("cap-cloud-version:", user, kind, key + ":" + revision(version))
+ history_key = record_id("cap-cloud-version:", user, kind, key + ":" + revision(version))
if db.get(ContentBlock, history_key) is None:
db.add(
ContentBlock(
@@ -332,8 +331,8 @@ def commit(user, kind, key, body):
except (ValueError, KeyError, TypeError, CapabilityError) as exc:
db.rollback()
raise HTTPException(422, detail="invalid capability files") from exc
- from core.agent_skills.cache_refresh import refresh_skill_caches
- from core.services.mcp_management_service import refresh_mcp_caches
+ from core.capabilities.skills.cache_refresh import refresh_skill_caches
+ from core.capabilities.mcp.management import refresh_mcp_caches
if applied:
try:
diff --git a/src/backend/features/desktop/edition.py b/src/backend/features/desktop/edition.py
new file mode 100644
index 00000000..8a438e48
--- /dev/null
+++ b/src/backend/features/desktop/edition.py
@@ -0,0 +1,23 @@
+"""Community desktop approval controls."""
+from typing import Literal
+from pydantic import BaseModel
+from fastapi import Depends
+from core.auth.backend import UserContext, get_current_user
+from core.infra.responses import success_response
+from features.desktop.local import grants
+
+def register_local_routes(router, require_local):
+ class ApprovalBody(BaseModel):
+ mode: Literal["strict", "standard", "full"]
+
+
+ @router.get("/approval-mode", summary="获取当前审批档(对话框权限栏)")
+ async def get_approval_mode(_user: UserContext = Depends(get_current_user)):
+ require_local()
+ return success_response(data={"mode": grants.get_approval_mode()})
+
+
+ @router.put("/approval-mode", summary="设置审批档")
+ async def set_approval_mode(body: ApprovalBody, _user: UserContext = Depends(get_current_user)):
+ require_local()
+ return success_response(data={"mode": grants.set_approval_mode(body.mode)}, message="已切换权限档")
diff --git a/src/backend/features/desktop/extension.py b/src/backend/features/desktop/extension.py
new file mode 100644
index 00000000..9c18f6af
--- /dev/null
+++ b/src/backend/features/desktop/extension.py
@@ -0,0 +1,4 @@
+"""Desktop capabilities are installed in cloud and local hosts."""
+NAME = "desktop"
+def setup():
+ return True
diff --git a/src/backend/features/desktop/gateway/__init__.py b/src/backend/features/desktop/gateway/__init__.py
new file mode 100644
index 00000000..bbfee548
--- /dev/null
+++ b/src/backend/features/desktop/gateway/__init__.py
@@ -0,0 +1 @@
+"""gateway package."""
diff --git a/src/backend/core/services/desktop_gateway_observability.py b/src/backend/features/desktop/gateway/observability.py
similarity index 98%
rename from src/backend/core/services/desktop_gateway_observability.py
rename to src/backend/features/desktop/gateway/observability.py
index 2480f9de..ee139282 100644
--- a/src/backend/core/services/desktop_gateway_observability.py
+++ b/src/backend/features/desktop/gateway/observability.py
@@ -10,8 +10,8 @@
from concurrent.futures import ThreadPoolExecutor
from core.db.engine import SessionLocal
-from core.db.models.observability import utcnow
-from core.services.desktop_observability import ingest
+from features.desktop.models import utcnow
+from features.desktop.observability.projection import ingest
from fastapi.routing import APIRoute
logger = logging.getLogger(__name__)
diff --git a/src/backend/features/desktop/local/__init__.py b/src/backend/features/desktop/local/__init__.py
new file mode 100644
index 00000000..11bccc04
--- /dev/null
+++ b/src/backend/features/desktop/local/__init__.py
@@ -0,0 +1 @@
+"""local package."""
diff --git a/src/backend/core/services/local_file_actions.py b/src/backend/features/desktop/local/file_actions.py
similarity index 99%
rename from src/backend/core/services/local_file_actions.py
rename to src/backend/features/desktop/local/file_actions.py
index 129fac6c..6166bcfa 100644
--- a/src/backend/core/services/local_file_actions.py
+++ b/src/backend/features/desktop/local/file_actions.py
@@ -10,7 +10,7 @@
import sys
from core.infra.proc import no_window_kwargs
-from core.services.native_app_icons import desktop_icon, windows_icon
+from features.desktop.local.native_app_icons import desktop_icon, windows_icon
def _run(arguments: list[str], *, input_text: str | None = None, timeout: int = 10) -> str:
diff --git a/src/backend/core/services/local_grant_service.py b/src/backend/features/desktop/local/grants.py
similarity index 100%
rename from src/backend/core/services/local_grant_service.py
rename to src/backend/features/desktop/local/grants.py
diff --git a/src/backend/core/services/local_management_migration.py b/src/backend/features/desktop/local/management_migration.py
similarity index 85%
rename from src/backend/core/services/local_management_migration.py
rename to src/backend/features/desktop/local/management_migration.py
index 4f8d506b..12a27de0 100644
--- a/src/backend/core/services/local_management_migration.py
+++ b/src/backend/features/desktop/local/management_migration.py
@@ -4,17 +4,19 @@
remain available; a removed installation is never reactivated by a legacy row.
Plugin-owned rows remain owned by their legacy plugin until that package migrates.
"""
+from sqlalchemy import inspect as inspect_model
import hashlib
import tempfile
from pathlib import Path
from sqlalchemy import inspect
-from core.capabilities import archive, registry
-from core.capabilities.paths import capabilities_enabled
-from core.capabilities.ref import local_ref
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
+from features.desktop.capabilities.paths import capabilities_enabled
+from core.capabilities.reference import local_ref
from core.db.models import AdminSkill, ContentBlock
-from core.services.management_package import serialized
-from core.services import local_skill_service as lifecycle
-from core.agent_skills.binary_files import decode_binary, is_binary_value
+from core.capabilities.plugins.management.package import serialized
+from features.desktop.local.skills import service as lifecycle
+from core.capabilities.skills.binary_files import decode_binary, is_binary_value
def _backup(db, row):
@@ -34,7 +36,7 @@ def migrate():
return 0
count = 0
with registry._session() as db:
- if not inspect(db.get_bind()).has_table(AdminSkill.__tablename__):
+ if not inspect(db.get_bind()).has_table(inspect_model(AdminSkill).local_table.name):
return 0
rows = db.query(AdminSkill).filter(AdminSkill.source_plugin.is_(None)).all()
for row in rows:
diff --git a/src/backend/features/desktop/local/manager_execution.py b/src/backend/features/desktop/local/manager_execution.py
new file mode 100644
index 00000000..9715d3d6
--- /dev/null
+++ b/src/backend/features/desktop/local/manager_execution.py
@@ -0,0 +1,51 @@
+"""Execute an installed manager contract after rechecking account and installation authority."""
+from core.harness.registry import execution_catalog as bridge_gateway
+from core.harness.registry import execution_identity as bridge_identity
+from core.harness.registry import execution_account as skill_account
+from core.capabilities.plugins.management.contract import declared, MANAGERS
+
+
+def execute(*, manager, server_id, headers, cloud_backed, tool_name, input_schema, arguments):
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
+ state = None
+ if cloud_backed and bridge_enabled():
+ state = bridge_identity.get_state()
+ bridge_identity.require_current_account(state)
+ user_id = skill_account.current_local_user_id()
+ context = bridge_gateway.bridge_context() or {}
+ server = next((s for s in context.get("servers", []) if s["server_id"] == server_id), None)
+ if not server or server.get("source_plugin") != manager:
+ raise PermissionError("management plugin is no longer installed or authorized")
+ # upload is a local contribution authorized by the same declared install capability.
+ op = "install_skill" if tool_name == "upload_skill_to_cloud" else tool_name
+ contract = next((t for t in server.get("tools", []) if t["name"] == op), None)
+ if contract is None or declared(contract, manager) != manager:
+ raise PermissionError("management plugin contract changed; synchronize capabilities")
+ else:
+ user_id = next((v for k, v in headers.items() if k.lower() == "x-current-user-id"), None)
+ from core.harness.registry.execution_catalog import authorizes_manager
+ if not authorizes_manager(user_id, server_id, manager):
+ raise PermissionError("local management plugin is not installed")
+ if not user_id:
+ raise PermissionError("authentication required")
+ from jsonschema import validate
+ validate(arguments, input_schema)
+ if state:
+ bridge_identity.require_current_account(state)
+ if tool_name == "upload_skill_to_cloud":
+ from features.desktop.local.skills.upload import upload
+ return upload(user_id, **arguments)
+ kind = MANAGERS[manager]
+ from features.desktop.local.skills import service as local_skill_service
+ from features.desktop.local.plugins import service as local_plugin_service
+ service = local_skill_service if kind == "skill" else local_plugin_service
+ action = tool_name.removesuffix("_" + kind)
+ if tool_name == "list_" + kind + "s":
+ return {"ok": True, "items": getattr(service, "list_" + kind + "s")(user_id)}
+ args = dict(arguments)
+ if "source" in args:
+ source = args.pop("source")
+ if source.get("kind") != "local_path":
+ raise ValueError("local installation requires source.kind=local_path")
+ args["source_path"] = source["path"]
+ return getattr(service, action)(user_id, **args)
diff --git a/src/backend/core/services/local_market_install.py b/src/backend/features/desktop/local/market_install.py
similarity index 61%
rename from src/backend/core/services/local_market_install.py
rename to src/backend/features/desktop/local/market_install.py
index f68fc3e0..71472510 100644
--- a/src/backend/core/services/local_market_install.py
+++ b/src/backend/features/desktop/local/market_install.py
@@ -1,23 +1,23 @@
"""Desktop marketplace packages enter the same device-owned installation lifecycle."""
-from core.plugins.management import packages as plugin_packages
-from core.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.management import packages as plugin_packages
+from core.capabilities.plugins.packaging import sources as plugin_sources
import base64
-from core.plugins.local import service as lifecycle
-from core.services.local_skill_editor import install_archive
+from features.desktop.local.plugins import service as lifecycle
+from features.desktop.local.skills.editor import install_archive
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
def install_plugin(db, user_id, slug, secrets):
- from core.services import marketplace_listing as listing
+ from features.marketplace import edition as listing
listing.ensure_item_visible(db, listing.KIND_PLUGIN, slug, user_id, resource="plugin")
if secrets:
raise BadRequestError(message="本机插件凭据请通过连接配置设置,不写入插件包")
- directory = plugin_sources._resolve_plugin_dir(slug)
+ directory = plugin_sources.resolve_plugin_dir(slug)
if directory is not None:
return lifecycle.install(user_id, str(directory.resolve()))
- row = plugin_packages._market_row(db, slug)
+ row = plugin_packages.market_row(db, slug)
if row is None:
raise ResourceNotFoundError("plugin", slug)
return install_archive(user_id, base64.b64decode(row.package_b64), "plugin")
diff --git a/src/backend/core/services/native_app_icons.py b/src/backend/features/desktop/local/native_app_icons.py
similarity index 100%
rename from src/backend/core/services/native_app_icons.py
rename to src/backend/features/desktop/local/native_app_icons.py
diff --git a/src/backend/core/plugins/local/__init__.py b/src/backend/features/desktop/local/plugins/__init__.py
similarity index 100%
rename from src/backend/core/plugins/local/__init__.py
rename to src/backend/features/desktop/local/plugins/__init__.py
diff --git a/src/backend/core/plugins/local/assets.py b/src/backend/features/desktop/local/plugins/assets.py
similarity index 92%
rename from src/backend/core/plugins/local/assets.py
rename to src/backend/features/desktop/local/plugins/assets.py
index b9a884fc..4a19a6ae 100644
--- a/src/backend/core/plugins/local/assets.py
+++ b/src/backend/features/desktop/local/plugins/assets.py
@@ -21,9 +21,9 @@
def _bundled_site_template() -> Optional[Path]:
- """安装包里的 React 站点模板(``docker/site-template/``)。"""
- # src/backend/core/plugins/local/assets.py → parents[5] = 仓库根
- candidate = Path(__file__).resolve().parents[5] / "docker" / "site-template"
+ """安装包里的 React 站点模板(``docker/sandbox/site-template/``)。"""
+ # src/backend/features/desktop/local/plugins/assets.py → parents[6] = 仓库根
+ candidate = Path(__file__).resolve().parents[6] / "docker" / "sandbox" / "site-template"
return candidate if (candidate / "init-react-site.sh").is_file() else None
@@ -42,7 +42,7 @@ def provision_site_template() -> bool:
"""
source = _bundled_site_template()
if source is None:
- logger.warning("[device-assets] 安装包内未找到站点模板(docker/site-template/)")
+ logger.warning("[device-assets] 安装包内未找到站点模板(docker/sandbox/site-template/)")
return False
home = site_template_home()
try:
@@ -90,7 +90,7 @@ def provision_present_plugins(db) -> List[str]:
也不会触发云端重新准备,但随版本发出的资产需要跟着更新。本机业务库的行和云端
同步下来的投影都算"这台机器上有"。
"""
- from core.capabilities import device_catalog
+ from core.harness.registry import device_catalog
from core.db.models import InstalledPlugin
# 只问登记表里那几个 slug 在不在,不把整张表和整份云端投影都物化出来。
diff --git a/src/backend/core/plugins/local/legacy_migration.py b/src/backend/features/desktop/local/plugins/legacy_migration.py
similarity index 87%
rename from src/backend/core/plugins/local/legacy_migration.py
rename to src/backend/features/desktop/local/plugins/legacy_migration.py
index 5aafbc5a..1c6e1450 100644
--- a/src/backend/core/plugins/local/legacy_migration.py
+++ b/src/backend/features/desktop/local/plugins/legacy_migration.py
@@ -5,12 +5,13 @@
import tempfile
from pathlib import Path
-from core.agent_skills.binary_files import decode_binary, is_binary_value
-from core.capabilities import archive, registry
+from core.capabilities.skills.binary_files import decode_binary, is_binary_value
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin
-from core.plugins.local import service as lifecycle
-from core.plugins.packaging import sources as plugin_sources
-from core.services.management_package import serialized
+from features.desktop.local.plugins import service as lifecycle
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.management.package import serialized
from extensions.paths import PLUGINS_DIR
@@ -36,7 +37,7 @@ def values(obj):
def _package(root, row, children, servers):
- from core.services.management_contract import MANAGERS
+ from core.capabilities.plugins.management.contract import MANAGERS
skill_ids, server_ids = {}, {}
official = row.source == "builtin" and row.slug in MANAGERS
@@ -49,15 +50,15 @@ def _package(root, row, children, servers):
if official:
package = PLUGINS_DIR / "marketplace" / row.slug
archive.write_files(root, {n: p.read_bytes() for n, p in archive.iter_files(package)})
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
np = normalize_plugin_dir(root)
skill_ids = {
- s.name: plugin_sources._make_skill_id(row.slug, s.name, row.owner_user_id)
+ s.name: plugin_sources.make_skill_id(row.slug, s.name, row.owner_user_id)
for s in np.skills
}
server_ids = {
- s.name: plugin_sources._make_server_id(row.slug, s.name, row.owner_user_id)
+ s.name: plugin_sources.make_server_id(row.slug, s.name, row.owner_user_id)
for s in np.mcp
}
return skill_ids, server_ids
@@ -105,15 +106,15 @@ def _package(root, row, children, servers):
@serialized
def migrate():
- from core.capabilities import device_catalog
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry import device_catalog
+ from core.harness.registry.run_capabilities import has_run_capabilities
from sqlalchemy import inspect
- if not capabilities_enabled():
+ if not has_run_capabilities():
return 0
count = 0
with registry._session() as db:
- if not inspect(db.get_bind()).has_table(InstalledPlugin.__tablename__):
+ if not inspect(db.get_bind()).has_table(inspect(InstalledPlugin).local_table.name):
return 0
for row in db.query(InstalledPlugin).all():
# Hybrid shared defaults are replaced by the authenticated cloud manifest.
diff --git a/src/backend/core/plugins/local/service.py b/src/backend/features/desktop/local/plugins/service.py
similarity index 85%
rename from src/backend/core/plugins/local/service.py
rename to src/backend/features/desktop/local/plugins/service.py
index ccd4392c..fa4fac87 100644
--- a/src/backend/core/plugins/local/service.py
+++ b/src/backend/features/desktop/local/plugins/service.py
@@ -4,14 +4,16 @@
import tempfile
from pathlib import Path
-from core.capabilities import archive, registry, store
-from core.capabilities.paths import LOCAL_PROFILE, require_root, revision_for_hash
-from core.capabilities.plugins import component_install_ids
-from core.capabilities.ref import local_ref
-from core.db.models import DeviceCapabilityInstallation
-from core.services import local_skill_service as skill_service
-from core.services.management_package import serialized, snapshot
-from sqlalchemy import update as sql_update
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import store
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from features.desktop.capabilities.paths import require_root, revision_for_hash
+from features.desktop.capabilities.plugins import component_install_ids
+from core.capabilities.reference import local_ref
+from core.harness.registry.local_publication import local_publication
+from features.desktop.local.skills import service as skill_service
+from core.capabilities.plugins.management.package import serialized, snapshot
def _owned(user_id, install_id, db=None):
@@ -58,16 +60,16 @@ def list_plugins(user_id):
def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None):
- from core.capabilities.mcp_json import _validate_local_server
- from core.plugins.packaging.importer import detect_manifest, normalize_plugin_dir
- from core.services.desktop_capability_protocol import entity_content_hash
- from core.services.marketplace_service import compute_install_id
+ from features.desktop.capabilities.mcp_json import _validate_local_server
+ from core.capabilities.plugins.packaging.importer import detect_manifest, normalize_plugin_dir
+ from core.capabilities.manifest_protocol import entity_content_hash
+ from core.capabilities.skills.installation import compute_install_id as compute_install_id
np = normalize_plugin_dir(root)
manifest = json.loads(detect_manifest(root)[1].read_text(encoding="utf-8"))
executor = manifest.get("extensions", {}).get("org.hugagent", {}).get("local_execution")
if executor:
- from core.services.management_contract import MANAGERS, VERSION
+ from core.capabilities.plugins.management.contract import MANAGERS, VERSION
if (
executor.get("id") != np.slug
@@ -127,7 +129,7 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None):
}
files = {"package/" + n: p.read_bytes() for n, p in archive.iter_files(root)}
# entity_content_hash represents binary assets using the existing wire encoding.
- from core.agent_skills.binary_files import encode_upload
+ from core.capabilities.skills.binary_files import encode_upload
files["plugin.json"] = json.dumps(definition, ensure_ascii=False, sort_keys=True)
digest = entity_content_hash(
@@ -139,16 +141,16 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None):
if existing and existing.state != "removed" and existing.resolved_revision
else None
)
- from core.capabilities.plugins import load_manifest
+ from features.desktop.capabilities.plugins import load_manifest
old_ui = load_manifest(previous).get("ui_contributions") if previous else None
candidate = bool(existing and existing.state != "removed" and (
(np.ui or {}).get("version") == 3 or (old_ui or {}).get("version") == 3))
if candidate:
if existing.payload.get("owner_user_id") != user_id:
raise PermissionError("plugin ownership conflict")
- from core.applications.plugin_candidate_storage import local_reserve
+ from core.harness.registry.package_sources import package_upgrade_policy
if revision_for_hash(digest) != existing.resolved_revision:
- local_reserve(existing, revision_for_hash(digest), files)
+ package_upgrade_policy().reserve_local(existing, revision_for_hash(digest), files)
comp = store.write_from_files("plugin", LOCAL_PROFILE, key, revision_for_hash(digest), files)
if not candidate:
children = _prepare_children(user_id, np, comp, skill_ids, server_ids)
@@ -158,15 +160,15 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None):
def _prepare_children(user_id, np, comp, skill_ids, server_ids):
children = []
- from core.agent_skills.registry import _load_skill_metadata_from_str
+ from core.capabilities.skills.registry import load_skill_metadata_from_str
for child in np.skills:
- _load_skill_metadata_from_str(child.skill_content, child.name)
+ load_skill_metadata_from_str(child.skill_content, child.name)
with tempfile.TemporaryDirectory(prefix="plugin-skills-") as tmp:
for child in np.skills:
child_key = skill_ids[child.name]
directory = Path(tmp) / child_key
- from core.capabilities.skills import device_view_dir, user_view_dir
- from core.plugins.packaging.skill_transform import files as component_files
+ from core.harness.registry.local_publication import local_publication
+ from core.capabilities.plugins.packaging.skill_transform import files as component_files
archive.write_files(
directory,
@@ -174,7 +176,7 @@ def _prepare_children(user_id, np, comp, skill_ids, server_ids):
child,
skill_ids,
server_ids,
- (user_view_dir(user_id) or device_view_dir()) / child_key,
+ (local_publication().user_view_dir(user_id) or local_publication().device_view_dir()) / child_key,
comp.path / "package",
),
)
@@ -244,7 +246,7 @@ def _publish(user_id, prepared, db, *, legacy_source=None):
if previous.created_at
else now
)
- db.get(DeviceCapabilityInstallation, iid).created_at = stamp
+ local_publication().stamp_installation(db, iid, stamp)
registry.upsert(
profile_id=LOCAL_PROFILE,
ref=local_ref("plugin", key),
@@ -298,16 +300,8 @@ def install(user_id, source_path):
def _claim(db, inst, expected_revision):
if not expected_revision or inst.resolved_revision != expected_revision:
raise ValueError("revision_conflict")
- result = db.execute(
- sql_update(DeviceCapabilityInstallation)
- .where(
- DeviceCapabilityInstallation.install_id == inst.install_id,
- DeviceCapabilityInstallation.resolved_revision == expected_revision,
- DeviceCapabilityInstallation.state != "removed",
- )
- .values(generation=DeviceCapabilityInstallation.generation + 1)
- )
- if result.rowcount != 1:
+ changed = local_publication().claim_revision(db, inst.install_id, expected_revision)
+ if changed != 1:
raise ValueError("revision_conflict")
@@ -322,8 +316,8 @@ def update(user_id, install_id, source_path, expected_revision):
with registry._session() as db:
current = _owned(user_id, install_id, db)
_claim(db, current, expected_revision)
- from core.applications.local_plugin_upgrades import prepare as prepare_upgrade
- pending = prepare_upgrade(db, current, prepared)
+ from core.harness.registry.package_sources import package_upgrade_policy
+ pending = package_upgrade_policy().prepare_local(db, current, prepared)
if pending is not None:
return pending
_publish(user_id, prepared, db)
@@ -345,7 +339,7 @@ def uninstall(user_id, install_id, expected_revision):
@serialized
def set_presentation(user_id, install_id, values):
- from core.plugins.management.market import _validate_icon
+ from core.capabilities.plugins.packaging.presentation import validate_icon as _validate_icon
with registry._session() as db:
inst = _owned(user_id, install_id, db)
diff --git a/src/backend/features/desktop/local/plugins/sidecars.py b/src/backend/features/desktop/local/plugins/sidecars.py
new file mode 100644
index 00000000..46440ad4
--- /dev/null
+++ b/src/backend/features/desktop/local/plugins/sidecars.py
@@ -0,0 +1,33 @@
+"""Start trusted bundled MCPs selected by owned local plugin definitions."""
+
+from __future__ import annotations
+
+from urllib.parse import urlsplit
+
+
+
+def native_bindings(configs):
+ from core.config.mcp_config import mcp_http_url as _mcp_http_url
+ from core.config.settings import settings
+ from extensions.mcp_servers._ports import PORTS
+
+ if not settings.deploy.is_local:
+ return set()
+ urls = {
+ _mcp_http_url(sid).rstrip("/"): (sid, port)
+ for sid, port in PORTS.items()
+ if urlsplit(_mcp_http_url(sid)).hostname in ("127.0.0.1", "localhost", "::1")
+ }
+ return {
+ urls[str(cfg.get("url") or "").rstrip("/")]
+ for cfg in configs.values()
+ if cfg.get("origin") == "local_plugin"
+ and cfg.get("execution_scope") == "local"
+ and cfg.get("owner_user_id")
+ and str(cfg.get("url") or "").rstrip("/") in urls
+ }
+
+
+async def ensure_native_servers(configs):
+ from core.infra.local_runtime.dynamic_servers import ensure_dynamic_servers
+ await ensure_dynamic_servers(native_bindings(configs))
diff --git a/src/backend/features/desktop/local/publication_provider.py b/src/backend/features/desktop/local/publication_provider.py
new file mode 100644
index 00000000..dc7ef5cb
--- /dev/null
+++ b/src/backend/features/desktop/local/publication_provider.py
@@ -0,0 +1,101 @@
+"""Desktop implementation of local skill publication and revision claims."""
+from features.desktop.capabilities import skill_paths, skill_publication, skill_views
+
+class DesktopLocalPublication:
+ def management_lock_path(self):
+ from features.desktop.capabilities.paths import meta_root
+ return meta_root() / "management.lock"
+
+ def provision_plugin_assets(self, slug):
+ from features.desktop.local.plugins.assets import provision_for
+ return provision_for(slug)
+
+ def migrate_local_plugins(self):
+ from features.desktop.local.plugins.legacy_migration import migrate
+ return migrate()
+
+ def migrate_local_skills(self):
+ from features.desktop.local.management_migration import migrate
+ return migrate()
+
+ async def ensure_native_servers(self, configs):
+ from features.desktop.local.plugins.sidecars import ensure_native_servers
+ return await ensure_native_servers(configs)
+
+ def publish_local_plugin(self, definition, *, owner_user_id):
+ from features.desktop.capabilities.plugins import publish_local_plugin
+ return publish_local_plugin(definition, owner_user_id=owner_user_id)
+
+ def remove_local_plugin(self, slug):
+ from features.desktop.capabilities.plugins import remove_local_plugin
+ return remove_local_plugin(slug)
+
+ def skill_backend(self, *, priority=150, local=False):
+ from features.desktop.skill_backend import CapabilityStoreBackend
+ return CapabilityStoreBackend(priority=priority, local=local)
+
+ def merge_skill_infos(self, groups, *, backend_hash):
+ from features.desktop.capabilities.skills import merge_skill_infos
+ return merge_skill_infos(groups, backend_hash=backend_hash)
+
+ def view_generation(self):
+ from features.desktop.capabilities.skills import view_generation
+ return view_generation()
+
+ def skill_root(self):
+ from features.desktop.capabilities.paths import kind_root
+ return kind_root("skill")
+
+ def merge_visible_agents(self, user_id, rows):
+ from features.desktop.capabilities.agents import merge_visible
+ return merge_visible(user_id, rows)
+
+ def account_agent_definition(self, agent_id):
+ from features.desktop.capabilities.agents import account_definition
+ return account_definition(agent_id)
+
+ def publish_local_agent(self, definition):
+ from features.desktop.capabilities.agents import publish_local_agent
+ return publish_local_agent(definition)
+
+ def remove_local_agent(self, agent_id):
+ from features.desktop.capabilities.agents import remove_local_agent
+ return remove_local_agent(agent_id)
+
+ def rebuild_device_view(self):
+ return skill_views.rebuild_device_view()
+
+ def rebuild_user_view(self, user_id):
+ return skill_views.rebuild_user_view(user_id)
+
+ def rebuild_views(self, user_id):
+ return skill_views.rebuild_views(user_id)
+
+ def remove_local_skill(self, skill_id):
+ return skill_publication.remove_local_skill(skill_id)
+
+ def prune_local(self, owners):
+ return skill_publication.prune_local(owners)
+
+ def publish_local_skill(self, *args, **kwargs):
+ return skill_publication.publish_local_skill(*args, **kwargs)
+
+ def device_view_dir(self):
+ return skill_paths.device_view_dir()
+
+ def user_view_dir(self, user_id):
+ return skill_paths.user_view_dir(user_id)
+
+ def stamp_installation(self, db, install_id, stamp):
+ from features.desktop.models import DeviceCapabilityInstallation
+ db.get(DeviceCapabilityInstallation, install_id).created_at = stamp
+
+ def claim_revision(self, db, install_id, expected_revision):
+ from features.desktop.models import DeviceCapabilityInstallation
+ from sqlalchemy import update
+ result = db.execute(update(DeviceCapabilityInstallation).where(
+ DeviceCapabilityInstallation.install_id == install_id,
+ DeviceCapabilityInstallation.resolved_revision == expected_revision,
+ DeviceCapabilityInstallation.state != "removed",
+ ).values(generation=DeviceCapabilityInstallation.generation + 1))
+ return result.rowcount
diff --git a/src/backend/features/desktop/local/skills/__init__.py b/src/backend/features/desktop/local/skills/__init__.py
new file mode 100644
index 00000000..8d571cfb
--- /dev/null
+++ b/src/backend/features/desktop/local/skills/__init__.py
@@ -0,0 +1 @@
+"""skills package."""
diff --git a/src/backend/core/services/local_skill_editor.py b/src/backend/features/desktop/local/skills/editor.py
similarity index 86%
rename from src/backend/core/services/local_skill_editor.py
rename to src/backend/features/desktop/local/skills/editor.py
index f4a1e8dc..54078751 100644
--- a/src/backend/core/services/local_skill_editor.py
+++ b/src/backend/features/desktop/local/skills/editor.py
@@ -1,18 +1,20 @@
"""Frontend skill editing uses the same immutable installation lifecycle as tools."""
import tempfile
from pathlib import Path
-from core.capabilities import archive, registry, store
-from core.services import local_skill_service as lifecycle
-from core.agent_skills.binary_files import encode_upload, is_binary_value
-from core.agent_skills.registry import _split_frontmatter
-from core.services.skill_management_service import build_skill_content, validate_skill_file_path
+from core.capabilities import archive as archive
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import store
+from features.desktop.local.skills import service as lifecycle
+from core.capabilities.skills.binary_files import encode_upload, is_binary_value
+from core.capabilities.skills.registry import split_frontmatter
+from core.capabilities.skills.management import build_skill_content, validate_skill_file_path
def frontend_errors(function):
from functools import wraps
@wraps(function)
def call(*args, **kwargs):
from fastapi import HTTPException
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
try:
return function(*args, **kwargs)
except PermissionError as exc:
@@ -38,8 +40,8 @@ def read(user_id, key):
@frontend_errors
def detail(user_id, key):
inst, files = read(user_id, key)
- meta, body = _split_frontmatter(files["SKILL.md"].decode("utf-8"))
- from core.services.skill_management_service import extract_mcp_server_ids
+ meta, body = split_frontmatter(files["SKILL.md"].decode("utf-8"))
+ from core.capabilities.skills.management import extract_mcp_server_ids
return {"id": key, "install_id": inst.install_id, "revision": inst.resolved_revision,
"display_name": inst.payload.get("presentation", {}).get("display_name") or inst.display_name,
"description": inst.description, "instructions": body.strip(), "tags": _list(meta.get("tags")),
@@ -73,12 +75,12 @@ def save(user_id, body):
if previous is None or previous.entry_file.read_bytes() != files["SKILL.md"]:
from fastapi import HTTPException
raise HTTPException(409, detail="技能已更新,请重新打开编辑器后保存")
- old, _ = _split_frontmatter(files["SKILL.md"].decode("utf-8"))
+ old, _ = split_frontmatter(files["SKILL.md"].decode("utf-8"))
else:
inst, files, old = None, {}, {}
# Names selected in the UI are already authorized connector identities. Runtime
# dependency resolution remains the authority on readiness, including missing MCPs.
- from core.services.skill_management_service import extract_mcp_server_ids
+ from core.capabilities.skills.management import extract_mcp_server_ids
ids = body.mcp_server_ids
if ids is None:
ids = extract_mcp_server_ids(files.get("SKILL.md", b"").decode("utf-8"))
@@ -103,7 +105,7 @@ def file_get(user_id, key, filename):
filename = validate_skill_file_path(filename)
if filename == "SKILL.md" or filename not in files:
raise ValueError("extra file not found")
- from core.agent_skills.binary_files import is_binary_value
+ from core.capabilities.skills.binary_files import is_binary_value
value = encode_upload(filename, files[filename])
binary = is_binary_value(value)
return {"filename": filename, "content": "" if binary else value, "is_binary": binary}
@@ -130,7 +132,7 @@ def file_change(user_id, key, filename, content=None):
@frontend_errors
def export(user_id, key):
_, files = read(user_id, key)
- from core.services.marketplace_service import build_skill_zip
+ from features.marketplace.skill_archives import build_skill_zip
return build_skill_zip(key, files.pop("SKILL.md").decode("utf-8"), {n: encode_upload(n, v) for n, v in files.items()})
@@ -152,7 +154,7 @@ def icon(user_id, key, value):
def install_archive(user_id, raw, kind="skill"):
if len(raw) > 50 * 1024 * 1024:
raise ValueError("package exceeds 50MB limit")
- from core.plugins.local import service as local_plugin_service
+ from features.desktop.local.plugins import service as local_plugin_service
with tempfile.TemporaryDirectory(prefix="manager-upload-") as tmp:
path = Path(tmp) / "package.zip"
path.write_bytes(raw)
diff --git a/src/backend/core/services/local_skill_service.py b/src/backend/features/desktop/local/skills/service.py
similarity index 81%
rename from src/backend/core/services/local_skill_service.py
rename to src/backend/features/desktop/local/skills/service.py
index 680b576a..db56f467 100644
--- a/src/backend/core/services/local_skill_service.py
+++ b/src/backend/features/desktop/local/skills/service.py
@@ -1,11 +1,13 @@
"""Device-owned skill lifecycle over the capability index and immutable versions."""
+from features.desktop.capabilities import skills
from pathlib import Path
from sqlalchemy import update as sql_update
-from core.capabilities import registry, store, skills
-from core.capabilities.paths import LOCAL_PROFILE, require_root, revision_for_hash
-from core.capabilities.ref import local_ref
-from core.db.models import DeviceCapabilityInstallation
-from core.services.management_package import snapshot, serialized
+from features.desktop.capabilities import registry, store, skills
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from features.desktop.capabilities.paths import require_root, revision_for_hash
+from core.capabilities.reference import local_ref
+from features.desktop.models import DeviceCapabilityInstallation
+from core.capabilities.plugins.management.package import snapshot, serialized
def _owned(user_id, install_id, db=None):
@@ -21,8 +23,8 @@ def _owned(user_id, install_id, db=None):
def _result(inst):
- from core.capabilities.readiness import file_readiness
- from core.capabilities.management_context import management_context
+ from features.desktop.capabilities.readiness import file_readiness
+ from features.desktop.capabilities.management_context import management_context
comp = store.get("skill", LOCAL_PROFILE, inst.key, inst.resolved_revision)
return {"ok": True, "install_id": inst.install_id, "skill_id": inst.key,
"name": inst.display_name, "revision": inst.resolved_revision,
@@ -42,17 +44,17 @@ def list_skills(user_id):
def _prepare(user_id, root, key=None):
- from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter
- from core.services.marketplace_service import compute_install_id, _rewrite_frontmatter_name
- from core.services.desktop_capability_protocol import skill_content_hash
- from core.agent_skills.binary_files import encode_upload
+ from core.capabilities.skills.registry import load_skill_metadata_from_str, split_frontmatter
+ from core.capabilities.skills.installation import compute_install_id as compute_install_id, rewrite_frontmatter_name as _rewrite_frontmatter_name
+ from core.capabilities.manifest_protocol import skill_content_hash
+ from core.capabilities.skills.binary_files import encode_upload
md = (root / "SKILL.md").read_text(encoding="utf-8")
- fm, _ = _split_frontmatter(md)
+ fm, _ = split_frontmatter(md)
original = str(fm.get("name") or "")
- _load_skill_metadata_from_str(md, original)
+ load_skill_metadata_from_str(md, original)
key = key or compute_install_id(original, user_id)
md = _rewrite_frontmatter_name(md, key)
- meta = _load_skill_metadata_from_str(md, key)
+ meta = load_skill_metadata_from_str(md, key)
from core.capabilities.archive import iter_files
files = {name: file.read_bytes() for name, file in iter_files(root)}
files["SKILL.md"] = md
@@ -64,14 +66,14 @@ def _prepare(user_id, root, key=None):
def _invalidate():
- from core.config.catalog_runtime import invalidate_runtime_catalog_cache
- from core.config.catalog_resolver import invalidate_capability_cache
+ from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache
+ from core.capabilities.catalog.resolver import invalidate_capability_cache
from prompts.prompt_runtime import invalidate_prompt_cache
invalidate_capability_cache()
invalidate_prompt_cache()
skills.bump_view_generation()
invalidate_runtime_catalog_cache()
- from core.agent_skills.loader import get_skill_loader
+ from core.capabilities.skills.loader_factory import get_skill_loader
get_skill_loader(reset=True)
diff --git a/src/backend/core/services/local_skill_upload.py b/src/backend/features/desktop/local/skills/upload.py
similarity index 74%
rename from src/backend/core/services/local_skill_upload.py
rename to src/backend/features/desktop/local/skills/upload.py
index f7352b45..4baf159d 100644
--- a/src/backend/core/services/local_skill_upload.py
+++ b/src/backend/features/desktop/local/skills/upload.py
@@ -1,25 +1,29 @@
"""Explicit device-to-cloud private skill registration with durable retry identity."""
+from features.desktop.bridge import identity as bridge_identity
+from features.desktop.capabilities import skill_account
import hashlib
-from core.capabilities import registry, store, archive
-from core.capabilities.paths import LOCAL_PROFILE
-from core.agent_skills.binary_files import encode_upload
-from core.services.management_package import serialized
-from core.services.local_skill_service import _owned
+from features.desktop.capabilities import registry
+from features.desktop.capabilities import store
+from core.capabilities import archive as archive
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from core.capabilities.skills.binary_files import encode_upload
+from core.capabilities.plugins.management.package import serialized
+from features.desktop.local.skills.service import _owned
@serialized
def upload(user_id, install_id, expected_revision, expected_cloud_revision=None):
- from core.capabilities import skills
- from core.capabilities.change_sync import cloud_request
- from core.services import desktop_cloud_bridge as bridge
- state = bridge.get_state()
- bridge.require_current_account(state)
- if not skills.account_authorized_for(user_id):
+ from features.desktop.capabilities import skills
+ from features.desktop.capabilities.change_sync import cloud_request
+ from features.desktop.bridge import client as bridge
+ state = bridge_identity.get_state()
+ bridge_identity.require_current_account(state)
+ if not skill_account.account_authorized_for(user_id):
raise PermissionError("current cloud account does not own this local session")
inst = _owned(user_id, install_id)
if not expected_revision or inst.resolved_revision != expected_revision:
raise ValueError("revision_conflict")
- profile = skills.current_account_profile()
+ profile = skill_account.current_account_profile()
links = dict(inst.payload.get("cloud_uploads") or {})
link = dict(links.get(profile) or {})
comp = store.get("skill", LOCAL_PROFILE, inst.key, expected_revision)
@@ -29,7 +33,7 @@ def upload(user_id, install_id, expected_revision, expected_cloud_revision=None)
if n != ".inventory.json"}
target = link.get("key") or ("device-" + hashlib.sha256((profile + install_id).encode()).hexdigest()[:32])
# Rewrite identity once to the stable cloud destination, never derive a target from a display name.
- from core.services.marketplace_service import _rewrite_frontmatter_name
+ from core.capabilities.skills.installation import rewrite_frontmatter_name as _rewrite_frontmatter_name
files["SKILL.md"] = _rewrite_frontmatter_name(files["SKILL.md"], target)
identity = hashlib.sha256((profile + install_id + expected_revision + str(expected_cloud_revision)).encode()).hexdigest()
if link.get("local_revision") == expected_revision and link.get("revision"):
@@ -49,13 +53,13 @@ def upload(user_id, install_id, expected_revision, expected_cloud_revision=None)
pending = {"request_id": identity, "expected_revision": remote["revision"],
"expected_model_version": remote.get("model_version"), "create_only": not remote.get("exists")}
links[profile] = {**link, "key": target, "pending": pending}
- with bridge.account_scope(state):
+ with bridge_identity.account_scope(state):
registry.set_state(install_id, inst.state, payload_update={"cloud_uploads": links})
body = {**pending, "files": files}
response = cloud_request(state, "POST", "skill", target, body)
if not response.get("applied"):
raise ValueError("cloud saved package but did not register it")
- with bridge.account_scope(state):
+ with bridge_identity.account_scope(state):
links[profile] = {"key": target, "revision": response["revision"], "local_revision": expected_revision}
registry.set_state(install_id, inst.state, payload_update={"cloud_uploads": links})
return {"ok": True, "install_id": install_id, "local_revision": expected_revision,
diff --git a/src/backend/core/services/local_snapshot_service.py b/src/backend/features/desktop/local/snapshots.py
similarity index 100%
rename from src/backend/core/services/local_snapshot_service.py
rename to src/backend/features/desktop/local/snapshots.py
diff --git a/src/backend/core/db/models/capability.py b/src/backend/features/desktop/models.py
similarity index 60%
rename from src/backend/core/db/models/capability.py
rename to src/backend/features/desktop/models.py
index 90e81be7..73a2fd7e 100644
--- a/src/backend/core/db/models/capability.py
+++ b/src/backend/features/desktop/models.py
@@ -1,22 +1,15 @@
-"""Device-side capability installation index.
-
-These tables live in the device's existing business database (no second SQLite),
-so an install transaction and its business-data side effects can share one
-transaction. On cloud deployments they exist but stay empty — the file store is
-disabled there and nothing writes to them.
-"""
+"""Desktop installation indexes and durable observation delivery records."""
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
-
from datetime import datetime
-
from core.db.engine import Base
from core.db.models import JSONType
from sqlalchemy import Boolean, Column, Index, Integer, String, Text
+from datetime import datetime, timezone
+from sqlalchemy import JSON, BigInteger, Boolean, Column, Index, Integer, String, Text
INSTALL_STATES = ("pending", "preparing", "ready", "failed", "removed")
-
class DeviceCapabilityInstallation(Base):
"""One (profile, kind, key) on this device: intent, resolved revision and state.
@@ -59,7 +52,6 @@ class DeviceCapabilityInstallation(Base):
Index("idx_dev_cap_inst_state", "state"),
)
-
class DeviceCapabilityNamePreference(Base):
"""The user's explicit answer to "which same-named candidate wins". Device-only."""
@@ -72,7 +64,6 @@ class DeviceCapabilityNamePreference(Base):
chosen_by = Column(String(64), nullable=True)
updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
-
class DeviceCapabilityTransaction(Base):
"""Install transaction log: explains what is committed and what is not after a crash."""
@@ -89,7 +80,6 @@ class DeviceCapabilityTransaction(Base):
__table_args__ = (Index("idx_dev_cap_tx_install", "install_id"),)
-
class DeviceCapabilityComponent(Base):
"""Ownership edge: a plugin installation → the component installations it requires."""
@@ -104,3 +94,56 @@ class DeviceCapabilityComponent(Base):
Index("idx_dev_cap_comp_owner", "owner_install_id"),
Index("idx_dev_cap_comp_component", "component_install_id"),
)
+
+def utcnow():
+ return datetime.now(timezone.utc).isoformat()
+
+class DesktopOutbox(Base):
+ __tablename__ = "desktop_observability_outbox"
+ id = Column(String(64), primary_key=True)
+ cloud_base = Column(String(512), nullable=False)
+ subject = Column(String(64), nullable=False)
+ device_id = Column(String(128), nullable=False)
+ local_user_id = Column(String(64), nullable=False)
+ kind = Column(String(24), nullable=False)
+ object_id = Column(String(128), nullable=False)
+ revision = Column(BigInteger, nullable=False, default=1)
+ pending = Column(Boolean, nullable=False, default=True)
+ deleted = Column(Boolean, nullable=False, default=False)
+ digest = Column(String(64), nullable=False, default="")
+ updated_at = Column(String(40), nullable=False, default=utcnow)
+ __table_args__ = (
+ Index("idx_desktop_outbox_pending", "cloud_base", "subject", "device_id", "pending"),
+ )
+
+class DesktopRecord(Base):
+ __tablename__ = "desktop_observability_records"
+ id = Column(String(64), primary_key=True)
+ user_id = Column(String(64), nullable=False)
+ device_id = Column(String(128), nullable=False)
+ kind = Column(String(24), nullable=False)
+ object_id = Column(String(128), nullable=False)
+ revision = Column(BigInteger, nullable=False)
+ chat_id = Column(String(128), nullable=False, default="")
+ run_id = Column(String(128), nullable=False, default="")
+ observation = Column(String(16), nullable=False, default="desktop")
+ payload = Column(JSON, nullable=False)
+ received_at = Column(String(40), nullable=False, default=utcnow)
+ __table_args__ = (
+ Index("idx_desktop_records_identity", "kind", "observation", "user_id", "device_id", "object_id", unique=True),
+ Index("idx_desktop_records_user_kind", "user_id", "kind", "received_at"),
+ Index("idx_desktop_records_chat", "user_id", "device_id", "chat_id"),
+ )
+
+class DesktopSyncDevice(Base):
+ __tablename__ = "desktop_observability_devices"
+ id = Column(String(64), primary_key=True)
+ user_id = Column(String(64), nullable=False)
+ device_id = Column(String(128), nullable=False)
+ last_seen = Column(String(40), nullable=False, default=utcnow)
+ pending = Column(Integer, nullable=False, default=0)
+ capture_errors = Column(Integer, nullable=False, default=0)
+ reconciling = Column(Boolean, nullable=False, default=False)
+ __table_args__ = (Index("idx_desktop_devices_user", "user_id"),)
+
+__all__ = ['DeviceCapabilityInstallation', 'DeviceCapabilityNamePreference', 'DeviceCapabilityTransaction', 'DeviceCapabilityComponent', 'DesktopOutbox', 'DesktopRecord', 'DesktopSyncDevice', 'INSTALL_STATES']
diff --git a/src/backend/features/desktop/observability/__init__.py b/src/backend/features/desktop/observability/__init__.py
new file mode 100644
index 00000000..afeb11f1
--- /dev/null
+++ b/src/backend/features/desktop/observability/__init__.py
@@ -0,0 +1 @@
+"""observability package."""
diff --git a/src/backend/core/services/desktop_observation_context.py b/src/backend/features/desktop/observability/context.py
similarity index 78%
rename from src/backend/core/services/desktop_observation_context.py
rename to src/backend/features/desktop/observability/context.py
index e0c8d8e1..d4fdab05 100644
--- a/src/backend/core/services/desktop_observation_context.py
+++ b/src/backend/features/desktop/observability/context.py
@@ -3,13 +3,13 @@
import uuid
from contextvars import ContextVar
-model_call_id = ContextVar("desktop_model_call_id", default="")
+from core.harness.log_context import model_call_id
def context_headers(*, model=False):
from core.infra.logging import chat_id_var, trace_id_var
- from core.llm.model_usage import CURRENT_MODEL_USAGE
- from core.services.log_service import _current_message_id
+ from core.models.usage import CURRENT_MODEL_USAGE
+ from core.harness.log_context import current_message_id as _current_message_id
values = {
"x-observation-chat-id": chat_id_var.get(),
@@ -24,7 +24,7 @@ def context_headers(*, model=False):
model_call_id.set(call_id)
values["x-observation-call-id"] = call_id
else:
- from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT
+ from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT
effect = CURRENT_TOOL_EFFECT.get()
if effect:
diff --git a/src/backend/core/services/desktop_observability_delivery.py b/src/backend/features/desktop/observability/delivery.py
similarity index 86%
rename from src/backend/core/services/desktop_observability_delivery.py
rename to src/backend/features/desktop/observability/delivery.py
index b7018dea..b19f01ec 100644
--- a/src/backend/core/services/desktop_observability_delivery.py
+++ b/src/backend/features/desktop/observability/delivery.py
@@ -1,13 +1,14 @@
"""HTTP delivery boundary for durable desktop observations."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
import logging
import time
import httpx
from core.infra.responses import generate_trace_id
-from core.services import desktop_cloud_bridge as bridge
+from features.desktop.bridge import client as bridge
logger = logging.getLogger(__name__)
@@ -15,9 +16,9 @@
async def deliver_batch(state, batch, *, transport=None):
# Headers and the rejection snapshot must name the same exact token, even
# if the shell rotates credentials while a queued batch is being prepared.
- with bridge.account_scope(state):
- credential = dict(bridge.current_credentials(state))
- headers = bridge.cloud_headers(credential)
+ with bridge_identity.account_scope(state):
+ credential = dict(bridge_identity.current_credentials(state))
+ headers = bridge_identity.cloud_headers(credential)
trace_id = generate_trace_id()
headers["X-Trace-ID"] = trace_id
async with httpx.AsyncClient(
@@ -48,7 +49,7 @@ async def deliver_batch(state, batch, *, transport=None):
raise ValueError("invalid desktop observation acknowledgements")
return body
except Exception as exc:
- bridge.note_rejected_credential(credential, exc)
+ bridge_identity.note_rejected_credential(credential, exc)
# Neither exception text nor remote fields are safe diagnostic data.
# The locally generated trace also reaches the cloud request logger.
logger.warning(
diff --git a/src/backend/core/services/observability_projection.py b/src/backend/features/desktop/observability/fields.py
similarity index 96%
rename from src/backend/core/services/observability_projection.py
rename to src/backend/features/desktop/observability/fields.py
index 696bbd86..6e85bf08 100644
--- a/src/backend/core/services/observability_projection.py
+++ b/src/backend/features/desktop/observability/fields.py
@@ -4,7 +4,7 @@
aggregates use the same field precedence as the serialized management record.
"""
-from core.db.models.observability import DesktopRecord
+from features.desktop.models import DesktopRecord
from sqlalchemy import case, func
from sqlalchemy.orm import aliased
diff --git a/src/backend/core/services/desktop_observability.py b/src/backend/features/desktop/observability/projection.py
similarity index 98%
rename from src/backend/core/services/desktop_observability.py
rename to src/backend/features/desktop/observability/projection.py
index 4861be9e..736c9737 100644
--- a/src/backend/core/services/desktop_observability.py
+++ b/src/backend/features/desktop/observability/projection.py
@@ -7,12 +7,8 @@
import math
import re
-from core.db.models.observability import DesktopRecord, DesktopSyncDevice, utcnow
-from core.services.observability_projection import (
- counterpart_identity,
- desktop_counterpart,
- gateway_counterpart,
-)
+from features.desktop.models import DesktopRecord, DesktopSyncDevice, utcnow
+from features.desktop.observability.fields import counterpart_identity, desktop_counterpart, gateway_counterpart
from sqlalchemy import func, or_, select
KINDS = {"session", "message", "run", "tool", "skill", "agent", "usage", "model"}
diff --git a/src/backend/core/services/desktop_observability_sync.py b/src/backend/features/desktop/observability/sync.py
similarity index 96%
rename from src/backend/core/services/desktop_observability_sync.py
rename to src/backend/features/desktop/observability/sync.py
index 2f74a5f7..959c7d48 100644
--- a/src/backend/core/services/desktop_observability_sync.py
+++ b/src/backend/features/desktop/observability/sync.py
@@ -1,6 +1,7 @@
"""Local durable change capture and independently scheduled, bounded cloud delivery."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
import asyncio
import json
@@ -17,8 +18,8 @@
ToolCallLog,
UserShadow,
)
-from core.db.models.observability import DesktopOutbox, utcnow
-from core.services.desktop_observability import FIELDS, insert_for, key, public_payload
+from features.desktop.models import DesktopOutbox, utcnow
+from features.desktop.observability.projection import FIELDS, insert_for, key, public_payload
from sqlalchemy import event, func, or_, select, update
logger = logging.getLogger(__name__)
@@ -338,10 +339,10 @@ def __init__(self, factory, *, interval=3.0, transport=None):
self.reconciled_at = 0.0
async def sync_once(self):
- from core.services import desktop_cloud_bridge as bridge
- from core.services.desktop_observability_delivery import deliver_batch
+ from features.desktop.bridge import client as bridge
+ from features.desktop.observability.delivery import deliver_batch
- state, identity = bridge.get_state(), bridge.get_identity_state()
+ state, identity = bridge_identity.get_state(), bridge_identity.get_identity_state()
if not state or not identity or not identity.get("device_id"):
return
account = key(identity["cloud_base"], identity["subject"], identity["device_id"])
@@ -366,7 +367,7 @@ def prepare():
batch = await asyncio.to_thread(prepare)
body = await deliver_batch(state, batch, transport=self.transport)
- bridge.require_current_account(state)
+ bridge_identity.require_current_account(state)
result = body.get("data", {})
# Accept acknowledgements only for this exact sent batch.
allowed = {(e["kind"], e["object_id"], e["revision"]) for e in batch["events"]}
diff --git a/src/backend/core/plugins/runtime/desktop_resolution.py b/src/backend/features/desktop/plugin_resolution.py
similarity index 85%
rename from src/backend/core/plugins/runtime/desktop_resolution.py
rename to src/backend/features/desktop/plugin_resolution.py
index ee1748ce..df798883 100644
--- a/src/backend/core/plugins/runtime/desktop_resolution.py
+++ b/src/backend/features/desktop/plugin_resolution.py
@@ -1,6 +1,9 @@
"""Progressive plugin runtime: desktop resolution."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
+from features.desktop.capabilities import skill_account
+from features.desktop.capabilities import skills
import logging
import os
@@ -9,15 +12,15 @@
logger = logging.getLogger(__name__)
-from core.plugins.runtime import models as plugin_runtime_models
-from core.plugins.runtime import resolution as plugin_resolution
+from core.capabilities.plugins.runtime import models as plugin_runtime_models
+from core.capabilities.plugins.runtime import resolution as plugin_resolution
def prepare_desktop_plugin_skill_defaults(user_id, skill_ids):
"""Include enabled cloud plugin intentions before ready-only skill filtering."""
- from core.capabilities import skills
- from core.capabilities.plugins import enabled_cloud_skill_intents
- from core.capabilities.preparation import ensure_cloud_ready
+ from features.desktop.capabilities import skills
+ from features.desktop.capabilities.plugins import enabled_cloud_skill_intents
+ from features.desktop.capabilities.preparation import ensure_cloud_ready
selected = set(skill_ids) | enabled_cloud_skill_intents(user_id)
ensure_cloud_ready(user_id, skill_keys=sorted(selected))
@@ -40,17 +43,17 @@ def resolve_desktop_progressive_plugins(
Only model-facing skills and MCP connections are delayed until load_plugin.
The device registry, not legacy cloud InstalledPlugin rows, owns identities.
"""
- from core.capabilities import registry, skills
- from core.capabilities.dependency import Context, Inspector, _identifier
- from core.capabilities.paths import LOCAL_PROFILE
+ from features.desktop.capabilities import registry, skills
+ from features.desktop.capabilities.dependency import Context, Inspector, _identifier
+ from core.capabilities.identity_rules import LOCAL_PROFILE
- profile = skills.current_account_profile() if skills.account_authorized_for(user_id) else None
+ profile = skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None
result = plugin_runtime_models.ProgressiveResolution()
eligible: List[plugin_runtime_models.DeferredPlugin] = []
allowed_skills, allowed_mcp = set(enabled_skill_ids or []), set(enabled_mcp_ids or [])
active = {item for item in (activated_ids or []) if item}
- from core.capabilities.errors import CapabilityError
- from core.capabilities.preparation import ensure_cloud_ready
+ from core.harness.capability_errors import CapabilityError
+ from features.desktop.capabilities.preparation import ensure_cloud_ready
for skill in sorted(allowed_skills):
try:
@@ -68,7 +71,7 @@ def resolve_desktop_progressive_plugins(
# Cloud projections and explicitly owned local packages are both valid.
# Legacy shared local bundles must stay hidden in hybrid mode: their MCP
# endpoints belong to a different execution plane.
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
hybrid = device_catalog.active() and profile
allowed_profiles = (LOCAL_PROFILE, profile)
@@ -83,7 +86,7 @@ def resolve_desktop_progressive_plugins(
if hybrid and row.profile_id == LOCAL_PROFILE:
if row.payload.get("owner_user_id") != user_id:
continue
- from core.capabilities.local_plugin_runtime import enabled_for
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
if not enabled_for(row, user_id):
continue
@@ -110,10 +113,10 @@ def resolve_desktop_progressive_plugins(
advertised_skills & allowed_skills or advertised_mcp & allowed_mcp
):
continue
- from core.services import desktop_cloud_bridge, desktop_cloud_bundles
+ from features.desktop.bridge import client as desktop_cloud_bridge, bundles as desktop_cloud_bundles
results = desktop_cloud_bundles.prepare(
- desktop_cloud_bridge.get_state(), [row.install_id]
+ bridge_identity.get_state(), [row.install_id]
)
if not results or not results[0]["ok"]:
result.unavailable_skill_ids.update(advertised_skills & allowed_skills)
@@ -146,7 +149,7 @@ def record_component(entry, required):
)
inspector.visit({"kind": "plugin", "id": row.install_id}, row.profile_id)
report = inspector.report()
- from core.capabilities.errors import IntegrityFailed, PackageMissing
+ from core.harness.capability_errors import IntegrityFailed, PackageMissing
nodes = [node for node in report["nodes"] if node["kind"] in ("plugin", "agent")]
for error in report["errors"]:
@@ -240,10 +243,10 @@ def inspect_plugin_or_skip(prepared):
)
unavailable_invoked = result.unavailable_skill_ids.intersection(invoked_skill_ids or ())
if unavailable_invoked:
- from core.capabilities.errors import PackageMissing
+ from core.harness.capability_errors import PackageMissing
raise PackageMissing(
"selected skill belongs to a plugin with unavailable MCP servers",
runtime_name=", ".join(sorted(unavailable_invoked)),
)
- return plugin_resolution._finalize_resolution(result, eligible)
+ return plugin_resolution.finalize_resolution(result, eligible)
diff --git a/src/backend/features/desktop/run_capabilities/__init__.py b/src/backend/features/desktop/run_capabilities/__init__.py
new file mode 100644
index 00000000..8bed212b
--- /dev/null
+++ b/src/backend/features/desktop/run_capabilities/__init__.py
@@ -0,0 +1 @@
+"""run capabilities package."""
diff --git a/src/backend/core/llm/factory/selection/device_preparation.py b/src/backend/features/desktop/run_capabilities/preparation.py
similarity index 83%
rename from src/backend/core/llm/factory/selection/device_preparation.py
rename to src/backend/features/desktop/run_capabilities/preparation.py
index 29e5bd81..1eff6ba5 100644
--- a/src/backend/core/llm/factory/selection/device_preparation.py
+++ b/src/backend/features/desktop/run_capabilities/preparation.py
@@ -1,11 +1,13 @@
"""Agent assembly phase: device preparation. """
from __future__ import annotations
+from core.capabilities.skills import loader_factory as loader_factory_module
+from core.capabilities.plugins.runtime.models import progressive_plugin_loading_enabled
import asyncio
-import core.agent_skills.loader as skill_loader
-from core.capabilities.paths import capabilities_enabled
+import core.capabilities.skills.loader as skill_loader
+from features.desktop.capabilities.paths import capabilities_enabled
from core.llm.factory.models import RequiredCapabilities, StickyCapabilities
from core.llm.factory.request import AgentRequest
from core.llm.factory.selection import capabilities as factory_capabilities
@@ -37,7 +39,7 @@ async def device_preparation(
and _api_scope is None
and _eval_scope is None
):
- from core.plugins import runtime as _desktop_plugins
+ from features.desktop import plugin_resolution as _desktop_plugins
_desktop_progressive = await asyncio.to_thread(
_desktop_plugins.resolve_desktop_progressive_plugins,
@@ -45,7 +47,7 @@ async def device_preparation(
enabled_skill_ids=(
request.enabled_skill_ids
if request.enabled_skill_ids is not None
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
),
enabled_mcp_ids=[
key
@@ -71,33 +73,33 @@ async def device_preparation(
for sid in (
request.enabled_skill_ids
if request.enabled_skill_ids is not None
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
if sid not in _desktop_progressive.unavailable_skill_ids
]
if (
not _desktop_progressive.directory
or request.turbo_mode
- or not _desktop_plugins.progressive_plugin_loading_enabled()
+ or not progressive_plugin_loading_enabled()
):
_desktop_progressive = None
- from core.capabilities import runtime as capability_runtime
+ from features.desktop.capabilities import runtime as capability_runtime
_log.info("[factory] +%s progressive definitions resolved", _elapsed())
_prepared_capabilities = None
if capabilities_enabled() and not request.disable_tools:
- _caps_loader = skill_loader.get_skill_loader()
+ _caps_loader = loader_factory_module.get_skill_loader()
_caps_skill_ids = (
request.enabled_skill_ids
if request.enabled_skill_ids is not None
- else factory_capabilities._effective_main_available_skills()
+ else factory_capabilities.effective_main_available_skills()
)
def _materialize_selected_skills():
for _sid in _caps_skill_ids or []:
- from core.capabilities.errors import CapabilityError
+ from core.harness.capability_errors import CapabilityError
try:
_caps_loader.get_skill_dir(_sid)
diff --git a/src/backend/features/desktop/run_capabilities/provider.py b/src/backend/features/desktop/run_capabilities/provider.py
new file mode 100644
index 00000000..7a40e13d
--- /dev/null
+++ b/src/backend/features/desktop/run_capabilities/provider.py
@@ -0,0 +1,118 @@
+"""Desktop implementation of the kernel's per-run capability contracts."""
+
+
+class DesktopRunCapabilityProvider:
+ def cloud_plugin_selection(self, plugin_id, user_id):
+ from features.desktop.capabilities.invocation import cloud_plugin_selection
+ return cloud_plugin_selection(plugin_id, user_id=user_id)
+
+ def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids):
+ from features.desktop.capabilities.invocation import resolve_explicit_ids
+ return resolve_explicit_ids(user_id, skill_ids, mcp_ids)
+
+ def plugin_skill_defaults(self, user_id, skill_ids):
+ from features.desktop.plugin_resolution import prepare_desktop_plugin_skill_defaults
+ return prepare_desktop_plugin_skill_defaults(user_id, skill_ids)
+
+ def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None):
+ from features.desktop.capabilities.preparation import ensure_cloud_ready
+ return ensure_cloud_ready(user_id, skill_keys=skill_keys or (), plugin_keys=plugin_keys or ())
+
+ def available_skill_names(self, names, *, user_id):
+ from features.desktop.capabilities.skills import filter_available_names
+ return filter_available_names(names, user_id=user_id)
+
+ def plugin_binding_ids(self, plugin_ids, *, user_id):
+ from features.desktop.capabilities.plugins import cloud_binding_ids
+ return cloud_binding_ids(plugin_ids, user_id=user_id)
+
+ async def prepare(self, request, **selection):
+ from features.desktop.run_capabilities.preparation import device_preparation
+ return await device_preparation(request, **selection)
+
+
+class PreparedCapabilityOperations:
+ def skill_loader(self):
+ from features.desktop.capabilities.runtime import frozen_loader
+ return frozen_loader(self)
+
+ def bind_mcp(self, servers, resolution):
+ from features.desktop.capabilities.runtime import bind_mcp
+ return bind_mcp(self, servers, resolution)
+
+ def preflight(self, **context):
+ from features.desktop.capabilities.runtime import preflight
+ return preflight(self, **context)
+
+ def tool_guard(self, *, scope_id):
+ from features.desktop.capabilities.runtime import get
+ run = get(self.run_id, scope_id=scope_id) or self
+ return run.validate_tool_call
+
+ def validate_selected_skills(self, skill_ids):
+ from core.harness.capability_errors import PackageMissing
+ missing = set(skill_ids or []) - set(self.bindings)
+ if missing:
+ raise PackageMissing("selected skills are absent from this prepared run", details={"skills": sorted(missing)})
+
+ def validate(self, *, user_id):
+ from features.desktop.capabilities.runtime import validate
+ return validate(self, user_id=user_id)
+
+ def record_tool_scope(self, tool_call_id, tool_name):
+ from features.desktop.capabilities.runtime import record_tool_scope
+ return record_tool_scope(self, tool_call_id, tool_name)
+
+ def child_scope(self, kind, *identities):
+ from core.harness.registry.capability_runtime import child_scope
+ return child_scope(self.scope_id, kind, *identities)
+
+ def validate_tool_call(self, server_id):
+ from features.desktop.capabilities import runtime
+ from core.harness.capability_errors import CapabilityError
+ from core.capabilities.identity_rules import LOCAL_PROFILE
+ try:
+ runtime.validate(self)
+ latest = runtime.get(self.run_id, scope_id=self.scope_id) or self
+ if "mcp:" + server_id in latest.unavailable:
+ return False
+ binding = self.mcp_bindings.get(server_id)
+ if binding is not None:
+ from features.desktop.capabilities import registry
+
+ relationships = latest.dependency_report.get("connector_parents", {})
+ parents = {
+ *relationships.get(server_id, []),
+ *relationships.get(binding["install_id"].split(":", 2)[2], []),
+ }
+ for parent_id in parents:
+ parent = registry.get(parent_id)
+ if parent is None or not parent.enabled or parent.state == "removed":
+ return False
+ owner = parent.payload.get("owner_user_id")
+ if owner and str(owner) != self.user_id:
+ return False
+ if binding is None or not binding.get("authorization_checked"):
+ return True
+ _, profile, key = binding["install_id"].split(":", 2)
+ if profile == "local-json":
+ from features.desktop.capabilities.mcp_json import local_server_configs
+
+ return key in local_server_configs()
+ if profile == LOCAL_PROFILE:
+ from core.capabilities.mcp.service import McpServerConfigService
+
+ service = McpServerConfigService.get_instance()
+ return key in {
+ **service.get_all_servers(enabled_only=True),
+ **service.get_owned_servers(self.user_id, enabled_only=True),
+ }
+ from features.desktop.bridge.gateway import cloud_gateway_mcp_configs
+
+ return server_id in cloud_gateway_mcp_configs([server_id])
+ except (CapabilityError, OSError):
+ return False
+
+ def release(self):
+ """Run snapshots remain durable with history under the existing retention policy."""
+ return None
diff --git a/src/backend/core/agent_skills/backends/capability_store.py b/src/backend/features/desktop/skill_backend.py
similarity index 86%
rename from src/backend/core/agent_skills/backends/capability_store.py
rename to src/backend/features/desktop/skill_backend.py
index e43d2f20..ac9eca43 100644
--- a/src/backend/core/agent_skills/backends/capability_store.py
+++ b/src/backend/features/desktop/skill_backend.py
@@ -11,8 +11,8 @@
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
-from .filesystem import FilesystemBackend
-from .protocol import SkillFileInfo
+from core.capabilities.skills.backends.filesystem import FilesystemBackend
+from core.capabilities.skills.backends.protocol import SkillFileInfo
class CapabilityStoreBackend:
@@ -29,7 +29,7 @@ def priority(self) -> int:
return self._priority
def _profile(self) -> Optional[str]:
- from core.capabilities.skills import current_account_profile
+ from features.desktop.capabilities.skill_account import current_account_profile
return "local" if self._local else current_account_profile()
@@ -37,13 +37,13 @@ def _scope_user(self) -> Optional[str]:
"""The account a private local entry must belong to (shared entries: nobody)."""
if not self._local:
return None
- from core.capabilities.skills import current_local_user_id
+ from features.desktop.capabilities.skill_account import current_local_user_id
return current_local_user_id()
def _installations(self):
- from core.capabilities import registry
- from core.capabilities.paths import KIND_SKILL
+ from features.desktop.capabilities import registry
+ from core.capabilities.identity_rules import KIND_SKILL
profile = self._profile()
if not profile:
@@ -62,8 +62,8 @@ def change_token(self) -> Tuple[Any, ...]:
Read on every skill lookup, so it must not repeat the listing it guards:
the registry answers "did anything move" with one aggregate.
"""
- from core.capabilities import registry
- from core.capabilities.paths import KIND_SKILL
+ from features.desktop.capabilities import registry
+ from core.capabilities.identity_rules import KIND_SKILL
profile = self._profile()
if not profile:
@@ -71,8 +71,8 @@ def change_token(self) -> Tuple[Any, ...]:
return (profile, self._scope_user(), registry.source_token(kind=KIND_SKILL, profile_id=profile))
def _dir_for(self, inst) -> Optional[Path]:
- from core.capabilities import store
- from core.capabilities.paths import KIND_SKILL
+ from features.desktop.capabilities import store
+ from core.capabilities.identity_rules import KIND_SKILL
comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "")
return comp.path if comp else None
diff --git a/src/backend/features/evaluation/README.md b/src/backend/features/evaluation/README.md
new file mode 100644
index 00000000..52237227
--- /dev/null
+++ b/src/backend/features/evaluation/README.md
@@ -0,0 +1,7 @@
+# 评测执行隔离
+
+提供 eval_ 会话的受限执行上下文、工具和独立沙箱租约。对外保留评测沙箱 API、admission 与 sandboxes 服务;runtime 管理执行范围,sandbox_provider 管理租约授权、进程和容器操作。
+
+`extension.setup` 通过现有 register_sandbox_route 注册 eval_ 前缀。每次选择提供方都核对 OpenSandbox 配置,禁止回退到普通沙箱;扩展未注册时保留拒绝策略。FULL 与 CE 均安装本模块。租约使用已有临时状态接口,无新增 ORM 表。
+
+依赖内核的沙箱协议、临时状态、配置、工具和运行取消接口;没有向其他 features 模块的依赖。直接运行取消引用按计划 P6c 的取消注册点处理,当前批次保持原行为。
diff --git a/src/backend/features/evaluation/__init__.py b/src/backend/features/evaluation/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/evaluation/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/services/evaluation_admission.py b/src/backend/features/evaluation/admission.py
similarity index 93%
rename from src/backend/core/services/evaluation_admission.py
rename to src/backend/features/evaluation/admission.py
index d1297c13..ed0634c3 100644
--- a/src/backend/core/services/evaluation_admission.py
+++ b/src/backend/features/evaluation/admission.py
@@ -2,7 +2,7 @@
from fastapi import HTTPException
from core.config.settings import settings
-import core.sandbox.providers.evaluation.binding as evaluation_binding
+import features.evaluation.sandbox_provider.binding as evaluation_binding
from core.sandbox.errors import SandboxError
diff --git a/src/backend/features/evaluation/api/__init__.py b/src/backend/features/evaluation/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/evaluation/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/evaluation_sandboxes.py b/src/backend/features/evaluation/api/evaluation_sandboxes.py
similarity index 98%
rename from src/backend/api/routes/v1/evaluation_sandboxes.py
rename to src/backend/features/evaluation/api/evaluation_sandboxes.py
index 02d4d340..486a2b03 100644
--- a/src/backend/api/routes/v1/evaluation_sandboxes.py
+++ b/src/backend/features/evaluation/api/evaluation_sandboxes.py
@@ -45,7 +45,7 @@ class ExecCommand(BaseModel):
async def _call(name: str, *args, **kwargs):
- from core.services import evaluation_sandbox_service as service
+ from features.evaluation import sandboxes as service
from core.sandbox.errors import SandboxError
try:
return await getattr(service, name)(*args, **kwargs)
diff --git a/src/backend/features/evaluation/extension.py b/src/backend/features/evaluation/extension.py
new file mode 100644
index 00000000..0bd30058
--- /dev/null
+++ b/src/backend/features/evaluation/extension.py
@@ -0,0 +1,23 @@
+"""Reserved evaluation sessions use a strict OpenSandbox provider."""
+from core.harness.registry import register_sandbox_route, sandbox_route
+
+NAME = "evaluation"
+
+
+def evaluation_provider():
+ from core.config.settings import settings
+ from core.sandbox.errors import SandboxError
+ from features.evaluation.sandbox_provider.provider import get_evaluation_provider
+
+ if settings.sandbox.provider != "opensandbox":
+ raise SandboxError("Evaluation requires OpenSandbox; fallback is forbidden")
+ return get_evaluation_provider()
+
+
+def setup():
+ from core.harness.registry.evaluation import register_evaluation_policy
+ from features.evaluation.policy import EvaluationPolicy
+ register_evaluation_policy(EvaluationPolicy())
+ if sandbox_route("eval_") is None:
+ register_sandbox_route("eval_", evaluation_provider)
+ return True
diff --git a/src/backend/features/evaluation/models.py b/src/backend/features/evaluation/models.py
new file mode 100644
index 00000000..132640c9
--- /dev/null
+++ b/src/backend/features/evaluation/models.py
@@ -0,0 +1 @@
+"""Evaluation leases use ephemeral state and add no ORM models."""
diff --git a/src/backend/features/evaluation/policy.py b/src/backend/features/evaluation/policy.py
new file mode 100644
index 00000000..3444b07d
--- /dev/null
+++ b/src/backend/features/evaluation/policy.py
@@ -0,0 +1,27 @@
+"""Bind the evaluation feature to neutral execution-policy hooks."""
+from features.evaluation.runtime import runtime
+
+
+class EvaluationPolicy:
+ def apply_context(self, context):
+ return runtime.apply_evaluation_scope(context)
+
+ def parse_scope(self, **identity):
+ return runtime.parse_evaluation_scope(**identity)
+
+ def project_agent(self, agent):
+ return runtime.evaluation_user_agent(agent)
+
+ def current_paths(self):
+ if runtime.CURRENT_EVALUATION_SCOPE.get() is not None:
+ from features.evaluation.runtime import paths
+ return paths
+ return None
+
+ def register_tools(self, toolkit, scope, **options):
+ from features.evaluation.runtime.tools import register_evaluation_tools
+ register_evaluation_tools(toolkit, scope, **options)
+
+ async def authorize(self, request, user_id, *, agent_scoped=False):
+ from features.evaluation.admission import authorize
+ await authorize(request, user_id, agent_scoped=agent_scoped)
diff --git a/src/backend/features/evaluation/runtime/__init__.py b/src/backend/features/evaluation/runtime/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/evaluation/runtime/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/llm/evaluation_paths.py b/src/backend/features/evaluation/runtime/paths.py
similarity index 100%
rename from src/backend/core/llm/evaluation_paths.py
rename to src/backend/features/evaluation/runtime/paths.py
diff --git a/src/backend/core/llm/evaluation_process_tools.py b/src/backend/features/evaluation/runtime/process_tools.py
similarity index 87%
rename from src/backend/core/llm/evaluation_process_tools.py
rename to src/backend/features/evaluation/runtime/process_tools.py
index f766a5e7..2f0b7adf 100644
--- a/src/backend/core/llm/evaluation_process_tools.py
+++ b/src/backend/features/evaluation/runtime/process_tools.py
@@ -1,5 +1,6 @@
"""Process tools owned by an evaluation lease, without ordinary-chat state."""
-from core.llm.tools._tool_helpers import _resp_json
+from core.llm.factory.tools.definitions import function_tool
+from core.llm.tools._tool_helpers import resp_json as _resp_json
def register_evaluation_process_tools(toolkit, scope):
"""An evaluation controller owns completion and destroys its task sandbox."""
@@ -27,5 +28,5 @@ async def write_stdin(session_id: str, chars: str = "",
chars=chars, yield_time_ms=yield_time_ms)
return _resp_json(result)
- toolkit.register_tool_function(write_stdin, namesake_strategy="override")
- toolkit.register_tool_function(Bash, namesake_strategy="override")
+ toolkit.append(function_tool(write_stdin))
+ toolkit.append(function_tool(Bash))
diff --git a/src/backend/core/llm/evaluation_runtime.py b/src/backend/features/evaluation/runtime/runtime.py
similarity index 100%
rename from src/backend/core/llm/evaluation_runtime.py
rename to src/backend/features/evaluation/runtime/runtime.py
diff --git a/src/backend/core/llm/evaluation_tools.py b/src/backend/features/evaluation/runtime/tools.py
similarity index 57%
rename from src/backend/core/llm/evaluation_tools.py
rename to src/backend/features/evaluation/runtime/tools.py
index 5d6c278b..6620b8fa 100644
--- a/src/backend/core/llm/evaluation_tools.py
+++ b/src/backend/features/evaluation/runtime/tools.py
@@ -2,7 +2,7 @@
from functools import wraps
from inspect import signature
-from core.llm.evaluation_runtime import CURRENT_EVALUATION_SCOPE
+from features.evaluation.runtime.runtime import CURRENT_EVALUATION_SCOPE
_DESCRIPTIONS = {
@@ -16,36 +16,31 @@
}
-class _EvaluationCollector:
- def __init__(self, target, scope):
- self.target = target
- self.scope = scope
+def scoped_tool(tool, scope):
+ from agentscope.tool import FunctionTool
+ from core.llm.factory.tools.definitions import function_tool
+ func = tool._func
+ original_signature = signature(func)
- def register_tool_function(self, func, **options):
- original_signature = signature(func)
+ @wraps(func)
+ async def scoped(*args, **kwargs):
+ from core.config.local_mode import local_mode_enabled
+ from core.llm.tools._common import resp_json
+ if local_mode_enabled():
+ return resp_json({"error": "Evaluation requires container execution, not local mode"})
+ bound = original_signature.bind(*args, **kwargs)
+ if bound.arguments.get("register_as_artifact"):
+ return resp_json({"error": "Evaluation outputs must remain in the task sandbox"})
+ token = CURRENT_EVALUATION_SCOPE.set(scope)
+ try:
+ return await func(*args, **kwargs)
+ finally:
+ CURRENT_EVALUATION_SCOPE.reset(token)
- @wraps(func)
- async def scoped(*args, **kwargs):
- from core.config.local_mode import local_mode_enabled
- from core.llm.tools._common import resp_json
- if local_mode_enabled():
- return resp_json({"error": "Evaluation requires container execution, not local mode"})
- bound = original_signature.bind(*args, **kwargs)
- if bound.arguments.get("register_as_artifact"):
- return resp_json({"error": "Evaluation outputs must remain in the task sandbox"})
- token = CURRENT_EVALUATION_SCOPE.set(self.scope)
- try:
- return await func(*args, **kwargs)
- finally:
- CURRENT_EVALUATION_SCOPE.reset(token)
-
- # Tool schemas retain the native parameters and behavioral contracts.
- scoped.__doc__ = (
- _DESCRIPTIONS.get(func.__name__, "") + " "
- f"Native {func.__name__} in the current evaluation sandbox only. "
- "No account workspace, historical artifacts, or host filesystem access."
- )
- self.target.register_tool_function(scoped, **options)
+ scoped.__doc__ = (_DESCRIPTIONS.get(tool.name, "") + " "
+ + f"Native {tool.name} in the current evaluation sandbox only. "
+ + "No account workspace, historical artifacts, or host filesystem access.")
+ return function_tool(scoped, dispatch_spec=tool.dispatch_spec)
def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=True, vision_mode="none"):
@@ -56,7 +51,7 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru
)
if local_mode_enabled():
raise ValueError("Evaluation requires container execution, not local mode")
- collector = _EvaluationCollector(toolkit, scope)
+ collector = []
common = dict(
chat_id=scope.session_id, sandbox_session_id=scope.session_id,
user_id=scope.owner_user_id,
@@ -64,7 +59,7 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru
token = CURRENT_EVALUATION_SCOPE.set(scope)
try:
if not read_only and allow_bash:
- from core.llm.evaluation_process_tools import register_evaluation_process_tools
+ from features.evaluation.runtime.process_tools import register_evaluation_process_tools
register_evaluation_process_tools(collector, scope)
state = ReadStateTracker()
register_read(collector, **common, state=state, vision_mode=vision_mode)
@@ -75,3 +70,5 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru
register_edit(collector, **common, state=state, interactive=False)
finally:
CURRENT_EVALUATION_SCOPE.reset(token)
+
+ toolkit.extend(scoped_tool(tool, scope) for tool in collector)
diff --git a/src/backend/core/sandbox/providers/evaluation/__init__.py b/src/backend/features/evaluation/sandbox_provider/__init__.py
similarity index 100%
rename from src/backend/core/sandbox/providers/evaluation/__init__.py
rename to src/backend/features/evaluation/sandbox_provider/__init__.py
diff --git a/src/backend/core/sandbox/providers/evaluation/binding.py b/src/backend/features/evaluation/sandbox_provider/binding.py
similarity index 100%
rename from src/backend/core/sandbox/providers/evaluation/binding.py
rename to src/backend/features/evaluation/sandbox_provider/binding.py
diff --git a/src/backend/core/sandbox/providers/evaluation/process_guard.py b/src/backend/features/evaluation/sandbox_provider/process_guard.py
similarity index 100%
rename from src/backend/core/sandbox/providers/evaluation/process_guard.py
rename to src/backend/features/evaluation/sandbox_provider/process_guard.py
diff --git a/src/backend/core/sandbox/providers/evaluation/provider.py b/src/backend/features/evaluation/sandbox_provider/provider.py
similarity index 98%
rename from src/backend/core/sandbox/providers/evaluation/provider.py
rename to src/backend/features/evaluation/sandbox_provider/provider.py
index 57ef7664..76d807fb 100644
--- a/src/backend/core/sandbox/providers/evaluation/provider.py
+++ b/src/backend/features/evaluation/sandbox_provider/provider.py
@@ -13,7 +13,7 @@
from contextlib import asynccontextmanager
from datetime import timedelta
-import core.sandbox.providers.evaluation.binding as binding
+import features.evaluation.sandbox_provider.binding as binding
from core.sandbox.errors import SandboxConnectError, SandboxError
from core.sandbox.protocol import ProcessResult, SandboxAdminCapabilities
@@ -290,7 +290,7 @@ async def freeze(self, session_id, owner_user_id, *, timeout=65):
await asyncio.sleep(0.1)
if status.running is not False:
raise SandboxError("Evaluation command stop is unconfirmed")
- from core.sandbox.providers.evaluation.process_guard import clear_processes
+ from features.evaluation.sandbox_provider.process_guard import clear_processes
await clear_processes(sandbox, item.protected_processes)
return await binding.finish_freeze(session_id, owner_user_id)
diff --git a/src/backend/core/services/evaluation_sandbox_service.py b/src/backend/features/evaluation/sandboxes.py
similarity index 90%
rename from src/backend/core/services/evaluation_sandbox_service.py
rename to src/backend/features/evaluation/sandboxes.py
index e949147d..adc07545 100644
--- a/src/backend/core/services/evaluation_sandbox_service.py
+++ b/src/backend/features/evaluation/sandboxes.py
@@ -7,6 +7,7 @@
from core.config.settings import settings
from core.sandbox.errors import SandboxError
+from core.db.threads import run_in_session
MAX_TRANSFER = 64 * 1024 * 1024
MAX_OUTPUT = 1024 * 1024
@@ -31,7 +32,7 @@ def session_id(lease_id: str) -> str:
async def _owned(lease_id, owner, *, allow_expired=False):
- import core.sandbox.providers.evaluation.binding as bindings
+ import features.evaluation.sandbox_provider.binding as bindings
return await bindings.assert_owner(session_id(lease_id), owner, allow_expired=allow_expired)
@@ -72,10 +73,10 @@ def _safe_attempt_id(attempt_id: str) -> str:
async def create(owner, *, image, attempt_id="", ttl_seconds=7200, cpu_count=2, memory_mb=4096):
+ import features.evaluation.sandbox_provider.binding as bindings
+ from features.evaluation.sandbox_provider.process_guard import capture_baseline
from opensandbox import Sandbox
from opensandbox.models.execd import RunCommandOpts
- import core.sandbox.providers.evaluation.binding as bindings
- from core.sandbox.providers.evaluation.process_guard import capture_baseline
if settings.sandbox.provider != "opensandbox":
raise SandboxError("opensandbox_required")
sandbox = await Sandbox.create(
@@ -84,7 +85,7 @@ async def create(owner, *, image, attempt_id="", ttl_seconds=7200, cpu_count=2,
resource={"cpu": str(cpu_count), "memory": f"{memory_mb}Mi"},
entrypoint=["bash", "-lc", "exec sleep infinity"],
metadata={"hugagent-evaluation": "true", "attempt-id": _safe_attempt_id(attempt_id)},
- volumes=None, env=None, skip_health_check=True,
+ volumes=None, env=None, skip_health_check=False,
)
try:
# All native tools use /workspace; Ageval uses /attempt/workspace.
@@ -166,19 +167,20 @@ async def download(lease_id, owner, path):
async def freeze(lease_id, owner):
- import core.sandbox.providers.evaluation.binding as bindings
- from core.sandbox.providers.evaluation.provider import get_evaluation_provider
- from core.db.engine import SessionLocal
+ import features.evaluation.sandbox_provider.binding as bindings
from core.db.models import ChatRun
- from orchestration.chat_run_executor import cancel_run
+ from core.harness.registry.run_cancellation import cancel_run
+ from features.evaluation.sandbox_provider.provider import get_evaluation_provider
sid = session_id(lease_id)
await bindings.assert_owner(sid, owner)
await bindings.begin_freeze(sid, owner)
- with SessionLocal() as db:
+ def _active_runs(db):
runs = db.query(ChatRun.run_id).filter(
ChatRun.chat_id == sid, ChatRun.user_id == owner,
ChatRun.status.in_(("pending", "running", "needs_attention")),
).all()
+ return runs
+ runs = await run_in_session(_active_runs)
for (run_id,) in runs:
await cancel_run(run_id, user_id=owner)
await get_evaluation_provider().freeze(sid, owner)
@@ -186,9 +188,9 @@ async def freeze(lease_id, owner):
async def destroy(lease_id, owner):
+ import features.evaluation.sandbox_provider.binding as bindings
+ from features.evaluation.sandbox_provider.provider import get_evaluation_provider
from opensandbox.exceptions import SandboxApiException
- import core.sandbox.providers.evaluation.binding as bindings
- from core.sandbox.providers.evaluation.provider import get_evaluation_provider
binding = await _owned(lease_id, owner, allow_expired=True)
if binding.destroyed:
return {"phase": "closed"}
diff --git a/src/backend/features/hosted_apps/README.md b/src/backend/features/hosted_apps/README.md
new file mode 100644
index 00000000..bd07894b
--- /dev/null
+++ b/src/backend/features/hosted_apps/README.md
@@ -0,0 +1,9 @@
+# 托管应用
+
+管理独立应用数据库、应用项目、集合、版本、发布、MCP 与内部数据接口,以及插件应用资源的准入和清理。HTTP 路由由中央路由表保持原顺序及权限策略;数据库配置通过 config 提供,独立初始化命令为 `python -m features.hosted_apps.hosting_setup`。
+
+应用数据表仍使用独立 store.metadata,不并入平台 Base。模块通过同步 schema 生命周期回调初始化受管理的本机 SQLite 应用库,执行位置保持在站点存储退役之后、平台 schema 调和之前。API、CLI 和容器入口均注册此回调,注册幂等;初始化异常仍阻断启动。云端数据库必须显式配置及预置,不自动创建。桥接模式的跳过条件保持不变。
+
+公开入口为 data、projects、publication、deployments、store、schema 等实际模块。允许 hosted_apps → sites:应用发布关联站点。当前桌面转发仍引用原始桌面桥接实现,属于 routes.json 虚拟归属识别的历史依赖,P5 须通过公开适配接口消除反向循环。已删除废弃 application_site_conversion,不保留兼容转发。
+
+依赖 features.desktop 提供本机认证及能力文件锁,桌面不反向依赖托管应用。执行位置由宿主策略接口提供。
diff --git a/src/backend/features/hosted_apps/__init__.py b/src/backend/features/hosted_apps/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/hosted_apps/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/features/hosted_apps/api/__init__.py b/src/backend/features/hosted_apps/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/hosted_apps/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/application_transports.py b/src/backend/features/hosted_apps/api/application_transports.py
similarity index 68%
rename from src/backend/api/routes/v1/application_transports.py
rename to src/backend/features/hosted_apps/api/application_transports.py
index 4ca4c287..08425906 100644
--- a/src/backend/api/routes/v1/application_transports.py
+++ b/src/backend/features/hosted_apps/api/application_transports.py
@@ -2,16 +2,16 @@
import json
-from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.application_data import ApplicationDataService
-from core.services.application_operations import perform_operation
-from core.services.application_schema import InternalMCPPublish, InternalOperation, RecordBatch
-from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from features.hosted_apps.data import ApplicationDataService
+from features.hosted_apps.operations import perform_operation
+from features.hosted_apps.schema import InternalMCPPublish, InternalOperation, RecordBatch
+from fastapi import APIRouter, Header, HTTPException, Request
from sqlalchemy import select
-from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
+from core.db.threads import run_in_session
+
internal_router = APIRouter(prefix="/v1/internal/applications", tags=["internal-applications"])
public_router = APIRouter(prefix="/site", tags=["application-data"])
mcp_router = APIRouter(prefix="/applications-mcp", tags=["hosted-mcp"])
@@ -21,15 +21,14 @@
async def internal_operation(
body: InternalOperation,
x_internal_token: str | None = Header(None),
- db: Session = Depends(get_db),
):
- from api.routes.v1.internal_site_auth import _check_internal_token
+ from features.sites.api.internal_site_auth import _check_internal_token
_check_internal_token(x_internal_token)
- from core.services.desktop_cloud_bridge import bridge_enabled
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
if bridge_enabled():
- from core.services.desktop_site_publish import forward_local_site_tool
+ from features.sites.desktop_publish import forward_local_site_tool
arguments = body.model_dump(exclude={"user_id", "chat_id"})
if not arguments["payload"]:
@@ -42,7 +41,7 @@ async def internal_operation(
chat_id=body.chat_id,
)
)
- result = await run_in_threadpool(perform_operation, body, db)
+ result = await run_in_session(lambda db: perform_operation(body, db))
return success_response(data=result)
@@ -50,14 +49,13 @@ async def internal_operation(
async def internal_publish_mcp(
body: InternalMCPPublish,
x_internal_token: str | None = Header(None),
- db: Session = Depends(get_db),
):
- from api.routes.v1.internal_site_auth import _check_internal_token
- from core.services.desktop_cloud_bridge import bridge_enabled
+ from features.sites.api.internal_site_auth import _check_internal_token
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
_check_internal_token(x_internal_token)
if bridge_enabled():
- from core.services.desktop_site_publish import forward_local_site_tool
+ from features.sites.desktop_publish import forward_local_site_tool
result = await forward_local_site_tool(
"publish_mcp",
@@ -66,34 +64,30 @@ async def internal_publish_mcp(
chat_id=body.chat_id,
)
else:
- from core.services.application_projects import ApplicationProjectService
- from core.services.application_publication import ApplicationPublicationService
-
- publication = ApplicationPublicationService(db)
- result = await run_in_threadpool(
- publication.publish,
- body.app_id,
- body.user_id,
- body.tools,
- install_personal=True,
- chat_id=body.chat_id,
+ from features.hosted_apps.projects import ApplicationProjectService
+ from features.hosted_apps.publication import ApplicationPublicationService
+
+ result = await run_in_session(
+ lambda db: ApplicationPublicationService(db).publish(
+ body.app_id,
+ body.user_id,
+ body.tools,
+ install_personal=True,
+ chat_id=body.chat_id,
+ )
)
return success_response(data=result)
@public_router.post("/{slug}/__api/data/{table}")
-async def visitor_insert(slug: str, table: str, request: Request, db: Session = Depends(get_db)):
- from api.routes.sites_serve import (
- _api_json,
- _client_ip,
- _load_authorized_site,
- )
- from core.services.application_store import applications
-
- from core.services.site_rate_limit import count_attempt
+async def visitor_insert(slug: str, table: str, request: Request):
+ from features.sites.api.sites_serve import _api_json, _client_ip, _load_authorized_site
+ from features.hosted_apps.store import applications
+
+ from features.sites.rate_limit import count_attempt
await count_attempt(_client_ip(request), slug)
- site, gate = await _load_authorized_site(slug, request, db, require_unlock=True)
+ site, gate = await _load_authorized_site(slug, request, require_unlock=True)
if gate is not None:
return _api_json(
{"detail": "Site access authorization is required"},
@@ -134,18 +128,12 @@ def submit():
@public_router.api_route("/{slug}/__api/data/{table}", methods=["GET", "PUT"])
-async def visitor_collection(
- slug: str, table: str, request: Request, db: Session = Depends(get_db)
-):
- from api.routes.sites_serve import (
- _api_json,
- _client_ip,
- _load_authorized_site,
- )
- from core.services.application_collections import read_collection, replace_collection
- from core.services.application_store import applications
-
- site, gate = await _load_authorized_site(slug, request, db, require_unlock=True)
+async def visitor_collection(slug: str, table: str, request: Request):
+ from features.sites.api.sites_serve import _api_json, _client_ip, _load_authorized_site
+ from features.hosted_apps.collections import read_collection, replace_collection
+ from features.hosted_apps.store import applications
+
+ site, gate = await _load_authorized_site(slug, request, require_unlock=True)
if gate is not None:
return _api_json(
{"detail": "Site access authorization is required"},
@@ -153,7 +141,7 @@ async def visitor_collection(
)
payload = None
if request.method == "PUT":
- from core.services.site_rate_limit import count_attempt
+ from features.sites.rate_limit import count_attempt
await count_attempt(_client_ip(request), slug)
body = bytearray()
async for chunk in request.stream():
@@ -190,6 +178,6 @@ def operation():
@mcp_router.api_route("/{app_id}", methods=["GET", "POST", "DELETE", "OPTIONS"])
async def hosted_mcp(app_id: str, request: Request):
- from core.services.application_mcp import ApplicationMCPResponse
+ from features.hosted_apps.mcp import ApplicationMCPResponse
return ApplicationMCPResponse()
diff --git a/src/backend/api/routes/v1/applications.py b/src/backend/features/hosted_apps/api/applications.py
similarity index 76%
rename from src/backend/api/routes/v1/applications.py
rename to src/backend/features/hosted_apps/api/applications.py
index 3aca0df6..0ed5455f 100644
--- a/src/backend/api/routes/v1/applications.py
+++ b/src/backend/features/hosted_apps/api/applications.py
@@ -7,31 +7,20 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.application_data import ApplicationDataService
-from core.services.application_operations import create_application
-from core.services.application_publication import (
- ApplicationPublicationService,
-)
-from core.services.application_schema import (
- ApplicationDefinition,
- MCPPublish,
- MCPRollback,
- PatchRecord,
- RecordBatch,
- RestoreCollection,
- TableDefinition,
-)
+from features.hosted_apps.data import ApplicationDataService
+from features.hosted_apps.operations import create_application
+from features.hosted_apps.publication import ApplicationPublicationService
+from features.hosted_apps.schema import ApplicationDefinition, MCPPublish, MCPRollback, PatchRecord, RecordBatch, RestoreCollection, TableDefinition
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import Response
from sqlalchemy.orm import Session
-from starlette.concurrency import run_in_threadpool
router = APIRouter(prefix="/v1/applications", tags=["Applications"])
@router.get("")
def list_applications(user: UserContext = Depends(get_current_user)):
- from core.config.application_hosting import application_hosting_settings
+ from features.hosted_apps.config import application_hosting_settings
from core.config.settings import settings
if not application_hosting_settings.database_url and not settings.deploy.is_local:
@@ -56,16 +45,16 @@ def detail(app_id: str, user: UserContext = Depends(get_current_user)):
@router.post("/{app_id}/editor")
-async def open_editor(
+def open_editor(
app_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
service = ApplicationDataService()
projects = ApplicationProjectService(db, service.engine)
- return success_response(data=await run_in_threadpool(projects.editor, app_id, user.user_id))
+ return success_response(data=projects.editor(app_id, user.user_id))
@router.post("/{app_id}/tables")
@@ -133,7 +122,7 @@ def export_records(
table: str,
user: UserContext = Depends(get_current_user),
):
- from core.services.application_export import export_csv
+ from features.hosted_apps.export import export_csv
from fastapi.responses import StreamingResponse
return StreamingResponse(
export_csv(ApplicationDataService(), app_id, user.user_id, table),
@@ -143,29 +132,29 @@ def export_records(
@router.post("/{app_id}/mcp")
-async def publish_mcp(
+def publish_mcp(
app_id: str,
body: MCPPublish,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
publication = ApplicationPublicationService(db)
- receipt = await run_in_threadpool(publication.publish, app_id, user.user_id, body.tools)
+ receipt = publication.publish(app_id, user.user_id, body.tools)
return success_response(data=receipt)
@router.post("/{app_id}/mcp/project")
-async def publish_project_mcp(
+def publish_project_mcp(
app_id: str,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
publication = ApplicationPublicationService(db)
- receipt = await run_in_threadpool(publication.publish, app_id, user.user_id, from_project=True)
+ receipt = publication.publish(app_id, user.user_id, from_project=True)
return success_response(data=receipt)
@@ -177,52 +166,52 @@ def revoke_mcp(
@router.post("/{app_id}/mcp/rollback")
-async def rollback_mcp(
+def rollback_mcp(
app_id: str,
body: MCPRollback,
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
publication = ApplicationPublicationService(db)
- receipt = await run_in_threadpool(publication.rollback, app_id, user.user_id, body.version)
+ receipt = publication.rollback(app_id, user.user_id, body.version)
return success_response(data=receipt)
@router.delete("/{app_id}/tables/{table}/records/{record_id}")
def erase_record(app_id: str, table: str, record_id: str,
version: int = Query(..., ge=1),
user: UserContext = Depends(get_current_user)):
- from core.services.application_lifecycle import delete_record
+ from features.hosted_apps.lifecycle import delete_record
return success_response(data=delete_record(ApplicationDataService(), app_id, user.user_id, table, record_id, version))
@router.delete("/{app_id}/tables/{table}")
def erase_table(app_id: str, table: str, user: UserContext = Depends(get_current_user)):
- from core.services.application_lifecycle import delete_table
+ from features.hosted_apps.lifecycle import delete_table
return success_response(data=delete_table(ApplicationDataService(), app_id, user.user_id, table))
@router.delete("/{app_id}")
def erase_application(app_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)):
- from core.services.application_lifecycle import delete_application
+ from features.hosted_apps.lifecycle import delete_application
return success_response(data=delete_application(ApplicationDataService(), db, app_id, user.user_id))
@router.get("/{app_id}/tables/{table}/history")
def recovery_history(app_id: str, table: str, user: UserContext = Depends(get_current_user)):
- from core.services.application_history import list_history
+ from features.hosted_apps.history import list_history
return success_response(data=list_history(ApplicationDataService(), app_id, user.user_id, table))
@router.post("/{app_id}/tables/{table}/history/{history_id}/restore")
def restore_collection(app_id: str, table: str, history_id: str, body: RestoreCollection,
user: UserContext = Depends(get_current_user)):
- from core.services.application_history import restore
+ from features.hosted_apps.history import restore
return success_response(data=restore(ApplicationDataService(), app_id, user.user_id, table, history_id, body.revision))
@router.get("/{app_id}/tables/{table}/collection")
def owner_collection(app_id: str, table: str, user: UserContext = Depends(get_current_user)):
- from core.services.application_collections import snapshot
- from core.services.application_store import owned_application
- from core.services.application_relational import table_definition
- from core.services.application_sql_policy import application_role
+ from features.hosted_apps.collections import snapshot
+ from features.hosted_apps.store import owned_application
+ from features.hosted_apps.relational import table_definition
+ from features.hosted_apps.sql_policy import application_role
service = ApplicationDataService()
with service.engine.begin() as connection:
app = owned_application(connection, app_id, user.user_id)
diff --git a/src/backend/core/services/application_collections.py b/src/backend/features/hosted_apps/collections.py
similarity index 92%
rename from src/backend/core/services/application_collections.py
rename to src/backend/features/hosted_apps/collections.py
index 5c36744f..f9886a46 100644
--- a/src/backend/core/services/application_collections.py
+++ b/src/backend/features/hosted_apps/collections.py
@@ -5,9 +5,9 @@
from uuid import uuid4
from core.infra.time import utc_now
-from core.services.application_relational import json_record, table_definition, validated_row
-from core.services.application_sql_policy import application_role
-from core.services.application_store import applications, owned_application, write_connection
+from features.hosted_apps.relational import json_record, table_definition, validated_row
+from features.hosted_apps.sql_policy import application_role
+from features.hosted_apps.store import applications, owned_application, write_connection
from fastapi import HTTPException
from sqlalchemy import delete, func, insert, select
from sqlalchemy.exc import IntegrityError
@@ -89,7 +89,7 @@ def replace_collection(service, app_id, owner, name, revision, rows, *, owner_re
raise HTTPException(409, "Collection changed; reload before saving")
if not previous["items"] and not rows:
return previous
- from core.services.application_history import archive
+ from features.hosted_apps.history import archive
archive(connection, app_id, name, previous["items"])
with application_role(connection, app_id):
connection.execute(delete(table))
diff --git a/src/backend/core/config/application_hosting.py b/src/backend/features/hosted_apps/config.py
similarity index 87%
rename from src/backend/core/config/application_hosting.py
rename to src/backend/features/hosted_apps/config.py
index 2cb8e9d9..664d2269 100644
--- a/src/backend/core/config/application_hosting.py
+++ b/src/backend/features/hosted_apps/config.py
@@ -2,7 +2,7 @@
from dataclasses import dataclass, field
-from core.config.settings import _env
+from core.config.settings import env_value as _env
@dataclass(frozen=True)
diff --git a/src/backend/core/services/application_data.py b/src/backend/features/hosted_apps/data.py
similarity index 95%
rename from src/backend/core/services/application_data.py
rename to src/backend/features/hosted_apps/data.py
index 8ceff56c..905da848 100644
--- a/src/backend/core/services/application_data.py
+++ b/src/backend/features/hosted_apps/data.py
@@ -7,22 +7,10 @@
from uuid import uuid4
from core.infra.time import utc_now
-from core.services.application_relational import json_record, table_definition, validated_row
-from core.services.application_schema import ApplicationDefinition, RecordBatch, TableDefinition
-from core.services.application_sql_policy import (
- application_role,
- application_table,
- provision_table,
-)
-from core.services.application_store import (
- application_engine,
- application_sources,
- applications,
- operations,
- owned_application,
- require_store,
- write_connection,
-)
+from features.hosted_apps.relational import json_record, table_definition, validated_row
+from features.hosted_apps.schema import ApplicationDefinition, RecordBatch, TableDefinition
+from features.hosted_apps.sql_policy import application_role, application_table, provision_table
+from features.hosted_apps.store import application_engine, application_sources, applications, operations, owned_application, require_store, write_connection
from fastapi import HTTPException
from sqlalchemy import func, insert, select, update
from sqlalchemy.exc import IntegrityError
diff --git a/src/backend/core/services/application_deployments.py b/src/backend/features/hosted_apps/deployments.py
similarity index 92%
rename from src/backend/core/services/application_deployments.py
rename to src/backend/features/hosted_apps/deployments.py
index 6eead246..7edfffe4 100644
--- a/src/backend/core/services/application_deployments.py
+++ b/src/backend/features/hosted_apps/deployments.py
@@ -5,16 +5,9 @@
import hashlib
from core.infra.time import utc_now
-from core.services.application_relational import table_definition
-from core.services.application_schema import ToolDefinition
-from core.services.application_store import (
- application_engine,
- applications,
- mcp_deployments,
- owned_application,
- require_store,
- write_connection,
-)
+from features.hosted_apps.relational import table_definition
+from features.hosted_apps.schema import ToolDefinition
+from features.hosted_apps.store import application_engine, applications, mcp_deployments, owned_application, require_store, write_connection
from fastapi import HTTPException
from sqlalchemy import insert, select, update
diff --git a/src/backend/core/services/application_export.py b/src/backend/features/hosted_apps/export.py
similarity index 89%
rename from src/backend/core/services/application_export.py
rename to src/backend/features/hosted_apps/export.py
index 91cc08b6..8a5c2ba5 100644
--- a/src/backend/core/services/application_export.py
+++ b/src/backend/features/hosted_apps/export.py
@@ -2,9 +2,9 @@
import csv
import io
import json
-from core.services.application_store import owned_application
-from core.services.application_relational import table_definition, json_record
-from core.services.application_sql_policy import application_role
+from features.hosted_apps.store import owned_application
+from features.hosted_apps.relational import table_definition, json_record
+from features.hosted_apps.sql_policy import application_role
from sqlalchemy import select
def csv_cell(value):
diff --git a/src/backend/features/hosted_apps/extension.py b/src/backend/features/hosted_apps/extension.py
new file mode 100644
index 00000000..deded1ce
--- /dev/null
+++ b/src/backend/features/hosted_apps/extension.py
@@ -0,0 +1,17 @@
+"""Hosted application schema lifecycle composition."""
+NAME = "hosted_apps"
+
+
+def initialize_schema():
+ from features.hosted_apps.store import initialize_local_store
+ initialize_local_store()
+
+
+def register_schema():
+ from core.harness.registry.schema_lifecycle import register_schema_lifecycle
+ register_schema_lifecycle(NAME, initialize_schema)
+
+
+def setup():
+ register_schema()
+ return True
diff --git a/src/backend/core/services/application_history.py b/src/backend/features/hosted_apps/history.py
similarity index 91%
rename from src/backend/core/services/application_history.py
rename to src/backend/features/hosted_apps/history.py
index 7286fd17..e3802569 100644
--- a/src/backend/core/services/application_history.py
+++ b/src/backend/features/hosted_apps/history.py
@@ -2,7 +2,7 @@
import json
from uuid import uuid4
from core.infra.time import utc_now
-from core.services.application_store import metadata, owned_application
+from features.hosted_apps.store import metadata, owned_application
from fastapi import HTTPException
from sqlalchemy import Boolean, Column, DateTime, JSON, String, Table, delete, insert, select
@@ -33,7 +33,7 @@ def archive(connection, app_id, name, rows):
connection.execute(delete(collection_history).where(collection_history.c.id.in_(retained)))
def list_history(service, app_id, owner, name):
- from core.services.application_relational import table_definition, json_record
+ from features.hosted_apps.relational import table_definition, json_record
with service.engine.connect() as connection:
table_definition(owned_application(connection, app_id, owner), name)
rows = connection.execute(select(
@@ -43,8 +43,8 @@ def list_history(service, app_id, owner, name):
return {"items": [json_record(row) for row in rows]}
def restore(service, app_id, owner, name, history_id, revision):
- from core.services.application_collections import replace_collection
- from core.services.application_relational import table_definition
+ from features.hosted_apps.collections import replace_collection
+ from features.hosted_apps.relational import table_definition
with service.engine.connect() as connection:
definition = table_definition(owned_application(connection, app_id, owner), name)
rows = connection.scalar(select(collection_history.c.rows).where(
diff --git a/src/backend/core/services/application_hosting_setup.py b/src/backend/features/hosted_apps/hosting_setup.py
similarity index 96%
rename from src/backend/core/services/application_hosting_setup.py
rename to src/backend/features/hosted_apps/hosting_setup.py
index f40b5769..54febd57 100644
--- a/src/backend/core/services/application_hosting_setup.py
+++ b/src/backend/features/hosted_apps/hosting_setup.py
@@ -2,13 +2,13 @@
import os
from sqlalchemy import create_engine, text
from sqlalchemy.engine import make_url
-from core.services.application_store import application_engine, initialize_store
+from features.hosted_apps.store import application_engine, initialize_store
def provision_owner(url):
parsed = make_url(url)
if parsed.get_backend_name() != "postgresql" or parsed.username != "application_admin":
raise RuntimeError("Provisioning requires the dedicated application_admin connection")
- from core.config.application_hosting import application_hosting_settings
+ from features.hosted_apps.config import application_hosting_settings
from core.db.engine import DATABASE_URL
target = make_url(application_hosting_settings.database_url)
platform = make_url(DATABASE_URL)
diff --git a/src/backend/core/services/application_industry.py b/src/backend/features/hosted_apps/industry.py
similarity index 95%
rename from src/backend/core/services/application_industry.py
rename to src/backend/features/hosted_apps/industry.py
index 20aaac74..47fa449d 100644
--- a/src/backend/core/services/application_industry.py
+++ b/src/backend/features/hosted_apps/industry.py
@@ -4,12 +4,7 @@
import httpx
from core.infra.time import utc_now
-from core.services.application_schema import (
- ColumnDefinition,
- IndustryImport,
- RecordBatch,
- TableDefinition,
-)
+from features.hosted_apps.schema import ColumnDefinition, IndustryImport, RecordBatch, TableDefinition
from fastapi import HTTPException
@@ -41,7 +36,7 @@ def fetch_enterprises(keyword: str, limit: int) -> list[dict]:
def import_enterprises(service, db, app_id: str, owner: str, payload: dict) -> dict:
from core.config.settings import settings
- from core.plugins.management.queries import list_installed
+ from core.capabilities.plugins.management.queries import list_installed
if settings.edition.edition != "ee":
raise HTTPException(403, "Industry knowledge center requires the enterprise edition")
diff --git a/src/backend/core/services/application_lifecycle.py b/src/backend/features/hosted_apps/lifecycle.py
similarity index 88%
rename from src/backend/core/services/application_lifecycle.py
rename to src/backend/features/hosted_apps/lifecycle.py
index e90dc184..e3a48767 100644
--- a/src/backend/core/services/application_lifecycle.py
+++ b/src/backend/features/hosted_apps/lifecycle.py
@@ -1,11 +1,11 @@
"""Owner deletion with publication serialization and complete quota reclamation."""
from fastapi import HTTPException
from sqlalchemy import delete, select, text, update
-from core.services.application_store import applications, operations, application_sources, mcp_deployments, owned_application, write_connection
-from core.services.application_history import collection_history
-from core.services.application_relational import table_definition
-from core.services.application_sql_policy import application_role
-from core.services.application_publication import publication_lock
+from features.hosted_apps.store import applications, operations, application_sources, mcp_deployments, owned_application, write_connection
+from features.hosted_apps.history import collection_history
+from features.hosted_apps.relational import table_definition
+from features.hosted_apps.sql_policy import application_role
+from features.hosted_apps.publication import publication_lock
def delete_record(service, app_id, owner, name, record_id, version):
with write_connection(service.engine) as connection:
@@ -39,7 +39,7 @@ def delete_table(service, app_id, owner, name):
def delete_application(service, db, app_id, owner):
from core.db.models import AdminMcpServer
- from core.services.mcp_management_service import refresh_mcp_caches
+ from core.capabilities.mcp.management import refresh_mcp_caches
with publication_lock(service.engine, app_id):
with write_connection(service.engine) as connection:
connection.execute(select(applications.c.id).where(applications.c.id == app_id).with_for_update())
@@ -60,7 +60,7 @@ def delete_application(service, db, app_id, owner):
connection.execute(text(f'DROP ROLE IF EXISTS "app_role_{app["id"]}"'))
else:
for raw in app["tables"].values():
- from core.services.application_schema import TableDefinition
+ from features.hosted_apps.schema import TableDefinition
service._table(app_id, TableDefinition.model_validate(raw)).drop(connection)
for table in (operations, mcp_deployments, application_sources, collection_history):
connection.execute(delete(table).where(table.c.app_id == app_id))
diff --git a/src/backend/core/services/application_mcp.py b/src/backend/features/hosted_apps/mcp.py
similarity index 82%
rename from src/backend/core/services/application_mcp.py
rename to src/backend/features/hosted_apps/mcp.py
index f616b9ce..a87e7ab6 100644
--- a/src/backend/core/services/application_mcp.py
+++ b/src/backend/features/hosted_apps/mcp.py
@@ -9,11 +9,12 @@
import re
from typing import Annotated, Any
-from core.services.application_data import ApplicationDataService
-from core.services.application_schema import ToolDefinition
-from core.services.application_store import applications
+from features.hosted_apps.data import ApplicationDataService
+from features.hosted_apps.schema import ToolDefinition
+from features.hosted_apps.store import applications
from fastapi import HTTPException
-from mcp.server.fastmcp import FastMCP
+from mcp.server.mcpserver import MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
from mcp.types import ToolAnnotations
from pydantic import Field
from sqlalchemy import select
@@ -58,8 +59,8 @@ def query(**arguments: Any) -> dict:
offset=arguments.get("offset", 0),
)
- # FastMCP needs an explicit typed signature rather than opaque **kwargs.
- from core.services.application_relational import table_definition
+ # MCPServer needs an explicit typed signature rather than opaque **kwargs.
+ from features.hosted_apps.relational import table_definition
fields = {c.name: c for c in table_definition(app, definition.table).columns}
types = {"text": str, "integer": int, "number": float, "boolean": bool, "date": str}
@@ -88,6 +89,20 @@ async def invoke(**arguments) -> dict[str, Any]:
return invoke
+class QueryMCPServer(MCPServer):
+ async def call_tool(self, name, arguments, context=None):
+ import jsonschema
+
+ schemas = {tool.name: tool.input_schema for tool in await self.list_tools()}
+ if name in schemas:
+ schema = {**schemas[name], "additionalProperties": False}
+ try:
+ jsonschema.validate(arguments, schema)
+ except jsonschema.ValidationError:
+ raise ToolError("Invalid query arguments") from None
+ return await super().call_tool(name, arguments, context)
+
+
class ApplicationMCPGateway:
"""Each HTTP call reloads persisted authorization, so revocation works across workers."""
@@ -112,7 +127,7 @@ async def __call__(self, scope, receive, send):
{"detail": error.detail}, error.status_code, headers={"WWW-Authenticate": "Bearer"}
)(scope, receive, send)
return
- server = FastMCP(app["title"], host="0.0.0.0", stateless_http=True, json_response=True)
+ server = QueryMCPServer(app["title"])
for raw in app["tools"]:
definition = ToolDefinition.model_validate(raw)
server.add_tool(
@@ -126,10 +141,7 @@ async def __call__(self, scope, receive, send):
openWorldHint=False,
),
)
- tool = server._tool_manager.get_tool(definition.name)
- tool.fn_metadata.arg_model.model_config["extra"] = "forbid"
- tool.fn_metadata.arg_model.model_rebuild(force=True)
- transport = server.streamable_http_app()
+ transport = server.streamable_http_app(host="0.0.0.0", stateless_http=True, json_response=True)
inner_scope = {**scope, "path": "/mcp", "raw_path": b"/mcp", "root_path": ""}
async with server.session_manager.run():
await transport(inner_scope, receive, send)
diff --git a/src/backend/features/hosted_apps/models.py b/src/backend/features/hosted_apps/models.py
new file mode 100644
index 00000000..92f6424a
--- /dev/null
+++ b/src/backend/features/hosted_apps/models.py
@@ -0,0 +1,5 @@
+"""Hosted application tables live in their separate application database.
+
+The platform model loader imports this module independently of runtime switches;
+application metadata is deliberately not attached to the platform Base.
+"""
diff --git a/src/backend/core/services/application_operations.py b/src/backend/features/hosted_apps/operations.py
similarity index 84%
rename from src/backend/core/services/application_operations.py
rename to src/backend/features/hosted_apps/operations.py
index 694144fa..57f3dc40 100644
--- a/src/backend/core/services/application_operations.py
+++ b/src/backend/features/hosted_apps/operations.py
@@ -1,15 +1,8 @@
"""Validated application actions shared by the agent callback and owner creation."""
-from core.services.application_data import ApplicationDataService
-from core.services.application_publication import ApplicationPublicationService
-from core.services.application_schema import (
- ApplicationDefinition,
- InsertOperation,
- InternalOperation,
- MCPPublish,
- QueryOperation,
- TableDefinition,
-)
+from features.hosted_apps.data import ApplicationDataService
+from features.hosted_apps.publication import ApplicationPublicationService
+from features.hosted_apps.schema import ApplicationDefinition, InsertOperation, InternalOperation, MCPPublish, QueryOperation, TableDefinition
from fastapi import HTTPException
from pydantic import ValidationError
from sqlalchemy.orm import Session
@@ -19,13 +12,13 @@ def create_application(
definition: ApplicationDefinition, owner: str, db: Session, chat_id: str = ""
) -> dict:
if definition.site_id:
- from core.services.site_service import SiteService
+ from features.sites.service import SiteService
SiteService(db).get_owned(definition.site_id, owner, required="admin")
service = ApplicationDataService()
app = service.create(owner, definition)
if definition.kind == "mcp":
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
editor = ApplicationProjectService(db, service.engine).editor(app["id"], owner, chat_id)
app["project_id"] = editor["project_id"]
@@ -43,7 +36,7 @@ def perform_operation(body: InternalOperation, db: Session) -> dict:
ApplicationDefinition.model_validate(payload), body.user_id, db, body.chat_id
)
if body.action == "source":
- from core.services.application_projects import ApplicationProjectService
+ from features.hosted_apps.projects import ApplicationProjectService
return ApplicationProjectService(db, service.engine).editor(
body.app_id, body.user_id, body.chat_id
@@ -86,7 +79,7 @@ def perform_operation(body: InternalOperation, db: Session) -> dict:
body.app_id, body.user_id
)
if body.action == "import_industry":
- from core.services.application_industry import import_enterprises
+ from features.hosted_apps.industry import import_enterprises
return import_enterprises(service, db, body.app_id, body.user_id, payload)
except ValidationError as error:
diff --git a/src/backend/core/services/application_projects.py b/src/backend/features/hosted_apps/projects.py
similarity index 95%
rename from src/backend/core/services/application_projects.py
rename to src/backend/features/hosted_apps/projects.py
index 088a3c3c..06a5cb55 100644
--- a/src/backend/core/services/application_projects.py
+++ b/src/backend/features/hosted_apps/projects.py
@@ -9,11 +9,11 @@
import json
-from core.auth.permissions_iface import resolve_project_permission
+from core.auth.edition import resolve_project_permission
from core.db.models import ChatSession, Project, UserFolder
from core.infra.time import utc_now
-from core.services.application_store import application_sources, owned_application
-from core.services.project_source import ProjectSourceService
+from features.hosted_apps.store import application_sources, owned_application
+from core.projects.source import ProjectSourceService
from fastapi import HTTPException
from sqlalchemy import insert, select
@@ -27,7 +27,7 @@ def __init__(self, db, engine):
def editor(self, app_id, owner, chat_id=""):
- from core.services.application_publication import publication_lock
+ from features.hosted_apps.publication import publication_lock
with publication_lock(self.engine, app_id):
project, app = self.ensure(app_id, owner)
@@ -128,7 +128,7 @@ def _provision(self, app, owner, project_id):
# These are system-provisioned projects, with stable identity and metadata.
# Folder creation still uses the ordinary owner lock, validation and audit.
from core.db.repository import AuditLogRepository
- from core.services.user_folder_service import UserFolderService
+ from core.projects.user_folders import UserFolderService
folder_name = "MCP-" + app["id"]
folder = (
diff --git a/src/backend/core/services/application_publication.py b/src/backend/features/hosted_apps/publication.py
similarity index 92%
rename from src/backend/core/services/application_publication.py
rename to src/backend/features/hosted_apps/publication.py
index df192a02..6a003beb 100644
--- a/src/backend/core/services/application_publication.py
+++ b/src/backend/features/hosted_apps/publication.py
@@ -17,8 +17,8 @@
from core.db.models import AdminMcpServer
from core.infra.time import utc_now
-from core.services.application_deployments import ApplicationDeploymentService
-from core.services.application_store import mcp_deployments, owned_application
+from features.hosted_apps.deployments import ApplicationDeploymentService
+from features.hosted_apps.store import mcp_deployments, owned_application
from fastapi import HTTPException
from sqlalchemy import select, text
@@ -30,7 +30,7 @@ def publication_lock(engine, app_id):
if not re.fullmatch(r"[0-9a-f]{32}", app_id):
raise HTTPException(404, "Application not found")
if engine.dialect.name == "sqlite":
- from core.capabilities.lockfile import LockTimeout, locked
+ from core.infra.file_lock import LockTimeout, locked
try:
with locked(Path(engine.url.database + ".mcp-publication.lock")):
@@ -73,7 +73,7 @@ def publish(
):
from core.auth.capabilities import resolve_user_capabilities
from core.config.settings import settings
- from core.services import mcp_management_service as management
+ from core.capabilities.mcp import management as management
with publication_lock(self.engine, app_id):
with self.engine.connect() as connection:
@@ -82,8 +82,8 @@ def publish(
"can_add_mcp"
):
raise HTTPException(403, "Administrator has not enabled personal MCP creation")
- from core.services.application_projects import ApplicationProjectService
- from core.services.application_schema import MCPProjectDefinition
+ from features.hosted_apps.projects import ApplicationProjectService
+ from features.hosted_apps.schema import MCPProjectDefinition
from pydantic import ValidationError
projects = ApplicationProjectService(self.db, self.engine)
@@ -140,7 +140,6 @@ def publish(
owner_user_id=owner,
created_by=owner,
created_at=utc_now(),
- is_stable=False,
)
self.db.add(row)
row.display_name = app["title"]
@@ -162,7 +161,7 @@ def publish(
self.db.commit()
management.refresh_mcp_caches()
ok, _ = asyncio.run(
- management.probe_mcp_connectivity(row, self.db, timeout_seconds=15)
+ management.probe_mcp_connectivity(row, validate=True, timeout_seconds=15)
)
if ok:
row.is_enabled = enable
@@ -201,7 +200,7 @@ def publish(
return receipt
def revoke(self, app_id, owner):
- from core.services.mcp_management_service import refresh_mcp_caches
+ from core.capabilities.mcp.management import refresh_mcp_caches
with publication_lock(self.engine, app_id):
with self.engine.connect() as connection:
@@ -217,7 +216,7 @@ def revoke(self, app_id, owner):
return result
def rollback(self, app_id, owner, version):
- from core.services.application_schema import ToolDefinition
+ from features.hosted_apps.schema import ToolDefinition
with self.engine.connect() as connection:
owned_application(connection, app_id, owner)
diff --git a/src/backend/core/services/application_relational.py b/src/backend/features/hosted_apps/relational.py
similarity index 98%
rename from src/backend/core/services/application_relational.py
rename to src/backend/features/hosted_apps/relational.py
index 67d7d558..5d0ca0a9 100644
--- a/src/backend/core/services/application_relational.py
+++ b/src/backend/features/hosted_apps/relational.py
@@ -5,7 +5,7 @@
import math
from datetime import date
-from core.services.application_schema import TableDefinition
+from features.hosted_apps.schema import TableDefinition
from fastapi import HTTPException
from sqlalchemy import (
JSON,
diff --git a/src/backend/features/hosted_apps/resources/__init__.py b/src/backend/features/hosted_apps/resources/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/hosted_apps/resources/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/features/hosted_apps/resources/admission.py b/src/backend/features/hosted_apps/resources/admission.py
new file mode 100644
index 00000000..3d82d2a5
--- /dev/null
+++ b/src/backend/features/hosted_apps/resources/admission.py
@@ -0,0 +1,159 @@
+"""Application resource admission uses the existing worker, quota and janitor."""
+
+import hashlib
+import json
+import time
+
+from fastapi import HTTPException
+
+from core.capabilities.plugins.resources import service, store
+from core.capabilities.plugins.resources.owners import application_binding
+from core.capabilities.plugins.resources.reservation import lock_owner
+from core.db.threads import run_in_session
+
+
+def reserve(db, selected, module_id, user_id, binding):
+ """One idempotent admission attempt under the owner lock (see ``service._admit``)."""
+ lock_owner(db, user_id)
+ row = db.get(store.PluginResource, binding["resource_id"])
+ if row is None:
+ return "reserve", None # caller retains the quota/admission lock
+ if (
+ row.user_id != user_id
+ or row.request_digest != binding["request_digest"]
+ or row.application_key != binding["application_key"]
+ or row.instance_id != binding["instance_id"]
+ or row.install_id != selected.install_id
+ or row.revision != selected.revision
+ or row.module_id != module_id
+ ):
+ db.rollback()
+ raise HTTPException(409, "idempotency_conflict")
+ if row.status == "active":
+ service.authorized(db, row.resource_id, user_id)
+ db.commit()
+ return "adopted", row
+ if row.status != "starting" or row.expires_at <= time.time():
+ db.commit()
+ raise HTTPException(410, "resource_closed_no_replay")
+ db.commit()
+ return "wait", None
+
+
+def _record_quota_denied(db, user_id, item, instance_id):
+ from core.harness.registry.resource_owners import resource_owner
+
+ resource_owner().record(db, user_id, "resource", "quota_denied", item=item, instance_id=instance_id)
+
+
+async def create(
+ user_id, key, instance_id, install_id, revision, execution_scope, module_id, idempotency_key
+):
+ if not idempotency_key or len(idempotency_key) > 128:
+ raise HTTPException(400, "idempotency_key_required")
+ item = await run_in_session(
+ application_binding, user_id, key, install_id, revision, execution_scope, module_id
+ )
+ payload = [key, instance_id, install_id, revision, execution_scope, module_id]
+ digest = hashlib.sha256(json.dumps(payload).encode()).hexdigest()
+ resource_id = hashlib.sha256(
+ json.dumps([user_id, key, instance_id, idempotency_key]).encode()
+ ).hexdigest()
+ try:
+ return await service.create(
+ module_id=module_id,
+ user_id=user_id,
+ chat_id=None,
+ slug=item["binding"]["slug"],
+ install_id=install_id,
+ _application={
+ "application_key": key,
+ "instance_id": instance_id,
+ "revision": revision,
+ "scope": execution_scope,
+ "request_digest": digest,
+ "resource_id": resource_id,
+ },
+ )
+ except HTTPException as error:
+ if error.status_code == 429:
+ try:
+ await run_in_session(_record_quota_denied, user_id, item, instance_id)
+ except Exception:
+ pass # The original quota denial remains authoritative if telemetry storage fails.
+ raise
+
+
+def owned(db, user_id, key, instance_id, resource_id, *, allow_closed=False):
+ row = store.get(db, resource_id, user_id)
+ if row.application_key != key or row.instance_id != instance_id or row.chat_id is not None:
+ raise HTTPException(404, "resource_unavailable")
+ if allow_closed:
+ # Revoked grants stop all use; authenticated owners may still release their worker.
+ return row
+ return service.authorized(db, resource_id, user_id)
+
+
+def _extend_lease(db, user_id, key, instance_id, resource_id):
+ """Extend a foreground lease; returns True when the background lease owns it instead."""
+ from core.harness.registry.resource_owners import resource_owner
+
+ row = owned(db, user_id, key, instance_id, resource_id)
+ if resource_owner().authorize(db, row):
+ return True
+ updated = (
+ db.query(store.PluginResource)
+ .filter_by(resource_id=row.resource_id, status="active")
+ .filter(store.PluginResource.expires_at > time.time())
+ .update({"expires_at": time.time() + 90}, synchronize_session=False)
+ )
+ db.commit()
+ if updated != 1:
+ raise HTTPException(410, "resource_expired")
+ return False
+
+
+def _public(db, user_id, resource_id):
+ return store.public(store.get(db, resource_id, user_id))
+
+
+async def renew(user_id, key, instance_id, resource_id):
+ from core.capabilities.plugins.resources.deadlines import arm
+ from core.harness.registry.resource_owners import resource_owner
+
+ if await run_in_session(_extend_lease, user_id, key, instance_id, resource_id):
+ await resource_owner().renew(user_id, key, instance_id, resource_id)
+ else:
+ await arm(resource_id)
+ return await run_in_session(_public, user_id, resource_id)
+
+
+def _application_rows(db, user_id, application_key, instance_id, install_id):
+ query = db.query(store.PluginResource).filter(
+ store.PluginResource.user_id == user_id,
+ store.PluginResource.application_key.is_not(None),
+ store.PluginResource.status != "closed",
+ )
+ for name, value in (
+ ("application_key", application_key),
+ ("instance_id", instance_id),
+ ("install_id", install_id),
+ ):
+ if value is not None:
+ query = query.filter(getattr(store.PluginResource, name) == value)
+ rows = query.all()
+ db.expunge_all()
+ return rows
+
+
+async def close_application_resources(
+ user_id, *, application_key=None, instance_id=None, install_id=None
+):
+ rows = await run_in_session(_application_rows, user_id, application_key, instance_id, install_id)
+ failures = []
+ for row in rows:
+ try:
+ await service.close(row)
+ except Exception:
+ failures.append(row.resource_id)
+ return {"closed": not failures, "retry_resource_ids": failures}
diff --git a/src/backend/features/hosted_apps/resources/cleanup.py b/src/backend/features/hosted_apps/resources/cleanup.py
new file mode 100644
index 00000000..50912f0a
--- /dev/null
+++ b/src/backend/features/hosted_apps/resources/cleanup.py
@@ -0,0 +1,71 @@
+"""Revoke application access before stopping a worker; retain failed cleanup."""
+
+import asyncio
+
+from fastapi import HTTPException
+
+from core.capabilities.plugins.resources import store
+from core.capabilities.plugins.resources.owners import runtime_session
+from core.db.threads import run_in_session
+
+
+def _begin(db, resource_id):
+ """Revoke the worker's tickets; return a detached row and its target, or None."""
+ from core.harness.registry.resource_owners import resource_owner
+
+ row = db.get(store.PluginResource, resource_id)
+ if row is None or row.status == "closed":
+ return None
+ row.status = "closing"
+ db.query(store.PluginResourceTicket).filter_by(resource_id=resource_id).delete()
+ db.query(store.PluginResourceAssetTicket).filter_by(resource_id=resource_id).delete()
+ db.commit()
+ try:
+ target = store.descriptor(row)
+ except HTTPException:
+ resource_owner().observe(db, row, "release", "release_failed")
+ raise HTTPException(503, "resource_cleanup_credentials_unavailable") from None
+ db.expunge(row)
+ return row, target
+
+
+def _finish(db, resource_id, outcome):
+ from core.harness.registry.resource_owners import resource_owner
+
+ row = db.get(store.PluginResource, resource_id)
+ if row is None:
+ return
+ if outcome == "ok":
+ row.status = "closed"
+ db.commit()
+ resource_owner().observe(db, row, "release", outcome)
+
+
+async def close(row):
+ """Stop an application worker. Database steps and network waits never overlap."""
+ from core.capabilities.plugins.resources.service import request
+ from core.sandbox.factory import get_sandbox_provider
+
+ begun = await run_in_session(_begin, row.resource_id)
+ if begun is None:
+ return
+ row, target = begun
+ if target.get("process_id"):
+ try:
+ await asyncio.wait_for(request(row, "/shutdown", {}), 5)
+ except (HTTPException, TimeoutError):
+ provider = get_sandbox_provider()
+ if target.get("provider") != provider.name:
+ await run_in_session(_finish, row.resource_id, "release_failed")
+ raise HTTPException(503, "resource_cleanup_scope_unavailable")
+ result = await provider.write_stdin(
+ target["process_id"],
+ sandbox_session_id=runtime_session(row),
+ user_id=row.user_id,
+ chars="\x03",
+ yield_time_ms=1000,
+ )
+ if not result or result.get("exit_code") is None:
+ await run_in_session(_finish, row.resource_id, "release_failed")
+ raise HTTPException(503, "resource_cleanup_pending")
+ await run_in_session(_finish, row.resource_id, "ok")
diff --git a/src/backend/core/services/application_schema.py b/src/backend/features/hosted_apps/schema.py
similarity index 100%
rename from src/backend/core/services/application_schema.py
rename to src/backend/features/hosted_apps/schema.py
diff --git a/src/backend/core/services/application_sql_policy.py b/src/backend/features/hosted_apps/sql_policy.py
similarity index 96%
rename from src/backend/core/services/application_sql_policy.py
rename to src/backend/features/hosted_apps/sql_policy.py
index 94f39fc2..6c96edfd 100644
--- a/src/backend/core/services/application_sql_policy.py
+++ b/src/backend/features/hosted_apps/sql_policy.py
@@ -4,7 +4,7 @@
from contextlib import contextmanager
-from core.services.application_relational import sql_table
+from features.hosted_apps.relational import sql_table
from sqlalchemy import text
diff --git a/src/backend/core/services/application_store.py b/src/backend/features/hosted_apps/store.py
similarity index 94%
rename from src/backend/core/services/application_store.py
rename to src/backend/features/hosted_apps/store.py
index 6b57c3e5..bd7bff35 100644
--- a/src/backend/core/services/application_store.py
+++ b/src/backend/features/hosted_apps/store.py
@@ -67,7 +67,7 @@
@lru_cache(maxsize=1)
def application_engine() -> Engine:
- from core.config.application_hosting import application_hosting_settings
+ from features.hosted_apps.config import application_hosting_settings
from core.config.settings import settings
url = application_hosting_settings.database_url
@@ -112,14 +112,14 @@ def application_engine() -> Engine:
def initialize_store(engine: Engine) -> None:
"""Explicit provisioning entry point; runtime requests never create registry tables."""
- from core.services.application_history import collection_history
+ from features.hosted_apps.history import collection_history
metadata.create_all(engine)
def initialize_local_store() -> None:
"""Provision the managed local SQLite store at startup, never cloud databases."""
- from core.auth.desktop_bridge import bridge_enabled
- from core.config.application_hosting import application_hosting_settings
+ from features.desktop.auth.bridge import bridge_enabled
+ from features.hosted_apps.config import application_hosting_settings
from core.config.settings import settings
if (
@@ -132,7 +132,7 @@ def initialize_local_store() -> None:
def require_store(engine: Engine) -> None:
- from core.services.application_history import collection_history
+ from features.hosted_apps.history import collection_history
try:
ready = all(
inspect(engine).has_table(table.name) for table in (applications, application_sources, metadata.tables["hosted_collection_history"])
diff --git a/src/backend/features/installed.py b/src/backend/features/installed.py
new file mode 100644
index 00000000..89dc92c1
--- /dev/null
+++ b/src/backend/features/installed.py
@@ -0,0 +1,3 @@
+"""Community modules; runtime switches do not affect schema registration."""
+
+INSTALLED_FEATURES: tuple[str, ...] = ("personal_agent", "features.evaluation", "features.automation", "features.sites", "features.hosted_apps", "integrations.dingtalk", "integrations.lark", "integrations.email", "integrations.yida", "integrations.channels", "features.workbench", "features.knowledge", "features.marketplace", "features.desktop")
diff --git a/src/backend/features/knowledge/README.md b/src/backend/features/knowledge/README.md
new file mode 100644
index 00000000..88b9e510
--- /dev/null
+++ b/src/backend/features/knowledge/README.md
@@ -0,0 +1,11 @@
+# 知识库
+
+拥有知识空间、文档、分块、媒体资产、Wiki 模型、权限、索引队列和外接知识库路由。FULL 与 CE 均安装模型;用户删除清理随模型注册,独立于运行开关。CE 通过 edition.py 选择权限、外部 provider 和请求模型。
+
+解析、分块、语义切分、媒体描述与索引分别归属对应模块。向量存储的客户端、索引写入和检索各自维护职责;共享 embedding/reranking 客户端位于 core.models,由所有领域复用。缓存仍为进程内唯一所有者,调用方引用明确模块,不保留旧路径转发。
+
+核心通过 catalog_options 取得管理选项,通过 retrieval_context 取得外部 MCP 请求上下文。模型请求头由宿主注册的环境解析器处理,知识库和模型层不直接依赖桌面实现。实际桌面身份校验与设备头逻辑保持在原桌面模块,P5 随其目录归属移动。
+
+API 按原端点顺序组合为一个中央路由入口。依赖 core 的模型、存储、认证、数据库及通用解析接口。独立模型加载与空库升级必须验证,测试使用隔离数据库和模拟外部模型/向量服务。
+
+版本差异统一由 edition.py 选择权限、外接 provider、检索、请求模型与初始授权;CE 只覆盖此入口。
diff --git a/src/backend/core/kb/__init__.py b/src/backend/features/knowledge/__init__.py
similarity index 64%
rename from src/backend/core/kb/__init__.py
rename to src/backend/features/knowledge/__init__.py
index 90d8da71..ed3a59e1 100644
--- a/src/backend/core/kb/__init__.py
+++ b/src/backend/features/knowledge/__init__.py
@@ -1,5 +1,5 @@
"""Knowledge-base subsystem: provider seam, document parser/chunker, and vector store.
Relocated from the top-level ``utils/`` package (where it was mislabeled as generic
-utilities) into ``core`` alongside core.content.kb_processing and core.services.kb_service.
+utilities) into ``core`` alongside features.knowledge.processing and features.knowledge.service.
"""
diff --git a/src/backend/features/knowledge/api/__init__.py b/src/backend/features/knowledge/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/knowledge/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/features/knowledge/api/kb.py b/src/backend/features/knowledge/api/kb.py
new file mode 100644
index 00000000..cdaa17ab
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb.py
@@ -0,0 +1,7 @@
+"""Compose knowledge routes in the original endpoint order."""
+from fastapi import APIRouter
+from features.knowledge.api import kb_assets, kb_spaces, kb_documents, kb_mutations, kb_chunks
+
+router = APIRouter()
+for group in (kb_assets, kb_spaces, kb_documents, kb_mutations, kb_chunks):
+ router.include_router(group.router, prefix="/v1/catalog/kb", tags=["Knowledge Base"])
diff --git a/src/backend/features/knowledge/api/kb_assets.py b/src/backend/features/knowledge/api/kb_assets.py
new file mode 100644
index 00000000..45c8df82
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_assets.py
@@ -0,0 +1,147 @@
+"""kb assets responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+router = APIRouter()
+from typing import Optional
+from core.auth.backend import UserContext, get_current_user
+from core.files.content.file_validation import validate_kb_file
+from core.db.engine import get_db
+from core.infra.exceptions import BadRequestError, FileTooLargeError, ResourceNotFoundError
+from core.infra.responses import success_response
+from core.files.storage import get_storage
+from fastapi import Depends, File, Form, Path, UploadFile
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge.api import kb_helpers
+from features.knowledge.api import kb_state
+
+@router.get("/assets/{asset_id}", summary="读取知识库媒体资产")
+def get_kb_asset(
+ asset_id: str = Path(..., description="Asset ID"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """按 ID 读取知识库中的媒体资产(版面解析切出的图片等)原始字节。
+
+ 资产的可见性完全跟随所属知识库空间:能读该库才能读它的图。图片链接会出现在分块正文
+ 与检索结果里,故按 ID 直取,不经过库/文档路径。
+ """
+ from fastapi.responses import Response
+
+ from features.knowledge.models import KBAsset
+ from features.knowledge.repository import KBRepository
+
+ asset = db.query(KBAsset).filter(KBAsset.asset_id == asset_id).first()
+ if not asset:
+ raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
+
+ kb_space = KBRepository(db).get_space(asset.kb_id)
+ if not kb_space or not kb_helpers.can_read_space(db, kb_space, user.user_id):
+ # Same shape as a missing asset: an unauthorised caller must not be able to
+ # probe which asset ids exist.
+ raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
+
+ try:
+ data = get_storage().download_bytes(asset.storage_key)
+ except Exception as exc:
+ logger.warning("Asset %s read failed: %s", asset_id, exc)
+ raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id)
+
+ return Response(
+ content=data,
+ media_type=asset.mime_type or "application/octet-stream",
+ headers={"Cache-Control": "private, max-age=86400"},
+ )
+
+
+@router.post("/preview-chunks", summary="预览文档分块效果")
+async def preview_chunks(
+ file: UploadFile = File(..., description="Document file to preview"),
+ chunk_method: str = Form(
+ "structured", description="Chunking method: structured|recursive|embedding_semantic|laws|qa"
+ ),
+ parent_chunk_size: int = Form(1024, description="Parent chunk size in tokens"),
+ child_chunk_size: int = Form(128, description="Child chunk size in tokens"),
+ overlap_tokens: int = Form(20, description="Overlap between child chunks"),
+ parent_child_indexing: bool = Form(
+ True, description="Enable parent-child indexing; False = flat chunks only"
+ ),
+ separators: Optional[str] = Form(
+ None, description="父分块分隔符,JSON 数组字符串;为空用默认层级"
+ ),
+ child_separators: Optional[str] = Form(
+ None, description="子分块分隔符,JSON 数组字符串;为空走定长滑窗"
+ ),
+ user: UserContext = Depends(get_current_user),
+):
+ """预览文档的分块效果,不落库、不写入向量库。上传文件并按指定分块方法(structured/recursive/embedding_semantic/laws/qa)与父子分块参数解析,返回父块及其子块预览,用于创建知识库前调参。文件大小受 MAX_PREVIEW_SIZE(20MB)限制。"""
+ content = await file.read()
+ if len(content) > kb_state.MAX_PREVIEW_SIZE:
+ raise FileTooLargeError(max_size=kb_state.MAX_PREVIEW_SIZE, actual_size=len(content))
+ if len(content) == 0:
+ raise BadRequestError(message="File is empty", data={"filename": file.filename})
+
+ if not file.filename:
+ raise BadRequestError(message="Filename is required", data={"field": "file"})
+ _, mime_type = validate_kb_file(file.filename, content)
+
+ from features.knowledge.parser_tokens import count_tokens as _count_tokens
+ from features.knowledge.kb_parser import parse_and_chunk
+ from features.knowledge.parser_chunks import parse_separators_form
+
+ _separators = parse_separators_form(separators)
+ _child_separators = parse_separators_form(child_separators)
+
+ embed_fn = None
+ if chunk_method == "embedding_semantic":
+ from core.models.embedding import embed_batch
+
+ embed_fn = embed_batch
+
+ parent_chunks = parse_and_chunk(
+ content,
+ mime_type,
+ chunk_method=chunk_method,
+ parent_size=parent_chunk_size,
+ child_size=child_chunk_size,
+ overlap=overlap_tokens,
+ embed_fn=embed_fn,
+ separators=_separators,
+ child_separators=_child_separators,
+ )
+
+ total_children = sum(len(pc.children) for pc in parent_chunks) if parent_child_indexing else 0
+
+ chunks_preview = []
+ for idx, pc in enumerate(parent_chunks):
+ if parent_child_indexing:
+ children_preview = [
+ {
+ "index": c.index,
+ "content": c.content[:200] + ("..." if len(c.content) > 200 else ""),
+ }
+ for c in pc.children[:5]
+ ]
+ children_count = len(pc.children)
+ else:
+ children_preview = []
+ children_count = 0
+ chunks_preview.append(
+ {
+ "index": idx,
+ "content": pc.content,
+ "token_count": _count_tokens(pc.content),
+ "children_count": children_count,
+ "children_preview": children_preview,
+ }
+ )
+
+ return success_response(
+ data={
+ "total_chunks": len(parent_chunks),
+ "total_children": total_children,
+ "chunks": chunks_preview,
+ },
+ message="Chunk preview generated successfully",
+ )
diff --git a/src/backend/features/knowledge/api/kb_chunks.py b/src/backend/features/knowledge/api/kb_chunks.py
new file mode 100644
index 00000000..2cef7385
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_chunks.py
@@ -0,0 +1,175 @@
+"""kb chunks responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+router = APIRouter()
+from typing import Optional
+from features.knowledge.edition import UpdateChunkRequest
+from core.auth.backend import UserContext, get_current_user
+from core.db.engine import get_db
+from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
+from core.infra.responses import paginated_response, success_response
+from features.knowledge.service import KBService
+from fastapi import Depends, Path, Query
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge.api import kb_helpers
+
+@router.get("/{kb_id}/chunks", summary="获取文档分块列表(含标签和问题)")
+def list_chunks(
+ kb_id: str = Path(..., description="KB space ID"),
+ document_id: Optional[str] = Query(None, description="Filter by document ID"),
+ page: int = Query(1, ge=1),
+ page_size: int = Query(20, ge=1, le=100),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """分页获取知识库分块列表,可选按 document_id 过滤。每个分块返回完整正文内容、标签和问题列表,供分块管理界面查看与编辑。仅空间所有者可访问。"""
+ from features.knowledge.models import KBChunk
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+ if not kb_space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ # Public KBs are readable by all logged-in users (chunks are read-only viewing)
+ if not kb_helpers.can_read_space(db, kb_space, user.user_id):
+ raise AccessDeniedError(message="Access denied", reason="Not the KB owner")
+
+ query = db.query(KBChunk).filter(KBChunk.kb_id == kb_id)
+ if document_id:
+ query = query.filter(KBChunk.document_id == document_id)
+ total = query.count()
+ chunks = (
+ query.order_by(KBChunk.chunk_index).offset((page - 1) * page_size).limit(page_size).all()
+ )
+
+ items = [
+ {
+ "chunk_id": c.chunk_id,
+ "document_id": c.document_id,
+ "chunk_index": c.chunk_index,
+ "content": c.content,
+ "tags": c.tags or [],
+ "questions": c.questions or [],
+ }
+ for c in chunks
+ ]
+ return paginated_response(items=items, page=page, page_size=page_size, total_items=total)
+
+
+@router.get("/{kb_id}/chunks/{chunk_id}/children", summary="获取父块下的子块列表")
+def list_chunk_children(
+ kb_id: str = Path(..., description="KB space ID"),
+ chunk_id: str = Path(..., description="Parent chunk ID"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """返回某父块在向量库里的子块(父子分块模式)。扁平模式无子块时返回空列表。仅空间可读者可访问。"""
+ from features.knowledge.models import KBChunk
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+ if not kb_space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ if not kb_helpers.can_read_space(db, kb_space, user.user_id):
+ raise AccessDeniedError(message="Access denied", reason="Not the KB owner")
+
+ parent = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first()
+ if not parent:
+ raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id)
+
+ from features.knowledge.vector_indexing import get_children_for_parent
+
+ children = get_children_for_parent(chunk_id)
+ return success_response(
+ data={"children": children, "count": len(children)},
+ message="Chunk children retrieved successfully",
+ )
+
+
+@router.patch("/{kb_id}/chunks/{chunk_id}", summary="更新分块内容/标签/问题")
+def update_chunk(
+ kb_id: str = Path(..., description="KB space ID"),
+ chunk_id: str = Path(..., description="Chunk ID"),
+ request: UpdateChunkRequest = ...,
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """更新指定分块的正文内容、标签和问题列表,并同步刷新其在向量库中的向量与索引。
+
+ 传入 ``content`` 时会对该分块重新向量化(dense + sparse);仅改标签时只刷新稀疏/标签
+ 索引;问题始终单独重建问题向量行。仅空间所有者(admin 权限)可操作,系统托管知识库禁止
+ 编辑分块;仅传入的字段会被更新。"""
+ from features.knowledge.models import KBChunk, KBDocument
+ from features.knowledge.repository import KBRepository
+ from features.knowledge.vector_indexing import reindex_chunk_content, reindex_chunk_tags, upsert_question_rows
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+ if not kb_space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ kb_helpers.require_kb_write(db, kb_space, user.user_id, "admin")
+ if KBService(db)._is_system_managed_space(kb_space):
+ raise AccessDeniedError(
+ message="Access denied", reason="System managed KB does not allow chunk edits"
+ )
+
+ chunk = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first()
+ if not chunk:
+ raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id)
+
+ content_changed = request.content is not None
+ if content_changed:
+ new_content = request.content.strip()
+ if not new_content:
+ raise BadRequestError(message="分块内容不能为空", data={"field": "content"})
+ chunk.content = new_content
+ if request.tags is not None:
+ chunk.tags = request.tags
+ if request.questions is not None:
+ chunk.questions = request.questions
+ db.commit()
+ db.refresh(chunk)
+
+ try:
+ doc = db.query(KBDocument).filter_by(document_id=chunk.document_id).first()
+ doc_title = doc.title if doc else ""
+
+ if content_changed:
+ # Content changed: re-vectorize the whole chunk (current tags already included)
+ reindex_chunk_content(
+ chunk_id=chunk_id,
+ content=chunk.content,
+ tags=chunk.tags or [],
+ user_id=user.user_id,
+ kb_id=kb_id,
+ document_id=chunk.document_id,
+ title=doc_title,
+ chunk_index=chunk.chunk_index,
+ )
+ elif request.tags is not None:
+ reindex_chunk_tags(chunk_id, chunk.content, chunk.tags)
+ if request.questions is not None:
+ upsert_question_rows(
+ parent_chunk_id=chunk_id,
+ questions=chunk.questions,
+ user_id=user.user_id,
+ kb_id=kb_id,
+ document_id=chunk.document_id,
+ title=doc_title,
+ chunk_index=chunk.chunk_index,
+ )
+ except Exception as exc:
+ logger.warning("Milvus re-index failed for chunk %s: %s", chunk_id, exc)
+
+ return success_response(
+ data={
+ "chunk_id": chunk.chunk_id,
+ "content": chunk.content,
+ "tags": chunk.tags,
+ "questions": chunk.questions,
+ },
+ message="Chunk updated successfully",
+ )
diff --git a/src/backend/features/knowledge/api/kb_documents.py b/src/backend/features/knowledge/api/kb_documents.py
new file mode 100644
index 00000000..3711db87
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_documents.py
@@ -0,0 +1,254 @@
+"""kb documents responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+router = APIRouter()
+import hashlib
+import json
+import os
+from typing import Optional
+from core.auth.backend import UserContext, get_current_user
+from core.files.content.file_validation import validate_kb_file
+from features.knowledge.wiki.config import index_modes_of
+from core.db.engine import get_db
+from core.infra.exceptions import AccessDeniedError, BadRequestError, FileTooLargeError, ResourceNotFoundError, StorageError
+from core.infra.responses import created_response, paginated_response, success_response
+from features.knowledge.service import KBService
+from core.files.storage import generate_storage_key, get_storage
+from fastapi import Depends, File, Form, Path, Query, UploadFile, status
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge.api import kb_helpers
+from features.knowledge.api import kb_state
+
+@router.post("/{kb_id}/documents", status_code=status.HTTP_201_CREATED, summary="上传文档到知识库")
+def upload_document(
+ kb_id: str = Path(..., description="KB space ID"),
+ file: UploadFile = File(..., description="Document file to upload"),
+ title: Optional[str] = Form(None, description="Document title (defaults to filename)"),
+ metadata: Optional[str] = Form(None, description="Additional metadata as JSON string"),
+ indexing_config: Optional[str] = Form(None, description="Indexing config as JSON string"),
+ chunk_method: Optional[str] = Form(None, description="Chunking method override"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """上传文档到指定知识库空间并异步建立索引。仅空间所有者可上传;文件大小受 MAX_FILE_SIZE(100MB)限制,可选传入标题、元数据、索引配置及分块方法覆盖。上传成功后通过后台任务完成解析、分块与向量化,立即返回 indexing_status=processing。"""
+ kb_service = KBService(db)
+
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+
+ if not kb_space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ kb_helpers.require_kb_write(db, kb_space, user.user_id, "edit")
+
+ if not file.filename:
+ raise BadRequestError(message="Filename is required", data={"field": "file"})
+
+ content = file.file.read()
+ file_size = len(content)
+
+ if file_size > kb_state.MAX_FILE_SIZE:
+ raise FileTooLargeError(max_size=kb_state.MAX_FILE_SIZE, actual_size=file_size)
+ if file_size == 0:
+ raise BadRequestError(message="File is empty", data={"filename": file.filename})
+
+ _, mime_type = validate_kb_file(file.filename, content)
+
+ checksum = hashlib.sha256(content).hexdigest()
+ env = os.getenv("ENVIRONMENT", "dev")
+ storage_key = generate_storage_key(
+ env=env, user_id=user.user_id, category="kb_documents", filename=file.filename
+ )
+
+ try:
+ storage = get_storage()
+ # 对象存储是阻塞的网络调用,直接在 async handler 里调会卡住整个事件循环——
+ # 批量上传时后端会连 SSE 和其它接口一起停摆。
+ storage.upload_bytes(content, storage_key)
+ logger.info("Document uploaded to storage: %s", storage_key)
+ except StorageError as e:
+ logger.error("Failed to upload document to storage: %s", e)
+ raise StorageError(
+ operation="upload", error=f"Failed to upload document: {e.data.get('error', str(e))}"
+ )
+
+ doc_title = title or file.filename
+
+ if metadata:
+ try:
+ json.loads(metadata)
+ except json.JSONDecodeError:
+ raise BadRequestError(message="Invalid metadata JSON", data={"field": "metadata"})
+
+ try:
+ document = kb_service.upload_document(
+ kb_id=kb_id,
+ user_id=user.user_id,
+ title=doc_title,
+ filename=file.filename,
+ size_bytes=file_size,
+ mime_type=mime_type,
+ storage_key=storage_key,
+ checksum=checksum,
+ )
+ except PermissionError as e:
+ raise AccessDeniedError(message="Access denied", reason=str(e))
+
+ _effective_chunk_method = chunk_method or kb_space.chunk_method or "semantic"
+ _upload_indexing_config = None
+ if indexing_config:
+ try:
+ _upload_indexing_config = json.loads(indexing_config)
+ except json.JSONDecodeError:
+ raise BadRequestError(
+ message="Invalid indexing_config JSON", data={"field": "indexing_config"}
+ )
+ _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {}
+ _indexing_config = _upload_indexing_config or _space_extra.get("indexing_config")
+
+ # 排进持久化索引队列,由常驻 worker 认领执行(见 core/kb/index_worker.py)。
+ # 早先这里挂的是 FastAPI BackgroundTasks:同步任务落进 Starlette 的共享请求线程池,
+ # 一次多选几十个文件就把池占满,后续请求连 handler 都进不去——前端一直转圈、网关
+ # 报 502;而且任务是进程内的,后端一重启就全丢,文档永远停在「索引中」。
+ kb_helpers.enqueue_document_indexing(
+ db,
+ document_id=document["document_id"],
+ user_id=user.user_id,
+ chunk_method=_effective_chunk_method,
+ indexing_config=_indexing_config,
+ index_modes=index_modes_of(kb_space),
+ )
+
+ return created_response(
+ data={**document, "indexing_status": "processing"},
+ message="Document uploaded successfully. Indexing in progress.",
+ )
+
+
+@router.get("/{kb_id}/documents", summary="获取知识库文档列表")
+def list_documents(
+ kb_id: str = Path(..., description="Local or external knowledge collection ID"),
+ page: int = Query(1, ge=1, description="Page number"),
+ page_size: int = Query(20, ge=1, le=100, description="Items per page"),
+ keyword: Optional[str] = Query(None, description="按标题/文件名搜索(全库范围,再分页)"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """分页获取知识库文档;非本地 ID 交由当前版本的外部知识提供方处理。"""
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+
+ if not kb_space:
+ from features.knowledge.edition import is_enabled, list_documents
+
+ if is_enabled():
+ result = list_documents(kb_id, page=page, limit=page_size, keyword=keyword or "")
+ response = paginated_response(
+ items=result.get("items", []),
+ page=result.get("page", page),
+ page_size=result.get("page_size", page_size),
+ total_items=result.get("total", 0),
+ message="Documents retrieved successfully",
+ )
+ from features.knowledge.edition import count_document_statuses
+
+ status_counts = count_document_statuses(
+ kb_id,
+ keyword=keyword or "",
+ initial_result=result,
+ )
+ if status_counts is not None:
+ response["data"]["status_counts"] = status_counts
+ return response
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+
+ # Public KBs (created in the admin console, owned by the system principal) are readable by all logged-in users
+ if not kb_helpers.can_read_space(db, kb_space, user.user_id):
+ raise AccessDeniedError(
+ message="Access denied", reason="Only the KB space owner can list documents"
+ )
+
+ documents, total = kb_repo.list_documents(kb_id, page, page_size, keyword=keyword)
+ items = [
+ {
+ "id": d.document_id,
+ "title": d.title,
+ "desc": d.filename,
+ "filename": d.filename,
+ "size": d.size_bytes,
+ "mime_type": d.mime_type,
+ "storage_key": d.storage_key,
+ "uploaded_at": d.uploaded_at.isoformat(),
+ "indexing_status": getattr(d, "indexing_status", "processing"),
+ }
+ for d in documents
+ ]
+ response = paginated_response(
+ items=items,
+ page=page,
+ page_size=page_size,
+ total_items=total,
+ message="Documents retrieved successfully",
+ )
+ response["data"]["status_counts"] = kb_repo.count_document_statuses(kb_id, keyword=keyword)
+ return response
+
+
+@router.get("/{kb_id}/documents/{document_id}", summary="获取知识库文档详情")
+def get_document_detail(
+ kb_id: str = Path(..., description="Local or external knowledge collection ID"),
+ document_id: str = Path(..., description="Document ID"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """获取文档详情;非本地 ID 交由当前版本的外部知识提供方处理。"""
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+
+ if not kb_space:
+ from features.knowledge.edition import get_document_detail, is_enabled
+
+ if is_enabled():
+ detail = get_document_detail(kb_id, document_id)
+ return success_response(data=detail, message="Document detail retrieved successfully")
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+
+ # Public KBs are readable by all logged-in users
+ if not kb_helpers.can_read_space(db, kb_space, user.user_id):
+ raise AccessDeniedError(
+ message="Access denied", reason="Only the KB space owner can view document details"
+ )
+
+ document = kb_repo.get_document(document_id)
+ if not document or document.kb_id != kb_id:
+ raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
+
+ from features.knowledge.models import KBChunk
+
+ chunks = (
+ db.query(KBChunk)
+ .filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id)
+ .order_by(KBChunk.chunk_index)
+ .all()
+ )
+ content = "\n\n".join(c.content for c in chunks) if chunks else ""
+
+ return success_response(
+ data={
+ "id": document.document_id,
+ "title": document.title,
+ "desc": document.filename,
+ "filename": document.filename,
+ "mime_type": document.mime_type,
+ "uploaded_at": document.uploaded_at.isoformat(),
+ "content": content,
+ },
+ message="Document detail retrieved successfully",
+ )
diff --git a/src/backend/features/knowledge/api/kb_helpers.py b/src/backend/features/knowledge/api/kb_helpers.py
new file mode 100644
index 00000000..c22a0bfc
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_helpers.py
@@ -0,0 +1,105 @@
+"""kb helpers responsibilities."""
+from __future__ import annotations
+from typing import Optional
+from core.infra.exceptions import AccessDeniedError
+from core.models.chat_models import get_summarize_model
+from core.llm.runtime.message_compat import extract_text_from_chat_response
+import logging
+logger = logging.getLogger(__name__)
+
+def can_read_space(db, space, user_id: str) -> bool:
+ """Read permission: owner / everyone for public KBs / grantees of a scoped-visibility KB (single-source-of-truth resolution)."""
+ from features.knowledge.edition import resolve_local_kb_level
+
+ return resolve_local_kb_level(db, user_id, space.kb_id) != "none"
+
+
+def require_kb_write(db, space, user_id: str, required: str) -> None:
+ """Write permission check: required is 'edit' (upload/reindex) or 'admin' (rename/delete/edit chunks).
+
+ Raises AccessDeniedError when insufficient. The owner is always admin, superadmins are always admin, granted users go by their level.
+ """
+ from features.knowledge.edition import has_kb_permission, resolve_local_kb_level
+
+ level = resolve_local_kb_level(db, user_id, space.kb_id)
+ if not has_kb_permission(level, required):
+ raise AccessDeniedError(message="Access denied", reason=f"requires kb level >= {required}")
+
+
+def enqueue_document_indexing(
+ db,
+ *,
+ document_id: str,
+ user_id: str,
+ chunk_method: str,
+ indexing_config: Optional[dict],
+ index_modes: Optional[list],
+) -> None:
+ """把文档排进持久化索引队列,交给常驻 worker 执行。
+
+ 索引不能再跑在请求线程池里:那是 Starlette 与所有同步依赖(``get_db``)共用的
+ 池子,批量上传会把它占满,导致整个后端在 handler 之前就排队。详见
+ :mod:`core.kb.index_worker`。
+ """
+ from features.knowledge.models import KBDocument
+ from features.knowledge import index_queue
+
+ doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first()
+ if doc is None:
+ logger.warning("Cannot enqueue indexing: document %s not found", document_id)
+ return
+ index_queue.enqueue(
+ db,
+ doc,
+ user_id=user_id,
+ chunk_method=chunk_method,
+ indexing_config=indexing_config,
+ index_modes=index_modes,
+ )
+
+
+def fallback_kb_description(name: str, description: str) -> str:
+ base_name = (name or "").strip()
+ base_desc = " ".join((description or "").split())
+ if base_desc:
+ return (
+ f"{base_name}相关资料,适合检索概念说明、政策信息、行业动态、业务规则和常见问题。"
+ f"当前范围包括:{base_desc[:120]}"
+ )[:220]
+ return (
+ f"{base_name}相关内容,适合检索概念说明、政策解读、业务规则、行业动态、"
+ f"典型案例和常见问题。"
+ )[:220]
+
+
+async def generate_kb_description(name: str, description: str) -> str:
+ prompt = f"""/no_think
+你是一名知识库配置助手。请根据知识库名称和现有简介,生成一段更利于智能路由判断的“知识库简介”。
+
+目标:
+1. 帮助系统快速判断什么问题应该路由到这个知识库
+2. 文本要像知识库简介,不是宣传文案
+3. 明确主题范围、文档类型、适合回答的问题类型
+4. 用中文,1 段即可,控制在 60 到 120 字
+5. 不要使用“本知识库可以帮助你”这类口语化表达
+6. 不要使用“该知识库聚焦”“本知识库涵盖”这类空泛起手式
+7. 直接输出简介正文,不要加标题、引号、序号或解释
+
+知识库名称:{name}
+现有简介:{description or "无"}
+"""
+
+ try:
+ model = get_summarize_model()
+ # AgentScope 2.0: model.__call__ takes list[Msg] (not list[dict])
+ from agentscope.message import Msg, TextBlock
+
+ result = await model(
+ messages=[Msg(name="user", role="user", content=[TextBlock(type="text", text=prompt)])]
+ )
+ polished = extract_text_from_chat_response(result).strip()
+ polished = " ".join(polished.split())
+ return polished or fallback_kb_description(name, description)
+ except Exception as exc:
+ logger.warning("Failed to polish KB description for %s: %s", name, exc)
+ return fallback_kb_description(name, description)
diff --git a/src/backend/features/knowledge/api/kb_mutations.py b/src/backend/features/knowledge/api/kb_mutations.py
new file mode 100644
index 00000000..33b8eee2
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_mutations.py
@@ -0,0 +1,117 @@
+"""kb mutations responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+router = APIRouter()
+from typing import Optional
+from features.knowledge.edition import ReindexRequest
+from core.auth.backend import UserContext, get_current_user
+from features.knowledge.wiki.config import index_modes_of
+from core.db.engine import get_db
+from core.infra.exceptions import ResourceNotFoundError
+from core.infra.responses import success_response
+from features.knowledge.service import KBService
+from fastapi import Depends, Path
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge.api import kb_helpers
+
+@router.delete("/{kb_id}", summary="删除知识库空间")
+def delete_kb_space(
+ kb_id: str = Path(..., description="KB space ID"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """删除指定知识库空间及其下属文档、分块与向量数据。仅空间所有者可操作;空间不存在或无权访问时返回 404。此操作不可恢复。"""
+ kb_service = KBService(db)
+ deleted = kb_service.delete_space(kb_id, user.user_id)
+ if not deleted:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ return success_response(data=None, message="Knowledge base deleted successfully")
+
+
+@router.delete("/{kb_id}/documents/{document_id}", summary="删除知识库文档")
+def delete_kb_document(
+ kb_id: str = Path(..., description="KB space ID"),
+ document_id: str = Path(..., description="Document ID"),
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """从知识库空间删除指定文档及其分块与向量数据。仅文档所属空间的所有者可操作;文档不存在或无权访问时返回 404。此操作不可恢复。"""
+ kb_service = KBService(db)
+ deleted = kb_service.delete_document(document_id, user.user_id)
+ if not deleted:
+ raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
+ return success_response(data=None, message="Document deleted successfully")
+
+
+@router.post("/{kb_id}/documents/{document_id}/reindex", summary="重新索引文档")
+def reindex_document(
+ kb_id: str = Path(..., description="KB space ID"),
+ document_id: str = Path(..., description="Document ID"),
+ request: Optional[ReindexRequest] = None,
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """对指定文档重新建立索引。会先清除该文档的旧分块及向量数据,再从存储重新下载原文,按可选的索引配置与分块方法异步重新解析、分块、向量化。仅空间所有者可操作,立即返回 indexing_status=processing。"""
+ from features.knowledge.models import KBChunk
+ from features.knowledge.repository import KBRepository
+
+ kb_repo = KBRepository(db)
+ kb_space = kb_repo.get_space(kb_id)
+ if not kb_space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ kb_helpers.require_kb_write(db, kb_space, user.user_id, "edit")
+
+ document = kb_repo.get_document(document_id)
+ if not document or document.kb_id != kb_id:
+ raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id)
+
+ db.query(KBChunk).filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id).delete()
+ document.indexing_status = "processing"
+ db.commit()
+
+ # 资产随分块一起重建:不清会在每次重索引后留下一份孤儿图(存储与表都涨)。
+ try:
+ from features.knowledge.kb_assets import purge_document_assets
+
+ purge_document_assets(db, document_id)
+ except Exception as exc:
+ logger.warning("Asset cleanup failed for reindex of %s: %s", document_id, exc)
+
+ try:
+ from features.knowledge.vector_indexing import delete_by_document
+
+ delete_by_document(document_id, user.user_id)
+ except Exception as exc:
+ logger.warning("Milvus cleanup failed for reindex of %s: %s", document_id, exc)
+
+ _idx_cfg = None
+ if request and request.indexing_config:
+ _idx_cfg = request.indexing_config.model_dump()
+ if not _idx_cfg:
+ _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {}
+ _idx_cfg = _space_extra.get("indexing_config")
+
+ _reindex_chunk_method = (
+ (request.chunk_method if request and request.chunk_method else None)
+ or kb_space.chunk_method
+ or "semantic"
+ )
+
+ # 与上传同一条路:排队 + 常驻 worker,原文由 worker 自己从存储取回。
+ # 这里不再预先 download_bytes——几十 MB 的原文既没必要在请求里读,也不该跟着
+ # 任务在内存里排队等着跑。
+ kb_helpers.enqueue_document_indexing(
+ db,
+ document_id=document_id,
+ user_id=user.user_id,
+ chunk_method=_reindex_chunk_method,
+ indexing_config=_idx_cfg,
+ index_modes=index_modes_of(kb_space),
+ )
+
+ return success_response(
+ data={"document_id": document_id, "indexing_status": "processing"},
+ message="Document re-indexing started",
+ )
diff --git a/src/backend/features/knowledge/api/kb_spaces.py b/src/backend/features/knowledge/api/kb_spaces.py
new file mode 100644
index 00000000..82b17cfe
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_spaces.py
@@ -0,0 +1,116 @@
+"""kb spaces responsibilities."""
+from __future__ import annotations
+from fastapi import APIRouter
+router = APIRouter()
+from features.knowledge.edition import CreateKBSpaceRequest, PolishKBDescriptionRequest, UpdateKBSpaceRequest
+from core.auth.backend import UserContext, get_current_user
+from core.config.settings import settings
+from features.knowledge.wiki.config import normalize_index_modes
+from core.db.engine import get_db
+from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError
+from core.infra.responses import created_response, success_response
+from features.knowledge.service import KBService
+from fastapi import Depends, status
+from sqlalchemy.orm import Session
+from features.knowledge.api import kb_helpers
+
+@router.post("", status_code=status.HTTP_201_CREATED, summary="创建知识库空间")
+def create_kb_space(
+ request: CreateKBSpaceRequest,
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """创建归属当前登录用户的知识库空间;可见性规则由当前版本决定。"""
+ visibility = (request.visibility or "private").strip()
+ if visibility not in ("private", "public"):
+ raise BadRequestError(
+ message="visibility 仅支持 private / public", data={"field": "visibility"}
+ )
+ if settings.edition.edition == "ce" and visibility != "private":
+ raise BadRequestError(
+ message="CE 仅支持私有知识库",
+ data={"field": "visibility", "allowed": ["private"]},
+ )
+
+ # Permission check: require the creation permission matching the chosen visibility
+ from core.auth.capabilities import resolve_user_capabilities
+
+ caps = resolve_user_capabilities(db, user.user_id)
+ required_cap = "can_create_public_kb" if visibility == "public" else "can_create_private_kb"
+ if not caps.get(required_cap, False):
+ label = "公有" if visibility == "public" else "私有"
+ raise AccessDeniedError(message="Access denied", reason=f"缺少「创建{label}知识库」权限")
+
+ kb_service = KBService(db)
+ metadata = dict(request.metadata or {})
+ if request.indexing_config:
+ metadata["indexing_config"] = request.indexing_config.model_dump()
+
+ # 索引模式:rag(向量检索)/ wiki(LLM Wiki 图谱),两者同选即 LLM-Wiki 知识库。
+ # 三种组合都会分块——wiki 省掉的是向量化,不是分块(引文标注要 chunk id、
+ # 回溯原文要按 chunk 直取)。
+ metadata["index_modes"] = normalize_index_modes(request.index_modes)
+ if request.wiki_config:
+ metadata["wiki_config"] = {
+ k: v for k, v in request.wiki_config.model_dump().items() if v is not None
+ }
+
+ from features.knowledge.edition import initial_visibility_grants
+
+ initial_grants = initial_visibility_grants(db, user.user_id, visibility)
+
+ space = kb_service.create_space(
+ user_id=user.user_id,
+ name=request.name,
+ description=request.description,
+ chunk_method=request.chunk_method or "semantic",
+ metadata=metadata,
+ visibility=visibility,
+ initial_grants=initial_grants,
+ granted_by=user.user_id,
+ )
+ return created_response(data=space, message="Knowledge base space created successfully")
+
+
+@router.post("/polish-description", summary="AI 生成知识库简介")
+async def polish_kb_description(
+ request: PolishKBDescriptionRequest,
+ user: UserContext = Depends(get_current_user),
+):
+ """根据知识库名称和现有简介,调用大模型生成一段更利于智能路由判断的知识库简介。名称为必填,模型失败时回退到本地模板生成。仅返回润色后的简介文本,不修改任何知识库记录。"""
+ name = request.name.strip()
+ description = (request.description or "").strip()
+ if not name:
+ raise BadRequestError(message="Knowledge base name is required", data={"field": "name"})
+
+ polished = await kb_helpers.generate_kb_description(name, description)
+ return success_response(
+ data={"description": polished},
+ message="Knowledge base description polished successfully",
+ )
+
+
+@router.patch("/{kb_id}", summary="更新知识库空间")
+def update_kb_space(
+ kb_id: str,
+ request: UpdateKBSpaceRequest,
+ user: UserContext = Depends(get_current_user),
+ db: Session = Depends(get_db),
+):
+ """更新指定知识库空间的名称、简介与索引模式。仅空间所有者可操作;空间不存在或无权访问时返回 404。
+
+ 改索引模式不会自动补建索引:新勾选 Wiki 后要调 `POST /{kb_id}/wiki/rebuild`
+ 才会把已有文档排进生成队列。
+ """
+ kb_service = KBService(db)
+ space = kb_service.update_space(
+ kb_id=kb_id,
+ user_id=user.user_id,
+ name=request.name,
+ description=request.description,
+ index_modes=request.index_modes,
+ wiki_config=(request.wiki_config.model_dump() if request.wiki_config else None),
+ )
+ if not space:
+ raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id)
+ return success_response(data=space, message="Knowledge base space updated successfully")
diff --git a/src/backend/features/knowledge/api/kb_state.py b/src/backend/features/knowledge/api/kb_state.py
new file mode 100644
index 00000000..dad6eb4f
--- /dev/null
+++ b/src/backend/features/knowledge/api/kb_state.py
@@ -0,0 +1,8 @@
+"""kb state responsibilities."""
+from __future__ import annotations
+
+
+MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB
+
+
+MAX_PREVIEW_SIZE = 20 * 1024 * 1024 # 20MB
diff --git a/src/backend/api/routes/v1/kb_wiki.py b/src/backend/features/knowledge/api/kb_wiki.py
similarity index 93%
rename from src/backend/api/routes/v1/kb_wiki.py
rename to src/backend/features/knowledge/api/kb_wiki.py
index faa7cf70..084013ed 100644
--- a/src/backend/api/routes/v1/kb_wiki.py
+++ b/src/backend/features/knowledge/api/kb_wiki.py
@@ -29,16 +29,16 @@
# 单页最多返回的 Wiki 页数,避免前端一次拉爆几千页的库
MAX_PAGE_SIZE = 200
# 图谱单次最多返回节点数——前端渲染性能的闸,与实现层共用 core 侧的定义
-from core.kb.wiki.config import MAX_GRAPH_NODES as MAX_GRAPH_LIMIT
+from features.knowledge.wiki.config import MAX_GRAPH_NODES as MAX_GRAPH_LIMIT
def _assert_readable(db: Session, user_id: str, kb_id: str) -> None:
"""自建库要过读权限;外接库的可见性由其实现模块的白名单负责。"""
- from core.kb.wiki_router import is_local_kb
+ from features.knowledge.wiki_router import is_local_kb
if not is_local_kb(kb_id):
return
- from core.auth.kb_permissions import resolve_local_kb_level
+ from features.knowledge.edition import resolve_local_kb_level
if resolve_local_kb_level(db, user_id, kb_id) == "none":
raise AccessDeniedError(message="Access denied", reason="无权访问该知识库")
@@ -50,7 +50,7 @@ def resolve_wiki(
db: Session = Depends(get_db),
):
"""取该知识库的 wiki 实现;无权访问 403,不支持 Wiki 则 404。"""
- from core.kb.wiki_router import wiki_module_for
+ from features.knowledge.wiki_router import wiki_module_for
_assert_readable(db, user.user_id, kb_id)
module = wiki_module_for(kb_id)
@@ -74,11 +74,11 @@ async def wiki_capability(
``supports_wiki`` 为真表示「平台上至少有一个 Wiki 源」——可能来自外接后端,
也可能来自任意一个勾选了 Wiki 的自建库。具体某个库有没有,看单库探测。
"""
- from core.kb.wiki_router import supports_wiki
+ from features.knowledge.wiki_router import supports_wiki
provider = ""
try:
- from core.kb.external_provider import get_provider_name
+ from features.knowledge.edition import get_provider_name
provider = get_provider_name()
except Exception: # noqa: BLE001 - 社区版下外接接缝可能整体缺席
@@ -97,7 +97,7 @@ def wiki_capability_for_kb(
db: Session = Depends(get_db),
):
"""单库探测。无权访问按「不具备」返回,不泄露该库是否存在。"""
- from core.kb.wiki_router import is_local_kb, supports_wiki
+ from features.knowledge.wiki_router import is_local_kb, supports_wiki
try:
_assert_readable(db, user.user_id, kb_id)
@@ -110,7 +110,7 @@ def wiki_capability_for_kb(
enabled = supports_wiki(kb_id)
data = {"kb_id": kb_id, "supports_wiki": enabled, "generating": False}
if enabled and is_local_kb(kb_id):
- from core.kb.wiki.jobs import kb_job_summary
+ from features.knowledge.wiki.jobs import kb_job_summary
data.update(kb_job_summary(db, kb_id))
return success_response(data=data, message="Wiki capability resolved")
@@ -279,10 +279,10 @@ def wiki_rebuild(
用于两种场景:建库后才勾上 Wiki(此前上传的文档需要补生成),以及生成结果
不理想时想换个粒度重来。需要写权限——这会实际消耗模型额度。
"""
- from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level
- from core.db.models import KBDocument
- from core.kb.wiki.jobs import enqueue_ingest
- from core.kb.wiki_router import is_local_kb, local_wiki_enabled
+ from features.knowledge.edition import has_kb_permission, resolve_local_kb_level
+ from features.knowledge.models import KBDocument
+ from features.knowledge.wiki.jobs import enqueue_ingest
+ from features.knowledge.wiki_router import is_local_kb, local_wiki_enabled
if not is_local_kb(kb_id):
raise BadRequestError("只有自建知识库支持重新生成 Wiki")
diff --git a/src/backend/features/knowledge/asset_captions.py b/src/backend/features/knowledge/asset_captions.py
new file mode 100644
index 00000000..e8bfc816
--- /dev/null
+++ b/src/backend/features/knowledge/asset_captions.py
@@ -0,0 +1,181 @@
+"""asset captions responsibilities."""
+from __future__ import annotations
+from typing import Sequence
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_assets_state
+
+def vision_available() -> bool:
+ """当前是否有可用的图像理解模型(视觉桥)。"""
+ try:
+ from core.models.vision import is_available
+
+ return is_available()
+ except Exception as exc: # noqa: BLE001 — 视觉桥不可用不该让索引失败
+ logger.debug("视觉桥可用性判定失败,按不可用处理: %s", exc)
+ return False
+
+
+def describe_batch(images: list, loop=None) -> list:
+ """把一批图交给视觉桥识别。返回与输入等长的结果列表(失败位为 None)。
+
+ 视觉桥是异步的,而索引是同步函数、跑在没有运行中事件循环的工作线程里。按可靠性
+ 从高到低三级取宿主循环——只有跑在**宿主循环**上,视觉桥的证据缓存与并发信号量才
+ 真的生效:
+
+ 1. ``loop``:常驻索引 worker 显式传进来的调度循环。这是当前的生产路径——索引跑在
+ worker 自己的 ``ThreadPoolExecutor`` 里(见 ``core/kb/index_worker.py``),那不是
+ anyio 的工作线程,第 2 级在这里拿不到东西。
+ 2. ``anyio.from_thread``:仍经 ``BackgroundTasks`` 进来的调用方(如我的空间同步产物
+ 入库)跑在 Starlette 的 anyio 线程池里,天然带着回宿主循环的通路。
+ 3. 临时循环 + **关掉证据缓存**:CLI / 脚本 / 测试。``core/infra/redis.py`` 的连接池是
+ 模块级全局、绑定首次使用它的循环,让一个转瞬即逝的临时循环抢先创建它,会把整个
+ 进程的 Redis 绑死在已销毁的循环上——那比"这次没吃到缓存"严重得多。
+ """
+ import asyncio
+ import functools
+
+ from core.models.vision import get_vision_bridge
+
+ bridge = get_vision_bridge()
+
+ if loop is not None and not loop.is_closed():
+ future = asyncio.run_coroutine_threadsafe(
+ bridge.describe_many(images, focus=kb_assets_state.CAPTION_FOCUS), loop
+ )
+ return future.result()
+
+ try:
+ import anyio.from_thread
+
+ return anyio.from_thread.run(
+ functools.partial(bridge.describe_many, images, focus=kb_assets_state.CAPTION_FOCUS)
+ )
+ except RuntimeError as exc:
+ logger.debug("视觉桥回宿主循环不可用,改用临时循环且不走缓存: %s", exc)
+
+ async def _uncached() -> list:
+ tasks = [
+ bridge.describe(data, mime, focus=kb_assets_state.CAPTION_FOCUS, use_cache=False)
+ for data, mime in images
+ ]
+ return list(await asyncio.gather(*tasks))
+
+ return asyncio.run(_uncached())
+
+
+def merge_text_content(existing: str, ocr_text: str) -> str:
+ """图注 + OCR 转写合成检索用文本,去重后截断。
+
+ 两者都留:图注是文档作者写的、往往点明了图的用途;OCR 是图里实际的字。
+ """
+ parts: list[str] = []
+ for part in ((existing or "").strip(), (ocr_text or "").strip()):
+ if part and part not in parts:
+ parts.append(part)
+ merged = "\n".join(parts)
+ return merged[:kb_assets_state.MAX_TEXT_CONTENT_CHARS]
+
+
+def evidence_digest(result) -> dict:
+ """留档进 ``extra_data`` 的证据摘要。
+
+ 刻意不存 ``layout.regions`` 与 ``ocr.lines``:整段文字已经进了 ``text_content``,
+ 再存一份逐行结构,一篇几百张图的文档会把 JSONB 撑得很难看。留下的是之后真会用到
+ 的部分——实体/关系(可喂本体与 Wiki 层)、不确定项(人工复核时看)、模型元数据
+ (换模型后判断哪些资产该重跑)。
+ """
+ evidence = result.evidence
+ return {
+ "summary": evidence.summary,
+ "scene": evidence.semantics.scene,
+ "intent": evidence.semantics.intent,
+ "entities": [e.model_dump(exclude_none=True) for e in evidence.semantics.entities],
+ "relations": [r.model_dump(exclude_none=True) for r in evidence.semantics.relations],
+ "uncertainty": list(evidence.uncertainty),
+ "meta": result.to_meta(),
+ }
+
+
+def generate_captions(db, assets: Sequence, *, loop=None) -> int:
+ """给资产补上图像理解结果:``caption`` 写概述,``text_content`` 并入 OCR 转写。
+
+ 走的是 ``core/vision`` 视觉桥——和对话里"纯文本主模型读图"完全同一条通路、同一个
+ ``vision`` 模型角色,因此知识库这边白得三样东西:按图片内容哈希的证据缓存(重新
+ 索引同一篇文档不用再付一次钱)、三协议与结构化输出降级、以及"该角色只能指派多模态
+ 模型"的硬约束。
+
+ 没有可用的视觉模型是正常状态,不是错误:资产保留文档自带的图注并照常入索引,之后
+ 配好 ``vision`` 角色再回填即可,**不用重新解析文档**。
+ """
+ if not assets:
+ return 0
+
+ if not vision_available():
+ logger.info("视觉桥不可用(未指派 vision 角色且主模型不支持读图),跳过图生文;图注仍会入索引")
+ for row in assets:
+ row.caption_status = "skipped"
+ db.commit()
+ return 0
+
+ from core.files.storage import get_storage
+
+ storage = get_storage()
+ images = [a for a in assets if a.kind == "image"]
+ budget, overflow = images[:kb_assets_state.MAX_CAPTIONS_PER_DOC], images[kb_assets_state.MAX_CAPTIONS_PER_DOC:]
+ # 没轮到的显式标注,让"上限生效"在数据里看得见,而不是像视觉模型什么都没返回。
+ for row in overflow + [a for a in assets if a.kind != "image"]:
+ row.caption_status = "skipped"
+ if overflow:
+ logger.warning(
+ "文档图片数 %d 超过单文档图生文上限 %d,超出部分仅用图注索引",
+ len(images),
+ kb_assets_state.MAX_CAPTIONS_PER_DOC,
+ )
+
+ done = 0
+ for offset in range(0, len(budget), kb_assets_state.CAPTION_BATCH):
+ chunk = budget[offset : offset + kb_assets_state.CAPTION_BATCH]
+ payloads: list = []
+ pending: list = []
+ for row in chunk:
+ try:
+ data = storage.download_bytes(row.storage_key)
+ except Exception as exc: # noqa: BLE001 — 单张读不出来不该拖垮整篇
+ logger.warning("资产 %s 读取失败,跳过图生文: %s", row.asset_id, exc)
+ row.caption_status = "failed"
+ continue
+ payloads.append((data, row.mime_type))
+ pending.append(row)
+
+ if not payloads:
+ continue
+
+ try:
+ results = describe_batch(payloads, loop)
+ except Exception as exc: # noqa: BLE001 — 整批失败也只降级,正文索引已经好了
+ logger.warning("视觉桥调用失败,该批 %d 张仅用图注索引: %s", len(pending), exc)
+ for row in pending:
+ row.caption_status = "failed"
+ continue
+
+ for row, result in zip(pending, results):
+ if result is None:
+ row.caption_status = "failed"
+ continue
+ evidence = result.evidence
+ summary = (evidence.summary or "").strip()
+ row.caption = summary
+ row.text_content = merge_text_content(row.text_content, evidence.ocr.full_text)
+ meta = dict(row.extra_data or {})
+ meta["vision"] = evidence_digest(result)
+ row.extra_data = meta
+ row.caption_status = "completed" if summary else "failed"
+ if summary:
+ done += 1
+
+ # 这里给 ``extra_data`` 赋的是**新字典**(不是原地改键),SQLAlchemy 的属性事件能
+ # 直接捕获,不需要 flag_modified —— 那个只有原地改 JSON 才必须。
+ db.commit()
+ logger.info("图生文完成 %d/%d", done, len(budget))
+ return done
diff --git a/src/backend/features/knowledge/asset_indexing.py b/src/backend/features/knowledge/asset_indexing.py
new file mode 100644
index 00000000..10d2d244
--- /dev/null
+++ b/src/backend/features/knowledge/asset_indexing.py
@@ -0,0 +1,88 @@
+"""asset indexing responsibilities."""
+from __future__ import annotations
+from typing import Sequence
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_assets_state
+
+def asset_index_text(asset, *, title: str = "") -> str:
+ """The text an asset is retrieved by: 视觉桥概述 + 图注/OCR 转写 + 文档标题。
+
+ Audio/video reuse this unchanged — ``text_content`` becomes the ASR transcript.
+ """
+ parts = [
+ (asset.caption or "").strip(),
+ (asset.text_content or "").strip(),
+ (title or "").strip(),
+ ]
+ return "\n".join(p for p in parts if p)
+
+
+def index_assets(
+ assets: Sequence, *, user_id: str, kb_id: str, document_id: str, title: str
+) -> int:
+ """Write one ``row_type="image"`` row per asset into the existing collection.
+
+ ``parent_chunk_id`` points at the chunk that contains the asset, so a hit here
+ resolves to that chunk's full text through the existing parent-fetch path. Assets
+ with no owning chunk (a standalone upload whose placeholder was lost) point at
+ themselves and fall back to the caption as their content.
+ """
+ if not assets:
+ return 0
+
+ from features.knowledge.kb_vector_state import TAGS_FIELD_MAX_BYTES, TITLE_FIELD_MAX_BYTES
+ from core.models.embedding import embed_batch
+ from features.knowledge.kb_vector import text_to_sparse
+ from features.knowledge.vector_fields import truncate_utf8
+ from features.knowledge.vector_indexing import upsert_rows
+
+ indexable = [(a, asset_index_text(a, title=title)) for a in assets]
+ indexable = [(a, t) for a, t in indexable if t.strip()]
+ if not indexable:
+ return 0
+
+ try:
+ vectors = embed_batch([t for _, t in indexable])
+ except Exception as exc:
+ logger.error("资产向量化失败,文档 %s 的图片不可检索: %s", document_id, exc)
+ return 0
+
+ rows = []
+ for (asset, text), vector in zip(indexable, vectors):
+ rows.append(
+ {
+ "chunk_id": f"{kb_assets_state.ASSET_ROW_PREFIX}{asset.asset_id}",
+ "parent_chunk_id": asset.chunk_id or f"{kb_assets_state.ASSET_ROW_PREFIX}{asset.asset_id}",
+ "row_type": kb_assets_state.ROW_TYPE_IMAGE,
+ "user_id": user_id,
+ "kb_id": kb_id,
+ "document_id": document_id,
+ "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES),
+ "content": truncate_utf8(text),
+ "tags_text": truncate_utf8(asset.text_content or "", TAGS_FIELD_MAX_BYTES),
+ "chunk_index": asset.asset_index or 0,
+ "dense_embedding": vector,
+ "sparse_embedding": text_to_sparse(text),
+ }
+ )
+
+ BATCH = 100
+ for i in range(0, len(rows), BATCH):
+ upsert_rows(rows[i : i + BATCH])
+ logger.info("文档 %s 写入 %d 条资产检索行", document_id, len(rows))
+ return len(rows)
+
+
+def mark_vector_state(db, assets: Sequence, space: str, state: str) -> None:
+ """Record that these assets are indexed into a vector space.
+
+ ``space`` is ``"text"`` for the caption/OCR rows written by :func:`index_assets`.
+ 路径 B 落地时用 ``"visual"``,与文本行互不覆盖。
+ """
+ for row in assets:
+ current = dict(row.vector_state or {})
+ current[space] = state
+ # 同上:赋新字典而不是原地改键,属性事件自然捕获得到。
+ row.vector_state = current
+ db.commit()
diff --git a/src/backend/features/knowledge/catalog_options.py b/src/backend/features/knowledge/catalog_options.py
new file mode 100644
index 00000000..08d50fc0
--- /dev/null
+++ b/src/backend/features/knowledge/catalog_options.py
@@ -0,0 +1,7 @@
+"""Knowledge options for the administrative agent capability picker."""
+from features.knowledge.models import KBSpace
+
+
+def list_options(db):
+ spaces = db.query(KBSpace).filter(KBSpace.deleted_at.is_(None)).order_by(KBSpace.created_at.desc()).all()
+ return [{"id": space.kb_id, "name": space.name, "description": space.description or ""} for space in spaces]
diff --git a/src/backend/core/kb/external_provider.py b/src/backend/features/knowledge/disabled_provider.py
similarity index 85%
rename from src/backend/core/kb/external_provider.py
rename to src/backend/features/knowledge/disabled_provider.py
index f522757d..1c81453e 100644
--- a/src/backend/core/kb/external_provider.py
+++ b/src/backend/features/knowledge/disabled_provider.py
@@ -37,8 +37,8 @@ def supports_wiki() -> bool:
"""社区版没有外接知识后端,因此这一路不提供 Wiki。
注意这**不代表社区版没有 Wiki**:自建知识库勾选 Wiki 索引模式后,页面由本地
- 管线生成,走 ``core.kb.wiki.local_provider``。要问「某个库有没有 Wiki」应当用
- ``core.kb.wiki_router.supports_wiki(kb_id)``,它会把两条来源都算上。
+ 管线生成,走 ``features.knowledge.wiki.local_provider``。要问「某个库有没有 Wiki」应当用
+ ``features.knowledge.wiki_router.supports_wiki(kb_id)``,它会把两条来源都算上。
"""
return False
diff --git a/src/backend/core/kb/external_retrieval.py b/src/backend/features/knowledge/disabled_retrieval.py
similarity index 100%
rename from src/backend/core/kb/external_retrieval.py
rename to src/backend/features/knowledge/disabled_retrieval.py
diff --git a/src/backend/core/kb/document_status.py b/src/backend/features/knowledge/document_status.py
similarity index 100%
rename from src/backend/core/kb/document_status.py
rename to src/backend/features/knowledge/document_status.py
diff --git a/src/backend/features/knowledge/edition.py b/src/backend/features/knowledge/edition.py
new file mode 100644
index 00000000..dac7fe11
--- /dev/null
+++ b/src/backend/features/knowledge/edition.py
@@ -0,0 +1,33 @@
+"""Community knowledge-base grant policy: resources are owner-only."""
+
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+
+def initial_visibility_grants(db: Session, user_id: str, visibility: str) -> list[tuple[str, str]]:
+ return []
+
+
+__all__ = ["initial_visibility_grants"]
+
+
+def grant_initial_access(db, kb_id, grants, *, granted_by):
+ if grants:
+ raise ValueError("Community knowledge bases do not support shared grants")
+
+
+def __getattr__(name):
+ if name in ('KBLevel', 'filter_accessible_kb_ids', 'get_accessible_local_kb_ids', 'get_accessible_local_kb_levels', 'get_dataset_levels', 'has_kb_permission', 'is_shared_visibility', 'level_to_caps', 'resolve_local_kb_level'):
+ import features.knowledge.personal_permissions as provider
+ return getattr(provider, name)
+ if name in ('get_allowed_collection_ids', 'get_document_detail', 'get_provider_cache_identity', 'get_provider_name', 'is_enabled', 'list_collections', 'list_documents', 'runtime_request_context', 'supports_wiki', 'wiki_module'):
+ import features.knowledge.disabled_provider as provider
+ return getattr(provider, name)
+ if name in ('DatasetRetrievalTimeoutError', 'DatasetRetrievalUnavailableError', 'MAX_RETRIEVE_TOKENS', 'RETRIEVE_MAX_CONCURRENCY', 'RETRIEVE_REQUEST_TIMEOUT_SECONDS', 'RETRIEVE_TOTAL_TIMEOUT_SECONDS', 'list_external_datasets', 'retrieve_dataset_content', 'retrieve_dataset_content_async'):
+ import features.knowledge.disabled_retrieval as provider
+ return getattr(provider, name)
+ if name in ('CreateKBSpaceRequest', 'IndexingConfig', 'PolishKBDescriptionRequest', 'ReindexRequest', 'UpdateChunkRequest', 'UpdateKBSpaceRequest'):
+ import features.knowledge.personal_request_models as provider
+ return getattr(provider, name)
+ raise AttributeError(name)
diff --git a/src/backend/features/knowledge/enrichment.py b/src/backend/features/knowledge/enrichment.py
new file mode 100644
index 00000000..096f66e0
--- /dev/null
+++ b/src/backend/features/knowledge/enrichment.py
@@ -0,0 +1,126 @@
+"""enrichment responsibilities."""
+from __future__ import annotations
+import os
+import re
+import logging
+logger = logging.getLogger(__name__)
+
+def strip_think_tags(text: str) -> str:
+ """Remove ... blocks and markdown formatting from LLM output."""
+ # Remove ... blocks (reasoning model output)
+ text = re.sub(r'.*? ', '', text, flags=re.DOTALL).strip()
+ # Remove markdown bold/italic markers
+ text = re.sub(r'\*+', '', text)
+ # Remove markdown headers
+ text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE)
+ # Remove bullet points
+ text = re.sub(r'^[\-\*]\s*', '', text, flags=re.MULTILINE)
+ return text.strip()
+
+
+def resolve_main_endpoint():
+ """main_agent 端点,带上它说的协议——KB 的两处抽取都得走对那条线。"""
+ from core.models.completions.single_turn import Endpoint
+
+ try:
+ from core.models.config import ModelConfigService
+
+ cfg = ModelConfigService.get_instance().resolve("main_agent")
+ if cfg and cfg.base_url and cfg.model_name:
+ return Endpoint.from_resolved(cfg)
+ except Exception:
+ pass
+ base_url, api_key, model_name = resolve_main_model_config()
+ return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name)
+
+
+def resolve_main_model_config() -> tuple[str, str, str]:
+ """Resolve main_agent model config from DB, with env fallback."""
+ try:
+ from core.models.config import ModelConfigService
+ cfg = ModelConfigService.get_instance().resolve("main_agent")
+ if cfg:
+ return cfg.base_url.rstrip("/"), cfg.api_key, cfg.model_name
+ except Exception:
+ pass
+ return (
+ os.getenv("MODEL_URL", "").rstrip("/"),
+ os.getenv("API_KEY", ""),
+ os.getenv("BASE_MODEL_NAME", ""),
+ )
+
+
+def call_main_model(
+ *, system: str, user: str, temperature: float, max_tokens: int, timeout: int = 30
+) -> str:
+ """跑一次主模型补全;未配置时返回空串,调用方按「这一步没产出」处理。"""
+ import requests as _requests
+
+ from core.models.completions.single_turn import build_request, parse_response, turns
+
+ endpoint = resolve_main_endpoint()
+ if not endpoint.base_url or not endpoint.model_name:
+ return ""
+ url, headers, payload = build_request(
+ endpoint, turns(user, system), temperature=temperature, max_tokens=max_tokens
+ )
+ resp = _requests.post(url, headers=headers, json=payload, timeout=timeout)
+ resp.raise_for_status()
+ return parse_response(endpoint, resp.json()).text
+
+
+def extract_keywords_llm(content: str, count: int) -> list[str]:
+ """Call configured LLM to extract keywords from a text chunk."""
+ try:
+ raw = strip_think_tags(
+ call_main_model(
+ system=(
+ "你是关键词提取工具。"
+ "输出规则:只输出关键词,用英文逗号分隔,不要编号,不要解释,不要分析过程。"
+ "示例输出:数字化转型,产业升级,人工智能"
+ ),
+ user=f"从以下文本提取{count}个关键词,直接输出逗号分隔的关键词:\n\n{content[:2000]}",
+ temperature=0.1,
+ max_tokens=200,
+ )
+ )
+ lines = [ln.strip() for ln in raw.split("\n") if ln.strip()]
+ best_line = raw
+ for line in reversed(lines):
+ if ("," in line or "," in line) and len(line) < 500:
+ best_line = line
+ break
+ keywords = [kw.strip() for kw in best_line.replace(",", ",").split(",") if kw.strip() and len(kw.strip()) < 30]
+ return keywords[:count]
+ except Exception as exc:
+ logger.warning("LLM keyword extraction failed: %s", exc)
+ return []
+
+
+def generate_questions_llm(content: str, count: int) -> list[str]:
+ """Call configured LLM to generate retrieval questions for a text chunk."""
+ try:
+ raw = strip_think_tags(
+ call_main_model(
+ system=(
+ "你是问题生成工具。"
+ "输出规则:每行一个问题,不要编号,不要解释,不要分析过程。直接输出问题本身。"
+ "示例输出:\n什么是数字化转型?\n如何申报产业升级项目?"
+ ),
+ user=f"根据以下文本生成{count}个检索问题,每行一个:\n\n{content[:2000]}",
+ temperature=0.3,
+ max_tokens=500,
+ )
+ )
+ questions = []
+ for q in raw.split("\n"):
+ q = q.strip().lstrip("0123456789.、))-— ").strip()
+ if not q or len(q) < 5:
+ continue
+ if q.startswith(("分析", "角色", "任务", "输出", "规则", "示例")):
+ continue
+ questions.append(q)
+ return questions[:count]
+ except Exception as exc:
+ logger.warning("LLM question generation failed: %s", exc)
+ return []
diff --git a/src/backend/features/knowledge/extension.py b/src/backend/features/knowledge/extension.py
new file mode 100644
index 00000000..0dc313a1
--- /dev/null
+++ b/src/backend/features/knowledge/extension.py
@@ -0,0 +1,19 @@
+"""Installed knowledge models and runtime composition."""
+NAME = "knowledge"
+def setup():
+ return True
+
+
+def register_catalog_options():
+ from core.harness.registry.catalog_options import register_catalog_options as register
+ from features.knowledge.catalog_options import list_options
+ register("kb_spaces", list_options)
+
+
+def register_retrieval_context():
+ from core.harness.registry.retrieval_context import register_retrieval_access_filter
+ from features.knowledge.edition import filter_accessible_kb_ids
+ register_retrieval_access_filter(filter_accessible_kb_ids)
+ from core.harness.registry.retrieval_context import register_retrieval_context as register
+ from features.knowledge import edition as external_provider
+ register(lambda selected_ids: external_provider.runtime_request_context(selected_ids))
diff --git a/src/backend/core/kb/index_queue.py b/src/backend/features/knowledge/index_queue.py
similarity index 99%
rename from src/backend/core/kb/index_queue.py
rename to src/backend/features/knowledge/index_queue.py
index 9450aaaf..8c1a74f8 100644
--- a/src/backend/core/kb/index_queue.py
+++ b/src/backend/features/knowledge/index_queue.py
@@ -31,7 +31,7 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Sequence
-from core.db.models import KBDocument
+from features.knowledge.models import KBDocument
from core.infra.logging import get_logger
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
@@ -231,7 +231,7 @@ def _params_from_space(db: Session, kb_id: str) -> Dict[str, Any]:
所以必须给出**合理**的默认——而不是硬编码一个可能与该库分块方式不符的值,
也不能把 ``user_id`` 留空(向量行按它打标)。
"""
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
space = db.query(KBSpace).filter(KBSpace.kb_id == kb_id).first()
if space is None:
diff --git a/src/backend/core/kb/index_worker.py b/src/backend/features/knowledge/index_worker.py
similarity index 97%
rename from src/backend/core/kb/index_worker.py
rename to src/backend/features/knowledge/index_worker.py
index bfcd81f5..ff6abdf7 100644
--- a/src/backend/core/kb/index_worker.py
+++ b/src/backend/features/knowledge/index_worker.py
@@ -10,7 +10,7 @@
``ThreadPoolExecutor``,并发上限由 ``KB_INDEX_CONCURRENCY`` 控制:索引再多也只会
排在自己的队里,碰不到请求线程池,也不会把 DB 连接池顶穿。
-队列语义与认领/心跳/回收见 :mod:`core.kb.index_queue`。
+队列语义与认领/心跳/回收见 :mod:`features.knowledge.index_queue`。
"""
from __future__ import annotations
@@ -24,7 +24,7 @@
from typing import Dict, Optional
from core.infra.logging import get_logger
-from core.kb import index_queue
+from features.knowledge import index_queue
logger = get_logger(__name__)
@@ -165,8 +165,8 @@ def _run_one(self, snapshot: dict, loop: "asyncio.AbstractEventLoop | None" = No
``loop`` 是调度这次索引的宿主事件循环,用于把异步子系统(当前是图像理解的
视觉桥)的协程投回主循环执行。
"""
- from core.content.kb_processing import vectorise_document_background
- from core.storage import get_storage
+ from features.knowledge.processing import vectorise_document_background
+ from core.files.storage import get_storage
document_id = snapshot["document_id"]
try:
diff --git a/src/backend/features/knowledge/kb_assets.py b/src/backend/features/knowledge/kb_assets.py
new file mode 100644
index 00000000..6a76d6f5
--- /dev/null
+++ b/src/backend/features/knowledge/kb_assets.py
@@ -0,0 +1,231 @@
+"""kb assets responsibilities."""
+from __future__ import annotations
+import os
+import uuid
+from typing import Iterable, Optional, Sequence
+from features.knowledge.parser_types import RawAsset
+from features.knowledge.parser_extract import image_extension_for_mime
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_assets_state
+
+def multimodal_indexing_enabled(indexing_config: Optional[dict]) -> bool:
+ """Whether to extract and index media for this document.
+
+ Per-KB opt-out via ``indexing_config.multimodal_indexing``; the global default
+ lives in ``KB_MULTIMODAL_INDEXING`` so an operator can turn the extra parse cost
+ off fleet-wide without touching each knowledge base.
+ """
+ if isinstance(indexing_config, dict) and indexing_config.get("multimodal_indexing") is not None:
+ return bool(indexing_config.get("multimodal_indexing"))
+ return (os.getenv("KB_MULTIMODAL_INDEXING", "true") or "").strip().lower() not in (
+ "false",
+ "0",
+ "no",
+ )
+
+
+def asset_public_url(asset_id: str) -> str:
+ """Frontend-resolvable URL for an asset. Matches the route in ``api/routes/v1/kb.py``.
+
+ The ``/api`` prefix mirrors the frontend's ``VITE_API_BASE_URL`` default (nginx
+ strips it before the backend sees the path). It is written into stored chunk text,
+ so a deployment that serves the API elsewhere overrides ``KB_ASSET_URL_PREFIX``
+ before indexing rather than rewriting rows afterwards.
+ """
+ prefix = (os.getenv("KB_ASSET_URL_PREFIX", "/api") or "").rstrip("/")
+ return f"{prefix}/v1/catalog/kb/assets/{asset_id}"
+
+
+def is_worth_indexing(asset: RawAsset) -> bool:
+ """Filter out layout noise (rules, bullets, logos) before it costs a vision call."""
+ if len(asset.data) < kb_assets_state.MIN_ASSET_BYTES:
+ return False
+ try:
+ import io
+
+ from PIL import Image # optional dependency (transitive via python-pptx)
+
+ with Image.open(io.BytesIO(asset.data)) as im:
+ width, height = im.size
+ if min(width, height) < kb_assets_state.MIN_ASSET_EDGE_PX:
+ return False
+ except Exception:
+ # Pillow missing or the bytes are not a decodable image — the byte-size floor
+ # above is the fallback filter; never drop an asset because probing failed.
+ pass
+ return True
+
+
+def locate_parent_chunk(parents: Sequence, placeholder: str) -> Optional[object]:
+ """Find the parent chunk whose text references this asset."""
+ for parent in parents:
+ if placeholder and placeholder in (parent.content or ""):
+ return parent
+ return None
+
+
+def rewrite_placeholder(parents: Sequence, placeholder: str, url: str) -> None:
+ """Point the markdown image link at the stored asset.
+
+ Before this, ```` in a chunk was a dead link — the layout
+ service's temp path, with the bytes discarded. Rewriting in both the parent text
+ and its children keeps the stored chunk, the embedded text, and what the UI renders
+ consistent.
+ """
+ needle = f"]({placeholder})"
+ replacement = f"]({url})"
+ for parent in parents:
+ if needle in (parent.content or ""):
+ parent.content = parent.content.replace(needle, replacement)
+ for child in getattr(parent, "children", []) or []:
+ if needle in (child.content or ""):
+ child.content = child.content.replace(needle, replacement)
+
+
+def persist_assets(
+ db,
+ *,
+ kb_id: str,
+ document_id: str,
+ user_id: str,
+ assets: Sequence[RawAsset],
+ parents: Sequence,
+) -> list:
+ """Store asset bytes, create ``kb_assets`` rows, and link them to parent chunks.
+
+ Must run **before** the parent chunks are embedded and written: it rewrites the
+ placeholder links inside ``parents`` in place, and the rewritten text is what gets
+ stored in PostgreSQL and vectorised.
+
+ Failures are per-asset — a storage error on one figure must not fail the document,
+ whose text is already parsed and perfectly usable.
+ """
+ from features.knowledge.models import KBAsset
+ from core.files.storage import generate_storage_key, get_storage
+
+ if not assets:
+ return []
+
+ env = os.getenv("ENVIRONMENT", "dev")
+ storage = get_storage()
+ created: list = []
+
+ for asset in assets:
+ if not is_worth_indexing(asset):
+ continue
+
+ asset_id = uuid.uuid4().hex[:32]
+ suffix = image_extension_for_mime(asset.mime_type) if asset.kind == "image" else ".bin"
+ storage_key = generate_storage_key(
+ env=env,
+ user_id=user_id,
+ category="kb_assets",
+ filename=f"{asset_id}{suffix}",
+ )
+ try:
+ storage.upload_bytes(asset.data, storage_key)
+ except Exception as exc:
+ logger.warning("资产 %s 落盘失败,跳过: %s", asset.placeholder, exc)
+ continue
+
+ parent = locate_parent_chunk(parents, asset.placeholder)
+ row = KBAsset(
+ asset_id=asset_id,
+ kb_id=kb_id,
+ document_id=document_id,
+ chunk_id=getattr(parent, "parent_id", None),
+ kind=asset.kind,
+ mime_type=asset.mime_type,
+ storage_key=storage_key,
+ size_bytes=len(asset.data),
+ asset_index=asset.index,
+ locator=asset.locator or {},
+ text_content=asset.text_content or "",
+ caption_status="pending",
+ vector_state={},
+ extra_data={"placeholder": asset.placeholder},
+ )
+ db.add(row)
+ created.append(row)
+ rewrite_placeholder(parents, asset.placeholder, asset_public_url(asset_id))
+
+ if created:
+ db.commit()
+ for row in created:
+ db.refresh(row)
+ logger.info("文档 %s 抽出并落盘 %d 个资产", document_id, len(created))
+ return created
+
+
+def purge_document_assets(db, document_id: str) -> int:
+ """Delete a document's assets (storage objects + rows). Used on re-index."""
+ from features.knowledge.models import KBAsset
+
+ rows = db.query(KBAsset).filter(KBAsset.document_id == document_id).all()
+ if not rows:
+ return 0
+
+ try:
+ from core.files.storage import get_storage
+
+ storage = get_storage()
+ for row in rows:
+ try:
+ storage.delete(row.storage_key)
+ except Exception as exc:
+ logger.warning("资产 %s 存储清理失败: %s", row.asset_id, exc)
+ except Exception as exc:
+ logger.warning("存储后端不可用,仅清理资产表: %s", exc)
+
+ count = db.query(KBAsset).filter(KBAsset.document_id == document_id).delete()
+ db.commit()
+ return count
+
+
+def fetch_assets_for_chunks(chunk_ids: Iterable[str], *, max_per_chunk: int = 4) -> dict:
+ """Map parent-chunk id → its assets, for attaching to retrieval results.
+
+ Also keyed by the synthetic ``a_`` parent used when an asset has no
+ owning chunk, so a hit on such a row still resolves to its own image.
+ """
+ ids = [c for c in dict.fromkeys(chunk_ids) if c]
+ if not ids:
+ return {}
+
+ direct = [c for c in ids if not c.startswith(kb_assets_state.ASSET_ROW_PREFIX)]
+ orphan_asset_ids = [c[len(kb_assets_state.ASSET_ROW_PREFIX) :] for c in ids if c.startswith(kb_assets_state.ASSET_ROW_PREFIX)]
+
+ try:
+ from core.db.engine import SessionLocal
+ from features.knowledge.models import KBAsset
+ from sqlalchemy import or_
+
+ with SessionLocal() as db:
+ clauses = []
+ if direct:
+ clauses.append(KBAsset.chunk_id.in_(direct))
+ if orphan_asset_ids:
+ clauses.append(KBAsset.asset_id.in_(orphan_asset_ids))
+ if not clauses:
+ return {}
+ rows = db.query(KBAsset).filter(or_(*clauses)).order_by(KBAsset.asset_index).all()
+ grouped: dict[str, list[dict]] = {}
+ for row in rows:
+ key = row.chunk_id or f"{kb_assets_state.ASSET_ROW_PREFIX}{row.asset_id}"
+ bucket = grouped.setdefault(key, [])
+ if len(bucket) >= max_per_chunk:
+ continue
+ bucket.append(
+ {
+ "asset_id": row.asset_id,
+ "kind": row.kind,
+ "url": asset_public_url(row.asset_id),
+ "caption": row.caption or row.text_content or "",
+ "locator": row.locator or {},
+ }
+ )
+ return grouped
+ except Exception as exc:
+ logger.warning("检索结果附图失败(不影响文本结果): %s", exc)
+ return {}
diff --git a/src/backend/features/knowledge/kb_assets_state.py b/src/backend/features/knowledge/kb_assets_state.py
new file mode 100644
index 00000000..214b609c
--- /dev/null
+++ b/src/backend/features/knowledge/kb_assets_state.py
@@ -0,0 +1,31 @@
+"""kb assets state responsibilities."""
+from __future__ import annotations
+import os
+
+ROW_TYPE_IMAGE = "image"
+
+
+ASSET_ROW_PREFIX = "a_"
+
+
+MIN_ASSET_BYTES = int(os.getenv("KB_ASSET_MIN_BYTES", "4096"))
+
+
+MAX_CAPTIONS_PER_DOC = int(os.getenv("KB_ASSET_CAPTION_MAX_PER_DOC", "200"))
+
+
+MIN_ASSET_EDGE_PX = int(os.getenv("KB_ASSET_MIN_EDGE_PX", "64"))
+
+
+CAPTION_BATCH = max(1, int(os.getenv("KB_ASSET_CAPTION_BATCH", "2")))
+
+
+MAX_TEXT_CONTENT_CHARS = max(200, int(os.getenv("KB_ASSET_TEXT_MAX_CHARS", "4000")))
+
+
+CAPTION_FOCUS = (
+ "这是文档中的插图,识别结果会被向量化用于知识库检索。请重点覆盖:"
+ "图的类型(照片/流程图/架构图/折线图/柱状图/表格截图等)、"
+ "图中出现的全部文字与数字(坐标轴、图例、标签、单位)、"
+ "以及图所反映的关键结论或趋势。读不出来的一律写进 uncertainty,不要猜。"
+)
diff --git a/src/backend/features/knowledge/kb_parser.py b/src/backend/features/knowledge/kb_parser.py
new file mode 100644
index 00000000..f4574932
--- /dev/null
+++ b/src/backend/features/knowledge/kb_parser.py
@@ -0,0 +1,116 @@
+"""kb parser responsibilities."""
+from __future__ import annotations
+from typing import Callable, Optional
+from features.knowledge import parser_chunks
+from features.knowledge import parser_extract
+from features.knowledge import parser_semantic
+from features.knowledge import parser_types
+
+def build_parent_child_chunks(
+ paragraphs: list[dict],
+ parent_size: int = 1024,
+ child_size: int = 128,
+ overlap: int = 20,
+ chunk_method: str = "semantic",
+ embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
+ separators: Optional[list[str]] = None,
+ child_separators: Optional[list[str]] = None,
+) -> list[parser_types.ParentChunk]:
+ """Build parent-child chunk pairs from extracted paragraphs.
+
+ Parent chunks: large semantic units (~parent_size tokens), stored in PostgreSQL,
+ returned to LLM as full context.
+ Child chunks: small slices (~child_size tokens), vectorised in Milvus for retrieval.
+
+ ``separators`` (parent-chunk separators): custom separator hierarchy that decides
+ parent-chunk boundaries; only effective on paths that go through recursive
+ splitting (the recursive method itself, and the recursive fallback of semantic
+ chunking for oversized parents); when empty, falls back to the default hierarchy.
+ ``child_separators`` (child-chunk separators): when a parent is split into
+ children, split on these separators (then pack by child_size); when empty,
+ children use the default fixed-length sliding-window split. The two are
+ independent and do not affect each other.
+ """
+ seps = parser_chunks.normalize_separators(separators)
+ csep = parser_chunks.normalize_separators(child_separators)
+ if chunk_method == "qa":
+ parents = parser_chunks.build_qa_chunks(paragraphs)
+ elif chunk_method == "laws":
+ parents = parser_chunks.build_law_chunks(paragraphs, parent_size, child_size, overlap)
+ elif chunk_method == "recursive":
+ parents = parser_chunks.build_recursive_chunks(paragraphs, parent_size, child_size, overlap, separators=seps)
+ elif chunk_method == "embedding_semantic":
+ parents = parser_semantic.build_embedding_semantic_chunks(
+ paragraphs, parent_size, child_size, overlap, embed_fn=embed_fn, separators=seps,
+ )
+ else:
+ # "semantic" / "structured" / default
+ parents = parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap)
+
+ # Child-chunk separators: re-split children uniformly after parents are built
+ # (overriding each builder's default fixed-length children), so we don't have to
+ # thread the same parameter into every builder — one post-processing step covers
+ # all chunking methods.
+ if csep:
+ parser_chunks.apply_child_separators(parents, child_size, csep)
+ return parents
+
+
+def parse_and_chunk(
+ file_bytes: bytes,
+ mime_type: str,
+ chunk_method: str = "semantic",
+ parent_size: int = 1024,
+ child_size: int = 128,
+ overlap: int = 20,
+ embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
+ separators: Optional[list[str]] = None,
+ child_separators: Optional[list[str]] = None,
+) -> list[parser_types.ParentChunk]:
+ """Full pipeline: extract text then build parent-child chunks."""
+ parents, _ = parse_and_chunk_rich(
+ file_bytes,
+ mime_type,
+ chunk_method=chunk_method,
+ parent_size=parent_size,
+ child_size=child_size,
+ overlap=overlap,
+ embed_fn=embed_fn,
+ separators=separators,
+ child_separators=child_separators,
+ with_assets=False,
+ )
+ return parents
+
+
+def parse_and_chunk_rich(
+ file_bytes: bytes,
+ mime_type: str,
+ chunk_method: str = "semantic",
+ parent_size: int = 1024,
+ child_size: int = 128,
+ overlap: int = 20,
+ embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
+ separators: Optional[list[str]] = None,
+ child_separators: Optional[list[str]] = None,
+ with_assets: bool = True,
+) -> tuple[list[parser_types.ParentChunk], list[parser_types.RawAsset]]:
+ """Same pipeline as :func:`parse_and_chunk`, also returning the document's media.
+
+ The assets are returned unpersisted; the caller links them to parent chunks and
+ stores the bytes (see ``core/kb/kb_assets.py``).
+ """
+ paragraphs, assets = parser_extract.extract_text_with_assets(
+ file_bytes, mime_type, with_assets=with_assets
+ )
+ parents = build_parent_child_chunks(
+ paragraphs,
+ parent_size=parent_size,
+ child_size=child_size,
+ overlap=overlap,
+ chunk_method=chunk_method,
+ embed_fn=embed_fn,
+ separators=separators,
+ child_separators=child_separators,
+ )
+ return parents, assets
diff --git a/src/backend/features/knowledge/kb_parser_state.py b/src/backend/features/knowledge/kb_parser_state.py
new file mode 100644
index 00000000..b0d082d2
--- /dev/null
+++ b/src/backend/features/knowledge/kb_parser_state.py
@@ -0,0 +1,34 @@
+"""kb parser state responsibilities."""
+from __future__ import annotations
+import re
+
+MIME_TO_SUFFIX: dict[str, str] = {
+ "application/pdf": ".pdf",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+ "application/msword": ".doc",
+ "text/plain": ".txt",
+ "text/markdown": ".txt",
+ "text/x-markdown": ".txt",
+ "text/csv": ".csv",
+ "application/json": ".txt",
+}
+
+
+MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)\)")
+
+
+IMAGE_MIME_TO_EXT = {
+ "image/png": ".png",
+ "image/jpeg": ".jpg",
+ "image/webp": ".webp",
+ "image/gif": ".gif",
+}
+
+
+HEADING_SOFT_RATIO = 0.5
+
+
+RECURSIVE_SEPARATORS = ["\n\n", "\n", "。", ".", ";", ";", ",", ",", " "]
+
+
+SEPARATOR_ESCAPES = {"\\n": "\n", "\\t": "\t", "\\r": "\r", "\\\\": "\\"}
diff --git a/src/backend/features/knowledge/kb_vector.py b/src/backend/features/knowledge/kb_vector.py
new file mode 100644
index 00000000..248797b0
--- /dev/null
+++ b/src/backend/features/knowledge/kb_vector.py
@@ -0,0 +1,149 @@
+"""kb vector responsibilities."""
+from __future__ import annotations
+import json
+from typing import Any
+from features.knowledge import kb_vector_state
+from features.knowledge import vector_client
+
+def hit_to_dict(ent, score: float) -> dict:
+ """Build the shared KB search-result row. ``ent`` is any entity accessor with
+ ``.get`` (a dict from Milvus Lite, or a Hit entity from hybrid search)."""
+ return {
+ "chunk_id": ent.get("chunk_id"),
+ "parent_chunk_id": ent.get("parent_chunk_id") or ent.get("chunk_id"),
+ "row_type": ent.get("row_type"),
+ "kb_id": ent.get("kb_id"),
+ "document_id": ent.get("document_id"),
+ "title": ent.get("title"),
+ "content": ent.get("content"),
+ "chunk_index": ent.get("chunk_index"),
+ "score": score,
+ }
+
+
+def hybrid_search(
+ user_id: str,
+ kb_ids: list[str],
+ query: str,
+ query_vec: list[float],
+ top_k: int = 10,
+ *,
+ public_kb_ids: list[str] | None = None,
+ timeout: float | None = None,
+) -> list[dict[str, Any]]:
+ """Hybrid search over dense + sparse vectors with RRF fusion.
+
+ Searches both chunk rows and question rows simultaneously.
+ Results are sorted by fused score; dedup by parent_chunk_id is done by the caller.
+
+ ``kb_ids`` are private spaces and are filtered by ``user_id`` (owner isolation).
+ ``public_kb_ids`` are admin-managed public spaces visible to everyone — they are
+ filtered by ``kb_id`` only (no owner check), since public KBs are global and
+ ``kb_id`` is globally unique.
+ """
+ from pymilvus import AnnSearchRequest, MilvusClient, RRFRanker
+
+ vector_client.get_or_create_collection(timeout=timeout)
+ client = vector_client.get_client()
+
+ clauses: list[str] = []
+ if kb_ids:
+ clauses.append(f'(user_id == "{user_id}" and kb_id in {json.dumps(kb_ids)})')
+ if public_kb_ids:
+ clauses.append(f"kb_id in {json.dumps(public_kb_ids)}")
+ if not clauses:
+ return []
+ expr = " or ".join(clauses)
+
+ output_fields = [
+ "chunk_id",
+ "parent_chunk_id",
+ "row_type",
+ "kb_id",
+ "document_id",
+ "title",
+ "content",
+ "chunk_index",
+ ]
+
+ # Milvus Lite (no-Docker local): no sparse field / hybrid search — fall back to
+ # a plain dense ANN search over the same collection. Same result shape.
+ if vector_client.is_lite():
+ lite_res = client.search(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ data=[query_vec],
+ anns_field="dense_embedding",
+ search_params={"metric_type": "IP"},
+ limit=top_k * 2,
+ filter=expr,
+ output_fields=output_fields,
+ timeout=timeout,
+ )
+ return [
+ hit_to_dict(hit.get("entity", hit), hit.get("distance", 0.0)) for hit in lite_res[0]
+ ]
+
+ # Dense vector search request
+ dense_req = AnnSearchRequest(
+ data=[query_vec],
+ anns_field="dense_embedding",
+ param={"metric_type": "IP", "params": {"nprobe": 10}},
+ limit=top_k * 3,
+ expr=expr,
+ )
+
+ # Sparse search request — bag-of-words vector
+ sparse_vec = text_to_sparse(query)
+ sparse_req = AnnSearchRequest(
+ data=[sparse_vec],
+ anns_field="sparse_embedding",
+ param={"metric_type": "IP"},
+ limit=top_k * 3,
+ expr=expr,
+ )
+
+ results = client.hybrid_search(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ reqs=[dense_req, sparse_req],
+ ranker=RRFRanker(k=60),
+ limit=top_k * 2,
+ output_fields=output_fields,
+ timeout=timeout,
+ )
+
+ return [hit_to_dict(hit.entity, hit.score) for hit in results[0]]
+
+
+def build_sparse_text(content: str, tags: list[str]) -> str:
+ """Build combined text for sparse vectorisation."""
+ if not tags:
+ return content
+ tag_str = " ".join(f"[{t}]" for t in tags)
+ return f"{content} {tag_str}"
+
+
+def text_to_sparse(text: str) -> dict[int, float]:
+ """Convert text to a bag-of-words sparse vector using term-frequency + hashing.
+
+ Compatible with Milvus SPARSE_FLOAT_VECTOR field (v2.4+).
+ Each unique token is hashed to a dimension index; value = TF.
+ """
+ import hashlib
+ import re
+ from collections import Counter
+
+ # Simple tokenisation: split on non-alphanumeric (works for CJK + Latin)
+ tokens = re.findall(r"[\w\u4e00-\u9fff]+", text.lower())
+ if not tokens:
+ return {0: 1.0}
+
+ counts = Counter(tokens)
+ total = sum(counts.values())
+
+ sparse: dict[int, float] = {}
+ for token, cnt in counts.items():
+ dim = int(hashlib.md5(token.encode()).hexdigest()[:8], 16) % kb_vector_state.SPARSE_DIM_SPACE
+ tf = cnt / total
+ sparse[dim] = sparse.get(dim, 0.0) + tf
+
+ return sparse if sparse else {0: 1.0}
diff --git a/src/backend/features/knowledge/kb_vector_state.py b/src/backend/features/knowledge/kb_vector_state.py
new file mode 100644
index 00000000..974365e0
--- /dev/null
+++ b/src/backend/features/knowledge/kb_vector_state.py
@@ -0,0 +1,27 @@
+"""kb vector state responsibilities."""
+from __future__ import annotations
+import os
+from typing import Optional
+
+COLLECTION_NAME = "hugagent_kb_private"
+
+
+TITLE_FIELD_MAX_BYTES = 500
+
+
+CONTENT_FIELD_MAX_BYTES = 4096
+
+
+TAGS_FIELD_MAX_BYTES = 1000
+
+
+
+
+
+
+
+
+VERIFIED_COLLECTION_DIM: Optional[int] = None
+
+
+SPARSE_DIM_SPACE = 100_000
diff --git a/src/backend/features/knowledge/models.py b/src/backend/features/knowledge/models.py
new file mode 100644
index 00000000..e61b0355
--- /dev/null
+++ b/src/backend/features/knowledge/models.py
@@ -0,0 +1,377 @@
+"""Knowledge spaces, source documents, assets, chunks and wiki metadata."""
+from core.db.engine import Base
+from core.db.utc_datetime import UTCDateTime
+from core.infra.time import utc_now
+from sqlalchemy import JSON, BigInteger, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text, text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import relationship
+JSONType = JSON().with_variant(JSONB(), "postgresql")
+
+
+class KBSpace(Base):
+ """Knowledge base space table."""
+
+ __tablename__ = "kb_spaces"
+
+ kb_id = Column(String(64), primary_key=True)
+ user_id = Column(
+ String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False
+ )
+ name = Column(String(255), nullable=False)
+ description = Column(Text)
+ document_count = Column(Integer, default=0)
+ total_size_bytes = Column(BigInteger, default=0)
+ visibility = Column(String(16), nullable=False, default="private")
+ chunk_method = Column(String(32), nullable=False, default="semantic")
+ extra_data = Column("metadata", JSONType, default={})
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ # Relationships
+ user = relationship("UserShadow")
+ documents = relationship("KBDocument", back_populates="kb_space", cascade="all, delete-orphan")
+
+ __table_args__ = (
+ CheckConstraint("length(name) >= 1 AND length(name) <= 255", name="kb_spaces_name_length"),
+ CheckConstraint("document_count >= 0", name="kb_spaces_document_count_check"),
+ CheckConstraint("total_size_bytes >= 0", name="kb_spaces_total_size_check"),
+ CheckConstraint(
+ "visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check"
+ ),
+ Index("idx_kb_spaces_user_id", "user_id"),
+ Index("idx_kb_spaces_updated_at", "updated_at"),
+ Index(
+ "idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None)
+ ),
+ Index("idx_kb_spaces_visibility", "visibility"),
+ )
+
+class KBDocument(Base):
+ """Knowledge base document table."""
+
+ __tablename__ = "kb_documents"
+
+ document_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ title = Column(String(500), nullable=False)
+ filename = Column(String(500), nullable=False)
+ size_bytes = Column(BigInteger, nullable=False)
+ mime_type = Column(String(100), nullable=False)
+ storage_key = Column(Text, nullable=False)
+ storage_url = Column(Text)
+ checksum = Column(String(64))
+ indexing_status = Column(
+ String(20), nullable=False, default="processing"
+ ) # processing | completed | failed
+ extra_data = Column("metadata", JSONType, default={})
+ uploaded_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ # Relationships
+ kb_space = relationship("KBSpace", back_populates="documents")
+ chunks = relationship("KBChunk", back_populates="document", cascade="all, delete-orphan")
+
+ __table_args__ = (
+ CheckConstraint("size_bytes > 0", name="kb_documents_size_check"),
+ CheckConstraint("length(filename) >= 1", name="kb_documents_filename_length"),
+ Index("idx_kb_documents_kb_id", "kb_id"),
+ Index("idx_kb_documents_uploaded_at", "uploaded_at"),
+ Index("idx_kb_documents_kb_uploaded", "kb_id", "uploaded_at"),
+ Index(
+ "idx_kb_documents_deleted",
+ "deleted_at",
+ postgresql_where=Column("deleted_at").isnot(None),
+ ),
+ Index("idx_kb_documents_metadata_gin", "metadata", postgresql_using="gin"),
+ )
+
+class KBChunk(Base):
+ """Knowledge base chunk table - stores parent chunks for context retrieval.
+
+ Each document is split into parent chunks (stored here) and child chunks
+ (vectorised in Milvus hugagent_kb_private collection). Retrieval finds child
+ chunks via vector search, then fetches the parent content from this table.
+ """
+
+ __tablename__ = "kb_chunks"
+
+ chunk_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ document_id = Column(
+ String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False
+ )
+ chunk_index = Column(Integer, nullable=False)
+ content = Column(
+ Text, nullable=False
+ ) # parent chunk original text, returned to the LLM on retrieval hit
+ tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"]
+ questions = Column(JSONType, default=list) # associated question list (array of strings)
+ char_start = Column(Integer)
+ char_end = Column(Integer)
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+
+ # Relationships
+ document = relationship("KBDocument", back_populates="chunks")
+
+ __table_args__ = (
+ Index("idx_kb_chunks_kb_id", "kb_id"),
+ Index("idx_kb_chunks_document_id", "document_id"),
+ Index("idx_kb_chunks_kb_doc", "kb_id", "document_id"),
+ )
+
+class KBAsset(Base):
+ """Non-text media cut out of a knowledge-base document (图片 / 后续音视频).
+
+ One row per figure the layout engine extracted (or per standalone uploaded image).
+ The bytes live in object storage under ``storage_key``; this table holds the
+ linkage (which document / which parent chunk), where it sits in the source
+ (``locator``), and the two text projections retrieval runs on:
+
+ ``text_content`` literal text carried by the medium — 图注 / OCR 文字;音视频接入后是 ASR 转写。
+ ``caption`` 语义描述,由 VLM 生成(未配置 vlm 角色时为空,检索退化为只用 text_content)。
+
+ Deliberately medium-agnostic so audio/video need no migration: ``kind`` widens,
+ ``locator`` holds ``{page_idx, bbox}`` for images and would hold ``{start_ms,
+ end_ms}`` for a transcript segment.
+
+ ``vector_state`` records which vector spaces this asset has been indexed into
+ (``{"text": "ok"}`` today). Path B — a true multimodal embedding in its own
+ collection — adds a second key here instead of another column.
+ """
+
+ __tablename__ = "kb_assets"
+
+ asset_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ document_id = Column(
+ String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False
+ )
+ # Owning parent chunk. Nullable: an asset whose placeholder cannot be located in
+ # any chunk is still worth keeping (it stays retrievable on its own row).
+ chunk_id = Column(String(64))
+ kind = Column(String(16), nullable=False, default="image") # image | audio | video
+ mime_type = Column(String(100), nullable=False, default="application/octet-stream")
+ storage_key = Column(Text, nullable=False)
+ size_bytes = Column(BigInteger, default=0)
+ asset_index = Column(Integer, nullable=False, default=0) # order within the document
+ locator = Column(JSONType, default=dict) # image: {page_idx, bbox}; av: {start_ms, end_ms}
+ caption = Column(Text) # VLM 语义描述
+ text_content = Column(Text) # 图注 / OCR / ASR 转写
+ caption_status = Column(
+ String(20), nullable=False, default="pending"
+ ) # pending | completed | skipped | failed
+ vector_state = Column(JSONType, default=dict) # {"text": "ok"} —— 路径 B 在此加键
+ extra_data = Column("metadata", JSONType, default={})
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+
+ __table_args__ = (
+ CheckConstraint("kind IN ('image', 'audio', 'video')", name="kb_assets_kind_check"),
+ Index("idx_kb_assets_kb_id", "kb_id"),
+ Index("idx_kb_assets_document_id", "document_id"),
+ Index("idx_kb_assets_chunk_id", "chunk_id"),
+ )
+
+WIKI_CATEGORY_MAX_DEPTH = 3
+
+WIKI_PAGE_TYPE_SUMMARY = "summary"
+
+WIKI_PAGE_TYPE_ENTITY = "entity"
+
+WIKI_PAGE_TYPE_CONCEPT = "concept"
+
+WIKI_PAGE_TYPE_INDEX = "index"
+
+WIKI_PAGE_TYPE_SYNTHESIS = "synthesis"
+
+WIKI_PAGE_TYPE_COMPARISON = "comparison"
+
+WIKI_PAGE_TYPES = (
+ WIKI_PAGE_TYPE_SUMMARY,
+ WIKI_PAGE_TYPE_ENTITY,
+ WIKI_PAGE_TYPE_CONCEPT,
+ WIKI_PAGE_TYPE_INDEX,
+ WIKI_PAGE_TYPE_SYNTHESIS,
+ WIKI_PAGE_TYPE_COMPARISON,
+)
+
+WIKI_KNOWLEDGE_PAGE_TYPES = (
+ WIKI_PAGE_TYPE_ENTITY,
+ WIKI_PAGE_TYPE_CONCEPT,
+ WIKI_PAGE_TYPE_SYNTHESIS,
+ WIKI_PAGE_TYPE_COMPARISON,
+)
+
+WIKI_PAGE_TYPE_LABELS = {
+ WIKI_PAGE_TYPE_CONCEPT: "概念",
+ WIKI_PAGE_TYPE_ENTITY: "实体",
+ WIKI_PAGE_TYPE_SYNTHESIS: "综述",
+ WIKI_PAGE_TYPE_COMPARISON: "对比",
+ WIKI_PAGE_TYPE_SUMMARY: "文档摘要",
+ WIKI_PAGE_TYPE_INDEX: "总索引",
+}
+
+WIKI_EDIT_SOURCE_PIPELINE = "pipeline"
+
+WIKI_EDIT_SOURCE_AGENT = "agent"
+
+WIKI_EDIT_SOURCE_USER = "user"
+
+class KBWikiFolder(Base):
+ """Wiki 目录节点(邻接表)。
+
+ ``parent_id`` 为空串表示根层。``path`` 是用 ``/`` 连接的名称链(物化路径),
+ 只为展示与筛选服务;归属关系的真源永远是 ``parent_id``。空目录允许独立存在,
+ 这样用户可以先搭骨架再往里填页。
+ """
+
+ __tablename__ = "kb_wiki_folders"
+
+ folder_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ parent_id = Column(String(64), nullable=False, default="")
+ name = Column(String(255), nullable=False)
+ path = Column(String(1024), nullable=False, default="")
+ depth = Column(Integer, nullable=False, default=0)
+ sort_order = Column(Integer, nullable=False, default=0)
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ Index("idx_kb_wiki_folders_kb_parent", "kb_id", "parent_id"),
+ Index(
+ "idx_kb_wiki_folders_unique_name",
+ "kb_id",
+ "parent_id",
+ "name",
+ unique=True,
+ postgresql_where=Column("deleted_at").is_(None),
+ ),
+ )
+
+class KBWikiPage(Base):
+ """一个 Wiki 页面。
+
+ ``folder_id`` 是页面归属目录的唯一真源;``category_path`` / ``wiki_path`` /
+ ``depth`` / ``sort_order`` 都是从目录链派生出来的缓存投影,重建目录时会一并刷新。
+
+ 血缘两列的语义要分清:``source_refs`` 是该页取材自哪些文档
+ (``kb_documents.document_id``),``chunk_refs`` 是具体哪几个父块支撑了页面内容
+ (``kb_chunks.chunk_id``)。回溯原文按后者直取,前者用于文档删除时定位受影响页。
+ """
+
+ __tablename__ = "kb_wiki_pages"
+
+ page_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ slug = Column(String(255), nullable=False)
+ title = Column(String(512), nullable=False, default="")
+ page_type = Column(String(32), nullable=False, default=WIKI_PAGE_TYPE_SUMMARY)
+ status = Column(String(32), nullable=False, default="published")
+ content = Column(Text, nullable=False, default="")
+ summary = Column(Text, nullable=False, default="")
+
+ folder_id = Column(String(64), nullable=False, default="")
+ category_path = Column(JSONType, default=list)
+ wiki_path = Column(String(1024), nullable=False, default="")
+ depth = Column(Integer, nullable=False, default=0)
+ sort_order = Column(Integer, nullable=False, default=0)
+
+ source_refs = Column(JSONType, default=list) # [document_id]
+ chunk_refs = Column(JSONType, default=list) # [chunk_id]
+ in_links = Column(JSONType, default=list) # [slug],反向链接
+ out_links = Column(JSONType, default=list) # [slug],正向链接
+ aliases = Column(JSONType, default=list)
+ page_metadata = Column(JSONType, default=dict)
+
+ version = Column(Integer, nullable=False, default=1)
+ last_edit_source = Column(String(16), nullable=False, default=WIKI_EDIT_SOURCE_PIPELINE)
+ last_editor_id = Column(String(64), nullable=False, default="")
+
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('draft', 'published', 'archived')", name="kb_wiki_pages_status_check"
+ ),
+ # slug 在单个知识库内唯一,跨库可以重复
+ Index(
+ "idx_kb_wiki_pages_kb_slug",
+ "kb_id",
+ "slug",
+ unique=True,
+ postgresql_where=Column("deleted_at").is_(None),
+ ),
+ Index("idx_kb_wiki_pages_kb_type", "kb_id", "page_type"),
+ Index("idx_kb_wiki_pages_folder", "kb_id", "folder_id"),
+ Index("idx_kb_wiki_pages_tree", "kb_id", "page_type", "wiki_path", "sort_order", "title"),
+ Index("idx_kb_wiki_pages_updated_at", "updated_at"),
+ )
+
+class KBWikiJob(Base):
+ """Wiki 生成作业。
+
+ Wiki 生成是分钟级到小时级的长任务,挂在请求生命周期上(BackgroundTasks)会在
+ 容器重启时丢一半,所以状态必须落库:worker 认领后写心跳,重启时按心跳超时回收
+ 僵尸作业重跑。``progress`` 供前端直接渲染进度,不用另开接口。
+ """
+
+ __tablename__ = "kb_wiki_jobs"
+
+ job_id = Column(String(64), primary_key=True)
+ kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False)
+ job_type = Column(String(20), nullable=False) # ingest | retract | finalize
+ payload = Column(JSONType, default=dict) # {"document_ids": [...]}
+ status = Column(String(20), nullable=False, default="pending")
+ attempt = Column(Integer, nullable=False, default=0)
+ max_attempts = Column(Integer, nullable=False, default=3)
+ last_error = Column(Text)
+ progress = Column(JSONType, default=dict) # {"stage": ..., "done": n, "total": m}
+
+ claimed_by = Column(String(64))
+ claimed_at = Column(UTCDateTime(timezone=True))
+ heartbeat_at = Column(UTCDateTime(timezone=True))
+
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ finished_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ CheckConstraint(
+ "job_type IN ('ingest', 'retract', 'finalize')", name="kb_wiki_jobs_type_check"
+ ),
+ CheckConstraint(
+ "status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')",
+ name="kb_wiki_jobs_status_check",
+ ),
+ Index("idx_kb_wiki_jobs_status", "status", "created_at"),
+ Index("idx_kb_wiki_jobs_kb", "kb_id", "status"),
+ # 同一知识库只保留一个未完成的 finalize:多篇文档摄入完只需要收尾一次。
+ # 条件里必须带 status,否则一次成功的 finalize 会永久占位、再也排不进下一次。
+ #
+ # 两个方言的 where 都要给:只写 postgresql_where 时,SQLite 会把谓词整个
+ # 丢掉、退化成 UNIQUE(kb_id)——那等于「一个知识库只能有一个作业」,本地
+ # 开发和测试库上批量摄入会直接撞唯一约束。
+ Index(
+ "idx_kb_wiki_jobs_pending_finalize",
+ "kb_id",
+ unique=True,
+ postgresql_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"),
+ sqlite_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"),
+ ),
+ )
+
+# Register with the schema, including when the installed feature is runtime-disabled.
+def _cleanup_user_knowledge(session, user):
+ for space in session.query(KBSpace).filter(KBSpace.user_id == user.user_id):
+ session.delete(space)
+
+
+from core.harness.registry import register_user_cleanup
+
+register_user_cleanup(_cleanup_user_knowledge)
diff --git a/src/backend/features/knowledge/parser_chunks.py b/src/backend/features/knowledge/parser_chunks.py
new file mode 100644
index 00000000..82256381
--- /dev/null
+++ b/src/backend/features/knowledge/parser_chunks.py
@@ -0,0 +1,278 @@
+"""parser chunks responsibilities."""
+from __future__ import annotations
+import json
+import re
+import uuid
+from typing import Optional
+from features.knowledge import kb_parser_state
+from features.knowledge import parser_tokens
+from features.knowledge import parser_types
+
+def apply_child_separators(parents: list[parser_types.ParentChunk], child_size: int, separators: list[str]) -> None:
+ """Re-split each parent's children by child_separators (replaces ``pc.children`` in place).
+
+ Splits the parent body with the recursive separators and packs by child_size,
+ aligning child boundaries to the user-specified separators.
+ """
+ for pc in parents:
+ texts = [s.strip() for s in recursive_split(pc.content, separators, child_size) if s.strip()]
+ pc.children = [
+ parser_types.ChildChunk(child_id=f"{pc.parent_id}_{i}", content=c, index=i)
+ for i, c in enumerate(texts or [pc.content])
+ ]
+
+
+def new_id() -> str:
+ return uuid.uuid4().hex[:16]
+
+
+def split_into_children(parent_text: str, child_size: int, overlap: int) -> list[str]:
+ """Split parent text into overlapping child chunks by token count."""
+ words = parent_text.split()
+ if not words:
+ return [parent_text]
+
+ # Use character-based approximation when tiktoken unavailable
+ chars_per_token = 2 # conservative for Chinese
+ child_chars = child_size * chars_per_token
+ overlap_chars = overlap * chars_per_token
+
+ children = []
+ start = 0
+ while start < len(parent_text):
+ end = start + child_chars
+ chunk = parent_text[start:end].strip()
+ if chunk:
+ children.append(chunk)
+ if end >= len(parent_text):
+ break
+ start = end - overlap_chars
+ return children if children else [parent_text]
+
+
+def make_parent_chunk(text: str, children_texts: list[str]) -> parser_types.ParentChunk:
+ pid = new_id()
+ children = [
+ parser_types.ChildChunk(child_id=f"{pid}_{i}", content=c, index=i)
+ for i, c in enumerate(children_texts)
+ ]
+ return parser_types.ParentChunk(parent_id=pid, content=text, children=children)
+
+
+def build_structured_chunks(
+ paragraphs: list[dict],
+ parent_size: int,
+ child_size: int,
+ overlap: int,
+) -> list[parser_types.ParentChunk]:
+ """Structure-aware chunking.
+
+ A heading is a "preferred break point", not a "forced break point": we only break
+ before the next heading once the buffer has accumulated to a certain fraction of
+ parent_size (``_HEADING_SOFT_RATIO``); otherwise consecutive subsections are
+ merged into the same parent chunk until it approaches parent_size. This keeps
+ the heading/paragraph-aligned structure awareness while making ``parent_size``
+ actually effective — heading-dense documents are no longer cut into lots of tiny
+ pieces, and parents can yield enough children.
+ """
+ result = []
+ buffer: list[str] = []
+ buffer_tokens = 0
+ soft_threshold = parent_size * kb_parser_state.HEADING_SOFT_RATIO
+
+ def _flush() -> None:
+ nonlocal buffer, buffer_tokens
+ parent_text = "\n".join(buffer)
+ result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap)))
+ buffer = []
+ buffer_tokens = 0
+
+ for para in paragraphs:
+ text = para["text"]
+ tokens = parser_tokens.count_tokens(text)
+
+ # Hard cap: adding this paragraph would exceed parent_size, so break off the
+ # accumulated content first
+ if buffer and buffer_tokens + tokens > parent_size:
+ _flush()
+ # Soft boundary: break before a heading only when enough content has
+ # accumulated (>= a fraction of parent_size); otherwise merge this
+ # subsection into the current parent so parent_size is the real determinant
+ # of parent-chunk size
+ elif buffer and para.get("heading_level") and buffer_tokens >= soft_threshold:
+ _flush()
+
+ buffer.append(text)
+ buffer_tokens += tokens
+
+ if buffer:
+ _flush()
+
+ return result
+
+
+def build_law_chunks(
+ paragraphs: list[dict],
+ parent_size: int,
+ child_size: int,
+ overlap: int,
+) -> list[parser_types.ParentChunk]:
+ """Split at article/chapter/section boundaries for legal documents."""
+ result = []
+ buffer: list[str] = []
+
+ article_pattern = re.compile(r'^第[一二三四五六七八九十百\d]+[条章节]')
+
+ for para in paragraphs:
+ text = para["text"]
+ is_boundary = bool(article_pattern.match(text)) or para.get("heading_level") in (1, 2)
+
+ if is_boundary and buffer:
+ parent_text = "\n".join(buffer)
+ result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap)))
+ buffer = []
+
+ buffer.append(text)
+
+ if buffer:
+ parent_text = "\n".join(buffer)
+ result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap)))
+
+ return result
+
+
+def build_qa_chunks(paragraphs: list[dict]) -> list[parser_types.ParentChunk]:
+ """Each Q&A pair becomes one parent chunk with one child."""
+ result = []
+ for para in paragraphs:
+ text = para["text"]
+ pid = new_id()
+ child = parser_types.ChildChunk(child_id=f"{pid}_0", content=text, index=0)
+ result.append(parser_types.ParentChunk(parent_id=pid, content=text, children=[child]))
+ return result
+
+
+def normalize_separators(seps: Optional[list[str]]) -> Optional[list[str]]:
+ """Normalize user-supplied custom separators into a usable hierarchy list.
+
+ - Restore escapes like ``\\n`` / ``\\t``;
+ - Drop empty strings, de-duplicate while preserving order;
+ - Return None when all empty (callers fall back to the ``_RECURSIVE_SEPARATORS`` default hierarchy).
+ """
+ if not seps:
+ return None
+ out: list[str] = []
+ seen: set[str] = set()
+ for raw in seps:
+ if raw is None:
+ continue
+ s = str(raw)
+ for token, real in kb_parser_state.SEPARATOR_ESCAPES.items():
+ s = s.replace(token, real)
+ if s == "" or s in seen:
+ continue
+ seen.add(s)
+ out.append(s)
+ return out or None
+
+
+def parse_separators_form(raw: Optional[str]) -> Optional[list[str]]:
+ """Parse custom separators passed as a JSON-array string in a multipart form into a list.
+
+ The preview endpoint takes Form params and cannot receive a list directly, so by
+ convention a JSON string is passed (e.g. ``["\\n\\n", "。"]``). On parse failure
+ or non-list input, silently return None (fall back to the default hierarchy)
+ instead of 500-ing over tuning input.
+ """
+ if not raw or not raw.strip():
+ return None
+ try:
+ data = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ if not isinstance(data, list):
+ return None
+ return normalize_separators([str(x) for x in data])
+
+
+def recursive_split(text: str, separators: list[str], max_tokens: int) -> list[str]:
+ """Recursively split text using a hierarchy of separators.
+
+ Tries to split on the first separator that produces segments.
+ Merges adjacent small segments up to max_tokens.
+ Over-sized segments recurse with the next separator level.
+ Final fallback: character-based truncation.
+ """
+ if parser_tokens.count_tokens(text) <= max_tokens:
+ return [text]
+
+ for i, sep in enumerate(separators):
+ parts = text.split(sep)
+ if len(parts) <= 1:
+ continue
+
+ # Merge adjacent small parts up to max_tokens
+ result: list[str] = []
+ current = ""
+ current_tokens = 0
+
+ for part in parts:
+ candidate = (current + sep + part) if current else part
+ candidate_tokens = parser_tokens.count_tokens(candidate)
+
+ if candidate_tokens <= max_tokens:
+ current = candidate
+ current_tokens = candidate_tokens
+ else:
+ if current:
+ result.append(current)
+ # If this single part exceeds max_tokens, recurse deeper
+ part_tokens = parser_tokens.count_tokens(part)
+ if part_tokens > max_tokens:
+ deeper = recursive_split(part, separators[i + 1:], max_tokens)
+ result.extend(deeper)
+ current = ""
+ current_tokens = 0
+ else:
+ current = part
+ current_tokens = part_tokens
+
+ if current:
+ result.append(current)
+
+ if result:
+ return result
+
+ # Final fallback: character-based truncation
+ chars_per_token = 2
+ chunk_chars = max_tokens * chars_per_token
+ result = []
+ start = 0
+ while start < len(text):
+ end = start + chunk_chars
+ chunk = text[start:end].strip()
+ if chunk:
+ result.append(chunk)
+ start = end
+ return result if result else [text]
+
+
+def build_recursive_chunks(
+ paragraphs: list[dict],
+ parent_size: int,
+ child_size: int,
+ overlap: int,
+ separators: Optional[list[str]] = None,
+) -> list[parser_types.ParentChunk]:
+ """Recursive multi-level separator chunking."""
+ # Join all paragraph text
+ full_text = "\n".join(p["text"] for p in paragraphs)
+ segments = recursive_split(full_text, separators or kb_parser_state.RECURSIVE_SEPARATORS, parent_size)
+
+ result = []
+ for seg in segments:
+ seg = seg.strip()
+ if not seg:
+ continue
+ result.append(make_parent_chunk(seg, split_into_children(seg, child_size, overlap)))
+ return result
diff --git a/src/backend/features/knowledge/parser_extract.py b/src/backend/features/knowledge/parser_extract.py
new file mode 100644
index 00000000..e0596acd
--- /dev/null
+++ b/src/backend/features/knowledge/parser_extract.py
@@ -0,0 +1,302 @@
+"""parser extract responsibilities."""
+from __future__ import annotations
+import re
+import uuid
+from typing import Optional
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_parser_state
+from features.knowledge import parser_types
+
+def extract_text(file_bytes: bytes, mime_type: str) -> list[dict]:
+ """Extract structured paragraphs from a document.
+
+ Delegates to core/file_parser.py for PDF, DOCX, DOC, TXT (uses the
+ external file-parser API for PDF, pandoc for DOCX, etc.).
+ Falls back to built-in xlsx/plain-text extraction for unsupported types.
+
+ Returns a list of paragraph dicts:
+ {"text": str, "heading_level": int|None, "element_type": str}
+ """
+ paragraphs, _ = extract_text_with_assets(file_bytes, mime_type, with_assets=False)
+ return paragraphs
+
+
+def extract_text_with_assets(
+ file_bytes: bytes,
+ mime_type: str,
+ *,
+ with_assets: bool = True,
+) -> tuple[list[dict], list[parser_types.RawAsset]]:
+ """Extract paragraphs and, when ``with_assets``, the media the document carries.
+
+ Asset extraction is opt-in because it costs a heavier parse call: the layout
+ service inlines every figure as base64 in the JSON response, which the text-only
+ callers have no use for. When the rich call fails for any reason we fall back to
+ the plain text path — a parse service that doesn't support the media params must
+ degrade to today's behaviour, never fail the document.
+ """
+ mt = (mime_type or "").lower()
+
+ # Standalone image upload. Without this branch an image falls through to
+ # ``_extract_plain_text``, which latin-1 decodes the binary into garbage
+ # paragraphs and indexes them — the upload allowlist accepts png/jpg/webp/gif
+ # (``core/files/content/file_validation.py``) so this path is reachable in production.
+ if mt.startswith("image/"):
+ return extract_standalone_image(file_bytes, mt, with_assets=with_assets)
+
+ # XLSX — keep built-in handler (file_parser doesn't support spreadsheets)
+ if mt in (
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel",
+ ):
+ return extract_xlsx(file_bytes), []
+
+ if with_assets and mt == "application/pdf":
+ rich = extract_pdf_rich(file_bytes)
+ if rich is not None:
+ return rich
+
+ # Try core/file_parser for all other known types
+ suffix = kb_parser_state.MIME_TO_SUFFIX.get(mt)
+ if suffix:
+ try:
+ from core.files.content.file_parser import parse_file
+ filename = f"upload{suffix}"
+ content = parse_file(file_bytes, filename)
+ if content:
+ return markdown_to_paragraphs(content), []
+ except RuntimeError:
+ # file_parser failed (e.g. service down) — fall through to built-in
+ pass
+
+ # Fallback: treat as plain text
+ return extract_plain_text(file_bytes), []
+
+
+def extract_standalone_image(
+ file_bytes: bytes,
+ mime_type: str,
+ *,
+ with_assets: bool,
+) -> tuple[list[dict], list[parser_types.RawAsset]]:
+ """An uploaded image is one asset with one synthetic paragraph holding its ref.
+
+ The paragraph exists so the normal chunker still produces a parent chunk: retrieval
+ fetches parent content from PostgreSQL, and an image-only document with no chunk
+ would be unreachable. The caption/description is attached to the asset, not
+ inlined here — it is not known until the VLM pass has run.
+ """
+ placeholder = f"asset://{uuid.uuid4().hex[:16]}"
+ paragraphs = [{"text": f"", "heading_level": None, "element_type": "image"}]
+ if not with_assets:
+ return paragraphs, []
+ asset = parser_types.RawAsset(
+ data=file_bytes,
+ placeholder=placeholder,
+ kind="image",
+ mime_type=mime_type,
+ index=0,
+ )
+ return paragraphs, [asset]
+
+
+def extract_pdf_rich(file_bytes: bytes) -> Optional[tuple[list[dict], list[parser_types.RawAsset]]]:
+ """Parse a PDF keeping its figures. Returns None to signal "fall back to text-only"."""
+ try:
+ from core.files.content.pdf_parser import parse_pdf_rich
+
+ result = parse_pdf_rich(file_bytes, "upload.pdf")
+ except Exception as exc:
+ logger.warning("PDF 富解析失败,退回纯文本解析: %s", exc)
+ return None
+
+ paragraphs = markdown_to_paragraphs(result.markdown)
+ if not result.images:
+ return paragraphs, []
+
+ # Layout metadata (caption / bbox / page) keyed by the same path the markdown
+ # references, so each figure's asset carries where it came from.
+ meta_by_path: dict[str, dict] = {}
+ for block in result.blocks:
+ if block.get("type") != "image":
+ continue
+ path = str(block.get("img_path") or "")
+ if path:
+ meta_by_path[path] = block
+
+ assets: list[parser_types.RawAsset] = []
+ seen: set[str] = set()
+ # Iterate in document order: the markdown's image links are the source of truth for
+ # ordering, and only images actually referenced by the text get an asset.
+ for idx, path in enumerate(kb_parser_state.MD_IMAGE_RE.findall(result.markdown)):
+ if path in seen:
+ continue
+ data = result.images.get(path)
+ if not data:
+ continue
+ seen.add(path)
+ meta = meta_by_path.get(path, {})
+ caption_parts = [
+ str(x).strip()
+ for x in (meta.get("image_caption") or []) + (meta.get("image_footnote") or [])
+ if str(x).strip()
+ ]
+ locator = {}
+ if meta.get("page_idx") is not None:
+ locator["page_idx"] = meta.get("page_idx")
+ if meta.get("bbox"):
+ locator["bbox"] = meta.get("bbox")
+ assets.append(
+ parser_types.RawAsset(
+ data=data,
+ placeholder=path,
+ kind="image",
+ mime_type=guess_image_mime(path),
+ text_content=" ".join(caption_parts),
+ locator=locator,
+ index=idx,
+ )
+ )
+
+ return paragraphs, assets
+
+
+def guess_image_mime(path: str) -> str:
+ lowered = path.lower()
+ for mime, ext in kb_parser_state.IMAGE_MIME_TO_EXT.items():
+ if lowered.endswith(ext):
+ return mime
+ if lowered.endswith(".jpeg"):
+ return "image/jpeg"
+ return "image/jpeg"
+
+
+def image_extension_for_mime(mime_type: str) -> str:
+ """Filename suffix for an image mime, used when building the storage key."""
+ return kb_parser_state.IMAGE_MIME_TO_EXT.get((mime_type or "").lower(), ".jpg")
+
+
+def markdown_to_paragraphs(text: str) -> list[dict]:
+ """Parse markdown text (from file_parser) into structured paragraph dicts."""
+ paragraphs: list[dict] = []
+ table_buffer: list[str] = []
+
+ def _flush_table():
+ if table_buffer:
+ paragraphs.append({
+ "text": "\n".join(table_buffer),
+ "heading_level": None,
+ "element_type": "table",
+ })
+ table_buffer.clear()
+
+ for line in text.splitlines():
+ stripped = line.strip()
+
+ # Empty line — flush table buffer, skip
+ if not stripped:
+ _flush_table()
+ continue
+
+ # Markdown table row
+ if stripped.startswith("|") and stripped.endswith("|"):
+ table_buffer.append(stripped)
+ continue
+
+ # Not a table row — flush any pending table
+ _flush_table()
+
+ # Markdown heading
+ md_match = re.match(r'^(#{1,6})\s+(.*)', stripped)
+ if md_match:
+ level = len(md_match.group(1))
+ paragraphs.append({
+ "text": stripped,
+ "heading_level": level,
+ "element_type": "heading",
+ })
+ continue
+
+ # Chinese legal/report heading patterns
+ heading_level = detect_heading_level(stripped)
+ if heading_level:
+ paragraphs.append({
+ "text": stripped,
+ "heading_level": heading_level,
+ "element_type": "heading",
+ })
+ continue
+
+ # Regular paragraph
+ paragraphs.append({
+ "text": stripped,
+ "heading_level": None,
+ "element_type": "paragraph",
+ })
+
+ _flush_table()
+ return paragraphs
+
+
+def extract_xlsx(file_bytes: bytes) -> list[dict]:
+ import io
+ try:
+ import openpyxl
+ except ImportError:
+ raise RuntimeError("openpyxl is required: pip install openpyxl")
+
+ wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True)
+ paragraphs = []
+ for sheet in wb.worksheets:
+ rows = list(sheet.iter_rows(values_only=True))
+ if not rows:
+ continue
+ header = [str(c or "") for c in rows[0]]
+ paragraphs.append({
+ "text": f"## Sheet: {sheet.title}",
+ "heading_level": 2,
+ "element_type": "heading",
+ })
+ md_lines = ["| " + " | ".join(header) + " |"]
+ md_lines.append("|" + "|".join("---" for _ in header) + "|")
+ for row in rows[1:]:
+ if all(c is None for c in row):
+ continue
+ md_lines.append("| " + " | ".join(str(c or "") for c in row) + " |")
+ paragraphs.append({
+ "text": "\n".join(md_lines),
+ "heading_level": None,
+ "element_type": "table",
+ })
+ return paragraphs
+
+
+def extract_plain_text(file_bytes: bytes) -> list[dict]:
+ """Fallback: decode as text and split into paragraphs."""
+ # Try common encodings
+ for enc in ("utf-8", "gbk", "latin-1"):
+ try:
+ text = file_bytes.decode(enc)
+ break
+ except UnicodeDecodeError:
+ continue
+ else:
+ text = file_bytes.decode("utf-8", errors="replace")
+
+ return markdown_to_paragraphs(text)
+
+
+def detect_heading_level(line: str) -> Optional[int]:
+ """Detect Chinese legal/report heading patterns."""
+ patterns = [
+ (r'^第[一二三四五六七八九十百]+章', 1),
+ (r'^第[一二三四五六七八九十百]+节', 2),
+ (r'^第[一二三四五六七八九十百\d]+条', 3),
+ (r'^[一二三四五六七八九十]+[、..]', 2),
+ (r'^\d+[、..]\s*\S', 3),
+ ]
+ for pattern, level in patterns:
+ if re.match(pattern, line):
+ return level
+ return None
diff --git a/src/backend/features/knowledge/parser_semantic.py b/src/backend/features/knowledge/parser_semantic.py
new file mode 100644
index 00000000..c582d1f4
--- /dev/null
+++ b/src/backend/features/knowledge/parser_semantic.py
@@ -0,0 +1,130 @@
+"""parser semantic responsibilities."""
+from __future__ import annotations
+import math
+from typing import Callable, Optional
+from features.knowledge import kb_parser_state
+from features.knowledge import parser_chunks
+from features.knowledge import parser_tokens
+from features.knowledge import parser_types
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+ """Compute cosine similarity between two vectors (pure Python)."""
+ dot = sum(x * y for x, y in zip(a, b))
+ norm_a = math.sqrt(sum(x * x for x in a))
+ norm_b = math.sqrt(sum(x * x for x in b))
+ if norm_a == 0 or norm_b == 0:
+ return 0.0
+ return dot / (norm_a * norm_b)
+
+
+def build_embedding_semantic_chunks(
+ paragraphs: list[dict],
+ parent_size: int,
+ child_size: int,
+ overlap: int,
+ embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None,
+ separators: Optional[list[str]] = None,
+) -> list[parser_types.ParentChunk]:
+ """Semantic chunking based on embedding similarity between adjacent blocks.
+
+ Algorithm:
+ 1. Merge paragraphs into minimum ~50-token blocks
+ 2. Batch-embed all blocks
+ 3. Compute cosine similarity between adjacent blocks
+ 4. Set breakpoints where similarity drops below P25 or is a local minimum
+ 5. Merge blocks between breakpoints into parent chunks
+ 6. Fallback to recursive splitting for oversized parents
+ """
+ if embed_fn is None:
+ # Fallback to structured chunking when no embed function available
+ return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap)
+
+ # Step 1: Merge paragraphs into minimum-size blocks (~50 tokens)
+ MIN_BLOCK_TOKENS = 50
+ blocks: list[str] = []
+ current_block = ""
+ current_tokens = 0
+
+ for para in paragraphs:
+ text = para["text"]
+ tokens = parser_tokens.count_tokens(text)
+ if current_block and current_tokens + tokens > MIN_BLOCK_TOKENS:
+ blocks.append(current_block)
+ current_block = text
+ current_tokens = tokens
+ else:
+ current_block = (current_block + "\n" + text) if current_block else text
+ current_tokens += tokens
+
+ if current_block:
+ blocks.append(current_block)
+
+ if len(blocks) <= 1:
+ # Single block — just use structured
+ return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap)
+
+ # Step 2: Batch embed
+ embeddings = embed_fn(blocks)
+
+ # Step 3: Compute adjacent cosine similarities
+ similarities: list[float] = []
+ for i in range(len(embeddings) - 1):
+ sim = cosine_similarity(embeddings[i], embeddings[i + 1])
+ similarities.append(sim)
+
+ if not similarities:
+ return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap)
+
+ # Step 4: Find breakpoints
+ sorted_sims = sorted(similarities)
+ p25_idx = max(0, len(sorted_sims) // 4 - 1)
+ p25_threshold = sorted_sims[p25_idx]
+
+ breakpoints: set[int] = set()
+ for i, sim in enumerate(similarities):
+ # Below P25 threshold
+ if sim <= p25_threshold:
+ breakpoints.add(i)
+ continue
+ # Local minimum: lower than both neighbors
+ is_local_min = True
+ if i > 0 and sim >= similarities[i - 1]:
+ is_local_min = False
+ if i < len(similarities) - 1 and sim >= similarities[i + 1]:
+ is_local_min = False
+ if is_local_min and sim < p25_threshold * 1.2:
+ breakpoints.add(i)
+
+ # Step 5: Merge blocks between breakpoints into parent chunks
+ result: list[parser_types.ParentChunk] = []
+ segment_start = 0
+
+ sorted_breaks = sorted(breakpoints)
+ # Add an artificial breakpoint at the end
+ sorted_breaks.append(len(blocks) - 1)
+
+ for bp in sorted_breaks:
+ segment_blocks = blocks[segment_start:bp + 1]
+ parent_text = "\n".join(segment_blocks).strip()
+ segment_start = bp + 1
+
+ if not parent_text:
+ continue
+
+ # Step 6: If parent is too large, use recursive split as fallback
+ if parser_tokens.count_tokens(parent_text) > parent_size * 1.5:
+ sub_segments = parser_chunks.recursive_split(parent_text, separators or kb_parser_state.RECURSIVE_SEPARATORS, parent_size)
+ for sub in sub_segments:
+ sub = sub.strip()
+ if sub:
+ result.append(parser_chunks.make_parent_chunk(sub, parser_chunks.split_into_children(sub, child_size, overlap)))
+ else:
+ result.append(parser_chunks.make_parent_chunk(parent_text, parser_chunks.split_into_children(parent_text, child_size, overlap)))
+
+ # Handle remaining blocks
+ if segment_start < len(blocks):
+ remaining = "\n".join(blocks[segment_start:]).strip()
+ if remaining:
+ result.append(parser_chunks.make_parent_chunk(remaining, parser_chunks.split_into_children(remaining, child_size, overlap)))
+
+ return result
diff --git a/src/backend/features/knowledge/parser_tokens.py b/src/backend/features/knowledge/parser_tokens.py
new file mode 100644
index 00000000..8a6fae1a
--- /dev/null
+++ b/src/backend/features/knowledge/parser_tokens.py
@@ -0,0 +1,26 @@
+"""parser tokens responsibilities."""
+from __future__ import annotations
+
+
+def get_tiktoken_enc():
+ """Return cached tiktoken cl100k_base encoder."""
+ global tiktoken_enc
+ if tiktoken_enc is None:
+ try:
+ import tiktoken
+ tiktoken_enc = tiktoken.get_encoding("cl100k_base")
+ except Exception:
+ pass
+ return tiktoken_enc
+
+
+def count_tokens(text: str) -> int:
+ """Approximate token count using tiktoken cl100k_base."""
+ enc = get_tiktoken_enc()
+ if enc is not None:
+ return len(enc.encode(text))
+ # Fallback: rough Chinese/English estimate
+ return max(1, len(text) // 2)
+
+
+tiktoken_enc = None
diff --git a/src/backend/features/knowledge/parser_types.py b/src/backend/features/knowledge/parser_types.py
new file mode 100644
index 00000000..ff72dc9c
--- /dev/null
+++ b/src/backend/features/knowledge/parser_types.py
@@ -0,0 +1,43 @@
+"""parser types responsibilities."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Optional
+
+@dataclass
+class ChildChunk:
+ child_id: str
+ content: str
+ index: int # position within parent
+
+
+@dataclass
+class ParentChunk:
+ parent_id: str
+ content: str # full parent text, returned to LLM on retrieval
+ children: list[ChildChunk]
+ char_start: Optional[int] = None
+ char_end: Optional[int] = None
+
+
+@dataclass
+class RawAsset:
+ """A non-text medium extracted from the source document, before persistence.
+
+ ``placeholder`` is the exact token this medium is referenced by inside the
+ paragraph text (``images/.jpg`` for a layout-extracted figure, a synthetic
+ ``asset://`` for a standalone upload). It is what links the medium back to
+ the parent chunk that contains it, and what gets rewritten to a real URL once the
+ asset has an id.
+
+ Medium-agnostic on purpose: ``kind`` widens to audio/video, and ``locator`` holds
+ ``{page_idx, bbox}`` for an image where a transcript segment would hold
+ ``{start_ms, end_ms}``.
+ """
+
+ data: bytes
+ placeholder: str
+ kind: str = "image"
+ mime_type: str = "application/octet-stream"
+ text_content: str = "" # 图注 / OCR(音视频接入后:ASR 转写)
+ locator: dict = field(default_factory=dict)
+ index: int = 0
diff --git a/src/backend/core/auth/kb_permissions.py b/src/backend/features/knowledge/personal_permissions.py
similarity index 97%
rename from src/backend/core/auth/kb_permissions.py
rename to src/backend/features/knowledge/personal_permissions.py
index 26102bdb..6250b73b 100644
--- a/src/backend/core/auth/kb_permissions.py
+++ b/src/backend/features/knowledge/personal_permissions.py
@@ -35,7 +35,7 @@ def _is_super_admin(db: Session, user_id: str) -> bool:
def get_accessible_local_kb_levels(db: Session, user_id: str) -> Dict[str, KBLevel]:
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
if not user_id:
return {}
diff --git a/src/backend/api/routes/v1/kb_models.py b/src/backend/features/knowledge/personal_request_models.py
similarity index 100%
rename from src/backend/api/routes/v1/kb_models.py
rename to src/backend/features/knowledge/personal_request_models.py
diff --git a/src/backend/core/content/kb_processing.py b/src/backend/features/knowledge/processing.py
similarity index 67%
rename from src/backend/core/content/kb_processing.py
rename to src/backend/features/knowledge/processing.py
index a2ffa8a0..30b10aad 100644
--- a/src/backend/core/content/kb_processing.py
+++ b/src/backend/features/knowledge/processing.py
@@ -1,147 +1,10 @@
-"""Knowledge base document processing — chunking, keyword extraction, vectorisation.
-
-Background task logic extracted from ``api/routes/v1/kb.py``.
-"""
-
+"""processing responsibilities."""
from __future__ import annotations
from core.infra.time import utc_now
-
-import logging
-import os
-import re
from typing import Optional
-
+import logging
logger = logging.getLogger(__name__)
-
-
-# ── Text helpers ────────────────────────────────────────────────────────────
-
-def strip_think_tags(text: str) -> str:
- """Remove ... blocks and markdown formatting from LLM output."""
- # Remove ... blocks (reasoning model output)
- text = re.sub(r'.*? ', '', text, flags=re.DOTALL).strip()
- # Remove markdown bold/italic markers
- text = re.sub(r'\*+', '', text)
- # Remove markdown headers
- text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE)
- # Remove bullet points
- text = re.sub(r'^[\-\*]\s*', '', text, flags=re.MULTILINE)
- return text.strip()
-
-
-# ── Model config helper ────────────────────────────────────────────────────
-
-def resolve_main_endpoint():
- """main_agent 端点,带上它说的协议——KB 的两处抽取都得走对那条线。"""
- from core.llm.single_turn import Endpoint
-
- try:
- from core.services.model_config import ModelConfigService
-
- cfg = ModelConfigService.get_instance().resolve("main_agent")
- if cfg and cfg.base_url and cfg.model_name:
- return Endpoint.from_resolved(cfg)
- except Exception:
- pass
- base_url, api_key, model_name = resolve_main_model_config()
- return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name)
-
-
-def resolve_main_model_config() -> tuple[str, str, str]:
- """Resolve main_agent model config from DB, with env fallback."""
- try:
- from core.services.model_config import ModelConfigService
- cfg = ModelConfigService.get_instance().resolve("main_agent")
- if cfg:
- return cfg.base_url.rstrip("/"), cfg.api_key, cfg.model_name
- except Exception:
- pass
- return (
- os.getenv("MODEL_URL", "").rstrip("/"),
- os.getenv("API_KEY", ""),
- os.getenv("BASE_MODEL_NAME", ""),
- )
-
-
-# ── LLM-powered enrichment ─────────────────────────────────────────────────
-
-def _call_main_model(
- *, system: str, user: str, temperature: float, max_tokens: int, timeout: int = 30
-) -> str:
- """跑一次主模型补全;未配置时返回空串,调用方按「这一步没产出」处理。"""
- import requests as _requests
-
- from core.llm.single_turn import build_request, parse_response, turns
-
- endpoint = resolve_main_endpoint()
- if not endpoint.base_url or not endpoint.model_name:
- return ""
- url, headers, payload = build_request(
- endpoint, turns(user, system), temperature=temperature, max_tokens=max_tokens
- )
- resp = _requests.post(url, headers=headers, json=payload, timeout=timeout)
- resp.raise_for_status()
- return parse_response(endpoint, resp.json()).text
-
-
-def extract_keywords_llm(content: str, count: int) -> list[str]:
- """Call configured LLM to extract keywords from a text chunk."""
- try:
- raw = strip_think_tags(
- _call_main_model(
- system=(
- "你是关键词提取工具。"
- "输出规则:只输出关键词,用英文逗号分隔,不要编号,不要解释,不要分析过程。"
- "示例输出:数字化转型,产业升级,人工智能"
- ),
- user=f"从以下文本提取{count}个关键词,直接输出逗号分隔的关键词:\n\n{content[:2000]}",
- temperature=0.1,
- max_tokens=200,
- )
- )
- lines = [ln.strip() for ln in raw.split("\n") if ln.strip()]
- best_line = raw
- for line in reversed(lines):
- if ("," in line or "," in line) and len(line) < 500:
- best_line = line
- break
- keywords = [kw.strip() for kw in best_line.replace(",", ",").split(",") if kw.strip() and len(kw.strip()) < 30]
- return keywords[:count]
- except Exception as exc:
- logger.warning("LLM keyword extraction failed: %s", exc)
- return []
-
-
-def generate_questions_llm(content: str, count: int) -> list[str]:
- """Call configured LLM to generate retrieval questions for a text chunk."""
- try:
- raw = strip_think_tags(
- _call_main_model(
- system=(
- "你是问题生成工具。"
- "输出规则:每行一个问题,不要编号,不要解释,不要分析过程。直接输出问题本身。"
- "示例输出:\n什么是数字化转型?\n如何申报产业升级项目?"
- ),
- user=f"根据以下文本生成{count}个检索问题,每行一个:\n\n{content[:2000]}",
- temperature=0.3,
- max_tokens=500,
- )
- )
- questions = []
- for q in raw.split("\n"):
- q = q.strip().lstrip("0123456789.、))-— ").strip()
- if not q or len(q) < 5:
- continue
- if q.startswith(("分析", "角色", "任务", "输出", "规则", "示例")):
- continue
- questions.append(q)
- return questions[:count]
- except Exception as exc:
- logger.warning("LLM question generation failed: %s", exc)
- return []
-
-
-# ── Background vectorisation ───────────────────────────────────────────────
+from features.knowledge import enrichment
def update_document_status(document_id: str, status: str, error: Optional[str] = None) -> None:
"""Update the indexing_status of a document in the DB.
@@ -154,7 +17,7 @@ def update_document_status(document_id: str, status: str, error: Optional[str] =
from sqlalchemy.orm.attributes import flag_modified
from core.db.engine import SessionLocal
- from core.db.models import KBDocument
+ from features.knowledge.models import KBDocument
db = SessionLocal()
try:
doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first()
@@ -205,21 +68,22 @@ def vectorise_document_background(
引文标注按 chunk 走、回溯原文按 chunk 直取,脱离分块无从谈起。
"""
try:
- from core.kb.kb_assets import multimodal_indexing_enabled
- from core.kb.kb_parser import parse_and_chunk_rich
- from core.kb.kb_vector import (
- get_or_create_collection, embed_batch, build_sparse_text, text_to_sparse,
- upsert_rows, upsert_question_rows, truncate_utf8,
- TITLE_FIELD_MAX_BYTES, TAGS_FIELD_MAX_BYTES,
- )
+ from features.knowledge.kb_assets import multimodal_indexing_enabled
+ from features.knowledge.kb_parser import parse_and_chunk_rich
+ from features.knowledge.vector_client import get_or_create_collection
+ from core.models.embedding import embed_batch
+ from features.knowledge.kb_vector import build_sparse_text, text_to_sparse
+ from features.knowledge.vector_indexing import upsert_rows, upsert_question_rows
+ from features.knowledge.vector_fields import truncate_utf8
+ from features.knowledge.kb_vector_state import TITLE_FIELD_MAX_BYTES, TAGS_FIELD_MAX_BYTES
from core.db.engine import SessionLocal
- from core.db.models import KBChunk
+ from features.knowledge.models import KBChunk
# 调用方手上都有 KBSpace,能传就传:批量上传时每篇再开一次 session 去
# 重读同一行,会和并发的索引任务一起把连接池顶满(实测 39 篇并发上传
# 时出现 QueuePool 超时导致索引失败)。
if index_modes is None:
- index_modes = _resolve_index_modes(kb_id)
+ index_modes = resolve_index_modes(kb_id)
want_vectors = "rag" in index_modes
want_wiki = "wiki" in index_modes
@@ -262,7 +126,7 @@ def vectorise_document_background(
# 资产必须在向量化之前落库:它会把父块正文里的图片占位符改写成真实资产 URL,
# 而入库的分块正文与被向量化的文本必须是同一份。
- stored_asset_ids = _persist_document_assets(
+ stored_asset_ids = persist_document_assets(
kb_id=kb_id,
document_id=document_id,
user_id=user_id,
@@ -293,11 +157,11 @@ def vectorise_document_background(
for pc_idx, pc in enumerate(parent_chunks):
chunk_tags: list[str] = []
if auto_keywords > 0:
- chunk_tags = extract_keywords_llm(pc.content, auto_keywords)
+ chunk_tags = enrichment.extract_keywords_llm(pc.content, auto_keywords)
chunk_questions: list[str] = []
if auto_questions > 0:
- chunk_questions = generate_questions_llm(pc.content, auto_questions)
+ chunk_questions = enrichment.generate_questions_llm(pc.content, auto_questions)
db_chunk = KBChunk(
chunk_id=pc.parent_id,
@@ -393,7 +257,7 @@ def vectorise_document_background(
# 图生文 + 资产检索行。放在文本索引之后:VLM 调用慢且可能没配,文本检索不该等它。
if stored_asset_ids and want_vectors:
- _index_document_assets(
+ index_document_assets(
loop=loop,
asset_ids=stored_asset_ids,
kb_id=kb_id,
@@ -406,14 +270,14 @@ def vectorise_document_background(
logger.info("Indexing completed for document %s", document_id)
if want_wiki:
- _enqueue_wiki_ingest(kb_id, document_id)
+ enqueue_wiki_ingest(kb_id, document_id)
except Exception as exc:
logger.error("Background vectorisation failed for document %s: %s", document_id, exc, exc_info=True)
update_document_status(document_id, "failed", error=f"{type(exc).__name__}: {exc}")
-def _persist_document_assets(
+def persist_document_assets(
*,
kb_id: str,
document_id: str,
@@ -430,7 +294,7 @@ def _persist_document_assets(
return []
try:
from core.db.engine import SessionLocal
- from core.kb.kb_assets import persist_assets
+ from features.knowledge.kb_assets import persist_assets
with SessionLocal() as db:
created = persist_assets(
@@ -447,7 +311,7 @@ def _persist_document_assets(
return []
-def _index_document_assets(
+def index_document_assets(
*,
loop,
asset_ids: list,
@@ -461,8 +325,9 @@ def _index_document_assets(
return
try:
from core.db.engine import SessionLocal
- from core.db.models import KBAsset
- from core.kb.kb_assets import generate_captions, index_assets, mark_vector_state
+ from features.knowledge.models import KBAsset
+ from features.knowledge.asset_captions import generate_captions
+ from features.knowledge.asset_indexing import index_assets, mark_vector_state
with SessionLocal() as db:
rows = db.query(KBAsset).filter(KBAsset.asset_id.in_(asset_ids)).all()
@@ -478,11 +343,11 @@ def _index_document_assets(
logger.warning("文档 %s 资产索引失败,图片不可检索: %s", document_id, exc)
-def _resolve_index_modes(kb_id: str) -> list[str]:
+def resolve_index_modes(kb_id: str) -> list[str]:
"""读该知识库的索引模式。查不到按仅 RAG 处理——历史库都是这个语义。"""
try:
from core.db.engine import SessionLocal
- from core.kb.wiki.config import index_modes_for_kb
+ from features.knowledge.wiki.config import index_modes_for_kb
with SessionLocal() as db:
return index_modes_for_kb(db, kb_id)
@@ -491,7 +356,7 @@ def _resolve_index_modes(kb_id: str) -> list[str]:
return ["rag"]
-def _enqueue_wiki_ingest(kb_id: str, document_id: str) -> None:
+def enqueue_wiki_ingest(kb_id: str, document_id: str) -> None:
"""索引完成后把文档排进 Wiki 生成队列。
入队失败不能反过来把文档判成索引失败——分块和向量都已经写好了,检索本身是
@@ -499,7 +364,7 @@ def _enqueue_wiki_ingest(kb_id: str, document_id: str) -> None:
"""
try:
from core.db.engine import SessionLocal
- from core.kb.wiki.jobs import enqueue_ingest
+ from features.knowledge.wiki.jobs import enqueue_ingest
with SessionLocal() as db:
enqueue_ingest(db, kb_id, [document_id])
diff --git a/src/backend/core/db/repository/kb.py b/src/backend/features/knowledge/repository.py
similarity index 97%
rename from src/backend/core/db/repository/kb.py
rename to src/backend/features/knowledge/repository.py
index 85a79638..82e7190d 100644
--- a/src/backend/core/db/repository/kb.py
+++ b/src/backend/features/knowledge/repository.py
@@ -11,8 +11,8 @@
from typing import Any, Dict, List, Optional
import sqlalchemy as sa
-from core.db.models import KBDocument, KBSpace
-from core.kb.document_status import classify_document_status, empty_document_status_counts
+from features.knowledge.models import KBDocument, KBSpace
+from features.knowledge.document_status import classify_document_status, empty_document_status_counts
from sqlalchemy import and_, desc, func, or_, select
from sqlalchemy.orm import Session
diff --git a/src/backend/core/services/kb_service.py b/src/backend/features/knowledge/service.py
similarity index 95%
rename from src/backend/core/services/kb_service.py
rename to src/backend/features/knowledge/service.py
index 54bd4fea..525ba503 100644
--- a/src/backend/core/services/kb_service.py
+++ b/src/backend/features/knowledge/service.py
@@ -7,12 +7,14 @@
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-from core.content.kb_processing import vectorise_document_background
-from core.db.models import KBChunk, KBDocument, UserShadow
-from core.kb.wiki.config import index_modes_of
-from core.db.repository import ArtifactRepository, AuditLogRepository, KBRepository
-from core.services.artifact_edition import can_access_artifact
-from core.storage import get_storage
+from features.knowledge.processing import vectorise_document_background
+from features.knowledge.models import KBChunk, KBDocument
+from core.db.models import UserShadow
+from features.knowledge.wiki.config import index_modes_of
+from core.db.repository import ArtifactRepository, AuditLogRepository
+from features.knowledge.repository import KBRepository
+from core.files.edition import can_access_artifact
+from core.files.storage import get_storage
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -42,7 +44,7 @@ def _has_kb_level(self, space: Any, user_id: str, required: str) -> bool:
"""
if not space:
return False
- from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level
+ from features.knowledge.edition import has_kb_permission, resolve_local_kb_level
return has_kb_permission(resolve_local_kb_level(self.db, user_id, space.kb_id), required)
@@ -192,8 +194,8 @@ def _enqueue_wiki_retract(self, space: Any, document_id: str) -> None:
会在下一次收尾时被当作死链清掉。
"""
try:
- from core.kb.wiki.config import wiki_enabled
- from core.kb.wiki.jobs import enqueue_retract
+ from features.knowledge.wiki.config import wiki_enabled
+ from features.knowledge.wiki.jobs import enqueue_retract
if space is None or not wiki_enabled(space):
return
@@ -239,14 +241,10 @@ def create_space(
if str(kind).strip() and str(principal_id).strip()
]
if grants:
- from core.db.repository import KBGrantRepository
-
- KBGrantRepository(self.db).bulk_grant(
- space.kb_id,
- "local",
- grants,
- "view",
- granted_by=granted_by or user_id,
+ from features.knowledge.edition import grant_initial_access
+
+ grant_initial_access(
+ self.db, space.kb_id, grants, granted_by=granted_by or user_id,
)
# Audit log
@@ -316,7 +314,7 @@ def update_space(
update_data["description"] = description
if index_modes is not None or wiki_config is not None:
- from core.kb.wiki.config import normalize_index_modes
+ from features.knowledge.wiki.config import normalize_index_modes
metadata = dict(space.extra_data or {}) if space else {}
if index_modes is not None:
diff --git a/src/backend/features/knowledge/vector_client.py b/src/backend/features/knowledge/vector_client.py
new file mode 100644
index 00000000..a727beb1
--- /dev/null
+++ b/src/backend/features/knowledge/vector_client.py
@@ -0,0 +1,166 @@
+"""vector client responsibilities."""
+from __future__ import annotations
+import os
+from typing import Any, Optional
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_vector_state
+from core.models import embedding as vector_embedding
+
+def milvus_uri() -> str:
+ from core.config.settings import settings
+
+ return os.getenv("MILVUS_URL") or settings.memory.milvus_url
+
+
+def is_lite() -> bool:
+ """True when the vector backend is embedded Milvus Lite (a local file uri).
+
+ Milvus Lite (no-Docker local profile) does **not** support SPARSE_FLOAT_VECTOR
+ / SPARSE_INVERTED_INDEX / hybrid search / IVF / scalar INVERTED indexes, so the
+ KB degrades to **dense-only** on it. A server uri (``http(s)://`` / ``tcp://``)
+ is a full Milvus and keeps the dense+sparse hybrid path byte-for-byte unchanged.
+ """
+ return not milvus_uri().startswith(("http://", "https://", "tcp://", "unix:"))
+
+
+def get_client():
+ """Return a MilvusClient connected to the configured Milvus instance."""
+ from pymilvus import MilvusClient
+
+ url = milvus_uri()
+ token = os.getenv("MILVUS_TOKEN", "")
+ if token:
+ return MilvusClient(uri=url, token=token)
+ return MilvusClient(uri=url)
+
+
+def upsert(client, data: list[dict[str, Any]]) -> None:
+ """Upsert helper that drops the ``sparse_embedding`` field on Milvus Lite.
+
+ On Lite the collection has no sparse field (see ``get_or_create_collection``),
+ and dynamic fields are disabled, so an extra ``sparse_embedding`` key would be
+ rejected. Stripping it here keeps every caller's row-building code identical
+ across both backends.
+ """
+ if is_lite():
+ data = [{k: v for k, v in row.items() if k != "sparse_embedding"} for row in data]
+ client.upsert(collection_name=kb_vector_state.COLLECTION_NAME, data=data)
+
+
+def collection_dense_dim(client, *, timeout: float | None = None) -> Optional[int]:
+ """Read the dense_embedding dimension of the existing collection.
+
+ Connection and schema-read errors deliberately propagate to the caller. Treating
+ an unavailable schema as a dimension mismatch used to trigger a destructive
+ collection rebuild.
+ """
+ desc = client.describe_collection(kb_vector_state.COLLECTION_NAME, timeout=timeout)
+ for f in desc.get("fields", []):
+ if f.get("name") == "dense_embedding":
+ dim = f.get("params", {}).get("dim")
+ return int(dim) if dim is not None else None
+ return None
+
+
+def get_or_create_collection(*, timeout: float | None = None) -> None:
+ """Idempotently create hugagent_kb_private (hybrid retrieval schema) at the current embedding model's real dimension.
+
+ New collections use the current embedding model's detected dimension. Existing
+ collections are never dropped implicitly: an unreadable schema or dimension
+ mismatch preserves all vectors and aborts the operation until an operator can
+ verify the embedding configuration and explicitly re-index the collection.
+ """
+ pass # State is owned by its responsibility module.
+ from pymilvus import DataType
+
+ client = get_client()
+ target_dim = vector_embedding.detect_embed_dim()
+
+ if client.has_collection(kb_vector_state.COLLECTION_NAME, timeout=timeout):
+ if kb_vector_state.VERIFIED_COLLECTION_DIM == target_dim:
+ return
+ try:
+ existing_dim = collection_dense_dim(client, timeout=timeout)
+ except Exception as exc:
+ message = (
+ "无法确认现有知识库集合的向量维度,已保留集合并中止本次操作;"
+ "请检查 Milvus 连接后重试。"
+ )
+ logger.error("%s 原因:%s", message, exc, exc_info=True)
+ raise RuntimeError(message) from exc
+ if existing_dim is None:
+ message = (
+ "无法确认现有知识库集合的向量维度,已保留集合并中止本次操作;"
+ "dense_embedding 字段或 dim 参数缺失。"
+ )
+ logger.error(message)
+ raise RuntimeError(message)
+ if existing_dim == target_dim:
+ kb_vector_state.VERIFIED_COLLECTION_DIM = target_dim
+ return
+ message = (
+ f"KB collection 维度不匹配(已存在={existing_dim},当前模型={target_dim}),"
+ "已保留集合并中止本次操作;请核实 embedding 配置并通过显式重建流程重新索引。"
+ )
+ logger.error(message)
+ raise RuntimeError(message)
+
+ lite = is_lite()
+
+ schema = client.create_schema(auto_id=False, enable_dynamic_field=False)
+ schema.add_field("chunk_id", DataType.VARCHAR, max_length=64, is_primary=True)
+ schema.add_field("parent_chunk_id", DataType.VARCHAR, max_length=64)
+ schema.add_field("row_type", DataType.VARCHAR, max_length=16) # "chunk" | "question"
+ schema.add_field("user_id", DataType.VARCHAR, max_length=64)
+ schema.add_field("kb_id", DataType.VARCHAR, max_length=64)
+ schema.add_field("document_id", DataType.VARCHAR, max_length=64)
+ schema.add_field("title", DataType.VARCHAR, max_length=kb_vector_state.TITLE_FIELD_MAX_BYTES)
+ schema.add_field("content", DataType.VARCHAR, max_length=kb_vector_state.CONTENT_FIELD_MAX_BYTES)
+ schema.add_field(
+ "tags_text", DataType.VARCHAR, max_length=kb_vector_state.TAGS_FIELD_MAX_BYTES
+ ) # BM25 augmentation
+ schema.add_field("chunk_index", DataType.INT64)
+ schema.add_field("dense_embedding", DataType.FLOAT_VECTOR, dim=target_dim)
+ if not lite:
+ schema.add_field("sparse_embedding", DataType.SPARSE_FLOAT_VECTOR)
+
+ index_params = client.prepare_index_params()
+ if lite:
+ # Milvus Lite: only a dense AUTOINDEX (FLAT under the hood). No sparse, no
+ # scalar INVERTED indexes — filtering falls back to brute force, fine at
+ # single-machine scale.
+ index_params.add_index(
+ field_name="dense_embedding",
+ index_type="AUTOINDEX",
+ metric_type="IP",
+ )
+ else:
+ index_params.add_index(
+ field_name="dense_embedding",
+ index_type="IVF_FLAT",
+ metric_type="IP",
+ params={"nlist": 128},
+ )
+ index_params.add_index(
+ field_name="sparse_embedding",
+ index_type="SPARSE_INVERTED_INDEX",
+ metric_type="IP",
+ )
+ # Scalar indexes for filtering acceleration
+ for field in ("user_id", "kb_id", "document_id", "row_type"):
+ index_params.add_index(field_name=field, index_type="INVERTED")
+
+ client.create_collection(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ schema=schema,
+ index_params=index_params,
+ timeout=timeout,
+ )
+ kb_vector_state.VERIFIED_COLLECTION_DIM = target_dim
+ logger.info(
+ "Created Milvus collection: %s (dim=%d, mode=%s)",
+ kb_vector_state.COLLECTION_NAME,
+ target_dim,
+ "lite/dense-only" if lite else "hybrid",
+ )
diff --git a/src/backend/features/knowledge/vector_fields.py b/src/backend/features/knowledge/vector_fields.py
new file mode 100644
index 00000000..c3fdaf17
--- /dev/null
+++ b/src/backend/features/knowledge/vector_fields.py
@@ -0,0 +1,16 @@
+"""vector fields responsibilities."""
+from __future__ import annotations
+from features.knowledge import kb_vector_state
+
+def truncate_utf8(text: str, max_bytes: int = kb_vector_state.CONTENT_FIELD_MAX_BYTES) -> str:
+ """Safe truncation by UTF-8 bytes (Chinese is 3 bytes/char); never cuts in the middle of a multi-byte character.
+
+ Historical bug: using ``text[:4096]`` truncated by "character"; 4096 Chinese chars ≈ 12KB bytes, still exceeding Milvus's
+ 4096 **byte** limit → ``length exceeds max length`` error.
+ """
+ if not text:
+ return ""
+ encoded = text.encode("utf-8")
+ if len(encoded) <= max_bytes:
+ return text
+ return encoded[:max_bytes].decode("utf-8", "ignore")
diff --git a/src/backend/features/knowledge/vector_indexing.py b/src/backend/features/knowledge/vector_indexing.py
new file mode 100644
index 00000000..52c50f24
--- /dev/null
+++ b/src/backend/features/knowledge/vector_indexing.py
@@ -0,0 +1,204 @@
+"""vector indexing responsibilities."""
+from __future__ import annotations
+from features.knowledge import kb_vector
+from typing import Any
+import logging
+logger = logging.getLogger(__name__)
+from features.knowledge import kb_vector
+from features.knowledge import kb_vector_state
+from features.knowledge import vector_client
+from core.models import embedding as vector_embedding
+from features.knowledge import vector_fields
+
+def upsert_rows(rows: list[dict[str, Any]]) -> None:
+ """Upsert a batch of rows into hugagent_kb_private.
+
+ Each row must include all schema fields. dense_embedding and sparse_embedding
+ must be pre-computed by the caller (use embed_text / build_sparse_text).
+ """
+ if not rows:
+ return
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ vector_client.upsert(client, rows)
+ logger.debug("Upserted %d rows into %s", len(rows), kb_vector_state.COLLECTION_NAME)
+
+
+def delete_by_document(document_id: str, user_id: str) -> None:
+ """Delete all Milvus rows (chunk + question) for a document."""
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ expr = f'document_id == "{document_id}" and user_id == "{user_id}"'
+ client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr)
+ logger.info("Deleted Milvus rows for document_id=%s user_id=%s", document_id, user_id)
+
+
+def delete_by_kb(kb_id: str, user_id: str) -> None:
+ """Delete all Milvus rows for an entire KB space."""
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ expr = f'kb_id == "{kb_id}" and user_id == "{user_id}"'
+ client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr)
+ logger.info("Deleted Milvus rows for kb_id=%s user_id=%s", kb_id, user_id)
+
+
+def get_children_for_parent(parent_chunk_id: str) -> list[dict]:
+ """Return the child-chunk rows of a given parent chunk in Milvus (for the "view chunks" parent→child display).
+
+ Child rows have ``chunk_id`` shaped like ``{pid}_{i}`` with ``parent_chunk_id == pid``; in flat mode a parent has only one
+ representative row (``chunk_id == parent_chunk_id``), which is filtered out —— i.e. flat mode returns an empty list.
+ Returns an empty list when Milvus is unavailable / the query fails, without affecting the parent-chunk list display.
+ """
+ try:
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ rows = client.query(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ # In flat mode a parent has only one representative row (chunk_id == parent_chunk_id); exclude it right in the query
+ filter=(
+ f'parent_chunk_id == "{parent_chunk_id}" and row_type == "chunk" '
+ f'and chunk_id != "{parent_chunk_id}"'
+ ),
+ output_fields=["chunk_id", "parent_chunk_id", "chunk_index", "content"],
+ limit=4096,
+ )
+ except Exception as exc:
+ logger.warning("get_children_for_parent(%s) failed: %s", parent_chunk_id, exc)
+ return []
+ children = [
+ {
+ "chunk_id": r.get("chunk_id"),
+ "chunk_index": r.get("chunk_index", 0),
+ "content": r.get("content", ""),
+ }
+ for r in rows
+ ]
+ children.sort(key=lambda x: x["chunk_index"])
+ return children
+
+
+def delete_by_chunk(parent_chunk_id: str) -> None:
+ """Delete all Milvus rows (chunk + question) for a single parent chunk."""
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ expr = f'parent_chunk_id == "{parent_chunk_id}"'
+ client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr)
+ logger.info("Deleted Milvus rows for parent_chunk_id=%s", parent_chunk_id)
+
+
+def reindex_chunk_tags(chunk_id: str, content: str, tags: list[str]) -> None:
+ """Re-compute and upsert sparse_embedding for a chunk row after tag changes."""
+ # Fetch the existing row to preserve all other fields
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ rows = client.query(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ filter=f'chunk_id == "{chunk_id}"',
+ output_fields=[
+ "chunk_id",
+ "parent_chunk_id",
+ "row_type",
+ "user_id",
+ "kb_id",
+ "document_id",
+ "title",
+ "chunk_index",
+ "dense_embedding",
+ ],
+ )
+ if not rows:
+ logger.warning("reindex_chunk_tags: chunk_id %s not found in Milvus", chunk_id)
+ return
+
+ row = rows[0]
+ sparse_text = kb_vector.build_sparse_text(content, tags)
+ row["content"] = vector_fields.truncate_utf8(content)
+ row["tags_text"] = vector_fields.truncate_utf8(" ".join(tags), kb_vector_state.TAGS_FIELD_MAX_BYTES)
+ row["sparse_embedding"] = kb_vector.text_to_sparse(sparse_text)
+ vector_client.upsert(client, [row])
+
+
+def reindex_chunk_content(
+ chunk_id: str,
+ content: str,
+ tags: list[str],
+ user_id: str,
+ kb_id: str,
+ document_id: str,
+ title: str,
+ chunk_index: int,
+) -> None:
+ """Re-embed a parent chunk after a manual content edit.
+
+ Removes the parent's existing chunk rows (child or flat) and upserts a single
+ fresh chunk row with recomputed dense + sparse embeddings. Question rows are
+ managed separately by :func:`upsert_question_rows`. Retrieval returns the parent
+ content from PostgreSQL, so this single representative row is sufficient for the
+ edited chunk to be matched and surfaced.
+ """
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+ # Drop existing chunk rows for this parent (keep questions; they are re-upserted separately)
+ client.delete(
+ collection_name=kb_vector_state.COLLECTION_NAME,
+ filter=f'parent_chunk_id == "{chunk_id}" and row_type == "chunk"',
+ )
+ sparse_text = kb_vector.build_sparse_text(content, tags or [])
+ row = {
+ "chunk_id": chunk_id,
+ "parent_chunk_id": chunk_id,
+ "row_type": "chunk",
+ "user_id": user_id,
+ "kb_id": kb_id,
+ "document_id": document_id,
+ "title": vector_fields.truncate_utf8(title, kb_vector_state.TITLE_FIELD_MAX_BYTES),
+ "content": vector_fields.truncate_utf8(content),
+ "tags_text": vector_fields.truncate_utf8(" ".join(tags or []), kb_vector_state.TAGS_FIELD_MAX_BYTES),
+ "chunk_index": chunk_index,
+ "dense_embedding": vector_embedding.embed_text(content),
+ "sparse_embedding": kb_vector.text_to_sparse(sparse_text),
+ }
+ vector_client.upsert(client, [row])
+
+
+def upsert_question_rows(
+ parent_chunk_id: str,
+ questions: list[str],
+ user_id: str,
+ kb_id: str,
+ document_id: str,
+ title: str,
+ chunk_index: int,
+) -> None:
+ """Delete existing question rows for a chunk and insert fresh ones."""
+ vector_client.get_or_create_collection()
+ client = vector_client.get_client()
+
+ # Delete old question rows
+ expr = f'parent_chunk_id == "{parent_chunk_id}" and row_type == "question"'
+ client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr)
+
+ if not questions:
+ return
+
+ # Embed all questions in one batch call
+ vecs = vector_embedding.embed_batch(questions)
+ rows = []
+ for i, (q, vec) in enumerate(zip(questions, vecs)):
+ rows.append(
+ {
+ "chunk_id": f"q_{parent_chunk_id}_{i}",
+ "parent_chunk_id": parent_chunk_id,
+ "row_type": "question",
+ "user_id": user_id,
+ "kb_id": kb_id,
+ "document_id": document_id,
+ "title": vector_fields.truncate_utf8(title, kb_vector_state.TITLE_FIELD_MAX_BYTES),
+ "content": vector_fields.truncate_utf8(q),
+ "tags_text": "",
+ "chunk_index": chunk_index,
+ "dense_embedding": vec,
+ "sparse_embedding": kb_vector.text_to_sparse(q),
+ }
+ )
+ vector_client.upsert(client, rows)
diff --git a/src/backend/core/kb/wiki/__init__.py b/src/backend/features/knowledge/wiki/__init__.py
similarity index 92%
rename from src/backend/core/kb/wiki/__init__.py
rename to src/backend/features/knowledge/wiki/__init__.py
index 3fbc3ed5..cdf25048 100644
--- a/src/backend/core/kb/wiki/__init__.py
+++ b/src/backend/features/knowledge/wiki/__init__.py
@@ -17,6 +17,6 @@
3. **Wiki 是地图不是第二检索源。** 答案与出处始终回到 kb_chunks 的原文。
"""
-from core.kb.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled
+from features.knowledge.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled
__all__ = ["WikiConfig", "resolve_wiki_config", "wiki_enabled"]
diff --git a/src/backend/core/kb/wiki/cite.py b/src/backend/features/knowledge/wiki/cite.py
similarity index 95%
rename from src/backend/core/kb/wiki/cite.py
rename to src/backend/features/knowledge/wiki/cite.py
index 71c5daf9..73e58726 100644
--- a/src/backend/core/kb/wiki/cite.py
+++ b/src/backend/features/knowledge/wiki/cite.py
@@ -13,11 +13,11 @@
import logging
from typing import Any, Dict, List, Optional, Sequence, Tuple
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.extract import CandidateItem, parse_candidates
-from core.kb.wiki.llm import LLMBudget, call_llm_json
-from core.kb.wiki.prompts import CHUNK_CITATION_PROMPT, append_business_instructions
-from core.kb.wiki.slug import normalize_slug
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.extract import CandidateItem, parse_candidates
+from features.knowledge.wiki.llm import LLMBudget, call_llm_json
+from features.knowledge.wiki.prompts import CHUNK_CITATION_PROMPT, append_business_instructions
+from features.knowledge.wiki.slug import normalize_slug
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/kb/wiki/config.py b/src/backend/features/knowledge/wiki/config.py
similarity index 99%
rename from src/backend/core/kb/wiki/config.py
rename to src/backend/features/knowledge/wiki/config.py
index 68771ca3..c27ef680 100644
--- a/src/backend/core/kb/wiki/config.py
+++ b/src/backend/features/knowledge/wiki/config.py
@@ -153,7 +153,7 @@ def wiki_enabled_kb_ids_map(db: Any, kb_ids: Sequence[str]) -> Dict[str, List[st
if not ids:
return {}
try:
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
rows = (
db.query(KBSpace.kb_id, KBSpace.extra_data)
diff --git a/src/backend/core/kb/wiki/dedup.py b/src/backend/features/knowledge/wiki/dedup.py
similarity index 94%
rename from src/backend/core/kb/wiki/dedup.py
rename to src/backend/features/knowledge/wiki/dedup.py
index 428a278f..143fa381 100644
--- a/src/backend/core/kb/wiki/dedup.py
+++ b/src/backend/features/knowledge/wiki/dedup.py
@@ -13,13 +13,13 @@
import logging
from typing import Dict, List, Optional, Sequence, Set, Tuple
-from core.db.models import KBWikiPage
-from core.evolution.similarity import _ngrams, lexical_similarity
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.extract import CandidateItem
-from core.kb.wiki.llm import LLMBudget, call_llm_json
-from core.kb.wiki.prompts import DEDUPLICATION_PROMPT
-from core.kb.wiki.slug import normalize_slug
+from features.knowledge.models import KBWikiPage
+from core.evolution.assessment.similarity import ngrams as _ngrams, lexical_similarity
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.extract import CandidateItem
+from features.knowledge.wiki.llm import LLMBudget, call_llm_json
+from features.knowledge.wiki.prompts import DEDUPLICATION_PROMPT
+from features.knowledge.wiki.slug import normalize_slug
logger = logging.getLogger(__name__)
@@ -36,7 +36,7 @@
def _similarity(left: str, right: str) -> float:
"""两个表面词的相似度。完全相同直接给 1.0。
- 底层复用 ``core.evolution.similarity.lexical_similarity``(字符 n-gram 的
+ 底层复用 ``core.evolution.assessment.similarity.lexical_similarity``(字符 n-gram 的
Jaccard),条目名很短所以取 n=2——三元组在四字名上几乎没有重叠可言。
"""
if not left or not right:
diff --git a/src/backend/core/kb/wiki/extract.py b/src/backend/features/knowledge/wiki/extract.py
similarity index 93%
rename from src/backend/core/kb/wiki/extract.py
rename to src/backend/features/knowledge/wiki/extract.py
index c45acd3f..76c215ee 100644
--- a/src/backend/core/kb/wiki/extract.py
+++ b/src/backend/features/knowledge/wiki/extract.py
@@ -11,14 +11,10 @@
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.llm import LLMBudget, call_llm_json
-from core.kb.wiki.prompts import (
- CANDIDATE_SLUG_PROMPT,
- append_business_instructions,
- granularity_guidance,
-)
-from core.kb.wiki.slug import normalize_slug
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.llm import LLMBudget, call_llm_json
+from features.knowledge.wiki.prompts import CANDIDATE_SLUG_PROMPT, append_business_instructions, granularity_guidance
+from features.knowledge.wiki.slug import normalize_slug
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/kb/wiki/finalize.py b/src/backend/features/knowledge/wiki/finalize.py
similarity index 92%
rename from src/backend/core/kb/wiki/finalize.py
rename to src/backend/features/knowledge/wiki/finalize.py
index 5212da3e..42be5e60 100644
--- a/src/backend/core/kb/wiki/finalize.py
+++ b/src/backend/features/knowledge/wiki/finalize.py
@@ -17,17 +17,12 @@
import logging
from typing import Dict, List, Set
-from core.db.models.kb_wiki import (
- WIKI_KNOWLEDGE_PAGE_TYPES,
- WIKI_PAGE_TYPE_INDEX,
- WIKI_PAGE_TYPE_LABELS,
- WIKI_PAGE_TYPE_SUMMARY,
-)
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.llm import LLMBudget, WikiLLMError, call_llm
-from core.kb.wiki.prompts import INDEX_INTRO_PROMPT, append_business_instructions
-from core.kb.wiki.slug import extract_link_slugs, strip_links_except
-from core.kb.wiki.store import all_pages, get_page, prune_empty_folders, upsert_page
+from features.knowledge.models import WIKI_KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_INDEX, WIKI_PAGE_TYPE_LABELS, WIKI_PAGE_TYPE_SUMMARY
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.llm import LLMBudget, WikiLLMError, call_llm
+from features.knowledge.wiki.prompts import INDEX_INTRO_PROMPT, append_business_instructions
+from features.knowledge.wiki.slug import extract_link_slugs, strip_links_except
+from features.knowledge.wiki.store import all_pages, get_page, prune_empty_folders, upsert_page
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/kb/wiki/jobs.py b/src/backend/features/knowledge/wiki/jobs.py
similarity index 99%
rename from src/backend/core/kb/wiki/jobs.py
rename to src/backend/features/knowledge/wiki/jobs.py
index 25508a0d..214adf3a 100644
--- a/src/backend/core/kb/wiki/jobs.py
+++ b/src/backend/features/knowledge/wiki/jobs.py
@@ -11,8 +11,8 @@
from datetime import datetime, timedelta, timezone
from typing import List, Optional, Sequence
-from core.db.models import KBWikiJob
-from core.kb.wiki.store import _new_id
+from features.knowledge.models import KBWikiJob
+from features.knowledge.wiki.store import _new_id
from sqlalchemy import func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
diff --git a/src/backend/core/kb/wiki/linkify.py b/src/backend/features/knowledge/wiki/linkify.py
similarity index 100%
rename from src/backend/core/kb/wiki/linkify.py
rename to src/backend/features/knowledge/wiki/linkify.py
diff --git a/src/backend/core/kb/wiki/llm.py b/src/backend/features/knowledge/wiki/llm.py
similarity index 96%
rename from src/backend/core/kb/wiki/llm.py
rename to src/backend/features/knowledge/wiki/llm.py
index 5e925fbc..171b695d 100644
--- a/src/backend/core/kb/wiki/llm.py
+++ b/src/backend/features/knowledge/wiki/llm.py
@@ -20,8 +20,8 @@
from typing import Any, Dict, Optional
import requests
-from core.llm._distill_shared import parse_strict_json, strip_think_blocks
-from core.llm.single_turn import Endpoint, build_request, parse_response, turns
+from core.infra.model_output import parse_strict_json, strip_think_blocks
+from core.models.completions.single_turn import Endpoint, build_request, parse_response, turns
logger = logging.getLogger(__name__)
@@ -46,7 +46,7 @@ def _resolve_endpoint() -> Endpoint:
Wiki 生成对「主模型在哪」必须给出同一个答案。
"""
try:
- from core.services.model_config import ModelConfigService
+ from core.models.config import ModelConfigService
svc = ModelConfigService.get_instance()
for role in ("kb_wiki", "main_agent"):
@@ -56,7 +56,7 @@ def _resolve_endpoint() -> Endpoint:
except Exception as exc: # noqa: BLE001 - 配置服务不可用时退到主模型,不阻断管线
logger.warning("Wiki 模型角色解析失败,回落主模型: %s", exc)
- from core.content.kb_processing import resolve_main_model_config
+ from features.knowledge.enrichment import resolve_main_model_config
base_url, api_key, model_name = resolve_main_model_config()
return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name)
@@ -109,7 +109,7 @@ def _repair_literal_newlines(text: str) -> str:
提示词已经禁止过,但模型仍会偶发违反,而这恰好是最常见的一种解析失败。逐字符
扫描并跟踪是否处在字符串里,只改字符串内部的控制字符。这一段是本模块相对
仓库通用解析器**额外**补的一层,其余能力(think 剥离、围栏、括号配平提取)
- 一律走 ``core.llm._distill_shared``。
+ 一律走 ``core.infra.model_output``。
"""
out = []
in_string = False
diff --git a/src/backend/features/knowledge/wiki/local_graph.py b/src/backend/features/knowledge/wiki/local_graph.py
new file mode 100644
index 00000000..f8739896
--- /dev/null
+++ b/src/backend/features/knowledge/wiki/local_graph.py
@@ -0,0 +1,156 @@
+"""local graph responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict, List, Sequence
+from core.db.engine import SessionLocal
+from features.knowledge.models import KBWikiPage
+from features.knowledge.models import WIKI_PAGE_TYPE_INDEX
+from features.knowledge.wiki.slug import readable_from_slug
+from sqlalchemy.orm import Session
+from features.knowledge.wiki import local_projection
+from features.knowledge.wiki import local_provider_state
+
+def resolve_link_titles(
+ db: Session, kb_id: str, slugs: Sequence[str], *, resolve: bool
+) -> List[Dict[str, str]]:
+ if not slugs:
+ return []
+ titles: Dict[str, KBWikiPage] = {}
+ if resolve:
+ rows = local_projection.base_query(db, kb_id).filter(KBWikiPage.slug.in_(list(slugs))).all()
+ titles = {row.slug: row for row in rows}
+ resolved = []
+ for slug in slugs:
+ row = titles.get(slug)
+ resolved.append(
+ {
+ "slug": slug,
+ "title": (row.title if row else "") or readable_from_slug(slug),
+ "page_type": (row.page_type if row else "") or slug.split("/", 1)[0],
+ }
+ )
+ return resolved
+
+
+def wiki_graph(
+ kb_id: str,
+ mode: str = "overview",
+ center: str = "",
+ depth: int = 1,
+ limit: int = 60,
+ types: str = "",
+) -> Dict[str, Any]:
+ """概念图谱。
+
+ ``overview`` 取全局连接度最高的节点,``ego`` 以 center 为中心展开 depth 层。
+ 大库有数千节点上万条边,前端必须靠这两种模式分批取,不能一次拉全量。
+ """
+ limit = max(1, min(int(limit or 60), local_provider_state.MAX_GRAPH_NODES))
+ depth = max(1, min(int(depth or 1), 3))
+ wanted = local_projection.split_types(types)
+
+ with SessionLocal() as db:
+ # 图谱只用到这四列;页面正文对邻接表毫无用处,却是最大的一列
+ query = db.query(
+ KBWikiPage.slug,
+ KBWikiPage.title,
+ KBWikiPage.page_type,
+ KBWikiPage.out_links,
+ ).filter(
+ KBWikiPage.kb_id == kb_id,
+ KBWikiPage.deleted_at.is_(None),
+ KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX,
+ )
+ if wanted:
+ query = query.filter(KBWikiPage.page_type.in_(wanted))
+ rows = query.all()
+
+ by_slug = {row.slug: row for row in rows}
+ adjacency: Dict[str, List[str]] = {
+ row.slug: [s for s in (row.out_links or []) if s in by_slug] for row in rows
+ }
+ # 边是无向展开用的,两个方向都要能走
+ undirected: Dict[str, set] = {slug: set(links) for slug, links in adjacency.items()}
+ for slug, links in adjacency.items():
+ for target in links:
+ undirected.setdefault(target, set()).add(slug)
+
+ if mode == "ego":
+ selected = ego_nodes(undirected, center, depth, limit)
+ if not selected:
+ return {
+ "nodes": [],
+ "edges": [],
+ "meta": {
+ "mode": "ego",
+ "center": center,
+ "depth": depth,
+ "truncated": False,
+ "total": len(rows),
+ "returned": 0,
+ "total_pages": len(rows),
+ },
+ }
+ else:
+ ranked = sorted(rows, key=lambda r: len(undirected.get(r.slug, ())), reverse=True)
+ selected = {r.slug for r in ranked[:limit]}
+
+ nodes = [
+ {
+ "slug": slug,
+ "title": by_slug[slug].title or readable_from_slug(slug),
+ "page_type": by_slug[slug].page_type,
+ "link_count": len(undirected.get(slug, ())),
+ }
+ for slug in selected
+ if slug in by_slug
+ ]
+ edges = []
+ seen_edges = set()
+ for slug in selected:
+ for target in adjacency.get(slug, []):
+ if target not in selected:
+ continue
+ key = tuple(sorted((slug, target)))
+ if key in seen_edges:
+ continue
+ seen_edges.add(key)
+ edges.append({"source": slug, "target": target})
+
+ return {
+ "nodes": nodes,
+ "edges": edges,
+ "meta": {
+ "mode": mode,
+ "center": center,
+ "depth": depth if mode == "ego" else None,
+ "truncated": len(rows) > len(selected),
+ # 前端类型声明读 total/returned;total_pages 是旧键,保留兼容
+ "total": len(rows),
+ "returned": len(nodes),
+ "total_pages": len(rows),
+ },
+ }
+
+
+def ego_nodes(adjacency: Dict[str, set], center: str, depth: int, limit: int) -> set:
+ """从 center 出发做 BFS,取 depth 层内的节点。"""
+ if not center or center not in adjacency:
+ return set()
+ selected = {center}
+ frontier = {center}
+ for _ in range(depth):
+ nxt = set()
+ for slug in frontier:
+ for neighbour in adjacency.get(slug, ()):
+ if neighbour not in selected:
+ nxt.add(neighbour)
+ if not nxt:
+ break
+ for slug in sorted(nxt, key=lambda s: len(adjacency.get(s, ())), reverse=True):
+ if len(selected) >= limit:
+ break
+ selected.add(slug)
+ frontier = nxt
+ if len(selected) >= limit:
+ break
+ return selected
diff --git a/src/backend/features/knowledge/wiki/local_projection.py b/src/backend/features/knowledge/wiki/local_projection.py
new file mode 100644
index 00000000..e4101c07
--- /dev/null
+++ b/src/backend/features/knowledge/wiki/local_projection.py
@@ -0,0 +1,51 @@
+"""local projection responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict, List, Sequence
+from features.knowledge.models import KBDocument, KBWikiPage
+from sqlalchemy.orm import Session
+from features.knowledge.wiki import local_provider_state
+
+def split_types(raw: str) -> List[str]:
+ return [t.strip() for t in str(raw or "").split(",") if t.strip()]
+
+
+def slim_page(page: KBWikiPage, *, with_content: bool = False) -> Dict[str, Any]:
+ """页面 → 精简结构。
+
+ 定位阶段只要标题/摘要/链接/血缘,正文按需带:一页动辄数千字,无差别塞进模型
+ 上下文会把「先定位再取回」的成本优势整个吃掉。
+ """
+ item: Dict[str, Any] = {
+ "slug": page.slug,
+ "title": page.title,
+ "page_type": page.page_type,
+ "type_label": local_provider_state.PAGE_TYPE_LABELS.get(page.page_type or "", ""),
+ "summary": page.summary or "",
+ "aliases": list(page.aliases or []),
+ "category_path": list(page.category_path or []),
+ "wiki_path": page.wiki_path or "",
+ "out_links": list(page.out_links or []),
+ "in_links": list(page.in_links or []),
+ "source_refs": list(page.source_refs or []),
+ "chunk_refs": list(page.chunk_refs or []),
+ "updated_at": page.updated_at.isoformat() if page.updated_at else None,
+ }
+ if with_content:
+ item["content"] = page.content or ""
+ return item
+
+
+def base_query(db: Session, kb_id: str):
+ return db.query(KBWikiPage).filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None))
+
+
+def document_titles(db: Session, kb_id: str, document_ids: Sequence[str]) -> Dict[str, str]:
+ ids = [d for d in dict.fromkeys(document_ids) if d]
+ if not ids:
+ return {}
+ rows = (
+ db.query(KBDocument.document_id, KBDocument.title, KBDocument.filename)
+ .filter(KBDocument.kb_id == kb_id, KBDocument.document_id.in_(ids))
+ .all()
+ )
+ return {row.document_id: (row.title or row.filename or row.document_id) for row in rows}
diff --git a/src/backend/features/knowledge/wiki/local_provider.py b/src/backend/features/knowledge/wiki/local_provider.py
new file mode 100644
index 00000000..ea36dd5c
--- /dev/null
+++ b/src/backend/features/knowledge/wiki/local_provider.py
@@ -0,0 +1,271 @@
+"""local provider responsibilities."""
+from __future__ import annotations
+import re
+from typing import Any, Dict, List
+from core.db.engine import SessionLocal
+from features.knowledge.models import KBChunk, KBWikiPage
+from features.knowledge.models import WIKI_PAGE_TYPE_INDEX, WIKI_PAGE_TYPE_SUMMARY
+from features.knowledge.wiki.finalize import INDEX_SLUG
+from features.knowledge.wiki.store import children_index, clean_category_path, descendant_folder_ids, get_page, load_folder_tree, page_counts_by_folder, subtree_counts, walk_folder_path
+from sqlalchemy import func, or_
+from features.knowledge.wiki import local_graph
+from features.knowledge.wiki import local_projection
+from features.knowledge.wiki import local_provider_state
+
+def wiki_stats(kb_id: str) -> Dict[str, Any]:
+ """规模统计:总页数、分类型页数、链接数、孤儿页数。
+
+ 只投影统计需要的三列——生成期间前端每 10 秒轮一次,把每页几 KB 的 ``content``
+ 拉出来纯属浪费。
+ """
+ with SessionLocal() as db:
+ rows = (
+ db.query(KBWikiPage.page_type, KBWikiPage.in_links, KBWikiPage.out_links)
+ .filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None))
+ .all()
+ )
+ by_type: Dict[str, int] = {}
+ total_links = 0
+ orphan = 0
+ for page_type, in_links, out_links in rows:
+ by_type[page_type] = by_type.get(page_type, 0) + 1
+ total_links += len(out_links or [])
+ if not (in_links or []) and not (out_links or []):
+ orphan += 1
+ return {
+ "total_pages": len(rows),
+ "pages_by_type": by_type,
+ "total_links": total_links,
+ "orphan_count": orphan,
+ }
+
+
+def wiki_list_pages(
+ kb_id: str,
+ page: int = 1,
+ page_size: int = 50,
+ page_type: str = "",
+ category_path: str = "",
+ category_depth: int = 0,
+) -> Dict[str, Any]:
+ """分页列出页面。
+
+ ``page_type`` 支持逗号分隔多选(「知识」视图把实体/概念/综述/对比合成一次
+ 请求,避免按类型扇出);``category_path`` 把范围收窄到某个目录子树。
+ """
+ page = max(1, int(page or 1))
+ page_size = max(1, int(page_size or 50))
+ with SessionLocal() as db:
+ query = local_projection.base_query(db, kb_id)
+ types = local_projection.split_types(page_type)
+ if types:
+ query = query.filter(KBWikiPage.page_type.in_(types))
+
+ labels = clean_category_path([category_path] if category_path else [])
+ if labels:
+ folder_id, _, _ = walk_folder_path(db, kb_id, labels, create=False)
+ if folder_id is None:
+ return {"pages": [], "total": 0, "page": page, "page_size": page_size}
+ query = query.filter(
+ KBWikiPage.folder_id.in_(descendant_folder_ids(db, kb_id, folder_id))
+ )
+
+ total = query.count()
+ rows = (
+ query.order_by(KBWikiPage.sort_order, KBWikiPage.title)
+ .offset((page - 1) * page_size)
+ .limit(page_size)
+ .all()
+ )
+ return {
+ "pages": [local_projection.slim_page(row) for row in rows],
+ "total": total,
+ "page": page,
+ "page_size": page_size,
+ }
+
+
+def wiki_folders(kb_id: str, parent_id: str = "", page_types: str = "") -> Dict[str, Any]:
+ """某一层的子目录,带**递归**页面计数,供前端逐层懒加载。
+
+ 两次查询搞定:整棵目录树 + 一次按目录分组的页数聚合,其余在内存里算。原来是
+ 每个子目录一次子树查询加一次计数,用户逐层展开时会反复付这个代价。
+ """
+ types = local_projection.split_types(page_types)
+ with SessionLocal() as db:
+ folders = load_folder_tree(db, kb_id)
+ if not folders:
+ return {"parent_id": parent_id or "", "folders": []}
+ totals = subtree_counts(folders, page_counts_by_folder(db, kb_id, types))
+ children = children_index(folders)
+ items = [
+ {
+ "id": folder.folder_id,
+ "name": folder.name,
+ "parent_id": folder.parent_id or "",
+ "page_count": totals.get(folder.folder_id, 0),
+ "has_children": bool(children.get(folder.folder_id)),
+ }
+ for folder in children.get(parent_id or "", [])
+ ]
+ return {"parent_id": parent_id or "", "folders": items}
+
+
+def wiki_index(kb_id: str, limit: int = 20, types: str = "") -> Dict[str, Any]:
+ """索引总览:一段总述 + 按类型分组的条目清单。"""
+ limit = max(1, int(limit or 20))
+ wanted = local_projection.split_types(types)
+ with SessionLocal() as db:
+ index_page = get_page(db, kb_id, INDEX_SLUG)
+ intro = ""
+ if index_page:
+ intro = str((index_page.page_metadata or {}).get("intro") or "")
+
+ groups = []
+ for page_type in [*local_provider_state.KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_SUMMARY]:
+ if wanted and page_type not in wanted:
+ continue
+ query = local_projection.base_query(db, kb_id).filter(KBWikiPage.page_type == page_type)
+ total = query.count()
+ if not total:
+ continue
+ rows = query.order_by(KBWikiPage.title).limit(limit).all()
+ groups.append(
+ {
+ "type": page_type,
+ "type_label": local_provider_state.PAGE_TYPE_LABELS.get(page_type, ""),
+ "total": total,
+ "items": [
+ {
+ "slug": row.slug,
+ "title": row.title,
+ "summary": row.summary or "",
+ "wiki_path": row.wiki_path or "",
+ }
+ for row in rows
+ ],
+ }
+ )
+ return {
+ "intro": intro,
+ "version": index_page.version if index_page else None,
+ "groups": groups,
+ }
+
+
+def wiki_search(kb_id: str, query: str, limit: int = 10) -> Dict[str, Any]:
+ """按标题/别名/摘要/正文命中页面。
+
+ 支持 ``资质|牌照`` 这样的交替写法——口语说法常常匹配不上书面术语,一次给多个
+ 说法能显著提高命中率。交替在 Python 侧拆成多个 LIKE,避免依赖数据库正则。
+ """
+ limit = max(1, min(int(limit or 10), local_provider_state.MAX_SEARCH_LIMIT))
+ terms = [t.strip() for t in re.split(r"[||]", str(query or "")) if t.strip()]
+ if not terms:
+ return {"pages": [], "total": 0}
+
+ with SessionLocal() as db:
+ conditions = []
+ for term in terms[:8]:
+ pattern = f"%{term.lower()}%"
+ conditions.append(func.lower(KBWikiPage.title).like(pattern))
+ conditions.append(func.lower(KBWikiPage.summary).like(pattern))
+ conditions.append(func.lower(KBWikiPage.content).like(pattern))
+ rows = (
+ local_projection.base_query(db, kb_id)
+ .filter(KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX)
+ .filter(or_(*conditions))
+ .order_by(KBWikiPage.page_type, KBWikiPage.title)
+ .limit(limit * 3)
+ .all()
+ )
+
+ # 别名命中在 SQL 里跨方言不好写,回到 Python 侧补一遍;同时按"命中位置"
+ # 排序——标题命中远比正文里出现一次更相关
+ lowered_terms = [t.lower() for t in terms]
+
+ def _rank(page: KBWikiPage) -> int:
+ title = (page.title or "").lower()
+ aliases = " ".join(page.aliases or []).lower()
+ summary = (page.summary or "").lower()
+ if any(t == title for t in lowered_terms):
+ return 0
+ if any(t in title for t in lowered_terms):
+ return 1
+ if any(t in aliases for t in lowered_terms):
+ return 2
+ if any(t in summary for t in lowered_terms):
+ return 3
+ return 4
+
+ ranked = sorted(rows, key=lambda p: (_rank(p), p.title or ""))[:limit]
+ return {"pages": [local_projection.slim_page(p) for p in ranked], "total": len(ranked)}
+
+
+def wiki_read_page(kb_id: str, slug: str, *, resolve_links: bool = True) -> Dict[str, Any]:
+ """读取整页(含正文、双链、血缘坐标)。
+
+ ``out_links`` 只有 slug,直接展示就是一串拼音,人和模型都读不出是什么。这里
+ 顺手把邻居的标题查回来贴上——同库一次 IN 查询就够。
+ """
+ with SessionLocal() as db:
+ page = get_page(db, kb_id, slug)
+ if page is None:
+ return {}
+ item = local_projection.slim_page(page, with_content=True)
+ item["related_pages"] = local_graph.resolve_link_titles(
+ db, kb_id, item["out_links"], resolve=resolve_links
+ )
+ return item
+
+
+def wiki_source_chunks(kb_id: str, slug: str, max_chunks: int = 6) -> Dict[str, Any]:
+ """按页面的 ``chunk_refs`` 直取原文分块。
+
+ 这是「地图 → 定位 → 顺坐标取回原文」这条链的最后一跳:**按 ID 直取**,不是
+ 再检索一次。页面与分块同库,一次 IN 查询就够。
+ """
+ max_chunks = max(1, min(int(max_chunks or 6), 20))
+ with SessionLocal() as db:
+ page = get_page(db, kb_id, slug)
+ if page is None:
+ return {"page": {}, "chunks": []}
+
+ item = local_projection.slim_page(page, with_content=True)
+ item["related_pages"] = local_graph.resolve_link_titles(db, kb_id, item["out_links"], resolve=True)
+
+ chunk_ids = list(page.chunk_refs or [])
+ rows: List[KBChunk] = []
+ if chunk_ids:
+ rows = (
+ db.query(KBChunk)
+ .filter(KBChunk.kb_id == kb_id, KBChunk.chunk_id.in_(chunk_ids))
+ .all()
+ )
+ # 保持页面记录的引用顺序:那是证据的呈现次序,不是数据库的返回次序
+ order = {cid: i for i, cid in enumerate(chunk_ids)}
+ rows.sort(key=lambda r: order.get(r.chunk_id, 10**6))
+
+ # chunk_refs 对不上(页面被改写过、或文档重新索引换了分块)时,退回该页
+ # 主要来源文档的前几块正文——给不出精确证据,至少不要空手而归
+ if not rows and (page.source_refs or []):
+ rows = (
+ db.query(KBChunk)
+ .filter(KBChunk.kb_id == kb_id, KBChunk.document_id == page.source_refs[0])
+ .order_by(KBChunk.chunk_index)
+ .limit(max_chunks)
+ .all()
+ )
+
+ titles = local_projection.document_titles(db, kb_id, [r.document_id for r in rows])
+ chunks = [
+ {
+ "chunk_id": row.chunk_id,
+ "document_id": row.document_id,
+ "document_title": titles.get(row.document_id, ""),
+ "content": row.content or "",
+ }
+ for row in rows[:max_chunks]
+ if (row.content or "").strip()
+ ]
+ return {"page": item, "chunks": chunks}
diff --git a/src/backend/features/knowledge/wiki/local_provider_state.py b/src/backend/features/knowledge/wiki/local_provider_state.py
new file mode 100644
index 00000000..e0244d4d
--- /dev/null
+++ b/src/backend/features/knowledge/wiki/local_provider_state.py
@@ -0,0 +1,15 @@
+"""local provider state responsibilities."""
+from __future__ import annotations
+from features.knowledge.models import WIKI_KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_LABELS
+from features.knowledge.wiki.config import MAX_GRAPH_NODES
+
+PAGE_TYPE_LABELS = WIKI_PAGE_TYPE_LABELS
+
+
+KNOWLEDGE_PAGE_TYPES = WIKI_KNOWLEDGE_PAGE_TYPES
+
+
+MAX_GRAPH_NODES = MAX_GRAPH_NODES
+
+
+MAX_SEARCH_LIMIT = 50
diff --git a/src/backend/core/kb/wiki/pipeline.py b/src/backend/features/knowledge/wiki/pipeline.py
similarity index 95%
rename from src/backend/core/kb/wiki/pipeline.py
rename to src/backend/features/knowledge/wiki/pipeline.py
index 0394f47d..a135f02b 100644
--- a/src/backend/core/kb/wiki/pipeline.py
+++ b/src/backend/features/knowledge/wiki/pipeline.py
@@ -22,21 +22,15 @@
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Sequence, Set
-from core.db.models import KBChunk, KBDocument
-from core.kb.wiki.cite import annotate_citations, apply_citations
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.dedup import apply_merges, resolve_duplicates
-from core.kb.wiki.extract import CandidateItem, extract_candidates
-from core.kb.wiki.llm import LLMBudget, WikiQuotaExceeded
-from core.kb.wiki.reduce import write_document_summary, write_entity_page
-from core.kb.wiki.slug import summary_slug
-from core.kb.wiki.store import (
- all_pages,
- archive_page,
- get_pages,
- pages_citing_document,
- upsert_page,
-)
+from features.knowledge.models import KBChunk, KBDocument
+from features.knowledge.wiki.cite import annotate_citations, apply_citations
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.dedup import apply_merges, resolve_duplicates
+from features.knowledge.wiki.extract import CandidateItem, extract_candidates
+from features.knowledge.wiki.llm import LLMBudget, WikiQuotaExceeded
+from features.knowledge.wiki.reduce import write_document_summary, write_entity_page
+from features.knowledge.wiki.slug import summary_slug
+from features.knowledge.wiki.store import all_pages, archive_page, get_pages, pages_citing_document, upsert_page
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -270,7 +264,7 @@ def _report(stage: str, done: int, total: int) -> None:
# ── 目录规划:只给尚未归档的页分配 ──────────────────────────────────────
assignments: Dict[str, List[str]] = {}
if not result.quota_exceeded:
- from core.kb.wiki.taxonomy import plan_taxonomy
+ from features.knowledge.wiki.taxonomy import plan_taxonomy
unfiled = [item for item in batch_items.values() if not existing_folder_ids.get(item.slug)]
try:
diff --git a/src/backend/core/kb/wiki/prompts.py b/src/backend/features/knowledge/wiki/prompts.py
similarity index 100%
rename from src/backend/core/kb/wiki/prompts.py
rename to src/backend/features/knowledge/wiki/prompts.py
diff --git a/src/backend/core/kb/wiki/reduce.py b/src/backend/features/knowledge/wiki/reduce.py
similarity index 93%
rename from src/backend/core/kb/wiki/reduce.py
rename to src/backend/features/knowledge/wiki/reduce.py
index 123ec80c..642fba14 100644
--- a/src/backend/core/kb/wiki/reduce.py
+++ b/src/backend/features/knowledge/wiki/reduce.py
@@ -15,17 +15,12 @@
import logging
from typing import Any, Dict, List, Optional, Sequence
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.extract import CandidateItem
-from core.kb.wiki.linkify import linkify_content
-from core.kb.wiki.llm import LLMBudget, WikiLLMError, call_llm, split_summary_line
-from core.kb.wiki.prompts import (
- DOCUMENT_SUMMARY_PROMPT,
- PAGE_MODIFY_SYSTEM_PROMPT,
- PAGE_MODIFY_USER_PROMPT,
- append_business_instructions,
-)
-from core.kb.wiki.slug import SlugHandles, extract_link_slugs, strip_links_except
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.extract import CandidateItem
+from features.knowledge.wiki.linkify import linkify_content
+from features.knowledge.wiki.llm import LLMBudget, WikiLLMError, call_llm, split_summary_line
+from features.knowledge.wiki.prompts import DOCUMENT_SUMMARY_PROMPT, PAGE_MODIFY_SYSTEM_PROMPT, PAGE_MODIFY_USER_PROMPT, append_business_instructions
+from features.knowledge.wiki.slug import SlugHandles, extract_link_slugs, strip_links_except
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/kb/wiki/slug.py b/src/backend/features/knowledge/wiki/slug.py
similarity index 100%
rename from src/backend/core/kb/wiki/slug.py
rename to src/backend/features/knowledge/wiki/slug.py
diff --git a/src/backend/core/kb/wiki/store.py b/src/backend/features/knowledge/wiki/store.py
similarity index 98%
rename from src/backend/core/kb/wiki/store.py
rename to src/backend/features/knowledge/wiki/store.py
index 99e202ae..154a85c3 100644
--- a/src/backend/core/kb/wiki/store.py
+++ b/src/backend/features/knowledge/wiki/store.py
@@ -10,12 +10,8 @@
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, List, Optional, Sequence
-from core.db.models import KBWikiFolder, KBWikiPage
-from core.db.models.kb_wiki import (
- WIKI_CATEGORY_MAX_DEPTH,
- WIKI_EDIT_SOURCE_PIPELINE,
- WIKI_PAGE_TYPE_ENTITY,
-)
+from features.knowledge.models import KBWikiFolder, KBWikiPage
+from features.knowledge.models import WIKI_CATEGORY_MAX_DEPTH, WIKI_EDIT_SOURCE_PIPELINE, WIKI_PAGE_TYPE_ENTITY
from sqlalchemy import func
from sqlalchemy.orm import Session
diff --git a/src/backend/core/kb/wiki/taxonomy.py b/src/backend/features/knowledge/wiki/taxonomy.py
similarity index 89%
rename from src/backend/core/kb/wiki/taxonomy.py
rename to src/backend/features/knowledge/wiki/taxonomy.py
index 4886fcc6..f534de3e 100644
--- a/src/backend/core/kb/wiki/taxonomy.py
+++ b/src/backend/features/knowledge/wiki/taxonomy.py
@@ -12,13 +12,13 @@
import logging
from typing import Any, Dict, List, Optional, Sequence
-from core.db.models import KBWikiFolder
-from core.kb.wiki.config import WikiConfig
-from core.kb.wiki.extract import CandidateItem
-from core.kb.wiki.llm import LLMBudget, call_llm_json
-from core.kb.wiki.prompts import TAXONOMY_PLAN_PROMPT
-from core.kb.wiki.slug import normalize_slug
-from core.kb.wiki.store import clean_category_path
+from features.knowledge.models import KBWikiFolder
+from features.knowledge.wiki.config import WikiConfig
+from features.knowledge.wiki.extract import CandidateItem
+from features.knowledge.wiki.llm import LLMBudget, call_llm_json
+from features.knowledge.wiki.prompts import TAXONOMY_PLAN_PROMPT
+from features.knowledge.wiki.slug import normalize_slug
+from features.knowledge.wiki.store import clean_category_path
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/kb/wiki/worker.py b/src/backend/features/knowledge/wiki/worker.py
similarity index 94%
rename from src/backend/core/kb/wiki/worker.py
rename to src/backend/features/knowledge/wiki/worker.py
index edf9abda..0e32914d 100644
--- a/src/backend/core/kb/wiki/worker.py
+++ b/src/backend/features/knowledge/wiki/worker.py
@@ -17,11 +17,11 @@
from typing import Optional
from core.infra.logging import get_logger
-from core.kb.wiki import jobs as job_queue
-from core.kb.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled
-from core.kb.wiki.finalize import finalize_kb_wiki
-from core.kb.wiki.llm import LLMBudget, WikiQuotaExceeded, wiki_model_configured
-from core.kb.wiki.pipeline import run_ingest, run_retract
+from features.knowledge.wiki import jobs as job_queue
+from features.knowledge.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled
+from features.knowledge.wiki.finalize import finalize_kb_wiki
+from features.knowledge.wiki.llm import LLMBudget, WikiQuotaExceeded, wiki_model_configured
+from features.knowledge.wiki.pipeline import run_ingest, run_retract
logger = get_logger(__name__)
@@ -71,7 +71,7 @@ async def _loop(self) -> None:
def _run_one(self) -> bool:
"""认领并执行一个作业。返回是否真的干了活(决定下一轮的等待时长)。"""
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
+ from features.knowledge.models import KBSpace
with SessionLocal() as db:
job = job_queue.claim_next_job(db, self.worker_id)
diff --git a/src/backend/core/kb/wiki_router.py b/src/backend/features/knowledge/wiki_router.py
similarity index 88%
rename from src/backend/core/kb/wiki_router.py
rename to src/backend/features/knowledge/wiki_router.py
index 32954456..dd25063f 100644
--- a/src/backend/core/kb/wiki_router.py
+++ b/src/backend/features/knowledge/wiki_router.py
@@ -30,7 +30,7 @@ def local_wiki_enabled(kb_id: str) -> bool:
return False
try:
from core.db.engine import SessionLocal
- from core.kb.wiki.config import wiki_enabled_for_kb
+ from features.knowledge.wiki.config import wiki_enabled_for_kb
with SessionLocal() as db:
return wiki_enabled_for_kb(db, kb_id)
@@ -44,12 +44,12 @@ def wiki_module_for(kb_id: str) -> Optional[ModuleType]:
if is_local_kb(kb_id):
if not local_wiki_enabled(kb_id):
return None
- from core.kb.wiki import local_provider
+ from features.knowledge.wiki import local_provider
return local_provider
try:
- from core.kb.external_provider import wiki_module
+ from features.knowledge.edition import wiki_module
return wiki_module()
except Exception as exc: # noqa: BLE001 - 社区版下外接接缝可能整体缺席
@@ -70,7 +70,7 @@ def supports_wiki(kb_id: str = "") -> bool:
def _external_supports_wiki() -> bool:
try:
- from core.kb.external_provider import supports_wiki as external_supports
+ from features.knowledge.edition import supports_wiki as external_supports
return bool(external_supports())
except Exception:
@@ -88,8 +88,8 @@ def _any_local_wiki_kb() -> bool:
"""
try:
from core.db.engine import SessionLocal
- from core.db.models import KBSpace
- from core.kb.wiki.config import INDEX_MODE_WIKI, normalize_index_modes
+ from features.knowledge.models import KBSpace
+ from features.knowledge.wiki.config import INDEX_MODE_WIKI, normalize_index_modes
with SessionLocal() as db:
rows = db.query(KBSpace.extra_data).filter(KBSpace.deleted_at.is_(None)).yield_per(200)
@@ -111,7 +111,7 @@ def wiki_capable_local_kb_ids(kb_ids: List[str]) -> List[str]:
return []
try:
from core.db.engine import SessionLocal
- from core.kb.wiki.config import INDEX_MODE_WIKI, wiki_enabled_kb_ids_map
+ from features.knowledge.wiki.config import INDEX_MODE_WIKI, wiki_enabled_kb_ids_map
with SessionLocal() as db:
modes = wiki_enabled_kb_ids_map(db, candidates)
@@ -123,7 +123,7 @@ def wiki_capable_local_kb_ids(kb_ids: List[str]) -> List[str]:
def capabilities_for_local_kb(space: Any) -> Dict[str, bool]:
"""把自建库的索引模式投影成能力位,供 catalog 与前端消费。"""
- from core.kb.wiki.config import rag_enabled, wiki_enabled
+ from features.knowledge.wiki.config import rag_enabled, wiki_enabled
has_wiki = wiki_enabled(space)
has_rag = rag_enabled(space)
diff --git a/src/backend/features/marketplace/README.md b/src/backend/features/marketplace/README.md
new file mode 100644
index 00000000..ccafeaac
--- /dev/null
+++ b/src/backend/features/marketplace/README.md
@@ -0,0 +1,15 @@
+# Marketplace
+
+Owns skill, agent, plugin and MCP listings, submissions, review, immutable versions, installation and export. Installed schemas load independently of runtime switches.
+
+API routes are registered by the host central route tables, preserving policy keys and order. `agent_versions` and `skill_versions` apply selected version content; catalog and installation modules contain the underlying operations. Callers use the concrete responsibility module; there are no legacy-path forwarding modules.
+
+Package resolution implements the core package-source interface through `package_source.MarketplacePackageSource`, registered by host composition. Plugin management callers use `plugin_market` directly for listing metadata. Knowledge selection and desktop resources remain host-owned interfaces; remaining legacy core domain references are migrated in P5/P6.
+
+State modules own bundle paths and shared constants. MCP curated configuration (`mcp_catalog`) is separate from listing queries (`mcp_listing`), authentication materialization, immutable version publication, installation, and revalidation. Models preserve the original table names and relationships, including listing visibility state.
+
+Validation: affected marketplace lifecycle, version conflict, archive safety and auth route tests; original route/schema snapshots; import and CE guards; CE empty database upgrade and release checks.
+
+The API host is initialized before direct route use; standalone feature-route imports retain the existing eager API initialization limitation.
+
+版本差异统一由 edition.py 选择列表策略、可见性及管理请求模型;CE 只覆盖此入口。
diff --git a/src/backend/features/marketplace/__init__.py b/src/backend/features/marketplace/__init__.py
new file mode 100644
index 00000000..86d226fc
--- /dev/null
+++ b/src/backend/features/marketplace/__init__.py
@@ -0,0 +1 @@
+"""marketplace package."""
diff --git a/src/backend/features/marketplace/agent_catalog.py b/src/backend/features/marketplace/agent_catalog.py
new file mode 100644
index 00000000..a51f6e56
--- /dev/null
+++ b/src/backend/features/marketplace/agent_catalog.py
@@ -0,0 +1,119 @@
+"""agent catalog responsibilities."""
+from __future__ import annotations
+import json
+from typing import Any, Dict, List, Optional
+from features.marketplace.models import AgentMarketSubmission
+from core.infra.exceptions import ResourceNotFoundError
+from features.marketplace import edition as ml
+from features.marketplace.agent_categories import AGENT_MARKETPLACE_CATEGORIES
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import agent_metadata
+from features.marketplace import agents_state
+
+def read_bundle(slug: str) -> Optional[Dict[str, Any]]:
+ """Read and validate a single preset sub-agent's ``agent.json`` (fault-tolerant; returns None if broken)."""
+ if not slug or "/" in slug or ".." in slug:
+ return None
+ path = agents_state.AGENT_MARKET_DIR / slug / agents_state.AGENT_JSON
+ if not path.is_file():
+ return None
+ try:
+ b = json.loads(path.read_text(encoding="utf-8"))
+ except Exception as exc: # noqa: BLE001
+ logger.warning("agent market bundle broken: %s (%s)", slug, exc)
+ return None
+ if not isinstance(b, dict) or not b.get("name"):
+ return None
+ b.setdefault("slug", slug)
+ return b
+
+
+def approved_submissions(db: Session) -> List[AgentMarketSubmission]:
+ return (
+ db.query(AgentMarketSubmission)
+ .filter(AgentMarketSubmission.status == "approved")
+ .order_by(AgentMarketSubmission.created_at.desc())
+ .all()
+ )
+
+
+def get_approved_submission(db: Session, slug: str) -> Optional[AgentMarketSubmission]:
+ return (
+ db.query(AgentMarketSubmission)
+ .filter(
+ AgentMarketSubmission.slug == slug,
+ AgentMarketSubmission.status == "approved",
+ )
+ .first()
+ )
+
+
+def list_marketplace_agents(
+ db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None
+) -> List[Dict[str, Any]]:
+ """List all marketplace sub-agents: presets (featured first) + community listings, layered with the enable/disable switch and visibility-scope filtering."""
+ items: List[Dict[str, Any]] = []
+ if agents_state.AGENT_MARKET_DIR.is_dir():
+ for child in sorted(agents_state.AGENT_MARKET_DIR.iterdir()):
+ if not child.is_dir():
+ continue
+ b = read_bundle(child.name)
+ if b:
+ items.append(agent_metadata.bundle_public_meta(b))
+ items.sort(key=lambda x: (0 if x["featured"] else 1, x["slug"]))
+ items.extend(agent_metadata.submission_public_meta(s) for s in approved_submissions(db))
+ items = ml.annotate_and_filter(
+ db,
+ ml.KIND_AGENT,
+ items,
+ id_key="slug",
+ include_disabled=include_disabled,
+ viewer_user_id=viewer_user_id,
+ )
+ return items
+
+
+def categories_from_items(items: List[Dict[str, Any]]) -> List[str]:
+ """Derive categories from an already-fetched marketplace list: the fixed 9 major categories first, legacy leftover categories appended after."""
+ seen: List[str] = list(AGENT_MARKETPLACE_CATEGORIES)
+ for it in items:
+ if it["category"] not in seen:
+ seen.append(it["category"])
+ return seen
+
+
+def list_categories(
+ db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None
+) -> List[str]:
+ """Marketplace category list (standalone entry point; the list endpoint should reuse categories_from_items to avoid a second scan)."""
+ return categories_from_items(
+ list_marketplace_agents(
+ db, include_disabled=include_disabled, viewer_user_id=viewer_user_id
+ )
+ )
+
+
+def get_agent_detail(db: Session, slug: str) -> Dict[str, Any]:
+ """Detail of a single marketplace sub-agent: metadata + prompt body + capability bindings (for the detail preview).
+
+ The preset directory takes priority; falls back to the community listing (approved submission) if not found.
+ """
+ b = read_bundle(slug)
+ if b:
+ data = agent_metadata.bundle_public_meta(b)
+ data["system_prompt"] = b.get("system_prompt") or ""
+ data["welcome_message"] = b.get("welcome_message") or ""
+ data["suggested_questions"] = list(b.get("suggested_questions") or [])
+ data["bindings"] = agent_metadata.bindings_of(b)
+ return data
+ sub = get_approved_submission(db, slug)
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_agent", slug)
+ data = agent_metadata.submission_public_meta(sub)
+ data["system_prompt"] = sub.system_prompt or ""
+ data["welcome_message"] = sub.welcome_message or ""
+ data["suggested_questions"] = list(sub.suggested_questions or [])
+ data["bindings"] = agent_metadata.normalize_binding_lists(sub.bindings_snapshot)
+ return data
diff --git a/src/backend/features/marketplace/agent_categories.py b/src/backend/features/marketplace/agent_categories.py
new file mode 100644
index 00000000..c6580899
--- /dev/null
+++ b/src/backend/features/marketplace/agent_categories.py
@@ -0,0 +1,38 @@
+"""Sub-agent marketplace categories (single source of truth).
+
+The sub-agent marketplace categories are **independent of the skill marketplace's 8
+categories** (role-positioning semantics fit agents better). The preset list, community
+listing applications, and admin review category changes may only take these 9 values;
+the frontend mirror is ``AGENT_MARKETPLACE_CATEGORIES`` in ``utils/constants.ts``.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from core.infra.exceptions import BadRequestError
+
+# Sub-agent marketplace fixed categories (stable order, frontend mirror must match)
+AGENT_MARKETPLACE_CATEGORIES: List[str] = [
+ "通用助手",
+ "职场办公",
+ "商业分析",
+ "数据分析",
+ "研发编程",
+ "翻译写作",
+ "创意设计",
+ "政策法务",
+ "教育科研",
+]
+
+DEFAULT_AGENT_CATEGORY = "通用助手"
+
+
+def validate_category(category: str) -> str:
+ """Validate and return a legal category; anything not in the fixed set 400s directly."""
+ category = (category or "").strip()
+ if category not in AGENT_MARKETPLACE_CATEGORIES:
+ raise BadRequestError(
+ message=f"请从固定分类中选择:{'、'.join(AGENT_MARKETPLACE_CATEGORIES)}"
+ )
+ return category
diff --git a/src/backend/features/marketplace/agent_installation.py b/src/backend/features/marketplace/agent_installation.py
new file mode 100644
index 00000000..2deef0c7
--- /dev/null
+++ b/src/backend/features/marketplace/agent_installation.py
@@ -0,0 +1,240 @@
+"""agent installation responsibilities."""
+from __future__ import annotations
+from features.marketplace import skill_installation
+from features.marketplace import skill_metadata
+from typing import Any, Dict, List, Optional
+from core.db.models import UserAgent
+from core.infra.exceptions import ResourceNotFoundError
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import agent_catalog
+from features.marketplace import agent_metadata
+from features.marketplace import agents_state
+
+def installed_slugs(db: Session, owner_user_id: Optional[str], slugs: List[str]) -> set:
+ """The subset of these marketplace slugs already installed (cloned) in the given scope (user / global admin)."""
+ if not slugs:
+ return set()
+ q = db.query(UserAgent.source_market_slug).filter(UserAgent.source_market_slug.in_(slugs))
+ if owner_user_id is None:
+ q = q.filter(UserAgent.owner_type == "admin")
+ else:
+ q = q.filter(UserAgent.owner_type == "user", UserAgent.user_id == owner_user_id)
+ return {row[0] for row in q.all()}
+
+
+def annotate_installed(
+ items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str]
+) -> List[Dict[str, Any]]:
+ """Add the ``installed`` flag to each marketplace list item (for the given scope)."""
+ if not items:
+ return items
+ installed = installed_slugs(db, owner_user_id, [it["slug"] for it in items])
+ for it in items:
+ it["installed"] = it["slug"] in installed
+ return items
+
+
+def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool:
+ return bool(installed_slugs(db, owner_user_id, [slug]))
+
+
+def resolve_market_entry(db: Session, slug: str) -> Dict[str, Any]:
+ """Normalize a preset bundle / community listing into a unified clone-source dict."""
+ b = agent_catalog.read_bundle(slug)
+ if b:
+ return {
+ "name": b.get("name") or slug,
+ "avatar": b.get("avatar") or "",
+ "description": b.get("description") or "",
+ "summary": b.get("summary") or "",
+ "system_prompt": b.get("system_prompt") or "",
+ "welcome_message": b.get("welcome_message") or "",
+ "suggested_questions": list(b.get("suggested_questions") or []),
+ "model_config": dict(b.get("model_config") or {}),
+ "bindings": agent_metadata.bindings_of(b),
+ "ontology_tags": [
+ str(tag)
+ for tag in (b.get("ontology_tags") or b.get("tags") or [])
+ if str(tag).startswith("ontology:")
+ ],
+ }
+ sub = agent_catalog.get_approved_submission(db, slug)
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_agent", slug)
+ return {
+ "name": sub.name or slug,
+ "avatar": sub.avatar or "",
+ "description": sub.description or "",
+ "summary": sub.summary or "",
+ "system_prompt": sub.system_prompt or "",
+ "welcome_message": sub.welcome_message or "",
+ "suggested_questions": list(sub.suggested_questions or []),
+ "model_config": dict(sub.model_config_snapshot or {}),
+ "bindings": agent_metadata.normalize_binding_lists(sub.bindings_snapshot),
+ "ontology_tags": [str(tag) for tag in (sub.tags or []) if str(tag).startswith("ontology:")],
+ }
+
+
+def resolve_bindings(
+ db: Session,
+ bindings: Dict[str, List[str]],
+ owner_user_id: Optional[str],
+ operator_name: Optional[str],
+) -> tuple[Dict[str, List[str]], Dict[str, List[str]]]:
+ """Resolve each marketplace binding id into the installer's scope; returns (final bindings, report).
+
+ - MCP / default skills / accessible KBs: bind directly if within ``available-resources``;
+ - marketplace skill slugs: if not in the available list, install as the installer's private skill and bind the returned id;
+ - plugin slugs: bind if already installed, otherwise install as private and then bind;
+ - unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``.
+ """
+ from core.capabilities.catalog.api import get_enabled_ids
+ from core.capabilities.plugins import management as plugin_service
+
+ from core.capabilities.agents.edition import UserAgentService
+
+ avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id)
+ avail_skill = {s["id"] for s in avail.get("skills", [])}
+ # available-resources only lists AdminMcpServer rows; but built-in MCP tools (e.g.
+ # database_query) are catalog-defined and loaded at runtime by catalog id,
+ # and don't necessarily have a same-named AdminMcpServer row. Merge the catalog-enabled
+ # built-in MCP ids into the "resolvable" set, so these legitimate built-in tools aren't
+ # misjudged as "unresolvable" and dropped (once bound, the runtime connects on demand).
+ avail_mcp = {s["id"] for s in avail.get("mcp_servers", [])} | set(get_enabled_ids("mcp"))
+ avail_plugin = {p["id"] for p in avail.get("plugins", [])}
+ avail_kb = {k["id"] for k in avail.get("kb_spaces", [])}
+
+ final: Dict[str, List[str]] = {
+ "skill_ids": [],
+ "mcp_server_ids": [],
+ "plugin_ids": [],
+ "kb_ids": [],
+ }
+ report: Dict[str, List[str]] = {
+ "bound": [],
+ "installed": [],
+ "dropped": [],
+ "needs_secret": [],
+ }
+
+ # MCP tools
+ for mid in bindings.get("mcp_server_ids", []):
+ if mid in avail_mcp:
+ final["mcp_server_ids"].append(mid)
+ report["bound"].append(f"mcp:{mid}")
+ else:
+ report["dropped"].append(f"mcp:{mid}")
+
+ # Skills (default skills bind directly; marketplace skills get auto-installed as private)
+ for sid in bindings.get("skill_ids", []):
+ if sid in avail_skill:
+ final["skill_ids"].append(sid)
+ report["bound"].append(f"skill:{sid}")
+ continue
+ try:
+ res = skill_installation.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={})
+ final["skill_ids"].append(res["id"])
+ report["installed"].append(f"skill:{sid}")
+ if skill_metadata.market_skill_requires_secrets(sid):
+ report["needs_secret"].append(f"skill:{sid}")
+ except Exception as exc: # noqa: BLE001
+ logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc)
+ report["dropped"].append(f"skill:{sid}")
+
+ # Plugins (bindings store the slug; bind the install_id if installed, otherwise auto-install)
+ for pslug in bindings.get("plugin_ids", []):
+ gid = f"{pslug}@global"
+ match = (
+ gid
+ if gid in avail_plugin
+ else next((p for p in avail_plugin if p.startswith(f"{pslug}@")), None)
+ )
+ if match:
+ final["plugin_ids"].append(match)
+ report["bound"].append(f"plugin:{pslug}")
+ continue
+ try:
+ res = plugin_service.install_plugin(
+ db, pslug, owner_user_id=owner_user_id, secrets={}, created_by=operator_name
+ )
+ final["plugin_ids"].append(res["install_id"])
+ report["installed"].append(f"plugin:{pslug}")
+ if res.get("required_secrets") or res.get("requires_secret"):
+ report["needs_secret"].append(f"plugin:{pslug}")
+ except Exception as exc: # noqa: BLE001
+ logger.info("agent clone: plugin binding unresolved %s (%s)", pslug, exc)
+ report["dropped"].append(f"plugin:{pslug}")
+
+ # Knowledge bases (keep only those the installer can access)
+ for kid in bindings.get("kb_ids", []):
+ if kid in avail_kb:
+ final["kb_ids"].append(kid)
+ report["bound"].append(f"kb:{kid}")
+ else:
+ report["dropped"].append(f"kb:{kid}")
+
+ return final, report
+
+
+def install_marketplace_agent(
+ db: Session,
+ slug: str,
+ *,
+ owner_user_id: Optional[str],
+ operator_name: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent)."""
+ from core.capabilities.agents.edition import UserAgentService
+
+ entry = resolve_market_entry(db, slug)
+ final_bindings, report = resolve_bindings(db, entry["bindings"], owner_user_id, operator_name)
+
+ mc = entry.get("model_config") or {}
+ data: Dict[str, Any] = {
+ "name": entry["name"],
+ "avatar": entry.get("avatar") or None,
+ "description": (entry.get("summary") or entry.get("description") or "")[:agents_state.MAX_DESC],
+ "system_prompt": entry.get("system_prompt") or "",
+ "welcome_message": entry.get("welcome_message") or "",
+ "suggested_questions": list(entry.get("suggested_questions") or []),
+ "skill_ids": final_bindings["skill_ids"],
+ "mcp_server_ids": final_bindings["mcp_server_ids"],
+ "plugin_ids": final_bindings["plugin_ids"],
+ "kb_ids": final_bindings["kb_ids"],
+ "source_market_slug": slug,
+ "ontology_tags": list(entry.get("ontology_tags") or []),
+ }
+ if mc.get("temperature") is not None:
+ data["temperature"] = mc["temperature"]
+ if mc.get("max_tokens") is not None:
+ data["max_tokens"] = mc["max_tokens"]
+ if mc.get("max_iters") is not None:
+ data["max_iters"] = mc["max_iters"]
+ if mc.get("timeout") is not None:
+ data["timeout"] = mc["timeout"]
+
+ owner_type = "admin" if owner_user_id is None else "user"
+ agent = UserAgentService(db).create(
+ user_id=owner_user_id,
+ operator_name=operator_name,
+ owner_type=owner_type,
+ data=data,
+ )
+ logger.info(
+ "agent_market_install: slug=%s owner=%s agent=%s bound=%d installed=%d dropped=%d",
+ slug,
+ owner_user_id or "global",
+ agent["agent_id"],
+ len(report["bound"]),
+ len(report["installed"]),
+ len(report["dropped"]),
+ )
+ return {
+ "agent_id": agent["agent_id"],
+ "slug": slug,
+ "owner": "self" if owner_user_id else "global",
+ "install_report": report,
+ "message": "子智能体已安装",
+ }
diff --git a/src/backend/features/marketplace/agent_metadata.py b/src/backend/features/marketplace/agent_metadata.py
new file mode 100644
index 00000000..fc879392
--- /dev/null
+++ b/src/backend/features/marketplace/agent_metadata.py
@@ -0,0 +1,101 @@
+"""agent metadata responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict, List, Optional
+from features.marketplace.models import AgentMarketSubmission
+from features.marketplace.agent_categories import DEFAULT_AGENT_CATEGORY
+
+def normalize_binding_lists(raw: Optional[Dict[str, Any]]) -> Dict[str, List[str]]:
+ """Normalize a bindings snapshot into the four id lists (missing ones become empty)."""
+ b = raw or {}
+ return {
+ "skill_ids": list(b.get("skill_ids") or []),
+ "mcp_server_ids": list(b.get("mcp_server_ids") or []),
+ "plugin_ids": list(b.get("plugin_ids") or []),
+ "kb_ids": list(b.get("kb_ids") or []),
+ }
+
+
+def bindings_of(d: Dict[str, Any]) -> Dict[str, List[str]]:
+ """Get the four binding id lists from a preset bundle's ``bindings``."""
+ return normalize_binding_lists(d.get("bindings"))
+
+
+def binding_counts(bindings: Dict[str, List[str]]) -> Dict[str, int]:
+ return {
+ "skill_count": len(bindings.get("skill_ids") or []),
+ "mcp_count": len(bindings.get("mcp_server_ids") or []),
+ "plugin_count": len(bindings.get("plugin_ids") or []),
+ "kb_count": len(bindings.get("kb_ids") or []),
+ }
+
+
+def make_market_meta(
+ *,
+ slug: str,
+ name: str,
+ avatar: str,
+ summary: str,
+ description: str,
+ category: str,
+ tags: Any,
+ version: str,
+ author: str,
+ source: str,
+ featured: bool,
+ deletable: bool,
+ bindings: Dict[str, List[str]],
+) -> Dict[str, Any]:
+ """Preset bundle / community listing → isomorphic public list metadata (without the system_prompt body)."""
+ return {
+ "slug": slug,
+ "name": name or slug,
+ "avatar": avatar or "",
+ "summary": summary or "",
+ "description": description or "",
+ "category": category or DEFAULT_AGENT_CATEGORY,
+ "tags": list(tags or []),
+ "version": version or "1.0.0",
+ "author": author,
+ "source": source,
+ "featured": featured,
+ "deletable": deletable,
+ **binding_counts(bindings),
+ }
+
+
+def bundle_public_meta(b: Dict[str, Any]) -> Dict[str, Any]:
+ # Presets (filesystem) ship with the repo; admins cannot delete them online (deletable=False).
+ return make_market_meta(
+ slug=b.get("slug"),
+ name=b.get("name"),
+ avatar=b.get("avatar"),
+ summary=b.get("summary"),
+ description=b.get("description"),
+ category=b.get("category"),
+ tags=b.get("tags"),
+ version=b.get("version"),
+ author=b.get("author") or "内置",
+ source="builtin",
+ featured=bool(b.get("featured")),
+ deletable=False,
+ bindings=bindings_of(b),
+ )
+
+
+def submission_public_meta(sub: AgentMarketSubmission) -> Dict[str, Any]:
+ # DB listing records (admin upload / user community submission) → admins can delete online (deletable=True).
+ return make_market_meta(
+ slug=sub.slug,
+ name=sub.name,
+ avatar=sub.avatar,
+ summary=sub.summary,
+ description=sub.description,
+ category=sub.category,
+ tags=sub.tags,
+ version=sub.version,
+ author=sub.submitter_name or sub.owner_user_id,
+ source="community",
+ featured=False,
+ deletable=True,
+ bindings=normalize_binding_lists(sub.bindings_snapshot),
+ )
diff --git a/src/backend/features/marketplace/agent_publishing.py b/src/backend/features/marketplace/agent_publishing.py
new file mode 100644
index 00000000..cfe7ac3d
--- /dev/null
+++ b/src/backend/features/marketplace/agent_publishing.py
@@ -0,0 +1,165 @@
+"""agent publishing responsibilities."""
+from __future__ import annotations
+import uuid
+from typing import Any, Dict, List, Optional
+from features.marketplace.models import AgentMarketSubmission
+from core.db.models import UserAgent
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.time import utc_now
+from features.marketplace import edition as ml
+from features.marketplace.agent_categories import validate_category
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_modified
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import agent_catalog
+from features.marketplace import agent_submission_data
+from features.marketplace import agents_state
+
+def publish_agent_to_market(
+ db: Session,
+ agent_id: str,
+ *,
+ category: str,
+ summary: str = "",
+ submitter_name: str = "管理员上传",
+) -> Dict[str, Any]:
+ """An admin lists an existing sub-agent directly on the marketplace as approved (no review).
+
+ If the same agent_id already has an admin listing record, update it (keeping the original slug); otherwise create a new one.
+ """
+ category = validate_category(category)
+ agent = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first()
+ if agent is None:
+ raise ResourceNotFoundError("user_agent", agent_id)
+
+ now = utc_now()
+ existing = (
+ db.query(AgentMarketSubmission)
+ .filter(
+ AgentMarketSubmission.agent_id == agent_id,
+ AgentMarketSubmission.owner_user_id == agents_state.ADMIN_UPLOAD_OWNER,
+ )
+ .first()
+ )
+ summary = (summary or "").strip() or (agent.description or "")
+ if existing is not None:
+ existing.name = agent.name
+ existing.avatar = agent.avatar
+ existing.description = agent.description or ""
+ existing.summary = summary
+ existing.category = category
+ existing.system_prompt = agent.system_prompt or ""
+ existing.welcome_message = agent.welcome_message or ""
+ existing.suggested_questions = list(agent.suggested_questions or [])
+ existing.model_config_snapshot = agent_submission_data.model_config_snapshot(agent)
+ existing.bindings_snapshot = agent_submission_data.bindings_snapshot(agent)
+ existing.tags = list((agent.extra_config or {}).get("ontology_tags") or [])
+ existing.status = "approved"
+ existing.review_note = ""
+ existing.reviewed_at = now
+ existing.updated_at = now
+ flag_modified(existing, "suggested_questions")
+ flag_modified(existing, "model_config_snapshot")
+ flag_modified(existing, "bindings_snapshot")
+ flag_modified(existing, "tags")
+ db.commit()
+ return {"slug": existing.slug, "action": "updated", "message": "子智能体市场内容已更新"}
+
+ sub = agent_submission_data.new_submission(
+ submission_id=f"aadm_{uuid.uuid4().hex[:16]}",
+ slug=agent_submission_data.derive_slug(db, agent.name),
+ agent_id=agent_id,
+ owner_user_id=agents_state.ADMIN_UPLOAD_OWNER,
+ submitter_name=submitter_name,
+ name=agent.name,
+ avatar=agent.avatar,
+ description=agent.description or "",
+ summary=summary,
+ category=category,
+ note="管理员上传,免审核直接上架",
+ system_prompt=agent.system_prompt or "",
+ welcome_message=agent.welcome_message or "",
+ suggested_questions=agent.suggested_questions,
+ model_config_snapshot=agent_submission_data.model_config_snapshot(agent),
+ bindings_snapshot=agent_submission_data.bindings_snapshot(agent),
+ tags=list((agent.extra_config or {}).get("ontology_tags") or []),
+ status="approved",
+ reviewed_at=now,
+ )
+ db.add(sub)
+ db.commit()
+ logger.info("agent_market_admin_published: agent=%s slug=%s", agent_id, sub.slug)
+ return {"slug": sub.slug, "action": "published", "message": "子智能体已上架市场"}
+
+
+def create_market_agent(
+ db: Session,
+ *,
+ name: str,
+ category: str,
+ avatar: str = "",
+ description: str = "",
+ summary: str = "",
+ system_prompt: str = "",
+ welcome_message: str = "",
+ suggested_questions: Optional[List[str]] = None,
+ bindings: Optional[Dict[str, Any]] = None,
+ model_config: Optional[Dict[str, Any]] = None,
+ tags: Optional[List[str]] = None,
+ submitter_name: str = "管理员上传",
+) -> Dict[str, Any]:
+ """An admin directly "creates" a sub-agent and lists it on the marketplace as approved (no global agent created).
+
+ Mirrors the skill side's "creating a skill lists it on the skill marketplace": the new
+ sub-agent enters the sub-agent marketplace (explicitly installable as a global / private
+ clone) and does not take effect globally right away. Content comes from form fields rather
+ than a source ``UserAgent``, hence a synthetic placeholder ``agent_id`` (installs always use
+ the snapshot and never look back at a source agent).
+ """
+ now = utc_now()
+ sub = agent_submission_data.new_submission(
+ submission_id=f"aadm_{uuid.uuid4().hex[:16]}",
+ slug=agent_submission_data.derive_slug(db, name),
+ agent_id=f"__market__{uuid.uuid4().hex[:12]}",
+ owner_user_id=agents_state.ADMIN_UPLOAD_OWNER,
+ submitter_name=submitter_name,
+ name=name,
+ avatar=avatar or "",
+ description=description or "",
+ summary=(summary or "").strip() or (description or ""),
+ category=validate_category(category),
+ note="管理员创建,直接上架",
+ system_prompt=system_prompt or "",
+ welcome_message=welcome_message or "",
+ suggested_questions=suggested_questions,
+ model_config_snapshot=model_config,
+ bindings_snapshot=bindings,
+ tags=tags,
+ status="approved",
+ reviewed_at=now,
+ )
+ db.add(sub)
+ db.commit()
+ logger.info("agent_market_admin_created: slug=%s name=%s", sub.slug, name)
+ return {"slug": sub.slug, "action": "created", "message": "子智能体已上架子智能体市场"}
+
+
+def delete_market_agent(db: Session, slug: str) -> Dict[str, Any]:
+ """An admin deletes one DB listing record from the marketplace. Presets ship with the repo and cannot be deleted online (400)."""
+ if agent_catalog.read_bundle(slug) is not None:
+ raise BadRequestError(message="预置子智能体随仓库发布,不可在线删除")
+ sub = db.query(AgentMarketSubmission).filter(AgentMarketSubmission.slug == slug).first()
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_agent", slug)
+ db.delete(sub)
+ db.commit()
+ logger.info("agent_market_deleted: slug=%s", slug)
+ return {"slug": slug, "deleted": True}
+
+
+def set_market_agent_enabled(
+ db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
+) -> Dict[str, Any]:
+ """List/delist a marketplace sub-agent (controls whether it shows in the marketplace; installed clones are unaffected)."""
+ return ml.set_listing_enabled(db, ml.KIND_AGENT, slug, enabled, updated_by=updated_by)
diff --git a/src/backend/features/marketplace/agent_submission_data.py b/src/backend/features/marketplace/agent_submission_data.py
new file mode 100644
index 00000000..409a0282
--- /dev/null
+++ b/src/backend/features/marketplace/agent_submission_data.py
@@ -0,0 +1,155 @@
+"""agent submission data responsibilities."""
+from __future__ import annotations
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+from features.marketplace.models import AgentMarketSubmission
+from core.db.models import UserAgent
+from core.infra.exceptions import ResourceNotFoundError
+from core.infra.time import utc_now
+from features.marketplace.agent_categories import DEFAULT_AGENT_CATEGORY
+from sqlalchemy.orm import Session
+from features.marketplace import agent_metadata
+from features.marketplace import agents_state
+
+def existing_slugs(db: Session) -> set:
+ """Marketplace slugs currently taken: preset directory + all submissions (including rejected; slugs are table-wide unique)."""
+ slugs = set()
+ if agents_state.AGENT_MARKET_DIR.is_dir():
+ slugs.update(c.name for c in agents_state.AGENT_MARKET_DIR.iterdir() if c.is_dir())
+ slugs.update(r[0] for r in db.query(AgentMarketSubmission.slug).all())
+ return slugs
+
+
+def derive_slug(db: Session, name: str) -> str:
+ """Derive the marketplace slug from the sub-agent name (ASCII-ized + unique); Chinese names fall back to agent-."""
+ base = re.sub(r"[^a-z0-9_-]+", "-", (name or "").lower()).strip("-")
+ if not base:
+ base = f"agent-{uuid.uuid4().hex[:8]}"
+ taken = existing_slugs(db)
+ slug = base
+ n = 2
+ while slug in taken:
+ slug = f"{base}-{n}"
+ n += 1
+ return slug
+
+
+def model_config_snapshot(agent: UserAgent) -> Dict[str, Any]:
+ return {
+ "temperature": float(agent.temperature) if agent.temperature is not None else None,
+ "max_tokens": agent.max_tokens,
+ "max_iters": agent.max_iters,
+ "timeout": agent.timeout,
+ }
+
+
+def bindings_snapshot(agent: UserAgent) -> Dict[str, List[str]]:
+ return {
+ "skill_ids": list(agent.skill_ids or []),
+ "mcp_server_ids": list(agent.mcp_server_ids or []),
+ "plugin_ids": list(agent.plugin_ids or []),
+ "kb_ids": list(agent.kb_ids or []),
+ }
+
+
+def submission_to_dict(
+ sub: AgentMarketSubmission, *, with_content: bool = False
+) -> Dict[str, Any]:
+ data = {
+ "submission_id": sub.submission_id,
+ "slug": sub.slug,
+ "agent_id": sub.agent_id,
+ "owner_user_id": sub.owner_user_id,
+ "submitter_name": sub.submitter_name or "",
+ "name": sub.name,
+ "avatar": sub.avatar or "",
+ "description": sub.description or "",
+ "summary": sub.summary or "",
+ "category": sub.category or DEFAULT_AGENT_CATEGORY,
+ "tags": list(sub.tags or []),
+ "version": sub.version or "1.0.0",
+ "note": sub.note or "",
+ "status": sub.status,
+ "review_note": sub.review_note or "",
+ "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None,
+ "created_at": sub.created_at.isoformat() if sub.created_at else None,
+ }
+ if with_content:
+ data["system_prompt"] = sub.system_prompt or ""
+ data["welcome_message"] = sub.welcome_message or ""
+ data["suggested_questions"] = list(sub.suggested_questions or [])
+ data["bindings"] = agent_metadata.normalize_binding_lists(sub.bindings_snapshot)
+ return data
+
+
+def owned_agent(db: Session, agent_id: str, owner_user_id: str) -> UserAgent:
+ """Fetch the sub-agent owned by this user (owner_type=user), otherwise 404."""
+ agent = (
+ db.query(UserAgent)
+ .filter(
+ UserAgent.agent_id == agent_id,
+ UserAgent.owner_type == "user",
+ UserAgent.user_id == owner_user_id,
+ )
+ .first()
+ )
+ if agent is None:
+ raise ResourceNotFoundError("user_agent", agent_id)
+ return agent
+
+
+def new_submission(
+ *,
+ submission_id: str,
+ slug: str,
+ agent_id: str,
+ owner_user_id: str,
+ name: str,
+ category: str,
+ status: str,
+ submitter_name: str = "",
+ avatar: Optional[str] = None,
+ description: str = "",
+ summary: str = "",
+ note: str = "",
+ system_prompt: str = "",
+ welcome_message: str = "",
+ suggested_questions: Optional[List[str]] = None,
+ model_config_snapshot: Optional[Dict[str, Any]] = None,
+ bindings_snapshot: Optional[Dict[str, Any]] = None,
+ tags: Optional[List[str]] = None,
+ reviewed_at: Optional[datetime] = None,
+) -> AgentMarketSubmission:
+ """Uniformly construct one ``AgentMarketSubmission`` (fixed fields and default normalization centralized here).
+
+ The three entry points (user submission ``submit_to_marketplace`` / the create branch of the
+ admin direct-upload ``publish_agent_to_market`` / admin create-and-list
+ ``create_market_agent``) share the same field set; callers pass only what differs.
+ """
+ now = utc_now()
+ return AgentMarketSubmission(
+ submission_id=submission_id,
+ slug=slug,
+ agent_id=agent_id,
+ owner_user_id=owner_user_id,
+ submitter_name=(submitter_name or "").strip(),
+ name=name,
+ avatar=avatar,
+ description=description or "",
+ summary=summary or "",
+ category=category,
+ tags=list(tags or []),
+ version="1.0.0",
+ note=(note or "").strip(),
+ system_prompt=system_prompt or "",
+ welcome_message=welcome_message or "",
+ suggested_questions=list(suggested_questions or []),
+ model_config_snapshot=dict(model_config_snapshot or {}),
+ bindings_snapshot=agent_metadata.normalize_binding_lists(bindings_snapshot),
+ status=status,
+ reviewed_at=reviewed_at,
+ created_at=now,
+ updated_at=now,
+ )
diff --git a/src/backend/features/marketplace/agent_submissions.py b/src/backend/features/marketplace/agent_submissions.py
new file mode 100644
index 00000000..de5d5d00
--- /dev/null
+++ b/src/backend/features/marketplace/agent_submissions.py
@@ -0,0 +1,160 @@
+"""agent submissions responsibilities."""
+from __future__ import annotations
+import uuid
+from typing import Any, Dict, List, Optional
+from features.marketplace.models import AgentMarketSubmission
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.time import utc_now
+from features.marketplace.agent_categories import validate_category
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import agent_submission_data
+
+def submit_to_marketplace(
+ db: Session,
+ agent_id: str,
+ *,
+ owner_user_id: str,
+ submitter_name: str = "",
+ note: str = "",
+ category: str = "",
+ summary: str = "",
+) -> Dict[str, Any]:
+ """A user submits their self-built sub-agent for marketplace listing (content snapshot, pending admin review)."""
+ agent = agent_submission_data.owned_agent(db, agent_id, owner_user_id)
+
+ active = (
+ db.query(AgentMarketSubmission)
+ .filter(
+ AgentMarketSubmission.agent_id == agent_id,
+ AgentMarketSubmission.owner_user_id == owner_user_id,
+ AgentMarketSubmission.status.in_(["pending", "approved"]),
+ )
+ .first()
+ )
+ if active is not None:
+ state = "待审核" if active.status == "pending" else "已上架"
+ raise HTTPException(status_code=409, detail=f"该子智能体已有{state}的上架申请")
+
+ sub = agent_submission_data.new_submission(
+ submission_id=f"asub_{uuid.uuid4().hex[:16]}",
+ slug=agent_submission_data.derive_slug(db, agent.name),
+ agent_id=agent_id,
+ owner_user_id=owner_user_id,
+ submitter_name=submitter_name,
+ name=agent.name,
+ avatar=agent.avatar,
+ description=agent.description or "",
+ summary=(summary or "").strip() or (agent.description or ""),
+ category=validate_category(category),
+ note=note,
+ system_prompt=agent.system_prompt or "",
+ welcome_message=agent.welcome_message or "",
+ suggested_questions=agent.suggested_questions,
+ model_config_snapshot=agent_submission_data.model_config_snapshot(agent),
+ bindings_snapshot=agent_submission_data.bindings_snapshot(agent),
+ tags=list((agent.extra_config or {}).get("ontology_tags") or []),
+ status="pending",
+ )
+ db.add(sub)
+ db.commit()
+ logger.info(
+ "agent_market_submission_created: id=%s agent=%s owner=%s slug=%s",
+ sub.submission_id,
+ agent_id,
+ owner_user_id,
+ sub.slug,
+ )
+ return agent_submission_data.submission_to_dict(sub)
+
+
+def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
+ rows = (
+ db.query(AgentMarketSubmission)
+ .filter(AgentMarketSubmission.owner_user_id == owner_user_id)
+ .order_by(AgentMarketSubmission.created_at.desc())
+ .all()
+ )
+ return [agent_submission_data.submission_to_dict(r) for r in rows]
+
+
+def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
+ """A user withdraws their own submission: pending/rejected can be deleted; approved requires an admin to delist."""
+ sub = (
+ db.query(AgentMarketSubmission)
+ .filter(
+ AgentMarketSubmission.submission_id == submission_id,
+ AgentMarketSubmission.owner_user_id == owner_user_id,
+ )
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("agent_market_submission", submission_id)
+ if sub.status == "approved":
+ raise BadRequestError(message="子智能体已上架,如需下架请联系管理员")
+ db.delete(sub)
+ db.commit()
+
+
+def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
+ """Admin view: all listing submissions, filterable by status (pending first, newest → oldest)."""
+ q = db.query(AgentMarketSubmission)
+ if status:
+ q = q.filter(AgentMarketSubmission.status == status)
+ rows = q.order_by(AgentMarketSubmission.created_at.desc()).all()
+ order = {"pending": 0, "approved": 1, "rejected": 2}
+ rows.sort(key=lambda r: order.get(r.status, 3))
+ return [agent_submission_data.submission_to_dict(r) for r in rows]
+
+
+def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
+ sub = (
+ db.query(AgentMarketSubmission)
+ .filter(AgentMarketSubmission.submission_id == submission_id)
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("agent_market_submission", submission_id)
+ return agent_submission_data.submission_to_dict(sub, with_content=True)
+
+
+def review_submission(
+ db: Session,
+ submission_id: str,
+ *,
+ approve: bool,
+ review_note: str = "",
+ category: str = "",
+) -> Dict[str, Any]:
+ """Admin review: approval lists it immediately; rejecting pending = refuse, rejecting approved = delist."""
+ sub = (
+ db.query(AgentMarketSubmission)
+ .filter(AgentMarketSubmission.submission_id == submission_id)
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("agent_market_submission", submission_id)
+ if approve and sub.status != "pending":
+ state = "已上架" if sub.status == "approved" else "已驳回"
+ raise HTTPException(
+ status_code=409,
+ detail=f"该申请{state},不能重复审核通过(如需重新上架请让用户重新提交)",
+ )
+ if not approve and sub.status == "rejected":
+ raise HTTPException(status_code=409, detail="该申请已是驳回状态")
+ sub.status = "approved" if approve else "rejected"
+ sub.review_note = (review_note or "").strip()
+ if approve and (category or "").strip():
+ sub.category = validate_category(category)
+ sub.reviewed_at = utc_now()
+ sub.updated_at = utc_now()
+ db.commit()
+ logger.info(
+ "agent_market_submission_reviewed: id=%s slug=%s status=%s",
+ sub.submission_id,
+ sub.slug,
+ sub.status,
+ )
+ return agent_submission_data.submission_to_dict(sub)
diff --git a/src/backend/core/services/marketplace_agent_versions.py b/src/backend/features/marketplace/agent_versions.py
similarity index 88%
rename from src/backend/core/services/marketplace_agent_versions.py
rename to src/backend/features/marketplace/agent_versions.py
index acc6b904..a25c413c 100644
--- a/src/backend/core/services/marketplace_agent_versions.py
+++ b/src/backend/features/marketplace/agent_versions.py
@@ -1,28 +1,33 @@
"""Version-aware sources for the existing agent marketplace API."""
from __future__ import annotations
+import features.marketplace.agent_catalog as market_owner_agent_catalog
+import features.marketplace.agent_installation as market_owner_agent_installation
+from features.marketplace.agents_state import MAX_DESC as _MAX_DESC
+import features.marketplace.skill_metadata as market_owner_skill_metadata
+import logging
+logger = logging.getLogger(__name__)
import json
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
-from . import agent_market_service as original
-from . import marketplace_version_store as store
-from .agent_market_service import _MAX_DESC, logger
+
+import features.marketplace.versions.store as store
+
+
-def __getattr__(name):
- return getattr(original, name)
def get_agent_detail(db, slug):
- native = original.get_agent_detail(db, slug)
+ native = market_owner_agent_catalog.get_agent_detail(db, slug)
return store.detail(db, "agent", slug, native)
def _resolve_market_entry(db, slug):
- native = original._resolve_market_entry(db, slug)
+ native = market_owner_agent_installation.resolve_market_entry(db, slug)
snapshot = store.active(db, "agent", slug)
return json.loads(store.decode(snapshot)["agent.json"]) if snapshot else native
@@ -40,10 +45,10 @@ def _resolve_bindings(
- plugin slugs: bind if already installed, otherwise install as private and then bind;
- unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``.
"""
- from core.config.catalog import get_enabled_ids
- from core.plugins import management as plugin_service
- from core.services import marketplace_skill_versions as mk
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.catalog.api import get_enabled_ids
+ from core.capabilities.plugins import management as plugin_service
+ from features.marketplace import skill_versions as mk
+ from core.capabilities.agents.edition import UserAgentService
avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id)
avail_skill = {s["id"] for s in avail.get("skills", [])}
@@ -87,7 +92,7 @@ def _resolve_bindings(
res = mk.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={})
final["skill_ids"].append(res["id"])
report["installed"].append(f"skill:{sid}")
- if mk.market_skill_requires_secrets(sid):
+ if market_owner_skill_metadata.market_skill_requires_secrets(sid):
report["needs_secret"].append(f"skill:{sid}")
except Exception as exc: # noqa: BLE001
logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc)
@@ -136,7 +141,7 @@ def install_marketplace_agent(
operator_name: Optional[str] = None,
) -> Dict[str, Any]:
"""Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent)."""
- from core.services.user_agent_service import UserAgentService
+ from core.capabilities.agents.edition import UserAgentService
entry = _resolve_market_entry(db, slug)
final_bindings, report = _resolve_bindings(db, entry["bindings"], owner_user_id, operator_name)
diff --git a/src/backend/features/marketplace/agents_state.py b/src/backend/features/marketplace/agents_state.py
new file mode 100644
index 00000000..2e084ae2
--- /dev/null
+++ b/src/backend/features/marketplace/agents_state.py
@@ -0,0 +1,14 @@
+"""agents state responsibilities."""
+from __future__ import annotations
+from extensions.paths import AGENTS_DIR
+
+AGENT_MARKET_DIR = AGENTS_DIR / "marketplace"
+
+
+AGENT_JSON = "agent.json"
+
+
+ADMIN_UPLOAD_OWNER = "__admin_upload__"
+
+
+MAX_DESC = 20
diff --git a/src/backend/features/marketplace/api/__init__.py b/src/backend/features/marketplace/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/marketplace/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/agent_marketplace.py b/src/backend/features/marketplace/api/agent_marketplace.py
similarity index 82%
rename from src/backend/api/routes/v1/agent_marketplace.py
rename to src/backend/features/marketplace/api/agent_marketplace.py
index fd136d97..44a07da2 100644
--- a/src/backend/api/routes/v1/agent_marketplace.py
+++ b/src/backend/features/marketplace/api/agent_marketplace.py
@@ -14,6 +14,14 @@
"""
from __future__ import annotations
+import features.marketplace.agent_catalog as market_owner_agent_catalog
+
+
+import features.marketplace.agent_installation as market_owner_agent_installation
+
+import features.marketplace.agent_submissions as market_owner_agent_submissions
+
+
import logging
@@ -26,8 +34,8 @@
from core.db.engine import get_db
from core.infra.exceptions import AccessDeniedError
from core.infra.responses import created_response, success_response
-from core.services import marketplace_agent_versions as am
-from core.services import marketplace_listing as ml
+from features.marketplace import agent_versions as am
+from features.marketplace import edition as ml
router = APIRouter(prefix="/v1/agent-marketplace", tags=["Agent Marketplace"])
logger = logging.getLogger(__name__)
@@ -44,9 +52,9 @@ def list_agents(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- items = am.list_marketplace_agents(db, viewer_user_id=str(user.user_id))
- items = am.annotate_installed(items, db, owner_user_id=str(user.user_id))
- return success_response(data={"items": items, "categories": am.categories_from_items(items)})
+ items = market_owner_agent_catalog.list_marketplace_agents(db, viewer_user_id=str(user.user_id))
+ items = market_owner_agent_installation.annotate_installed(items, db, owner_user_id=str(user.user_id))
+ return success_response(data={"items": items, "categories": market_owner_agent_catalog.categories_from_items(items)})
@router.get("/categories", summary="子智能体市场分类")
@@ -54,7 +62,7 @@ def list_categories(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- return success_response(data=am.list_categories(db, viewer_user_id=str(user.user_id)))
+ return success_response(data=market_owner_agent_catalog.list_categories(db, viewer_user_id=str(user.user_id)))
@router.get("/agents/{slug}", summary="子智能体市场详情")
@@ -65,7 +73,7 @@ def get_agent(
):
ml.ensure_item_visible(db, ml.KIND_AGENT, slug, str(user.user_id), resource="marketplace_agent")
data = am.get_agent_detail(db, slug)
- data["installed"] = am.is_installed(db, slug, owner_user_id=str(user.user_id))
+ data["installed"] = market_owner_agent_installation.is_installed(db, slug, owner_user_id=str(user.user_id))
return success_response(data=data)
@@ -104,7 +112,7 @@ def submit_agent(
db: Session = Depends(get_db),
):
_require_can_add_agent(str(user.user_id), db)
- result = am.submit_to_marketplace(
+ result = market_owner_agent_submissions.submit_to_marketplace(
db,
body.agent_id,
owner_user_id=str(user.user_id),
@@ -121,7 +129,7 @@ def list_my_submissions(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- return success_response(data={"items": am.list_my_submissions(db, str(user.user_id))})
+ return success_response(data={"items": market_owner_agent_submissions.list_my_submissions(db, str(user.user_id))})
@router.delete("/submissions/{submission_id}", summary="撤回上架申请")
@@ -130,5 +138,5 @@ def withdraw_submission(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- am.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id))
+ market_owner_agent_submissions.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id))
return success_response(data={"id": submission_id, "message": "申请已撤回"})
diff --git a/src/backend/api/routes/v1/catalog_ownership.py b/src/backend/features/marketplace/api/catalog_ownership.py
similarity index 94%
rename from src/backend/api/routes/v1/catalog_ownership.py
rename to src/backend/features/marketplace/api/catalog_ownership.py
index a520a2fa..db70fcce 100644
--- a/src/backend/api/routes/v1/catalog_ownership.py
+++ b/src/backend/features/marketplace/api/catalog_ownership.py
@@ -10,9 +10,10 @@ def _load_owned_capability_items(db, user_id: str) -> tuple:
injected only into that user's /v1/catalog response; the frontend shows the "mine"
badge and delete button based on ``owner == 'self'``.
"""
- from core.config.catalog_details import resolve_skill_detail
- from core.db.models import AdminMcpServer, AdminSkill, McpMarketInstallation
- from core.services.skill_icon_service import get_skill_icons
+ from core.capabilities.catalog.details import resolve_skill_detail
+ from core.db.models import AdminMcpServer, AdminSkill
+ from features.marketplace.models import McpMarketInstallation
+ from core.capabilities.skills.icons import get_skill_icons
skill_items: List[Dict[str, Any]] = []
mcp_items: List[Dict[str, Any]] = []
@@ -128,7 +129,7 @@ def _plugin_component_ids(db) -> tuple:
except Exception as exc: # noqa: BLE001
logger.warning("plugin component id load failed: %s", exc)
try:
- from core.plugins.management import builtin_plugin_component_ids
+ from core.capabilities.plugins.management import builtin_plugin_component_ids
fs_skill_ids, fs_mcp_ids = builtin_plugin_component_ids()
skill_ids |= fs_skill_ids
diff --git a/src/backend/api/routes/v1/marketplace.py b/src/backend/features/marketplace/api/marketplace.py
similarity index 84%
rename from src/backend/api/routes/v1/marketplace.py
rename to src/backend/features/marketplace/api/marketplace.py
index 55cb71b2..e2384e2f 100644
--- a/src/backend/api/routes/v1/marketplace.py
+++ b/src/backend/features/marketplace/api/marketplace.py
@@ -14,6 +14,13 @@
"""
from __future__ import annotations
+import features.marketplace.skill_catalog as market_owner_skill_catalog
+
+
+
+import features.marketplace.skill_submissions as market_owner_skill_submissions
+
+
import logging
from typing import Dict, Optional
@@ -26,8 +33,8 @@
from core.auth.capabilities import resolve_user_capabilities
from core.db.engine import get_db
from core.infra.responses import created_response, success_response
-from core.services import marketplace_skill_versions as mk
-from core.services import marketplace_listing as ml
+from features.marketplace import skill_versions as mk
+from features.marketplace import edition as ml
from core.infra.exceptions import AccessDeniedError
router = APIRouter(prefix="/v1/marketplace", tags=["Skill Marketplace"])
@@ -46,10 +53,10 @@ def list_skills(
db: Session = Depends(get_db),
):
"""列出全部市场技能(预置+社区上架,按可见范围过滤),并标注当前用户是否已安装。"""
- items = mk.list_marketplace_skills(db, viewer_user_id=str(user.user_id))
- items = mk.annotate_installed(items, db, owner_user_id=str(user.user_id))
+ items = market_owner_skill_catalog.list_marketplace_skills(db, viewer_user_id=str(user.user_id))
+ items = market_owner_skill_catalog.annotate_installed(items, db, owner_user_id=str(user.user_id))
return success_response(
- data={"items": items, "categories": mk.list_categories(db, viewer_user_id=str(user.user_id))}
+ data={"items": items, "categories": market_owner_skill_catalog.list_categories(db, viewer_user_id=str(user.user_id))}
)
@@ -58,7 +65,7 @@ def list_categories(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- return success_response(data=mk.list_categories(db, viewer_user_id=str(user.user_id)))
+ return success_response(data=market_owner_skill_catalog.list_categories(db, viewer_user_id=str(user.user_id)))
@router.get("/skills/{slug}", summary="技能市场详情")
@@ -69,7 +76,7 @@ def get_skill(
):
ml.ensure_item_visible(db, ml.KIND_SKILL, slug, str(user.user_id), resource="marketplace_skill")
data = mk.get_marketplace_skill(slug, db)
- data["installed"] = mk.is_installed(db, slug, owner_user_id=str(user.user_id))
+ data["installed"] = market_owner_skill_catalog.is_installed(db, slug, owner_user_id=str(user.user_id))
return success_response(data=data)
@@ -118,7 +125,7 @@ def submit_skill(
):
"""把自己的私有技能提交上架申请(内容快照),管理员审核通过后全员可安装。"""
_require_can_add_skill(str(user.user_id), db)
- result = mk.submit_to_marketplace(
+ result = market_owner_skill_submissions.submit_to_marketplace(
db,
body.skill_id,
owner_user_id=str(user.user_id),
@@ -135,7 +142,7 @@ def list_my_submissions(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- return success_response(data={"items": mk.list_my_submissions(db, str(user.user_id))})
+ return success_response(data={"items": market_owner_skill_submissions.list_my_submissions(db, str(user.user_id))})
@router.delete("/submissions/{submission_id}", summary="撤回上架申请")
@@ -144,5 +151,5 @@ def withdraw_submission(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- mk.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id))
+ market_owner_skill_submissions.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id))
return success_response(data={"id": submission_id, "message": "申请已撤回"})
diff --git a/src/backend/api/routes/v1/mcp_marketplace.py b/src/backend/features/marketplace/api/mcp_marketplace.py
similarity index 79%
rename from src/backend/api/routes/v1/mcp_marketplace.py
rename to src/backend/features/marketplace/api/mcp_marketplace.py
index 72cc4edb..efeca18f 100644
--- a/src/backend/api/routes/v1/mcp_marketplace.py
+++ b/src/backend/features/marketplace/api/mcp_marketplace.py
@@ -1,16 +1,22 @@
"""User-facing MCP marketplace: browse, install, and submit private remote MCPs."""
from __future__ import annotations
+from core.capabilities.mcp import oauth as oauth_module
+from features.marketplace import mcp_listing
+from features.marketplace import mcp_curated
+from features.marketplace import mcp_installation
+from features.marketplace import mcp_submissions
from typing import Dict, List, Optional
from core.auth.backend import UserContext, get_current_user
from core.auth.capabilities import resolve_user_capabilities
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.exceptions import AccessDeniedError
from core.infra.responses import created_response, success_response
-from core.services import mcp_marketplace_service as market
-from core.services import mcp_oauth_service
+
+from core.capabilities.mcp import oauth as mcp_oauth_service
from fastapi import APIRouter, Depends, Query, Request
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field
@@ -32,9 +38,9 @@ def list_items(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- market.ensure_curated_market_items(db)
+ mcp_curated.ensure_curated_market_items(db)
return success_response(
- data=market.list_market_items(
+ data=mcp_listing.list_market_items(
db,
owner_user_id=str(user.user_id),
viewer_user_id=str(user.user_id),
@@ -49,7 +55,7 @@ def get_item(
db: Session = Depends(get_db),
):
return success_response(
- data=market.get_market_item(
+ data=mcp_listing.get_market_item(
db,
slug,
owner_user_id=str(user.user_id),
@@ -77,25 +83,22 @@ async def start_oauth(
body: OAuthStartRequest,
request: Request,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
user_id = str(user.user_id)
- _require_can_add_mcp(user_id, db)
- market.get_market_item(
- db,
- body.slug,
- owner_user_id=user_id,
- viewer_user_id=user_id,
- )
+
+ def admit(db: Session) -> None:
+ _require_can_add_mcp(user_id, db)
+ mcp_listing.get_market_item(db, body.slug, owner_user_id=user_id, viewer_user_id=user_id)
+
+ await run_in_session(admit)
return success_response(
- data=await mcp_oauth_service.start_oauth_install(
- db,
+ data=await oauth_module.start_oauth_install(
slug=body.slug,
owner_user_id=user_id,
installed_by=user_id,
auth_method=body.auth_method,
credentials=body.credentials,
- callback_url_base=mcp_oauth_service.public_oauth_callback_url(request),
+ callback_url_base=oauth_module.public_oauth_callback_url(request),
client_id=body.client_id,
client_secret=body.client_secret,
)
@@ -108,7 +111,7 @@ async def oauth_status(
user: UserContext = Depends(get_current_user),
):
return success_response(
- data=await mcp_oauth_service.get_flow_status(flow_id, owner_user_id=str(user.user_id))
+ data=await oauth_module.get_flow_status(flow_id, owner_user_id=str(user.user_id))
)
@@ -118,7 +121,7 @@ async def cancel_oauth(
user: UserContext = Depends(get_current_user),
):
return success_response(
- data=await mcp_oauth_service.cancel_flow(
+ data=await oauth_module.cancel_flow(
flow_id,
owner_user_id=str(user.user_id),
)
@@ -135,13 +138,15 @@ async def oauth_callback(
flow_id: str = Query(...),
code: str = Query(""),
state: Optional[str] = Query(None),
+ iss: Optional[str] = Query(None),
error: str = Query(""),
error_description: str = Query(""),
):
- await mcp_oauth_service.complete_callback(
+ await oauth_module.complete_callback(
flow_id,
code=code,
state=state,
+ iss=iss,
error=error_description or error,
)
return HTMLResponse(
@@ -155,12 +160,10 @@ async def oauth_callback(
async def install_item(
body: InstallRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
user_id = str(user.user_id)
- _require_can_add_mcp(user_id, db)
- result = await market.install_market_item(
- db,
+ await run_in_session(lambda db: _require_can_add_mcp(user_id, db))
+ result = await mcp_installation.install_market_item(
body.slug,
owner_user_id=user_id,
installed_by=user_id,
@@ -183,12 +186,10 @@ class SubmissionRequest(BaseModel):
async def submit_item(
body: SubmissionRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
user_id = str(user.user_id)
- _require_can_add_mcp(user_id, db)
- result = await market.submit_to_marketplace(
- db,
+ await run_in_session(lambda db: _require_can_add_mcp(user_id, db))
+ result = await mcp_submissions.submit_to_marketplace(
body.source_server_id,
owner_user_id=user_id,
submitter_name=user.username or "",
@@ -206,7 +207,7 @@ def list_my_submissions(
user: UserContext = Depends(get_current_user),
db: Session = Depends(get_db),
):
- return success_response(data={"items": market.list_my_submissions(db, str(user.user_id))})
+ return success_response(data={"items": mcp_submissions.list_my_submissions(db, str(user.user_id))})
@router.delete("/submissions/{submission_id}", summary="撤回 MCP 上架申请")
@@ -217,5 +218,5 @@ def withdraw_submission(
):
user_id = str(user.user_id)
_require_can_add_mcp(user_id, db)
- market.withdraw_submission(db, submission_id, user_id)
+ mcp_submissions.withdraw_submission(db, submission_id, user_id)
return success_response(data={"submission_id": submission_id, "withdrawn": True})
diff --git a/src/backend/features/marketplace/core_catalog.py b/src/backend/features/marketplace/core_catalog.py
new file mode 100644
index 00000000..7bca80f8
--- /dev/null
+++ b/src/backend/features/marketplace/core_catalog.py
@@ -0,0 +1,108 @@
+"""Marketplace contribution to core capability selection and chat modes."""
+
+class MarketplaceCatalogProvider:
+ def plugin_metadata(self, db, slug):
+ from features.marketplace.plugin_market import resolve_market_meta
+ return resolve_market_meta(db, slug)
+
+ def suspended_mcp_ids(self, db, ids):
+ from features.marketplace.models import McpMarketInstallation
+ return {
+ row[0] for row in db.query(McpMarketInstallation.server_id).filter(
+ McpMarketInstallation.server_id.in_(ids),
+ McpMarketInstallation.status == "suspended",
+ ).all()
+ }
+
+ def list_marketplace_skills(self, db):
+ from features.marketplace import skill_catalog
+ return skill_catalog.list_marketplace_skills(db)
+
+ def list_marketplace_agents(self, db):
+ from features.marketplace import agent_catalog
+ return agent_catalog.list_marketplace_agents(db)
+
+ def list_plugins(self, db, user_id):
+ from features.marketplace import plugin_market
+ return plugin_market.list_plugins(db, user_id)
+
+ def install_marketplace_skill(self, db, slug, *, owner_user_id):
+ from features.marketplace import skill_installation
+ return skill_installation.install_marketplace_skill(db, slug, owner_user_id=owner_user_id)
+
+ def install_marketplace_agent(self, db, slug, *, owner_user_id):
+ from features.marketplace import agent_installation
+ return agent_installation.install_marketplace_agent(db, slug, owner_user_id=owner_user_id)
+
+ def market_package(self, db, slug):
+ from features.marketplace.models import PluginMarketPackage
+ if not slug:
+ return None
+ return db.query(PluginMarketPackage).filter(PluginMarketPackage.slug == slug).first()
+
+ def publish_package(self, db, slug, fields, *, created_by, now):
+ from features.marketplace.models import PluginMarketPackage
+ from sqlalchemy.orm.attributes import flag_modified
+ existing = self.market_package(db, slug)
+ if existing is not None:
+ for key, value in fields.items():
+ setattr(existing, key, value)
+ flag_modified(existing, "required_secrets")
+ action = "updated"
+ else:
+ db.add(PluginMarketPackage(slug=slug, created_at=now, created_by=created_by, **fields))
+ action = "published"
+ db.commit()
+ return action
+
+ def delete_package(self, db, slug):
+ from core.infra.exceptions import ResourceNotFoundError
+ row = self.market_package(db, slug)
+ if row is None:
+ raise ResourceNotFoundError("plugin_market_package", slug)
+ db.delete(row)
+ db.commit()
+
+ def excluded_skills(self, db, slug):
+ from features.marketplace.models import PluginMarketSkillExclusion
+ if db is None:
+ return set()
+ return {row[0] for row in db.query(PluginMarketSkillExclusion.skill_name).filter(PluginMarketSkillExclusion.slug == slug).all()}
+
+ def exclude_skill(self, db, slug, skill_name, *, created_by):
+ from features.marketplace.models import PluginMarketSkillExclusion
+ existing = db.query(PluginMarketSkillExclusion).filter(
+ PluginMarketSkillExclusion.slug == slug,
+ PluginMarketSkillExclusion.skill_name == skill_name,
+ ).first()
+ if existing is not None:
+ return False
+ db.add(PluginMarketSkillExclusion(slug=slug, skill_name=skill_name, created_by=created_by))
+ db.commit()
+ return True
+
+ def mcp_version(self, db, slug):
+ from features.marketplace.models import McpMarketItem, McpMarketVersion
+ from core.infra.exceptions import ResourceNotFoundError
+ item = db.query(McpMarketItem).filter(
+ McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None),
+ McpMarketItem.status == "active",
+ ).first()
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ version = db.query(McpMarketVersion).filter(McpMarketVersion.version_id == item.latest_version_id).first()
+ if not version:
+ raise ResourceNotFoundError("mcp_market_version", item.latest_version_id)
+ return item, version
+
+ def normalize_auth_config(self, config, schema):
+ from features.marketplace import mcp_validation
+ return mcp_validation.normalize_auth_config(config, schema)
+
+ def installation_secrets(self, version, credentials, existing, *, auth_method):
+ from features.marketplace import mcp_credentials
+ return mcp_credentials.installation_secrets(version, credentials, existing, auth_method=auth_method)
+
+ async def install_market_item(self, slug, **kwargs):
+ from features.marketplace import mcp_installation
+ return await mcp_installation.install_market_item(slug, **kwargs)
diff --git a/src/backend/features/marketplace/edition.py b/src/backend/features/marketplace/edition.py
new file mode 100644
index 00000000..818b0a82
--- /dev/null
+++ b/src/backend/features/marketplace/edition.py
@@ -0,0 +1,10 @@
+"""Edition policy selected by the installed distribution."""
+
+def __getattr__(name):
+ if name in ('get_hidden_item_ids', 'is_item_visible'):
+ import features.marketplace.personal_visibility as provider
+ return getattr(provider, name)
+ if name in ('KIND_AGENT', 'KIND_MCP', 'KIND_PLUGIN', 'KIND_SKILL', 'annotate_and_filter', 'ensure_item_visible', 'get_disabled_ids', 'set_listing_enabled'):
+ import features.marketplace.personal_listing as provider
+ return getattr(provider, name)
+ raise AttributeError(name)
diff --git a/src/backend/features/marketplace/extension.py b/src/backend/features/marketplace/extension.py
new file mode 100644
index 00000000..a9ee7edd
--- /dev/null
+++ b/src/backend/features/marketplace/extension.py
@@ -0,0 +1,4 @@
+"""Installed marketplace models and runtime composition."""
+NAME = "marketplace"
+def setup():
+ return True
diff --git a/src/backend/features/marketplace/mcp_administration.py b/src/backend/features/marketplace/mcp_administration.py
new file mode 100644
index 00000000..6b13d123
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_administration.py
@@ -0,0 +1,291 @@
+"""mcp administration responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from features.marketplace import edition as ml
+from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, refresh_mcp_caches
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_listing
+from features.marketplace import mcp_credentials
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+def set_market_enabled(
+ db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
+) -> Dict[str, Any]:
+ item = (
+ db.query(McpMarketItem)
+ .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ return ml.set_listing_enabled(db, ml.KIND_MCP, slug, enabled, updated_by=updated_by)
+
+
+def update_market_item(
+ db: Session,
+ slug: str,
+ *,
+ display_name: Optional[str] = None,
+ description: Optional[str] = None,
+ user_intro: Optional[str] = None,
+ category: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ icon: Optional[str] = None,
+ requires_auth: Optional[bool] = None,
+ credential_mode: Optional[str] = None,
+ managed_credentials: Optional[Dict[str, str]] = None,
+) -> Dict[str, Any]:
+ """Edit marketplace presentation metadata and its administrator-owned auth policy.
+
+ Tool schemas, transport, endpoint templates, and version numbers remain
+ immutable. Authentication can be disabled, delegated to each installer,
+ or backed by the administrator source server's encrypted credentials.
+ Secret values are never persisted in marketplace rows or returned by this
+ function. Presentation fields are propagated to existing installations.
+ """
+ item = (
+ db.query(McpMarketItem)
+ .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ version = mcp_versions.version_for_item(db, item)
+ previous_auth_schema = list(version.auth_schema or [])
+
+ if display_name is not None:
+ normalized_name = display_name.strip()
+ if not normalized_name:
+ raise BadRequestError(message="MCP 市场名称不能为空")
+ item.display_name = normalized_name
+ if description is not None:
+ item.description = description.strip()
+ if user_intro is not None:
+ item.user_intro = user_intro.strip() or None
+ if category is not None:
+ item.category = mcp_validation.validate_category(category)
+ if tags is not None:
+ item.tags = list(dict.fromkeys(str(tag).strip() for tag in tags if str(tag).strip()))
+ if icon is not None:
+ item.icon = icon.strip() or None
+
+ auth_schema = list(version.auth_schema or [])
+ auth_config = mcp_validation.normalize_auth_config(version.auth_config, auth_schema)
+ if requires_auth is False:
+ auth_schema = []
+ auth_config = mcp_validation.normalize_auth_config(
+ {
+ "default_method": "none",
+ "methods": [{"id": "none", "type": "none", "label": "无需认证"}],
+ "credential_mode": "installer",
+ },
+ auth_schema,
+ )
+ elif requires_auth is True and not any(
+ method["type"] != "none" for method in auth_config["methods"]
+ ):
+ auth_schema = [
+ {
+ "key": "Authorization",
+ "label": "Token / Authorization",
+ "target": "header",
+ "required": True,
+ "secret": True,
+ "prefix": "Bearer ",
+ }
+ ]
+ auth_config = mcp_validation.normalize_auth_config(
+ {
+ "default_method": "token",
+ "methods": [{"id": "token", "type": "token", "label": "Token"}],
+ "credential_mode": credential_mode or "installer",
+ },
+ auth_schema,
+ )
+
+ if credential_mode is not None:
+ if credential_mode not in {"installer", "admin"}:
+ raise BadRequestError(message="MCP 凭据提供方式无效")
+ if credential_mode == "admin" and not auth_schema:
+ raise BadRequestError(message="请先勾选需要 Token/Auth")
+ auth_config["credential_mode"] = credential_mode
+
+ version.auth_schema = auth_schema
+ version.auth_config = auth_config
+
+ admin_source = None
+ if item.source == "admin" and version.source_server_id:
+ admin_source = (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == version.source_server_id,
+ AdminMcpServer.owner_user_id.is_(None),
+ )
+ .first()
+ )
+ if admin_source is not None:
+ admin_source.extra_config = {
+ **dict(admin_source.extra_config or {}),
+ "market_auth_schema": auth_schema,
+ "market_auth_config": auth_config,
+ }
+
+ if auth_config["credential_mode"] == "admin":
+ if item.source != "admin" or not version.source_server_id:
+ raise BadRequestError(message="只有管理员发布的 MCP 才能统一配置 Token")
+ if admin_source is None:
+ raise BadRequestError(message="管理员发布源已不存在,无法统一配置 Token")
+ source_headers = decrypt_mcp_headers(admin_source.headers)
+ supplied = dict(managed_credentials or {})
+ allowed_keys = {str(field.get("key") or "") for field in auth_schema}
+ unknown_keys = sorted(key for key in supplied if key not in allowed_keys)
+ if unknown_keys:
+ raise BadRequestError(message=f"未知的 MCP 认证字段:{', '.join(unknown_keys)}")
+ for field in auth_schema:
+ key = str(field.get("key") or "").strip()
+ value = str(supplied.get(key) or "").strip()
+ if value:
+ source_headers[mcp_credentials.credential_storage_key(field)] = mcp_credentials.materialize_credential_value(
+ field, value
+ )
+ admin_source.headers = encrypt_mcp_headers(source_headers)
+ managed_headers = mcp_credentials.admin_managed_credential_headers(
+ db,
+ item,
+ version,
+ auth_method=str(auth_config["default_method"]),
+ )
+ if not managed_headers:
+ raise BadRequestError(message="请填写管理员统一 Token/Auth")
+ else:
+ managed_headers = {}
+
+ item.updated_at = utc_now()
+
+ installation_server_ids = [
+ str(row[0])
+ for row in db.query(McpMarketInstallation.server_id)
+ .filter(McpMarketInstallation.slug == slug)
+ .all()
+ ]
+ installations: List[AdminMcpServer] = []
+ if installation_server_ids:
+ installations = (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id.in_(installation_server_ids))
+ .all()
+ )
+ for server in installations:
+ server.display_name = item.display_name
+ server.description = item.description or ""
+ server.user_intro = item.user_intro
+ server.icon = item.icon
+ server.updated_at = item.updated_at
+ if requires_auth is False:
+ existing_headers = decrypt_mcp_headers(server.headers)
+ for field in previous_auth_schema:
+ existing_headers.pop(mcp_credentials.credential_storage_key(field), None)
+ server.headers = encrypt_mcp_headers(existing_headers)
+ elif managed_headers:
+ existing_headers = decrypt_mcp_headers(server.headers)
+ existing_headers.update(managed_headers)
+ server.headers = encrypt_mcp_headers(existing_headers)
+
+ db.commit()
+ refresh_mcp_caches()
+ result = mcp_listing.get_market_item(
+ db,
+ slug,
+ owner_user_id=None,
+ viewer_user_id=None,
+ admin=True,
+ )
+ result["updated_installations"] = len(installations)
+ return result
+
+
+def delete_market_item(
+ db: Session, slug: str, *, updated_by: Optional[str] = None
+) -> Dict[str, Any]:
+ item = (
+ db.query(McpMarketItem)
+ .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ version = mcp_versions.version_for_item(db, item)
+ if version.source_server_id:
+ source = (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id == version.source_server_id)
+ .first()
+ )
+ if source and (source.extra_config or {}).get("market_source_only"):
+ db.delete(source)
+ item.deleted_at = utc_now()
+ item.updated_at = utc_now()
+ db.commit()
+ ml.set_listing_enabled(db, ml.KIND_MCP, slug, False, updated_by=updated_by)
+ return {"slug": slug, "deleted": True}
+
+
+def set_suspended(
+ db: Session,
+ slug: str,
+ *,
+ suspended: bool,
+ reason: str = "",
+) -> Dict[str, Any]:
+ item = (
+ db.query(McpMarketItem)
+ .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ now = utc_now()
+ item.status = "suspended" if suspended else "active"
+ item.status_reason = (reason or "").strip() if suspended else None
+ item.updated_at = now
+ affected = 0
+ installs = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug).all()
+ if suspended:
+ server_ids = []
+ for install in installs:
+ install.status = "suspended"
+ install.updated_at = now
+ server_ids.append(install.server_id)
+ if server_ids:
+ affected = (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id.in_(server_ids))
+ .update({AdminMcpServer.is_enabled: False}, synchronize_session=False)
+ )
+ else:
+ server_ids = []
+ for install in installs:
+ if install.status != "suspended":
+ continue
+ install.status = "active"
+ install.updated_at = now
+ server_ids.append(install.server_id)
+ if server_ids:
+ affected = (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id.in_(server_ids))
+ .update({AdminMcpServer.is_enabled: True}, synchronize_session=False)
+ )
+ db.commit()
+ refresh_mcp_caches()
+ return {
+ "slug": slug,
+ "status": item.status,
+ "disabled_installations": affected if suspended else 0,
+ "restored_installations": affected if not suspended else 0,
+ }
diff --git a/src/backend/core/config/mcp_marketplace_catalog.py b/src/backend/features/marketplace/mcp_catalog.py
similarity index 100%
rename from src/backend/core/config/mcp_marketplace_catalog.py
rename to src/backend/features/marketplace/mcp_catalog.py
diff --git a/src/backend/features/marketplace/mcp_credentials.py b/src/backend/features/marketplace/mcp_credentials.py
new file mode 100644
index 00000000..33ad88b5
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_credentials.py
@@ -0,0 +1,130 @@
+"""mcp credentials responsibilities."""
+from __future__ import annotations
+import json
+from typing import Any, Dict, List, Optional
+from urllib.parse import parse_qsl, urlparse
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketItem, McpMarketVersion
+from core.infra.exceptions import BadRequestError
+from core.capabilities.mcp.management import decrypt_mcp_headers, mcp_oauth_bundle_storage_key, mcp_query_secret_storage_key, mcp_url_override_storage_key
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_validation
+
+def credential_storage_key(field: Dict[str, Any]) -> str:
+ target = str(field.get("target") or "header")
+ name = str(field.get("name") or field.get("key") or "").strip()
+ if target == "query":
+ return mcp_query_secret_storage_key(name)
+ if target == "url":
+ return mcp_url_override_storage_key()
+ return name
+
+
+def admin_credential_source(
+ db: Session,
+ item: McpMarketItem,
+ version: McpMarketVersion,
+) -> Optional[AdminMcpServer]:
+ if item.source != "admin" or not version.source_server_id:
+ return None
+ return (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == version.source_server_id,
+ AdminMcpServer.owner_user_id.is_(None),
+ )
+ .first()
+ )
+
+
+def admin_managed_credential_headers(
+ db: Session,
+ item: McpMarketItem,
+ version: McpMarketVersion,
+ *,
+ auth_method: str,
+) -> Dict[str, str]:
+ """Return an admin publisher's credentials without exposing them to marketplace APIs.
+
+ Marketplace versions remain credential-free. For an admin-published MCP,
+ the original global/source ``AdminMcpServer`` is the encrypted credential
+ authority. If it still has every required field for the selected token
+ method, user installations may copy those values into their own encrypted
+ server row without asking the user to enter the same token again.
+ """
+ auth_config = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or []))
+ if auth_config["credential_mode"] == "installer":
+ return {}
+ selected = next(
+ (method for method in auth_config["methods"] if method["id"] == auth_method),
+ None,
+ )
+ if selected is None or selected["type"] != "token":
+ return {}
+ fields = [
+ field
+ for field in (version.auth_schema or [])
+ if not field.get("methods") or auth_method in field.get("methods", [])
+ ]
+ if not fields:
+ return {}
+ source = admin_credential_source(db, item, version)
+ if source is None:
+ return {}
+
+ stored = decrypt_mcp_headers(source.headers)
+ source_query = dict(parse_qsl(urlparse(source.url or "").query, keep_blank_values=True))
+ managed: Dict[str, str] = {}
+ for field in fields:
+ storage_key = credential_storage_key(field)
+ value = str(stored.get(storage_key) or "")
+ if not value and str(field.get("target") or "header") == "query":
+ query_name = str(field.get("name") or field.get("key") or "").strip()
+ value = str(source_query.get(query_name) or "")
+ if value:
+ managed[storage_key] = value
+ elif field.get("required", True):
+ return {}
+ return managed
+
+
+def materialize_credential_value(field: Dict[str, Any], value: str) -> str:
+ prefix = str(field.get("prefix") or "")
+ if prefix and not value.lower().startswith(prefix.lower()):
+ return f"{prefix}{value}"
+ return value
+
+
+def installation_secrets(
+ version: McpMarketVersion,
+ credentials: Dict[str, str],
+ existing_headers: Dict[str, str] | None = None,
+ *,
+ auth_method: str,
+ oauth_bundle: Optional[Dict[str, Any]] = None,
+) -> Dict[str, str]:
+ stored: Dict[str, str] = {}
+ missing: List[str] = []
+ for field in version.auth_schema or []:
+ method_ids = [str(value) for value in (field.get("methods") or []) if str(value)]
+ if method_ids and auth_method not in method_ids:
+ continue
+ key = str(field.get("key") or "").strip()
+ storage_key = credential_storage_key(field)
+ if not key or not storage_key:
+ continue
+ value = str(credentials.get(key) or "").strip()
+ if value:
+ stored[storage_key] = materialize_credential_value(field, value)
+ elif storage_key in (existing_headers or {}):
+ stored[storage_key] = str((existing_headers or {})[storage_key])
+ elif field.get("required", True):
+ missing.append(str(field.get("label") or key))
+ if missing:
+ raise BadRequestError(message=f"请填写安装凭据:{', '.join(missing)}")
+ oauth_key = mcp_oauth_bundle_storage_key()
+ if oauth_bundle:
+ stored[oauth_key] = json.dumps(oauth_bundle, ensure_ascii=False, separators=(",", ":"))
+ elif auth_method == "oauth2" and oauth_key in (existing_headers or {}):
+ stored[oauth_key] = str((existing_headers or {})[oauth_key])
+ return stored
diff --git a/src/backend/features/marketplace/mcp_curated.py b/src/backend/features/marketplace/mcp_curated.py
new file mode 100644
index 00000000..698ecb80
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_curated.py
@@ -0,0 +1,127 @@
+"""mcp curated responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+from typing import List
+from features.marketplace.mcp_catalog import CURATED_MCP_MARKET_ITEMS
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion
+from features.marketplace import edition as ml
+from core.capabilities.mcp.management import tool_snapshot_hash
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_validation
+
+def ensure_curated_market_items(db: Session) -> List[str]:
+ """Seed credential-free official templates once, without resurrecting deletions."""
+ seeded: List[str] = []
+ updated = False
+ now = utc_now()
+ for definition in CURATED_MCP_MARKET_ITEMS:
+ slug = str(definition["slug"])
+ # Soft-deleted rows deliberately count as existing: an administrator's
+ # removal choice must survive application restarts.
+ existing_item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
+ if existing_item:
+ if existing_item.deleted_at is None:
+ existing_version = (
+ db.query(McpMarketVersion)
+ .filter(McpMarketVersion.version_id == existing_item.latest_version_id)
+ .first()
+ )
+ if existing_version and existing_version.approved_by == "system-curated":
+ schema = list(definition.get("auth_schema") or [])
+ auth_config = mcp_validation.normalize_auth_config(definition.get("auth_config"), schema)
+ notice = dict(existing_version.listing_notice or {})
+ notice.update(
+ {
+ "discovery_mode": "per_install",
+ "install_notice": str(definition.get("install_notice") or ""),
+ "docs_url": str(definition.get("docs_url") or ""),
+ }
+ )
+ description = str(definition.get("description") or "")
+ user_intro = str(definition.get("user_intro") or "")
+ icon = str(definition.get("icon") or "")
+ changed = (
+ list(existing_version.auth_schema or []) != schema
+ or dict(existing_version.auth_config or {}) != auth_config
+ or dict(existing_version.listing_notice or {}) != notice
+ or (existing_item.description or "") != description
+ or (existing_item.user_intro or "") != user_intro
+ or (existing_item.icon or "") != icon
+ )
+ if changed:
+ existing_version.auth_schema = schema
+ existing_version.auth_config = auth_config
+ existing_version.listing_notice = notice
+ existing_item.description = description
+ existing_item.user_intro = user_intro
+ existing_item.icon = icon
+ existing_item.updated_at = now
+ server_ids = [
+ row.server_id
+ for row in db.query(McpMarketInstallation)
+ .filter(McpMarketInstallation.slug == slug)
+ .all()
+ ]
+ if server_ids:
+ (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id.in_(server_ids))
+ .update({AdminMcpServer.icon: icon}, synchronize_session=False)
+ )
+ updated = True
+ continue
+ tools = list(definition.get("tools") or [])
+ listing_notice = {
+ "discovery_mode": "per_install",
+ "install_notice": str(definition.get("install_notice") or ""),
+ "docs_url": str(definition.get("docs_url") or ""),
+ }
+ version_id = f"mcpver_curated_{slug}"
+ item = McpMarketItem(
+ slug=slug,
+ display_name=str(definition["display_name"]),
+ description=str(definition.get("description") or ""),
+ user_intro=str(definition.get("user_intro") or ""),
+ category=mcp_validation.validate_category(str(definition.get("category") or "通用工具")),
+ tags=list(definition.get("tags") or []),
+ icon=str(definition.get("icon") or ""),
+ publisher_id=None,
+ publisher_name=str(definition.get("publisher_name") or "HugAgentOS"),
+ source="admin",
+ latest_version_id=version_id,
+ status="active",
+ status_reason=None,
+ last_verified_at=None,
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(item)
+ db.add(
+ McpMarketVersion(
+ version_id=version_id,
+ slug=slug,
+ version=mcp_validation.normalize_version(str(definition.get("version") or "1.0.0")),
+ transport=str(definition.get("transport") or "streamable_http"),
+ url=str(definition["url"]),
+ auth_schema=list(definition.get("auth_schema") or []),
+ auth_config=mcp_validation.normalize_auth_config(
+ definition.get("auth_config"),
+ list(definition.get("auth_schema") or []),
+ ),
+ tools_json=tools,
+ tool_hash=tool_snapshot_hash(tools),
+ listing_notice=listing_notice,
+ source_server_id=None,
+ approved_by="system-curated",
+ approved_at=now,
+ created_at=now,
+ )
+ )
+ seeded.append(slug)
+ if not seeded and not updated:
+ return []
+ db.commit()
+ for slug in seeded:
+ ml.set_listing_enabled(db, ml.KIND_MCP, slug, True, updated_by="system-curated")
+ return seeded
diff --git a/src/backend/features/marketplace/mcp_installation.py b/src/backend/features/marketplace/mcp_installation.py
new file mode 100644
index 00000000..d280e6d1
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_installation.py
@@ -0,0 +1,267 @@
+"""mcp installation responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import uuid
+from typing import Any, Dict, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from features.marketplace import edition as ml
+from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, materialize_mcp_http_connection, probe_mcp_connectivity, refresh_mcp_caches, tool_snapshot_hash, validate_remote_mcp_url
+from sqlalchemy.orm import Session
+from core.db.threads import run_in_session
+from features.marketplace import mcp_credentials
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+def _plan_installation(
+ db: Session,
+ slug: str,
+ owner_user_id: Optional[str],
+ installed_by: Optional[str],
+ credentials: Optional[Dict[str, str]],
+ auth_method: Optional[str],
+ oauth_bundle: Optional[Dict[str, Any]],
+) -> Dict[str, Any]:
+ """Resolve the version, credentials and target server; returns a transient candidate row."""
+ item = (
+ db.query(McpMarketItem)
+ .filter(
+ McpMarketItem.slug == slug,
+ McpMarketItem.deleted_at.is_(None),
+ McpMarketItem.status == "active",
+ )
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ if owner_user_id is not None:
+ if slug in ml.get_disabled_ids(db, ml.KIND_MCP):
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ ml.ensure_item_visible(db, ml.KIND_MCP, slug, owner_user_id, resource="mcp_market_item")
+ version = mcp_versions.version_for_item(db, item)
+ auth_config = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or []))
+ selected_method = str(auth_method or auth_config["default_method"])
+ selected = next(
+ (method for method in auth_config["methods"] if method["id"] == selected_method),
+ None,
+ )
+ if selected is None:
+ raise BadRequestError(message="不支持所选 MCP 认证方式")
+ if selected["type"] == "oauth2" and not oauth_bundle:
+ raise BadRequestError(message="请先完成 OAuth 登录")
+ if selected["type"] != "oauth2" and oauth_bundle:
+ raise BadRequestError(message="当前认证方式不能写入 OAuth 凭据")
+ existing = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug)
+ existing = (
+ existing.filter(McpMarketInstallation.owner_user_id.is_(None))
+ if owner_user_id is None
+ else existing.filter(McpMarketInstallation.owner_user_id == owner_user_id)
+ ).first()
+ existing_server = (
+ db.query(AdminMcpServer).filter(AdminMcpServer.server_id == existing.server_id).first()
+ if existing
+ else None
+ )
+ if existing_server is None and owner_user_id is None and version.source_server_id:
+ source_candidate = (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == version.source_server_id,
+ AdminMcpServer.owner_user_id.is_(None),
+ )
+ .first()
+ )
+ if source_candidate and (source_candidate.extra_config or {}).get("market_source_only"):
+ existing_server = source_candidate
+ old_headers = decrypt_mcp_headers(existing_server.headers) if existing_server else {}
+ managed_headers = mcp_credentials.admin_managed_credential_headers(
+ db,
+ item,
+ version,
+ auth_method=selected_method,
+ )
+ fallback_headers = {**managed_headers, **old_headers}
+ headers = mcp_credentials.installation_secrets(
+ version,
+ dict(credentials or {}),
+ fallback_headers,
+ auth_method=selected_method,
+ oauth_bundle=oauth_bundle,
+ )
+ runtime_url, _ = materialize_mcp_http_connection(version.url, headers)
+ now = utc_now()
+ server_id = existing_server.server_id if existing_server else f"mmcp_{uuid.uuid4().hex[:20]}"
+ candidate = AdminMcpServer(
+ server_id=server_id,
+ display_name=item.display_name,
+ description=item.description or "",
+ user_intro=item.user_intro,
+ transport=version.transport,
+ command=None,
+ args=[],
+ url=version.url,
+ env_vars={},
+ env_inherit=[],
+ headers=headers,
+ is_enabled=True,
+ sort_order=0,
+ extra_config={
+ **(
+ {
+ key: value
+ for key, value in dict(existing_server.extra_config or {}).items()
+ if key not in {"market_source_only", "market_source_slug"}
+ }
+ if existing_server
+ else {}
+ ),
+ "market_slug": slug,
+ "market_version": version.version,
+ "listing_notice": dict(version.listing_notice or {}),
+ "auth_method": selected_method,
+ },
+ tools_json=list(version.tools_json or []),
+ icon=item.icon,
+ owner_user_id=owner_user_id,
+ source_plugin=None,
+ created_by=installed_by,
+ created_at=existing_server.created_at if existing_server else now,
+ updated_at=now,
+ )
+
+ return {
+ "candidate": candidate,
+ "headers": headers,
+ "runtime_url": runtime_url,
+ "allow_private_network": mcp_validation.installs_admin_published_endpoint(
+ item, version, runtime_url
+ ),
+ "item_id": item.slug,
+ "version_id": version.version_id,
+ "install_id": existing.install_id if existing else None,
+ "existing_server_id": existing_server.server_id if existing_server else None,
+ "now": now,
+ }
+
+
+async def install_market_item(
+ slug: str,
+ *,
+ owner_user_id: Optional[str],
+ installed_by: Optional[str],
+ credentials: Optional[Dict[str, str]] = None,
+ auth_method: Optional[str] = None,
+ oauth_bundle: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ """Install a marketplace MCP: plan, probe the live server, then commit.
+
+ The URL check and probe run between two database steps, on a transient
+ candidate row, so nothing is held open while the remote server answers.
+ """
+ plan = await run_in_session(
+ _plan_installation, slug, owner_user_id, installed_by, credentials, auth_method, oauth_bundle
+ )
+ await validate_remote_mcp_url(
+ plan["runtime_url"],
+ allow_private_network=plan["allow_private_network"],
+ require_https=False,
+ )
+ ok, error = await probe_mcp_connectivity(plan["candidate"], validate=True)
+ if not ok:
+ raise BadRequestError(message=f"MCP 安装检测失败:{error}")
+ result = await run_in_session(_commit_installation, slug, owner_user_id, installed_by, plan)
+ refresh_mcp_caches()
+ return result
+
+
+def _commit_installation(
+ db: Session,
+ slug: str,
+ owner_user_id: Optional[str],
+ installed_by: Optional[str],
+ plan: Dict[str, Any],
+) -> Dict[str, Any]:
+ from features.marketplace.models import McpMarketVersion
+
+ candidate, headers, now = plan["candidate"], plan["headers"], plan["now"]
+ item = db.query(McpMarketItem).filter(McpMarketItem.slug == plan["item_id"]).one()
+ version = db.get(McpMarketVersion, plan["version_id"])
+ existing = db.get(McpMarketInstallation, plan["install_id"]) if plan["install_id"] else None
+ existing_server = (
+ db.query(AdminMcpServer).filter(AdminMcpServer.server_id == plan["existing_server_id"]).first()
+ if plan["existing_server_id"]
+ else None
+ )
+ server_id = candidate.server_id
+ discovery_mode = str((version.listing_notice or {}).get("discovery_mode") or "reviewed")
+ if discovery_mode == "per_install":
+ candidate.extra_config = {
+ **dict(candidate.extra_config or {}),
+ "discovery_mode": discovery_mode,
+ }
+ else:
+ actual_hash = tool_snapshot_hash(candidate.tools_json or [])
+ if actual_hash != version.tool_hash:
+ item.status = "changed"
+ item.status_reason = "远程工具或参数结构已变化,等待管理员重新审核"
+ item.updated_at = now
+ db.commit()
+ raise BadRequestError(message="远程 MCP 工具清单已发生变化,已暂停安装并等待重新审核")
+
+ server = existing_server or AdminMcpServer(server_id=server_id)
+ for field in (
+ "display_name",
+ "description",
+ "user_intro",
+ "transport",
+ "command",
+ "args",
+ "url",
+ "env_vars",
+ "env_inherit",
+ "is_enabled",
+ "sort_order",
+ "extra_config",
+ "tools_json",
+ "icon",
+ "owner_user_id",
+ "source_plugin",
+ "created_by",
+ "updated_at",
+ ):
+ setattr(server, field, getattr(candidate, field))
+ server.headers = encrypt_mcp_headers(headers)
+ if not existing_server:
+ server.created_at = now
+ db.add(server)
+ db.flush()
+ if existing:
+ previous_version_id = existing.version_id
+ existing.version_id = version.version_id
+ existing.status = "active"
+ existing.installed_by = installed_by
+ existing.updated_at = now
+ action = "updated" if previous_version_id != version.version_id else "existing"
+ else:
+ existing = McpMarketInstallation(
+ install_id=f"mcpins_{uuid.uuid4().hex}",
+ slug=slug,
+ version_id=version.version_id,
+ server_id=server.server_id,
+ owner_user_id=owner_user_id,
+ status="active",
+ installed_by=installed_by,
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(existing)
+ action = "installed"
+ db.commit()
+ return {
+ "install_id": existing.install_id,
+ "server_id": server.server_id,
+ "slug": slug,
+ "version": version.version,
+ "action": action,
+ }
diff --git a/src/backend/features/marketplace/mcp_listing.py b/src/backend/features/marketplace/mcp_listing.py
new file mode 100644
index 00000000..d2c1aebf
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_listing.py
@@ -0,0 +1,142 @@
+"""mcp catalog responsibilities."""
+from __future__ import annotations
+from typing import Any, Dict, Optional
+from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion
+from core.infra.exceptions import ResourceNotFoundError
+from features.marketplace import edition as ml
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_credentials
+from features.marketplace import mcp_state
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+def installed_slug_set(db: Session, owner_user_id: Optional[str]) -> set[str]:
+ query = db.query(McpMarketInstallation.slug).filter(
+ McpMarketInstallation.status.in_(["active", "suspended"])
+ )
+ query = (
+ query.filter(McpMarketInstallation.owner_user_id.is_(None))
+ if owner_user_id is None
+ else query.filter(McpMarketInstallation.owner_user_id == owner_user_id)
+ )
+ return {str(row[0]) for row in query.all()}
+
+
+def item_dict(
+ db: Session,
+ item: McpMarketItem,
+ version: McpMarketVersion,
+ *,
+ installed: bool = False,
+) -> Dict[str, Any]:
+ auth_schema = list(version.auth_schema or [])
+ auth_config = mcp_validation.normalize_auth_config(version.auth_config, auth_schema)
+ requires_auth = any(method["type"] != "none" for method in auth_config["methods"])
+ credentials_managed_by_admin = bool(
+ mcp_credentials.admin_managed_credential_headers(
+ db,
+ item,
+ version,
+ auth_method=str(auth_config["default_method"]),
+ )
+ )
+ if auth_config["credential_mode"] == "auto":
+ auth_config["credential_mode"] = "admin" if credentials_managed_by_admin else "installer"
+ supports_admin_credentials = bool(mcp_credentials.admin_credential_source(db, item, version))
+ return {
+ "slug": item.slug,
+ "display_name": item.display_name,
+ "description": item.description or "",
+ "summary": item.description or "",
+ "user_intro": item.user_intro or "",
+ "category": item.category,
+ "tags": list(item.tags or []),
+ "icon": item.icon or "",
+ "publisher_name": item.publisher_name or "",
+ "source": item.source,
+ "version": version.version,
+ "version_id": version.version_id,
+ "transport": version.transport,
+ "url_origin": mcp_validation.safe_origin(version.url),
+ "auth_schema": auth_schema,
+ "auth_config": auth_config,
+ "requires_auth": requires_auth,
+ "credentials_managed_by_admin": credentials_managed_by_admin,
+ "supports_admin_credentials": supports_admin_credentials,
+ "requires_user_credentials": requires_auth and not credentials_managed_by_admin,
+ "tools": list(version.tools_json or []),
+ "tool_count": len(version.tools_json or []),
+ "tool_hash": version.tool_hash,
+ "listing_notice": dict(version.listing_notice or {}),
+ "status": item.status,
+ "status_reason": item.status_reason or "",
+ "last_verified_at": item.last_verified_at.isoformat() if item.last_verified_at else None,
+ "installed": installed,
+ "deletable": True,
+ "created_at": item.created_at.isoformat() if item.created_at else None,
+ "updated_at": item.updated_at.isoformat() if item.updated_at else None,
+ }
+
+
+def list_market_items(
+ db: Session,
+ *,
+ owner_user_id: Optional[str],
+ viewer_user_id: Optional[str],
+ include_disabled: bool = False,
+) -> Dict[str, Any]:
+ query = db.query(McpMarketItem).filter(McpMarketItem.deleted_at.is_(None))
+ if not include_disabled:
+ query = query.filter(McpMarketItem.status == "active")
+ rows = query.order_by(McpMarketItem.updated_at.desc(), McpMarketItem.slug).all()
+ installed = installed_slug_set(db, owner_user_id)
+ items = [
+ item_dict(db, row, mcp_versions.version_for_item(db, row), installed=row.slug in installed)
+ for row in rows
+ ]
+ items = ml.annotate_and_filter(
+ db,
+ ml.KIND_MCP,
+ items,
+ id_key="slug",
+ include_disabled=include_disabled,
+ viewer_user_id=viewer_user_id,
+ )
+ categories = [
+ category
+ for category in mcp_state.MCP_MARKET_CATEGORIES
+ if any(i["category"] == category for i in items)
+ ]
+ return {"items": items, "categories": categories}
+
+
+def get_market_item(
+ db: Session,
+ slug: str,
+ *,
+ owner_user_id: Optional[str],
+ viewer_user_id: Optional[str],
+ admin: bool = False,
+) -> Dict[str, Any]:
+ item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
+ if not item or (not admin and item.status != "active"):
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ if not admin:
+ ml.ensure_item_visible(db, ml.KIND_MCP, slug, viewer_user_id, resource="mcp_market_item")
+ data = item_dict(
+ db,
+ item,
+ mcp_versions.version_for_item(db, item),
+ installed=slug in installed_slug_set(db, owner_user_id),
+ )
+ state = ml.annotate_and_filter(
+ db,
+ ml.KIND_MCP,
+ [data],
+ id_key="slug",
+ include_disabled=admin,
+ viewer_user_id=viewer_user_id,
+ )
+ if not state:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ return state[0]
diff --git a/src/backend/core/services/mcp_marketplace_monitor.py b/src/backend/features/marketplace/mcp_monitor.py
similarity index 55%
rename from src/backend/core/services/mcp_marketplace_monitor.py
rename to src/backend/features/marketplace/mcp_monitor.py
index 13daa508..65579e60 100644
--- a/src/backend/core/services/mcp_marketplace_monitor.py
+++ b/src/backend/features/marketplace/mcp_monitor.py
@@ -7,9 +7,9 @@
from contextlib import suppress
from core.config.settings import settings
-from core.db.engine import SessionLocal
-from core.db.models import McpMarketItem
-from core.services.mcp_marketplace_service import revalidate_market_item
+from core.db.threads import run_in_session
+from features.marketplace.models import McpMarketItem
+from features.marketplace.mcp_revalidation import revalidate_market_item
logger = logging.getLogger(__name__)
_task: asyncio.Task | None = None
@@ -19,29 +19,30 @@ def _interval_seconds() -> float:
return float(settings.server.mcp_market_revalidate_interval)
+def _monitored_slugs(db) -> list[str]:
+ return [
+ str(row[0])
+ for row in db.query(McpMarketItem.slug)
+ .filter(
+ McpMarketItem.deleted_at.is_(None),
+ McpMarketItem.status.in_(["active", "changed"]),
+ )
+ .all()
+ ]
+
+
async def _run() -> None:
interval = _interval_seconds()
await asyncio.sleep(min(60.0, interval))
while True:
try:
- with SessionLocal() as db:
- slugs = [
- str(row[0])
- for row in db.query(McpMarketItem.slug)
- .filter(
- McpMarketItem.deleted_at.is_(None),
- McpMarketItem.status.in_(["active", "changed"]),
+ for slug in await run_in_session(_monitored_slugs):
+ try:
+ await revalidate_market_item(slug)
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ "mcp_market_revalidate_failed", extra={"slug": slug, "error": str(exc)}
)
- .all()
- ]
- for slug in slugs:
- try:
- await revalidate_market_item(db, slug)
- except Exception as exc: # noqa: BLE001
- db.rollback()
- logger.warning(
- "mcp_market_revalidate_failed", extra={"slug": slug, "error": str(exc)}
- )
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
diff --git a/src/backend/features/marketplace/mcp_publishing.py b/src/backend/features/marketplace/mcp_publishing.py
new file mode 100644
index 00000000..42ff18fd
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_publishing.py
@@ -0,0 +1,193 @@
+"""mcp publishing responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import uuid
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from features.marketplace import edition as ml
+from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url
+from sqlalchemy.orm import Session
+from core.db.threads import run_in_session
+from features.marketplace import mcp_listing
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+@dataclass(frozen=True)
+class Publication:
+ """Marketplace listing options for one admin MCP server."""
+
+ category: str
+ version: str
+ summary: str = ""
+ tags: Optional[List[str]] = None
+ publisher_name: str = "管理员"
+ link_as_installation: bool = True
+ auth_schema_override: Optional[List[Dict[str, Any]]] = None
+ auth_config_override: Optional[Dict[str, Any]] = None
+ allow_deferred_discovery: bool = False
+
+
+def _source_server(db: Session, source_server_id: str) -> AdminMcpServer:
+ row = (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == source_server_id,
+ AdminMcpServer.owner_user_id.is_(None),
+ )
+ .first()
+ )
+ if not row:
+ raise ResourceNotFoundError("admin_mcp_server", source_server_id)
+ return row
+
+
+def _detached_source(db: Session, source_server_id: str) -> AdminMcpServer:
+ row = _source_server(db, source_server_id)
+ db.expunge(row)
+ return row
+
+
+async def check_publishable(row: AdminMcpServer, publication: Publication) -> List[Dict[str, Any]]:
+ """Validate a server for listing (URL policy + live probe); returns its discovered tools."""
+ if row.transport not in ("streamable_http", "sse") or not row.url:
+ raise BadRequestError(message="MCP 市场仅支持远程 HTTP/SSE 服务;stdio 请通过插件市场分发")
+ if (row.extra_config or {}).get("market_slug"):
+ raise BadRequestError(message="从 MCP 市场安装的实例不能重复发布")
+ await validate_remote_mcp_url(
+ row.url,
+ allow_private_network=True,
+ require_https=False,
+ )
+ ok, error = await probe_mcp_connectivity(row, validate=True)
+ if not ok and not publication.allow_deferred_discovery:
+ raise BadRequestError(message=f"MCP 发布检测失败:{error}")
+ tools = list(row.tools_json or [])
+ if not tools and not publication.allow_deferred_discovery:
+ raise BadRequestError(message="MCP 未发现任何工具,不能发布")
+ return tools
+
+
+async def publish_admin_server(source_server_id: str, publication: Publication) -> Dict[str, Any]:
+ row = await run_in_session(_detached_source, source_server_id)
+ tools = await check_publishable(row, publication)
+ return await run_in_session(
+ lambda db: record_publication(db, _source_server(db, source_server_id), tools, publication)
+ )
+
+
+def record_publication(
+ db: Session, row: AdminMcpServer, tools: List[Dict[str, Any]], publication: Publication
+) -> Dict[str, Any]:
+ """List a checked server in the marketplace and commit (``row`` belongs to ``db``)."""
+ category = publication.category
+ version = publication.version
+ summary = publication.summary
+ tags = publication.tags
+ publisher_name = publication.publisher_name
+ link_as_installation = publication.link_as_installation
+ auth_schema_override = publication.auth_schema_override
+ auth_config_override = publication.auth_config_override
+ allow_deferred_discovery = publication.allow_deferred_discovery
+ if tools:
+ row.tools_json = tools
+ source_server_id = row.server_id
+ existing = (
+ db.query(McpMarketItem)
+ .join(McpMarketVersion, McpMarketVersion.version_id == McpMarketItem.latest_version_id)
+ .filter(McpMarketVersion.source_server_id == source_server_id)
+ .first()
+ )
+ slug = existing.slug if existing else mcp_validation.new_slug(db, row.display_name, source_server_id)
+ template_url, inferred_auth_schema = mcp_validation.credential_free_connection(row)
+ stored_market_config = dict(row.extra_config or {})
+ auth_schema = list(
+ auth_schema_override
+ if auth_schema_override is not None
+ else stored_market_config.get("market_auth_schema") or inferred_auth_schema
+ )
+ auth_config = mcp_validation.normalize_auth_config(
+ (
+ auth_config_override
+ if auth_config_override is not None
+ else stored_market_config.get("market_auth_config")
+ ),
+ auth_schema,
+ )
+ item = mcp_versions.publish_snapshot(
+ db,
+ slug=slug,
+ display_name=row.display_name,
+ description=(summary or row.description or "").strip(),
+ user_intro=row.user_intro,
+ category=category,
+ tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())),
+ icon=row.icon,
+ publisher_id=None,
+ publisher_name=publisher_name,
+ source="admin",
+ version=version,
+ transport=row.transport,
+ url=template_url,
+ auth_schema=auth_schema,
+ auth_config=auth_config,
+ tools=tools,
+ source_server_id=row.server_id,
+ approved_by=publisher_name,
+ )
+ db.flush()
+ if allow_deferred_discovery:
+ market_version = mcp_versions.version_for_item(db, item)
+ market_version.listing_notice = {
+ **dict(market_version.listing_notice or {}),
+ "discovery_mode": "per_install",
+ "install_notice": "该 MCP 的完整工具清单将在用户完成认证后动态发现。",
+ }
+ # Existing global MCPs are linked as an installation. A server created
+ # through the marketplace-first flow remains a disabled source template
+ # until an administrator explicitly installs it from the market.
+ if link_as_installation:
+ installation = (
+ db.query(McpMarketInstallation)
+ .filter(McpMarketInstallation.server_id == row.server_id)
+ .first()
+ )
+ if installation is None:
+ db.add(
+ McpMarketInstallation(
+ install_id=f"mcpins_{uuid.uuid4().hex}",
+ slug=item.slug,
+ version_id=item.latest_version_id,
+ server_id=row.server_id,
+ owner_user_id=None,
+ status="active",
+ installed_by="admin",
+ created_at=utc_now(),
+ updated_at=utc_now(),
+ )
+ )
+ else:
+ installation.slug = item.slug
+ installation.version_id = item.latest_version_id
+ installation.status = "active"
+ installation.updated_at = utc_now()
+ else:
+ for installation in (
+ db.query(McpMarketInstallation)
+ .filter(McpMarketInstallation.server_id == row.server_id)
+ .all()
+ ):
+ db.delete(installation)
+ row.is_enabled = False
+ row.extra_config = {
+ **dict(row.extra_config or {}),
+ "market_source_only": True,
+ "market_source_slug": item.slug,
+ }
+ row.headers = encrypt_mcp_headers(decrypt_mcp_headers(row.headers))
+ db.commit()
+ ml.set_listing_enabled(db, ml.KIND_MCP, item.slug, True, updated_by=publisher_name)
+ refresh_mcp_caches()
+ return mcp_listing.item_dict(db, item, mcp_versions.version_for_item(db, item), installed=link_as_installation)
diff --git a/src/backend/features/marketplace/mcp_revalidation.py b/src/backend/features/marketplace/mcp_revalidation.py
new file mode 100644
index 00000000..b2b18809
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_revalidation.py
@@ -0,0 +1,146 @@
+"""mcp revalidation responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+from typing import Any, Dict, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketItem
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.capabilities.mcp.management import probe_mcp_connectivity, tool_snapshot_hash, validate_remote_mcp_url
+from core.db.threads import run_in_session
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_administration
+from features.marketplace import mcp_listing
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+
+def _listing(db: Session, slug: str):
+ item = (
+ db.query(McpMarketItem)
+ .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None))
+ .first()
+ )
+ if not item:
+ raise ResourceNotFoundError("mcp_market_item", slug)
+ return item, mcp_versions.version_for_item(db, item)
+
+
+def _settle(db: Session, item, version, status: str, reason: Optional[str]) -> Dict[str, Any]:
+ item.status = status
+ item.status_reason = reason
+ item.last_verified_at = utc_now()
+ item.updated_at = utc_now()
+ db.commit()
+ return mcp_listing.item_dict(db, item, version, installed=False)
+
+
+def _plan(db: Session, slug: str) -> Dict[str, Any]:
+ """Decide what has to be checked remotely; settles the cases that need no network."""
+ item, version = _listing(db, slug)
+ if str((version.listing_notice or {}).get("discovery_mode") or "") == "per_install":
+ # Curated templates have no shared provider credential, so their live
+ # tools are intentionally discovered with each installer's credential.
+ # Revalidation checks only a fixed public endpoint; private endpoint
+ # templates are validated when the user installs them.
+ has_endpoint_override = any(
+ str(field.get("target") or "") == "url" for field in version.auth_schema or []
+ )
+ if not has_endpoint_override:
+ return {"kind": "endpoint", "url": version.url}
+ return {"kind": "done", "result": _settle(db, item, version, "active", "个人端点将在安装时使用用户凭据验证")}
+ source = (
+ db.query(AdminMcpServer)
+ .filter(AdminMcpServer.server_id == version.source_server_id)
+ .first()
+ if version.source_server_id
+ else None
+ )
+ if source:
+ source_template_url, source_auth_schema = mcp_validation.credential_free_connection(source)
+ if (
+ source.transport != version.transport
+ or source_template_url.strip() != version.url.strip()
+ ):
+ return {"kind": "done", "result": _settle(
+ db, item, version, "changed", "原始 MCP 的连接地址或传输方式已变化,需发布新版本"
+ )}
+ auth_policy = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or []))
+ if auth_policy["credential_mode"] == "auto" and source_auth_schema != list(
+ version.auth_schema or []
+ ):
+ return {"kind": "done", "result": _settle(
+ db, item, version, "changed", "原始 MCP 的认证参数已变化,需发布新版本"
+ )}
+ db.expunge(source)
+ return {
+ "kind": "probe",
+ "url": version.url,
+ "allow_private_network": item.source == "admin",
+ "target": source or mcp_versions.version_probe_candidate(item, version),
+ "from_source": source is not None,
+ }
+
+
+def _record_endpoint(db: Session, slug: str, error: Optional[str]) -> Dict[str, Any]:
+ item, version = _listing(db, slug)
+ if error is None:
+ return _settle(db, item, version, "active", None)
+ return _settle(db, item, version, "suspended", f"官方端点安全复检失败:{error}")
+
+
+def _record_unsafe(db: Session, slug: str, error: str) -> Dict[str, Any]:
+ mcp_administration.set_suspended(db, slug, suspended=True, reason=f"远程地址安全复检失败:{error}")
+ item, version = _listing(db, slug)
+ return mcp_listing.item_dict(db, item, version, installed=False)
+
+
+def _record_probe(
+ db: Session, slug: str, ok: bool, error: str, discovered_tools, from_source: bool, accept_changes: bool
+) -> Dict[str, Any]:
+ item, version = _listing(db, slug)
+ if not ok:
+ prefix = "复检连接失败" if from_source else "原始 MCP 连接已不存在,版本端点复检失败"
+ return _settle(db, item, version, "changed", f"{prefix}:{error}")
+ if not discovered_tools:
+ return _settle(db, item, version, "changed", "复检连接成功,但远程 MCP 未返回任何工具,暂不接受新快照")
+ if tool_snapshot_hash(discovered_tools) == version.tool_hash:
+ return _settle(db, item, version, "active", None)
+ if accept_changes and item.source == "admin":
+ previous_version = version.version
+ accepted = mcp_versions.accept_revalidated_snapshot(db, item, version, discovered_tools)
+ result = mcp_listing.item_dict(db, item, accepted, installed=False)
+ result["snapshot_accepted"] = True
+ result["previous_version"] = previous_version
+ return result
+ return _settle(db, item, version, "changed", "远程工具或参数结构已变化,需发布新版本并重新审核")
+
+
+async def revalidate_market_item(slug: str, *, accept_changes: bool = False) -> Dict[str, Any]:
+ """Re-check a listing against its live endpoint and record the verdict.
+
+ Reading the listing, the remote checks and recording the result are three
+ separate steps; no session waits on the remote server.
+ """
+ plan = await run_in_session(_plan, slug)
+ if plan["kind"] == "done":
+ return plan["result"]
+ if plan["kind"] == "endpoint":
+ try:
+ await validate_remote_mcp_url(plan["url"], require_https=False)
+ error = None
+ except BadRequestError as exc:
+ error = str(exc)
+ return await run_in_session(_record_endpoint, slug, error)
+ try:
+ await validate_remote_mcp_url(
+ plan["url"],
+ allow_private_network=plan["allow_private_network"],
+ require_https=False,
+ )
+ except BadRequestError as exc:
+ return await run_in_session(_record_unsafe, slug, str(exc))
+ target = plan["target"]
+ ok, error = await probe_mcp_connectivity(target, validate=plan["from_source"])
+ return await run_in_session(
+ _record_probe, slug, ok, error, list(target.tools_json or []), plan["from_source"], accept_changes
+ )
diff --git a/src/backend/features/marketplace/mcp_state.py b/src/backend/features/marketplace/mcp_state.py
new file mode 100644
index 00000000..cc0cbdc1
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_state.py
@@ -0,0 +1,17 @@
+"""mcp state responsibilities."""
+from __future__ import annotations
+
+
+MCP_MARKET_CATEGORIES = [
+ "信息检索",
+ "数据分析",
+ "内容创作",
+ "办公协作",
+ "研发工具",
+ "业务系统",
+ "自动化",
+ "通用工具",
+]
+
+
+SECRET_QUERY_TERMS = ("key", "token", "secret", "password", "credential")
diff --git a/src/backend/features/marketplace/mcp_submissions.py b/src/backend/features/marketplace/mcp_submissions.py
new file mode 100644
index 00000000..77545fe8
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_submissions.py
@@ -0,0 +1,319 @@
+"""mcp submissions responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketSubmission
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from features.marketplace import edition as ml
+from core.capabilities.mcp.management import probe_mcp_connectivity, tool_snapshot_hash, validate_remote_mcp_url
+from sqlalchemy.orm import Session
+from core.db.threads import run_in_session
+from features.marketplace import mcp_validation
+from features.marketplace import mcp_versions
+
+def _store_tools(db: Session, server_id: str, tools) -> None:
+ """Keep the probed tool list on the source server, as a live probe of it would."""
+ source = db.query(AdminMcpServer).filter(AdminMcpServer.server_id == server_id).first()
+ if source is not None:
+ source.tools_json = tools
+
+
+def _submittable(db: Session, source_server_id: str, owner_user_id: str) -> AdminMcpServer:
+ """The owner's remote MCP, detached, once it may be submitted for listing."""
+ row = (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == source_server_id,
+ AdminMcpServer.owner_user_id == owner_user_id,
+ )
+ .first()
+ )
+ if not row:
+ raise ResourceNotFoundError("private_mcp_server", source_server_id)
+ if row.transport not in ("streamable_http", "sse") or not row.url:
+ raise BadRequestError(message="只有远程 HTTP/SSE MCP 可以申请上架")
+ installed_from_market = (
+ db.query(McpMarketInstallation.install_id)
+ .filter(McpMarketInstallation.server_id == row.server_id)
+ .first()
+ )
+ if (row.extra_config or {}).get("market_slug") or installed_from_market:
+ raise BadRequestError(message="从 MCP 市场安装的实例不能再次申请上架")
+ pending = (
+ db.query(McpMarketSubmission)
+ .filter(
+ McpMarketSubmission.source_server_id == source_server_id,
+ McpMarketSubmission.owner_user_id == owner_user_id,
+ McpMarketSubmission.status == "pending",
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .first()
+ )
+ if pending:
+ raise BadRequestError(message="该 MCP 已有待审核的上架申请")
+ db.expunge(row)
+ return row
+
+
+async def submit_to_marketplace(
+ source_server_id: str,
+ *,
+ owner_user_id: str,
+ submitter_name: str,
+ category: str,
+ version: str,
+ summary: str = "",
+ note: str = "",
+ tags: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+ row = await run_in_session(_submittable, source_server_id, owner_user_id)
+ await validate_remote_mcp_url(row.url, require_https=False)
+ ok, error = await probe_mcp_connectivity(row, validate=True)
+ if not ok:
+ raise BadRequestError(message=f"MCP 上架检测失败:{error}")
+ tools = list(row.tools_json or [])
+ if not tools:
+ raise BadRequestError(message="MCP 未发现任何工具,不能申请上架")
+
+ def record(db: Session) -> Dict[str, Any]:
+ return _record_submission(
+ db, row, tools, owner_user_id, submitter_name, category, version, summary, note, tags
+ )
+
+ return await run_in_session(record)
+
+
+def _record_submission(
+ db: Session, row, tools, owner_user_id, submitter_name, category, version, summary, note, tags
+) -> Dict[str, Any]:
+ _store_tools(db, row.server_id, tools)
+ template_url, auth_schema = mcp_validation.credential_free_connection(row)
+ now = utc_now()
+ submission = McpMarketSubmission(
+ submission_id=f"mcpsub_{uuid.uuid4().hex}",
+ slug=mcp_validation.source_slug(db, row, owner_user_id),
+ source_server_id=row.server_id,
+ owner_user_id=owner_user_id,
+ submitter_name=submitter_name or "",
+ display_name=row.display_name,
+ description=(summary or row.description or "").strip(),
+ user_intro=row.user_intro,
+ category=mcp_validation.validate_category(category),
+ tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())),
+ icon=row.icon,
+ version=mcp_validation.normalize_version(version),
+ transport=row.transport,
+ url=template_url,
+ auth_schema=auth_schema,
+ auth_config=mcp_validation.normalize_auth_config(None, auth_schema),
+ tools_json=tools,
+ tool_hash=tool_snapshot_hash(tools),
+ note=(note or "").strip(),
+ status="pending",
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(submission)
+ db.commit()
+ db.refresh(submission)
+ return submission_dict(submission)
+
+
+def submission_dict(row: McpMarketSubmission, *, detail: bool = False) -> Dict[str, Any]:
+ data: Dict[str, Any] = {
+ "submission_id": row.submission_id,
+ "slug": row.slug,
+ "source_server_id": row.source_server_id,
+ "owner_user_id": row.owner_user_id,
+ "submitter_name": row.submitter_name or "",
+ "display_name": row.display_name,
+ "description": row.description or "",
+ "category": row.category,
+ "tags": list(row.tags or []),
+ "icon": row.icon or "",
+ "version": row.version,
+ "transport": row.transport,
+ "url_origin": mcp_validation.safe_origin(row.url),
+ "auth_schema": list(row.auth_schema or []),
+ "auth_config": mcp_validation.normalize_auth_config(row.auth_config, list(row.auth_schema or [])),
+ "tool_count": len(row.tools_json or []),
+ "tool_hash": row.tool_hash,
+ "listing_notice": dict(row.listing_notice or {}),
+ "note": row.note or "",
+ "status": row.status,
+ "review_note": row.review_note or "",
+ "reviewed_by": row.reviewed_by,
+ "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None,
+ "created_at": row.created_at.isoformat() if row.created_at else None,
+ }
+ if detail:
+ data["user_intro"] = row.user_intro or ""
+ data["url"] = row.url
+ data["tools"] = list(row.tools_json or [])
+ return data
+
+
+def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
+ rows = (
+ db.query(McpMarketSubmission)
+ .filter(
+ McpMarketSubmission.owner_user_id == owner_user_id,
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .order_by(McpMarketSubmission.created_at.desc())
+ .all()
+ )
+ return [submission_dict(row) for row in rows]
+
+
+def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
+ row = (
+ db.query(McpMarketSubmission)
+ .filter(
+ McpMarketSubmission.submission_id == submission_id,
+ McpMarketSubmission.owner_user_id == owner_user_id,
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .first()
+ )
+ if not row:
+ raise ResourceNotFoundError("mcp_market_submission", submission_id)
+ if row.status != "pending":
+ raise BadRequestError(message="只有待审核申请可以撤回")
+ row.status = "withdrawn"
+ row.deleted_at = utc_now()
+ row.updated_at = utc_now()
+ db.commit()
+
+
+def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
+ query = db.query(McpMarketSubmission).filter(McpMarketSubmission.deleted_at.is_(None))
+ if status:
+ query = query.filter(McpMarketSubmission.status == status)
+ return [
+ submission_dict(row) for row in query.order_by(McpMarketSubmission.created_at.desc()).all()
+ ]
+
+
+def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
+ row = (
+ db.query(McpMarketSubmission)
+ .filter(
+ McpMarketSubmission.submission_id == submission_id,
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .first()
+ )
+ if not row:
+ raise ResourceNotFoundError("mcp_market_submission", submission_id)
+ return submission_dict(row, detail=True)
+
+
+def _reviewable(db: Session, submission_id: str, approve: bool):
+ row = (
+ db.query(McpMarketSubmission)
+ .filter(
+ McpMarketSubmission.submission_id == submission_id,
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .first()
+ )
+ if not row:
+ raise ResourceNotFoundError("mcp_market_submission", submission_id)
+ if (approve and row.status != "pending") or (
+ not approve and row.status not in ("pending", "approved")
+ ):
+ raise BadRequestError(message="该申请当前状态不可审核")
+ return row
+
+
+def _review_source(db: Session, submission_id: str):
+ """The applicant's MCP, detached, once it still matches the submission."""
+ row = _reviewable(db, submission_id, True)
+ source = (
+ db.query(AdminMcpServer)
+ .filter(
+ AdminMcpServer.server_id == row.source_server_id,
+ AdminMcpServer.owner_user_id == row.owner_user_id,
+ )
+ .first()
+ )
+ if not source:
+ raise BadRequestError(message="申请人的原始 MCP 已删除,不能通过审核")
+ source_template_url, source_auth_schema = mcp_validation.credential_free_connection(source)
+ if source.transport != row.transport or source_template_url.strip() != row.url.strip():
+ raise BadRequestError(message="MCP 连接地址或传输方式已变化,请申请人重新提交")
+ if source_auth_schema != list(row.auth_schema or []):
+ raise BadRequestError(message="MCP 认证参数已变化,请申请人重新提交")
+ db.expunge(source)
+ return source
+
+
+async def review_submission(
+ submission_id: str,
+ *,
+ approve: bool,
+ review_note: str = "",
+ category: str = "",
+ reviewed_by: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Approve (after a live re-check) or reject a listing request."""
+ tools = None
+ if approve:
+ source = await run_in_session(_review_source, submission_id)
+ await validate_remote_mcp_url(source.url or "", require_https=False)
+ ok, error = await probe_mcp_connectivity(source, validate=True)
+ if not ok:
+ raise BadRequestError(message=f"MCP 审核复检失败:{error}")
+ tools = list(source.tools_json or [])
+ return await run_in_session(
+ _record_review, submission_id, approve, tools, review_note, category, reviewed_by
+ )
+
+
+def _record_review(db: Session, submission_id, approve, tools, review_note, category, reviewed_by):
+ row = _reviewable(db, submission_id, approve)
+ now = utc_now()
+ if approve:
+ if tool_snapshot_hash(tools or []) != row.tool_hash:
+ raise BadRequestError(message="MCP 工具或参数结构已变化,请申请人重新提交")
+ _store_tools(db, row.source_server_id, tools)
+ mcp_versions.publish_snapshot(
+ db,
+ slug=row.slug,
+ display_name=row.display_name,
+ description=row.description,
+ user_intro=row.user_intro,
+ category=category or row.category,
+ tags=list(row.tags or []),
+ icon=row.icon,
+ publisher_id=row.owner_user_id,
+ publisher_name=row.submitter_name,
+ source="community",
+ version=row.version,
+ transport=row.transport,
+ url=row.url,
+ auth_schema=list(row.auth_schema or []),
+ auth_config=mcp_validation.normalize_auth_config(row.auth_config, list(row.auth_schema or [])),
+ tools=list(row.tools_json or []),
+ source_server_id=row.source_server_id,
+ approved_by=reviewed_by,
+ )
+ row.status = "approved"
+ else:
+ row.status = "rejected"
+ item = db.query(McpMarketItem).filter(McpMarketItem.slug == row.slug).first()
+ if item:
+ item.deleted_at = now
+ item.updated_at = now
+ ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, False, updated_by=reviewed_by)
+ row.review_note = (review_note or "").strip()
+ row.reviewed_by = reviewed_by
+ row.reviewed_at = now
+ row.updated_at = now
+ db.commit()
+ if approve:
+ ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, True, updated_by=reviewed_by)
+ return submission_dict(row, detail=True)
diff --git a/src/backend/features/marketplace/mcp_validation.py b/src/backend/features/marketplace/mcp_validation.py
new file mode 100644
index 00000000..6b16dcb4
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_validation.py
@@ -0,0 +1,166 @@
+"""mcp validation responsibilities."""
+from __future__ import annotations
+import re
+from typing import Any, Dict, List
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketItem, McpMarketSubmission, McpMarketVersion
+from core.infra.exceptions import BadRequestError
+from core.capabilities.mcp.management import auth_schema_from_headers
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_state
+
+def validate_category(category: str) -> str:
+ value = (category or "").strip()
+ if value not in mcp_state.MCP_MARKET_CATEGORIES:
+ raise BadRequestError(message=f"MCP 市场分类必须是:{', '.join(mcp_state.MCP_MARKET_CATEGORIES)}")
+ return value
+
+
+def normalize_auth_config(
+ auth_config: Any,
+ auth_schema: List[Dict[str, Any]],
+) -> Dict[str, Any]:
+ """Return a backwards-compatible none/token/oauth2 authentication contract."""
+ config = dict(auth_config or {}) if isinstance(auth_config, dict) else {}
+ normalized: List[Dict[str, Any]] = []
+ seen: set[str] = set()
+ for raw in config.get("methods") or []:
+ if not isinstance(raw, dict):
+ continue
+ method = dict(raw)
+ method_id = str(method.get("id") or method.get("type") or "").strip()
+ method_type = str(method.get("type") or "").strip()
+ if not method_id or method_id in seen or method_type not in {"none", "token", "oauth2"}:
+ continue
+ method["id"] = method_id
+ method["type"] = method_type
+ method["label"] = str(method.get("label") or method_type)
+ if method_type == "oauth2":
+ method["client_registration"] = str(
+ method.get("client_registration") or "dynamic_or_manual"
+ )
+ method["scopes"] = [str(scope) for scope in (method.get("scopes") or []) if str(scope)]
+ normalized.append(method)
+ seen.add(method_id)
+ if not normalized:
+ inferred = "token" if auth_schema else "none"
+ normalized = [
+ {"id": inferred, "type": inferred, "label": "Token" if auth_schema else "无需认证"}
+ ]
+ default_method = str(config.get("default_method") or normalized[0]["id"])
+ if default_method not in {method["id"] for method in normalized}:
+ default_method = str(normalized[0]["id"])
+ credential_mode = str(config.get("credential_mode") or "auto")
+ if credential_mode not in {"auto", "installer", "admin"}:
+ credential_mode = "auto"
+ return {
+ "default_method": default_method,
+ "methods": normalized,
+ "credential_mode": credential_mode,
+ }
+
+
+def endpoint_identity(url: str) -> tuple[str, str, int | None]:
+ """Scheme/host/port of an endpoint, ignoring path, query, and credentials."""
+ parsed = urlparse((url or "").strip())
+ scheme = (parsed.scheme or "").lower()
+ default_port = 443 if scheme == "https" else 80 if scheme == "http" else None
+ return scheme, (parsed.hostname or "").lower(), parsed.port or default_port
+
+
+def installs_admin_published_endpoint(
+ item: McpMarketItem,
+ version: McpMarketVersion,
+ runtime_url: str,
+) -> bool:
+ """Whether an install still targets the exact endpoint an admin put on the shelf.
+
+ Admin publishing already vets the endpoint and may legitimately point at the
+ deployment's own network (the built-in MCP container, a LAN service). The
+ SSRF boundary exists for endpoints the *installer* supplies, so it must key
+ on the endpoint's provenance rather than on who is installing: an unchanged
+ admin-published host stays trusted for everyone who installs it, while a
+ URL-target credential that redirects the install elsewhere is checked
+ strictly, whoever submits it.
+ """
+ if item.source != "admin":
+ return False
+ return endpoint_identity(runtime_url) == endpoint_identity(version.url)
+
+
+def credential_free_connection(row: AdminMcpServer) -> tuple[str, List[Dict[str, Any]]]:
+ """Strip likely query-string secrets and expose only their install-time schema."""
+ parsed = urlparse(row.url or "")
+ public_query: List[tuple[str, str]] = []
+ auth_schema = auth_schema_from_headers(row.headers)
+ known_keys = {str(field.get("key") or "") for field in auth_schema}
+ for name, value in parse_qsl(parsed.query, keep_blank_values=True):
+ normalized = re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper()
+ looks_secret = any(term in name.lower() for term in mcp_state.SECRET_QUERY_TERMS)
+ credential_key = f"QUERY_{normalized or 'SECRET'}"
+ if looks_secret and value:
+ if credential_key not in known_keys:
+ auth_schema.append(
+ {
+ "key": credential_key,
+ "label": name,
+ "target": "query",
+ "name": name,
+ "required": True,
+ "secret": True,
+ }
+ )
+ known_keys.add(credential_key)
+ continue
+ public_query.append((name, value))
+ template_url = urlunparse(parsed._replace(query=urlencode(public_query)))
+ return template_url, auth_schema
+
+
+def normalize_version(version: str) -> str:
+ value = (version or "1.0.0").strip()
+ if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z._+-]{0,49}", value):
+ raise BadRequestError(message="版本号格式无效")
+ return value
+
+
+def new_slug(db: Session, display_name: str, source_server_id: str) -> str:
+ ascii_slug = re.sub(r"[^a-z0-9]+", "-", display_name.lower()).strip("-")
+ base = (ascii_slug or f"mcp-{source_server_id[-12:].lower()}")[:100]
+ slug = base
+ suffix = 2
+ while (
+ db.query(McpMarketItem.slug).filter(McpMarketItem.slug == slug).first()
+ or db.query(McpMarketSubmission.slug)
+ .filter(
+ McpMarketSubmission.slug == slug,
+ McpMarketSubmission.status.in_(["pending", "approved"]),
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .first()
+ ):
+ slug = f"{base[:115]}-{suffix}"
+ suffix += 1
+ return slug
+
+
+def source_slug(db: Session, row: AdminMcpServer, owner_user_id: str) -> str:
+ previous = (
+ db.query(McpMarketSubmission.slug)
+ .filter(
+ McpMarketSubmission.source_server_id == row.server_id,
+ McpMarketSubmission.owner_user_id == owner_user_id,
+ McpMarketSubmission.deleted_at.is_(None),
+ )
+ .order_by(McpMarketSubmission.created_at.desc())
+ .first()
+ )
+ return str(previous[0]) if previous else new_slug(db, row.display_name, row.server_id)
+
+
+def safe_origin(url: str) -> str:
+ from urllib.parse import urlparse
+
+ parsed = urlparse(url or "")
+ return f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else ""
diff --git a/src/backend/features/marketplace/mcp_versions.py b/src/backend/features/marketplace/mcp_versions.py
new file mode 100644
index 00000000..a03be55b
--- /dev/null
+++ b/src/backend/features/marketplace/mcp_versions.py
@@ -0,0 +1,221 @@
+"""mcp versions responsibilities."""
+from __future__ import annotations
+from core.infra.time import utc_now
+import re
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminMcpServer
+from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.capabilities.mcp.management import refresh_mcp_caches, tool_snapshot_hash
+from sqlalchemy.orm import Session
+from features.marketplace import mcp_validation
+
+def version_for_item(db: Session, item: McpMarketItem) -> McpMarketVersion:
+ version = (
+ db.query(McpMarketVersion)
+ .filter(McpMarketVersion.version_id == item.latest_version_id)
+ .first()
+ )
+ if not version:
+ raise ResourceNotFoundError("mcp_market_version", item.latest_version_id)
+ return version
+
+
+def publish_snapshot(
+ db: Session,
+ *,
+ slug: str,
+ display_name: str,
+ description: str,
+ user_intro: Optional[str],
+ category: str,
+ tags: List[str],
+ icon: Optional[str],
+ publisher_id: Optional[str],
+ publisher_name: str,
+ source: str,
+ version: str,
+ transport: str,
+ url: str,
+ auth_schema: List[Dict[str, Any]],
+ auth_config: Optional[Dict[str, Any]],
+ tools: List[Dict[str, Any]],
+ source_server_id: Optional[str],
+ approved_by: Optional[str],
+) -> McpMarketItem:
+ if (
+ db.query(McpMarketVersion)
+ .filter(McpMarketVersion.slug == slug, McpMarketVersion.version == version)
+ .first()
+ ):
+ raise BadRequestError(message=f"市场条目 {slug} 已存在版本 {version},请提升版本号")
+ now = utc_now()
+ # Include a soft-deleted row so a later reviewed version can resurrect the
+ # same stable marketplace slug without colliding on the primary key.
+ item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first()
+ version_id = f"mcpver_{uuid.uuid4().hex}"
+ if item is None:
+ item = McpMarketItem(
+ slug=slug,
+ display_name=display_name,
+ description=description,
+ user_intro=user_intro,
+ category=mcp_validation.validate_category(category),
+ tags=tags,
+ icon=icon,
+ publisher_id=publisher_id,
+ publisher_name=publisher_name,
+ source=source,
+ latest_version_id=version_id,
+ status="active",
+ last_verified_at=now,
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(item)
+ db.flush()
+ else:
+ item.deleted_at = None
+ item.display_name = display_name
+ item.description = description
+ item.user_intro = user_intro
+ item.category = mcp_validation.validate_category(category)
+ item.tags = tags
+ item.icon = icon
+ item.publisher_id = publisher_id
+ item.publisher_name = publisher_name
+ item.latest_version_id = version_id
+ item.status = "active"
+ item.status_reason = None
+ item.last_verified_at = now
+ item.updated_at = now
+ db.add(
+ McpMarketVersion(
+ version_id=version_id,
+ slug=slug,
+ version=mcp_validation.normalize_version(version),
+ transport=transport,
+ url=url,
+ auth_schema=auth_schema,
+ auth_config=mcp_validation.normalize_auth_config(auth_config, auth_schema),
+ tools_json=tools,
+ tool_hash=tool_snapshot_hash(tools),
+ source_server_id=source_server_id,
+ approved_by=approved_by,
+ approved_at=now,
+ created_at=now,
+ )
+ )
+ return item
+
+
+def next_revalidated_version(db: Session, slug: str, current_version: str) -> str:
+ """Return a collision-free version for an administrator-accepted snapshot."""
+ semantic = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", current_version)
+ if semantic:
+ major, minor, patch = (int(value) for value in semantic.groups())
+ candidate = f"{major}.{minor}.{patch + 1}"
+ while (
+ db.query(McpMarketVersion.version_id)
+ .filter(
+ McpMarketVersion.slug == slug,
+ McpMarketVersion.version == candidate,
+ )
+ .first()
+ ):
+ patch += 1
+ candidate = f"{major}.{minor}.{patch + 1}"
+ return candidate
+
+ base = re.sub(r"[^0-9A-Za-z._+-]", "-", current_version).strip("-.") or "1.0.0"
+ sequence = 1
+ while True:
+ candidate = f"{base[:32]}+recheck.{sequence}"
+ if not (
+ db.query(McpMarketVersion.version_id)
+ .filter(
+ McpMarketVersion.slug == slug,
+ McpMarketVersion.version == candidate,
+ )
+ .first()
+ ):
+ return candidate
+ sequence += 1
+
+
+def version_probe_candidate(item: McpMarketItem, version: McpMarketVersion) -> AdminMcpServer:
+ """Build a credential-free probe target from an immutable reviewed version."""
+ return AdminMcpServer(
+ server_id=f"market_revalidate_{uuid.uuid4().hex}",
+ display_name=item.display_name,
+ description=item.description or "",
+ user_intro=item.user_intro,
+ transport=version.transport,
+ command=None,
+ args=[],
+ url=version.url,
+ env_vars={},
+ env_inherit=[],
+ headers={},
+ is_enabled=False,
+ sort_order=0,
+ extra_config={},
+ tools_json=list(version.tools_json or []),
+ icon=item.icon,
+ owner_user_id=None,
+ source_plugin=None,
+ created_by="market-revalidation",
+ )
+
+
+def accept_revalidated_snapshot(
+ db: Session,
+ item: McpMarketItem,
+ version: McpMarketVersion,
+ tools: List[Dict[str, Any]],
+) -> McpMarketVersion:
+ """Publish live drift as a new immutable version after explicit admin review."""
+ next_version = next_revalidated_version(db, item.slug, version.version)
+ publish_snapshot(
+ db,
+ slug=item.slug,
+ display_name=item.display_name,
+ description=item.description or "",
+ user_intro=item.user_intro,
+ category=item.category,
+ tags=list(item.tags or []),
+ icon=item.icon,
+ publisher_id=item.publisher_id,
+ publisher_name=item.publisher_name or "",
+ source=item.source,
+ version=next_version,
+ transport=version.transport,
+ url=version.url,
+ auth_schema=list(version.auth_schema or []),
+ auth_config=dict(version.auth_config or {}),
+ tools=tools,
+ source_server_id=version.source_server_id,
+ approved_by="admin-revalidation",
+ )
+ db.flush()
+ accepted = version_for_item(db, item)
+ if version.source_server_id:
+ (
+ db.query(McpMarketInstallation)
+ .filter(
+ McpMarketInstallation.slug == item.slug,
+ McpMarketInstallation.server_id == version.source_server_id,
+ McpMarketInstallation.version_id == version.version_id,
+ )
+ .update(
+ {
+ McpMarketInstallation.version_id: accepted.version_id,
+ McpMarketInstallation.updated_at: utc_now(),
+ },
+ synchronize_session=False,
+ )
+ )
+ db.commit()
+ refresh_mcp_caches()
+ return accepted
diff --git a/src/backend/features/marketplace/models.py b/src/backend/features/marketplace/models.py
new file mode 100644
index 00000000..b0375816
--- /dev/null
+++ b/src/backend/features/marketplace/models.py
@@ -0,0 +1,388 @@
+"""Marketplace submissions, listings, versions and installations."""
+from core.db.utc_datetime import UTCDateTime
+from core.infra.time import utc_now
+from datetime import datetime, timezone
+from core.db.engine import Base
+from core.db.edition import MarketplaceListingEditionFields
+from sqlalchemy import JSON, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint, text
+from sqlalchemy.dialects.postgresql import INET, JSONB
+from sqlalchemy.orm import mapped_column, relationship
+from core.db.edition import UserAgentEditionFields, user_agent_edition_table_args
+from sqlalchemy import JSON, BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint
+JSONType = JSON().with_variant(JSONB(), "postgresql")
+
+class McpMarketItem(Base):
+ """Installable MCP marketplace entry.
+
+ The marketplace stores a credential-free definition. Concrete credentials
+ and enablement live on the ``AdminMcpServer`` created for each installation.
+ ``latest_version_id`` points at an immutable ``McpMarketVersion`` snapshot.
+ """
+
+ __tablename__ = "mcp_market_items"
+
+ slug = Column(String(128), primary_key=True)
+ display_name = Column(String(255), nullable=False)
+ description = Column(Text, nullable=False, default="")
+ user_intro = Column(Text)
+ category = Column(String(64), nullable=False, default="通用工具")
+ tags = Column(JSONType, default=list)
+ icon = Column(String(500))
+ publisher_id = Column(String(64))
+ publisher_name = Column(String(255), nullable=False, default="")
+ source = Column(String(16), nullable=False, default="community")
+ latest_version_id = Column(String(64), nullable=False)
+ # active = installable; changed = remote tool drift awaiting review;
+ # suspended = security kill switch (derived installations are disabled).
+ status = Column(String(16), nullable=False, default="active")
+ status_reason = Column(Text)
+ last_verified_at = Column(UTCDateTime(timezone=True))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
+ updated_at = Column(
+ UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
+ )
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ CheckConstraint(
+ "source IN ('admin', 'community')",
+ name="mcp_market_items_source_check",
+ ),
+ CheckConstraint(
+ "status IN ('active', 'changed', 'suspended')",
+ name="mcp_market_items_status_check",
+ ),
+ Index("idx_mcp_market_items_status", "status", "updated_at"),
+ Index("idx_mcp_market_items_category", "category"),
+ )
+
+class McpMarketVersion(Base):
+ """Immutable, reviewed MCP marketplace version snapshot."""
+
+ __tablename__ = "mcp_market_versions"
+
+ version_id = Column(String(64), primary_key=True)
+ slug = Column(
+ String(128),
+ ForeignKey("mcp_market_items.slug", ondelete="CASCADE"),
+ nullable=False,
+ )
+ version = Column(String(50), nullable=False, default="1.0.0")
+ transport = Column(String(20), nullable=False, default="streamable_http")
+ url = Column(Text, nullable=False)
+ # [{key, label, target: "header" | "query" | "url", required, secret, ...}];
+ # values are never stored here.
+ auth_schema = Column(JSONType, default=list)
+ # Provider-neutral authentication contract. ``methods`` may contain
+ # none/token/oauth2 options; OAuth endpoint discovery follows MCP metadata.
+ auth_config = Column(JSONType, default=dict)
+ tools_json = Column(JSONType, default=list)
+ tool_hash = Column(String(64), nullable=False)
+ listing_notice = Column(JSONType, default=dict)
+ source_server_id = Column(String(100))
+ approved_by = Column(String(64))
+ approved_at = Column(UTCDateTime(timezone=True))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
+
+ __table_args__ = (
+ CheckConstraint(
+ "transport IN ('streamable_http', 'sse')",
+ name="mcp_market_versions_transport_check",
+ ),
+ UniqueConstraint("slug", "version", name="uq_mcp_market_versions_slug_version"),
+ Index("idx_mcp_market_versions_slug", "slug", "created_at"),
+ )
+
+class McpMarketSubmission(Base):
+ """Credential-free snapshot submitted by a user for MCP marketplace review."""
+
+ __tablename__ = "mcp_market_submissions"
+
+ submission_id = Column(String(64), primary_key=True)
+ slug = Column(String(128), nullable=False)
+ source_server_id = Column(String(100), nullable=False)
+ owner_user_id = Column(String(64), nullable=False)
+ submitter_name = Column(String(255), nullable=False, default="")
+ display_name = Column(String(255), nullable=False)
+ description = Column(Text, nullable=False, default="")
+ user_intro = Column(Text)
+ category = Column(String(64), nullable=False, default="通用工具")
+ tags = Column(JSONType, default=list)
+ icon = Column(String(500))
+ version = Column(String(50), nullable=False, default="1.0.0")
+ transport = Column(String(20), nullable=False)
+ url = Column(Text, nullable=False)
+ auth_schema = Column(JSONType, default=list)
+ auth_config = Column(JSONType, default=dict)
+ tools_json = Column(JSONType, default=list)
+ tool_hash = Column(String(64), nullable=False)
+ listing_notice = Column(JSONType, default=dict)
+ note = Column(Text, nullable=False, default="")
+ status = Column(String(16), nullable=False, default="pending")
+ review_note = Column(Text)
+ reviewed_by = Column(String(64))
+ reviewed_at = Column(UTCDateTime(timezone=True))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
+ updated_at = Column(
+ UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
+ )
+ deleted_at = Column(UTCDateTime(timezone=True))
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('pending', 'approved', 'rejected', 'withdrawn')",
+ name="mcp_market_submissions_status_check",
+ ),
+ CheckConstraint(
+ "transport IN ('streamable_http', 'sse')",
+ name="mcp_market_submissions_transport_check",
+ ),
+ Index("idx_mcp_market_submissions_status", "status", "created_at"),
+ Index("idx_mcp_market_submissions_owner", "owner_user_id", "created_at"),
+ Index("idx_mcp_market_submissions_slug", "slug"),
+ )
+
+class McpMarketInstallation(Base):
+ """Links a reviewed market version to the concrete installed MCP server."""
+
+ __tablename__ = "mcp_market_installations"
+
+ install_id = Column(String(64), primary_key=True)
+ slug = Column(
+ String(128),
+ ForeignKey("mcp_market_items.slug", ondelete="CASCADE"),
+ nullable=False,
+ )
+ version_id = Column(
+ String(64),
+ ForeignKey("mcp_market_versions.version_id", ondelete="RESTRICT"),
+ nullable=False,
+ )
+ server_id = Column(
+ String(100),
+ ForeignKey("admin_mcp_servers.server_id", ondelete="CASCADE"),
+ nullable=False,
+ unique=True,
+ )
+ # NULL means the admin installed it globally; otherwise it is private to the user.
+ owner_user_id = Column(String(64))
+ status = Column(String(16), nullable=False, default="active")
+ installed_by = Column(String(64))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False)
+ updated_at = Column(
+ UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False
+ )
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('active', 'suspended')",
+ name="mcp_market_installations_status_check",
+ ),
+ Index("idx_mcp_market_installations_owner", "owner_user_id", "created_at"),
+ Index("idx_mcp_market_installations_slug", "slug", "status"),
+ Index(
+ "uq_mcp_market_installations_user_slug",
+ "slug",
+ "owner_user_id",
+ unique=True,
+ postgresql_where=text("owner_user_id IS NOT NULL"),
+ sqlite_where=text("owner_user_id IS NOT NULL"),
+ ),
+ Index(
+ "uq_mcp_market_installations_global_slug",
+ "slug",
+ unique=True,
+ postgresql_where=text("owner_user_id IS NULL"),
+ sqlite_where=text("owner_user_id IS NULL"),
+ ),
+ )
+
+class MarketplaceSubmission(Base):
+ """A user's private skill "apply for listing on the skill marketplace" record (pending admin review).
+
+ At submission time a content snapshot of the source ``AdminSkill`` is
+ taken (skill_content / extra_files); review and listing are both based
+ on the snapshot — the user later modifying or deleting the original
+ skill does not affect listed content. Records with
+ ``status='approved'`` appear in the skill marketplace list
+ (source=community) and can be installed by everyone; the admin can
+ reject at any time (including delisting already-listed items).
+ """
+
+ __tablename__ = "marketplace_submissions"
+
+ submission_id = Column(String(64), primary_key=True)
+ # Marketplace slug / entry_name after listing (derived at submission time,
+ # globally unique, does not clash with the preset marketplace catalog)
+ slug = Column(String(128), nullable=False, unique=True)
+ skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id
+ owner_user_id = Column(String(64), nullable=False) # applicant
+ submitter_name = Column(
+ String(255), default=""
+ ) # applicant display name (denormalized, for display)
+
+ display_name = Column(String(255), nullable=False)
+ summary = Column(Text, default="")
+ user_intro = Column(Text) # explicit user-facing intro snapshot; NULL falls back to SKILL.md
+ category = Column(String(64), default="社区共享")
+ tags = Column(JSONType, default=list)
+ version = Column(String(50), default="1.0.0")
+ note = Column(Text, default="") # application note (for the admin)
+
+ skill_content = Column(Text, nullable=False) # SKILL.md snapshot
+ extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content}
+
+ status = Column(String(16), nullable=False, default="pending")
+ review_note = Column(Text) # rejection reason / review note
+ reviewed_at = Column(UTCDateTime(timezone=True))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('pending', 'approved', 'rejected')",
+ name="marketplace_submissions_status_check",
+ ),
+ Index("idx_marketplace_submissions_status", "status", "created_at"),
+ Index("idx_marketplace_submissions_owner", "owner_user_id"),
+ )
+
+class PluginMarketPackage(Base):
+ """Admin-uploaded plugin marketplace package (stored/listed in the DB).
+
+ Counterpart of the skill marketplace's ``MarketplaceSubmission``: a
+ plugin zip uploaded by the admin from the /admin console is no longer
+ installed globally right away, but persisted as a "market package" —
+ storing the original zip's base64 (``package_b64``) + a set of display
+ metadata (extracted by a single normalize pass). Once listed it appears
+ in the plugin marketplace list for admins/users to install explicitly;
+ installation unzips and re-runs the existing ``import_plugin`` chain.
+ It coexists with filesystem preset packages
+ (``extensions/plugins/{default,marketplace}//``); when resolving by
+ slug, **filesystem takes precedence, DB market package is the
+ fallback**. Install records still go into ``InstalledPlugin``; this
+ table is only an "installable source".
+ """
+
+ __tablename__ = "plugin_market_packages"
+
+ slug = Column(String(100), primary_key=True)
+ name = Column(String(255), nullable=False)
+ version = Column(String(50), nullable=False, default="1.0.0")
+ description = Column(Text, default="")
+ category = Column(String(64), default="")
+ # Library path / URL / inline data-URI (uploaded custom icons, capped at ~200KB by the service layer)
+ icon = Column(Text)
+ # Package kind: native / claude / codex (determined by normalize), display only
+ kind = Column(String(16), nullable=False, default="native")
+ skills_count = Column(Integer, nullable=False, default=0)
+ required_secrets = Column(JSONType, default=list)
+ has_admin_config = Column(Boolean, nullable=False, default=False)
+ package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ created_by = Column(String(64))
+
+ __table_args__ = (Index("idx_plugin_market_packages_category", "category"),)
+
+class PluginMarketSkillExclusion(Base):
+ """A single skill "deleted" by the admin from a plugin in the plugin marketplace (excluded by plugin slug + skill name).
+
+ The plugin marketplace originally only supported whole-package deletion;
+ this table lets the admin precisely remove a specific skill within a
+ plugin. The market list's ``skills_count``, market detail ``skills``,
+ and DB persistence at install time are all filtered by it — built-in
+ filesystem packages (whose source files cannot be modified at runtime)
+ and DB-listed packages are uniformly covered by this exclusion list.
+ Installed instances are unaffected (those are install snapshots, managed
+ separately via enable/disable and uninstall).
+ """
+
+ __tablename__ = "plugin_market_skill_exclusions"
+
+ slug = Column(String(100), primary_key=True)
+ skill_name = Column(String(100), primary_key=True)
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ created_by = Column(String(64))
+
+class MarketplaceListingState(MarketplaceListingEditionFields, Base):
+ """Marketplace listing switch: controls whether a plugin/skill is shown in the (plugin/skill) marketplace.
+
+ ``kind`` = ``plugin`` | ``skill``; ``item_id`` = plugin slug or skill
+ marketplace slug. **A missing row counts as enabled** (everything listed
+ by default); a row with ``enabled=false`` is written only when the admin
+ explicitly disables — so built-in/existing items are not wiped after an
+ upgrade. The user-facing marketplace shows only enabled items; the admin
+ console shows everything with toggles. Physical deletion (only for
+ uploaded DB items) still goes through the respective delete endpoints;
+ this table only governs "listing visibility".
+
+ """
+
+ __tablename__ = "marketplace_listing_states"
+
+ kind = Column(String(16), primary_key=True) # plugin | skill
+ item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug
+ enabled = Column(Boolean, nullable=False, default=True)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+ updated_by = Column(String(64))
+
+class AgentMarketSubmission(Base):
+ """Record of a user-created sub-agent's "apply for listing on the sub-agent marketplace" (pending admin review).
+
+ Mirrors the skill marketplace's ``MarketplaceSubmission``, but snapshots a
+ "sub-agent" rather than skill files: on submission, the source ``UserAgent`` is
+ content-snapshotted (prompt/welcome message/suggested questions/model config/
+ capability bindings); review and listing are both based on the snapshot — later
+ edits or deletion of the original agent do not affect listed content.
+ ``status='approved'`` records appear in the sub-agent marketplace list
+ (source=community) and can be "install-cloned" by everyone as a private sub-agent;
+ admins can reject at any time (including delisting already-listed items). Direct
+ admin uploads to the marketplace are distinguished by the sentinel owner
+ ``__admin_upload__``.
+ """
+
+ __tablename__ = "agent_market_submissions"
+
+ submission_id = Column(String(64), primary_key=True)
+ # Marketplace slug once listed (derived from name at submission, globally unique, never colliding with the preset marketplace catalog)
+ slug = Column(String(128), nullable=False, unique=True)
+ agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id
+ owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__)
+ submitter_name = Column(
+ String(255), default=""
+ ) # applicant display name (denormalized for display)
+
+ # Marketplace display metadata
+ name = Column(String(255), nullable=False)
+ avatar = Column(Text)
+ description = Column(Text, default="")
+ summary = Column(Text, default="")
+ category = Column(String(64), default="通用助手")
+ tags = Column(JSONType, default=list)
+ version = Column(String(50), default="1.0.0")
+ note = Column(Text, default="") # application note (for the admin)
+
+ # Sub-agent content snapshot (decoupled from the source UserAgent)
+ system_prompt = Column(Text, default="")
+ welcome_message = Column(Text, default="")
+ suggested_questions = Column(JSONType, default=list)
+ # {temperature, max_tokens, max_iters, timeout}
+ model_config_snapshot = Column(JSONType, default=dict)
+ # {skill_ids, mcp_server_ids, plugin_ids, kb_ids}
+ bindings_snapshot = Column(JSONType, default=dict)
+
+ status = Column(String(16), nullable=False, default="pending")
+ review_note = Column(Text) # rejection reason / review note
+ reviewed_at = Column(UTCDateTime(timezone=True))
+ created_at = Column(UTCDateTime(timezone=True), default=utc_now)
+ updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now)
+
+ __table_args__ = (
+ CheckConstraint(
+ "status IN ('pending', 'approved', 'rejected')",
+ name="agent_market_submissions_status_check",
+ ),
+ Index("idx_agent_market_submissions_status", "status", "created_at"),
+ Index("idx_agent_market_submissions_owner", "owner_user_id"),
+ )
diff --git a/src/backend/features/marketplace/package_source.py b/src/backend/features/marketplace/package_source.py
new file mode 100644
index 00000000..cf24de92
--- /dev/null
+++ b/src/backend/features/marketplace/package_source.py
@@ -0,0 +1,35 @@
+"""Marketplace package resolution with versioned, bundled and uploaded precedence."""
+import base64
+from core.infra.exceptions import ResourceNotFoundError
+from core.capabilities.plugins.management import packages as plugin_packages
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.packaging.archive import extract_plugin_zip
+from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
+
+
+class MarketplacePackageSource:
+ def normalize(self, db, slug):
+ """Fetch and normalize a plugin by slug: filesystem preset bundle first, DB-published package as fallback."""
+ from features.marketplace.versions.store import active, decode
+ from features.marketplace.versions.sources import normalized_plugin
+ if plugin_sources.resolve_plugin_dir(slug) is None and (
+ db is None or plugin_packages.market_row(db, slug) is None
+ ):
+ raise ResourceNotFoundError("plugin", slug)
+ snapshot = active(db, "plugin", slug)
+ if snapshot:
+ return normalized_plugin(decode(snapshot))
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
+ if plugin_dir is not None:
+ return normalize_plugin_dir(plugin_dir)
+ if db is not None:
+ row = plugin_packages.market_row(db, slug)
+ if row is not None:
+ with extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root:
+ return normalize_plugin_dir(plugin_root)
+ raise ResourceNotFoundError("plugin", slug)
+
+ def archive(self, db, slug):
+ from features.marketplace.versions.store import active, archive, decode
+ snapshot = active(db, "plugin", slug)
+ return archive(decode(snapshot)) if snapshot else None
diff --git a/src/backend/core/services/marketplace_listing.py b/src/backend/features/marketplace/personal_listing.py
similarity index 97%
rename from src/backend/core/services/marketplace_listing.py
rename to src/backend/features/marketplace/personal_listing.py
index cc697c5a..bd03c984 100644
--- a/src/backend/core/services/marketplace_listing.py
+++ b/src/backend/features/marketplace/personal_listing.py
@@ -9,7 +9,7 @@
from core.infra.time import utc_now
from typing import Any, Dict, List, Optional
-from core.db.models import MarketplaceListingState
+from features.marketplace.models import MarketplaceListingState
from sqlalchemy.orm import Session
KIND_PLUGIN = "plugin"
diff --git a/src/backend/core/auth/marketplace_visibility.py b/src/backend/features/marketplace/personal_visibility.py
similarity index 100%
rename from src/backend/core/auth/marketplace_visibility.py
rename to src/backend/features/marketplace/personal_visibility.py
diff --git a/src/backend/core/plugins/management/market.py b/src/backend/features/marketplace/plugin_market.py
similarity index 73%
rename from src/backend/core/plugins/management/market.py
rename to src/backend/features/marketplace/plugin_market.py
index 99cb0cd9..0b667847 100644
--- a/src/backend/core/plugins/management/market.py
+++ b/src/backend/features/marketplace/plugin_market.py
@@ -3,40 +3,21 @@
from __future__ import annotations
import logging
+from core.capabilities.plugins.packaging.presentation import validate_icon, overlay_market_meta
from typing import Any, Dict, List, Optional
-from core.db.models import (
- ContentBlock,
- InstalledPlugin,
- PluginMarketPackage,
- PluginMarketSkillExclusion,
-)
+from core.db.models import ContentBlock, InstalledPlugin
+from features.marketplace.models import PluginMarketPackage, PluginMarketSkillExclusion
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
-from core.plugins.management import packages as plugin_packages
-from core.plugins.packaging import definitions as plugin_definitions
-from core.plugins.packaging import sources as plugin_sources
+from core.capabilities.plugins.management import packages as plugin_packages
+from core.capabilities.plugins.packaging import definitions as plugin_definitions
+from core.capabilities.plugins.packaging import sources as plugin_sources
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
-def _validate_icon(icon: str) -> str:
- """Validate an icon value from the UI picker; returns the stripped value ('' = clear)."""
- icon = (icon or "").strip()
- if not icon:
- return ""
- if icon.startswith("data:"):
- if not icon.startswith("data:image/"):
- raise BadRequestError(message="图标 data URI 必须是 image 类型")
- if len(icon) > plugin_definitions.MAX_ICON_LEN:
- raise BadRequestError(message="图标过大(上传原图请控制在 80KB 以内)")
- return icon
- if not (icon.startswith("/") or icon.startswith("http://") or icon.startswith("https://")):
- raise BadRequestError(message="图标须从图标库选择或上传(不支持任意文本)")
- if len(icon) > plugin_definitions.MAX_ICON_URL_LEN:
- raise BadRequestError(message="图标地址过长")
- return icon
def _market_meta_overrides(db: Optional[Session]) -> Dict[str, Dict[str, Any]]:
@@ -56,21 +37,13 @@ def resolve_market_meta(db: Optional[Session], slug: str) -> Dict[str, Any]:
"""Effective display metadata for one market plugin: DB override → builtin seed."""
meta = dict(plugin_definitions.BUILTIN_PLUGIN_MARKET_META.get(slug) or {})
override = _market_meta_overrides(db).get(slug) or {}
- for k in plugin_definitions._META_KEYS:
+ for k in plugin_definitions.META_KEYS:
v = override.get(k)
if isinstance(v, str) and v.strip():
meta[k] = v.strip()
return meta
-def _overlay_market_meta(item: Dict[str, Any], meta: Dict[str, Any]) -> None:
- """Apply effective display metadata onto a market list/detail dict (in place)."""
- if meta.get("display_name"):
- item["name"] = meta["display_name"]
- if meta.get("category"):
- item["category"] = meta["category"]
- if meta.get("icon"):
- item["icon"] = meta["icon"]
def set_market_meta(
@@ -89,8 +62,8 @@ def set_market_meta(
market (filesystem bundle or uploaded DB package).
"""
if (
- plugin_sources._resolve_plugin_dir(slug) is None
- and plugin_packages._market_row(db, slug) is None
+ plugin_sources.resolve_plugin_dir(slug) is None
+ and plugin_packages.market_row(db, slug) is None
):
raise ResourceNotFoundError("plugin", slug)
row = (
@@ -101,7 +74,7 @@ def set_market_meta(
payload = dict(row.payload or {}) if row is not None else {}
entry = dict(payload.get(slug) or {})
if icon is not None:
- icon = _validate_icon(icon)
+ icon = validate_icon(icon)
for key, val in (("display_name", display_name), ("category", category), ("icon", icon)):
if val is None:
continue
@@ -142,8 +115,8 @@ def list_plugins(
"""
items: List[Dict[str, Any]] = []
seen_slugs: set = set()
- for child in plugin_sources._iter_plugin_dirs():
- meta = plugin_sources._scan_native_manifest(child)
+ for child in plugin_sources.iter_plugin_dirs():
+ meta = plugin_sources.scan_native_manifest(child)
if meta:
meta["source"] = "builtin"
items.append(meta)
@@ -152,17 +125,17 @@ def list_plugins(
for row in db.query(PluginMarketPackage).order_by(PluginMarketPackage.created_at.desc()).all():
if row.slug in seen_slugs:
continue
- items.append(plugin_packages._market_meta_dict(row))
+ items.append(plugin_packages.market_meta_dict(row))
seen_slugs.add(row.slug)
# Display metadata is UI configuration: DB override → builtin seed → manifest fallback
overrides = _market_meta_overrides(db)
for it in items:
meta = dict(plugin_definitions.BUILTIN_PLUGIN_MARKET_META.get(it["slug"]) or {})
- for k in plugin_definitions._META_KEYS:
+ for k in plugin_definitions.META_KEYS:
v = (overrides.get(it["slug"]) or {}).get(k)
if isinstance(v, str) and v.strip():
meta[k] = v.strip()
- _overlay_market_meta(it, meta)
+ overlay_market_meta(it, meta)
# Subtract skills the admin removed from the marketplace (aggregated once per slug) so skills_count reflects the real offering
excl_by_slug: Dict[str, set] = {}
for ex_slug, ex_name in db.query(
@@ -178,7 +151,7 @@ def list_plugins(
# Annotate installed status
if items:
install_ids = {
- it["slug"]: plugin_sources._make_plugin_install_id(it["slug"], owner_user_id)
+ it["slug"]: plugin_sources.make_plugin_install_id(it["slug"], owner_user_id)
for it in items
}
present = {
@@ -194,7 +167,7 @@ def list_plugins(
# entries visible to them; the admin panel sees everything with
# annotations. For user-side callers owner_user_id is the current browsing
# user, reused directly as the viewer.
- from core.services import marketplace_listing as ml
+ from features.marketplace import edition as ml
items = ml.annotate_and_filter(
db,
@@ -211,7 +184,7 @@ def set_plugin_market_enabled(
db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
) -> Dict[str, Any]:
"""Publish/unpublish a marketplace plugin (controls display in the plugin marketplace; does not affect installed instances)."""
- from core.services import marketplace_listing as ml
+ from features.marketplace import edition as ml
res = ml.set_listing_enabled(db, ml.KIND_PLUGIN, slug, enabled, updated_by=updated_by)
logger.info("plugin_market_listing: slug=%s enabled=%s", slug, enabled)
diff --git a/src/backend/features/marketplace/skill_archives.py b/src/backend/features/marketplace/skill_archives.py
new file mode 100644
index 00000000..06724ea2
--- /dev/null
+++ b/src/backend/features/marketplace/skill_archives.py
@@ -0,0 +1,49 @@
+"""skill archives responsibilities."""
+from __future__ import annotations
+import io
+import os
+import zipfile
+from pathlib import Path
+from typing import Any, Dict, List
+from core.capabilities.skills.binary_files import BINARY_EXTENSIONS, JUNK_BASENAMES, MAX_SINGLE_FILE, MAX_TOTAL, decode_binary, encode_binary, is_binary_value
+from core.capabilities.skills.deps.detector import detect_dependencies
+from core.capabilities.skills.registry import SkillSpecError, load_skill_metadata_from_str, split_frontmatter
+from fastapi import HTTPException
+from features.marketplace import skills_state
+
+
+
+def build_skill_zip(skill_id: str, skill_content: str, extra_files: Dict[str, str]) -> bytes:
+ """Pack a DB skill into a zip byte stream (inverse of ``parse_skill_zip``).
+
+ Layout is ``/SKILL.md`` + ``/``,
+ consistent with the upload/import convention, so an exported package can be
+ re-imported directly. base64 values in extra_files carrying the binary
+ marker are restored to their original bytes.
+ """
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+ zf.writestr(f"{skill_id}/SKILL.md", skill_content or "")
+ for rel, stored in (extra_files or {}).items():
+ data = decode_binary(stored) if is_binary_value(stored) else str(stored).encode("utf-8")
+ zf.writestr(f"{skill_id}/{rel}", data)
+ return buf.getvalue()
+
+
+def build_skill_zip_from_dir(skill_id: str, skill_dir: Path) -> bytes:
+ """Pack an on-disk skill directory (built-in/filesystem skills) into a zip byte stream.
+
+ Same layout as ``build_skill_zip``; skips version-control/cache directories
+ and junk files, and packs binary files as-is (no base64 round-trip,
+ lossless).
+ """
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+ for p in sorted(skill_dir.rglob("*")):
+ if not p.is_file() or p.name in JUNK_BASENAMES:
+ continue
+ rel = p.relative_to(skill_dir)
+ if skills_state.ZIP_SKIP_DIR_PARTS.intersection(rel.parts):
+ continue
+ zf.write(p, f"{skill_id}/{rel.as_posix()}")
+ return buf.getvalue()
diff --git a/src/backend/features/marketplace/skill_catalog.py b/src/backend/features/marketplace/skill_catalog.py
new file mode 100644
index 00000000..6abf2803
--- /dev/null
+++ b/src/backend/features/marketplace/skill_catalog.py
@@ -0,0 +1,184 @@
+"""skill catalog responsibilities."""
+from __future__ import annotations
+from core.capabilities.skills import installation as package_installation
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminSkill
+from core.infra.exceptions import ResourceNotFoundError
+from sqlalchemy.orm import Session
+from features.marketplace import skill_metadata
+from features.marketplace import skills_state
+
+def list_marketplace_skills(
+ db: Optional[Session] = None,
+ *,
+ include_disabled: bool = False,
+ viewer_user_id: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+ """List all market skill metadata: built-in default skills + preloaded (featured first, downloads descending) + community submissions.
+
+ ``include_disabled``: the admin console passes True (sees everything +
+ ``market_enabled`` annotation); the user side defaults to False (only sees
+ listed items, and scoped items invisible to ``viewer_user_id`` are filtered
+ by visibility). See ``marketplace_listing`` for the delist switch and
+ visibility scopes.
+ """
+ items: List[Dict[str, Any]] = []
+ if skills_state.MARKETPLACE_DIR.is_dir():
+ for child in sorted(skills_state.MARKETPLACE_DIR.iterdir()):
+ if not child.is_dir():
+ continue
+ m = skill_metadata.read_manifest(child.name)
+ if m:
+ items.append(skill_metadata.public_meta(m))
+ items.sort(key=lambda x: (0 if x["featured"] else 1, -x["downloads"], x["slug"]))
+ # Built-in default skills come first (built-in resident, shown with priority).
+ items = skill_metadata.default_skill_metas() + items
+ if db is not None:
+ items.extend(skill_metadata.submission_public_meta(s) for s in skill_metadata.approved_submissions(db))
+ from features.marketplace import edition as ml
+ items = ml.annotate_and_filter(
+ db, ml.KIND_SKILL, items, id_key="slug",
+ include_disabled=include_disabled, viewer_user_id=viewer_user_id,
+ )
+ return items
+
+
+def set_marketplace_skill_enabled(
+ db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None
+) -> Dict[str, Any]:
+ """List/delist a market skill (controls whether it shows in the skill marketplace; installed instances are unaffected)."""
+ from features.marketplace import edition as ml
+ return ml.set_listing_enabled(db, ml.KIND_SKILL, slug, enabled, updated_by=updated_by)
+
+
+def list_categories(
+ db: Optional[Session] = None,
+ *,
+ include_disabled: bool = False,
+ viewer_user_id: Optional[str] = None,
+) -> List[str]:
+ """Marketplace category list: the 8 fixed categories first (stable order), legacy leftover categories appended after."""
+ seen: List[str] = list(skills_state.MARKETPLACE_CATEGORIES)
+ for it in list_marketplace_skills(
+ db, include_disabled=include_disabled, viewer_user_id=viewer_user_id
+ ):
+ if it["category"] not in seen:
+ seen.append(it["category"])
+ return seen
+
+
+def get_marketplace_skill(slug: str, db: Optional[Session] = None) -> Dict[str, Any]:
+ """Single market skill detail: metadata + file list + SKILL.md body (for the detail preview).
+
+ The preloaded directory takes priority; when not found, fall back to the
+ community listing (approved submission).
+ """
+ # Built-in default skills: read extensions/skills/default/; the body comes from SKILL.md, the file list from the directory.
+ if slug in skills_state.DEFAULT_SKILL_MARKET:
+ meta = skill_metadata.default_skill_public_meta(slug)
+ if meta is None:
+ raise ResourceNotFoundError("marketplace_skill", slug)
+ pkg_dir = skills_state.DEFAULT_SKILLS_DIR / slug
+ files: List[Dict[str, Any]] = []
+ for p in sorted(pkg_dir.rglob("*")):
+ if not p.is_file():
+ continue
+ rel = p.relative_to(pkg_dir).as_posix()
+ if rel == skills_state.SKILL_MD_NAME:
+ continue
+ files.append({"path": rel, "size": p.stat().st_size})
+ skill_md = pkg_dir / skills_state.SKILL_MD_NAME
+ meta["files"] = files
+ meta["instructions"] = package_installation.strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) if skill_md.is_file() else ""
+ return meta
+ m = skill_metadata.read_manifest(slug)
+ if not m:
+ sub = skill_metadata.get_approved_submission(db, slug) if db is not None else None
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_skill", slug)
+ data = skill_metadata.submission_public_meta(sub)
+ data["files"] = [
+ {"path": rel, "size": len(str(content).encode("utf-8"))}
+ for rel, content in sorted((sub.extra_files or {}).items())
+ if rel != skills_state.REQUIRED_SECRETS_SNAPSHOT # internal sentinel, not part of the installed content
+ ]
+ data["instructions"] = package_installation.strip_frontmatter(sub.skill_content or "")
+ return data
+ pkg_dir = skills_state.MARKETPLACE_DIR / slug
+ data = skill_metadata.public_meta(m)
+
+ files: List[Dict[str, Any]] = []
+ for p in sorted(pkg_dir.rglob("*")):
+ if not p.is_file() or p.name == skills_state.MANIFEST_NAME:
+ continue
+ rel = p.relative_to(pkg_dir).as_posix()
+ if rel == skills_state.SKILL_MD_NAME: # SKILL.md is returned separately as instructions; don't duplicate it in the file list
+ continue
+ files.append({"path": rel, "size": p.stat().st_size})
+ body = ""
+ skill_md = pkg_dir / skills_state.SKILL_MD_NAME
+ if skill_md.is_file():
+ body = package_installation.strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore"))
+ data["files"] = files
+ data["instructions"] = body
+ return data
+
+
+def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool:
+ """Whether this market skill has been installed by the given scope (global / current user)."""
+ if slug in skills_state.DEFAULT_SKILL_MARKET:
+ return True # built-in default skills are globally resident; considered always "installed"
+ m = skill_metadata.read_manifest(slug)
+ if not m:
+ if skill_metadata.get_approved_submission(db, slug) is None:
+ return False
+ entry_name = slug
+ else:
+ entry_name = m["entry_name"]
+ install_id = package_installation.compute_install_id(entry_name, owner_user_id)
+ q = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id)
+ if owner_user_id is None:
+ q = q.filter(AdminSkill.owner_user_id.is_(None))
+ else:
+ q = q.filter(AdminSkill.owner_user_id == owner_user_id)
+ return q.first() is not None
+
+
+def annotate_installed(
+ items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str]
+) -> List[Dict[str, Any]]:
+ """Annotate each market list item with an ``installed`` flag (for the given scope). One batched query decides all."""
+ if not items:
+ return items
+ install_ids = {
+ it["slug"]: package_installation.compute_install_id(it["entry_name"], owner_user_id) for it in items
+ }
+ q = db.query(AdminSkill.skill_id, AdminSkill.dep_status)
+ if owner_user_id is None:
+ q = q.filter(AdminSkill.owner_user_id.is_(None))
+ else:
+ q = q.filter(AdminSkill.owner_user_id == owner_user_id)
+ q = q.filter(AdminSkill.skill_id.in_(list(install_ids.values())))
+ status_map = {sid: dep for sid, dep in q.all()}
+ # Rejected skills carry the admin's reason, surfaced to the user ("Rejected by admin: ...").
+ rejected_ids = [sid for sid, dep in status_map.items() if dep == "rejected"]
+ reason_map: Dict[str, Optional[str]] = {}
+ if rejected_ids:
+ from core.capabilities.skills.deps.requests import get_reject_reason
+ for sid in rejected_ids:
+ reason_map[sid] = get_reject_reason(db, sid)
+ for it in items:
+ # Built-in default skills are globally resident and always available to
+ # everyone — mark them "installed/ready" directly (the frontend button
+ # shows "already built-in"); no AdminSkill lookup, no install flow.
+ if it.get("builtin"):
+ it["installed"] = True
+ it["dep_status"] = "ready"
+ it["dep_reason"] = None
+ continue
+ iid = install_ids[it["slug"]]
+ it["installed"] = iid in status_map
+ # dep_status: 'installing' = dependencies installing / 'rejected' = admin declined / 'ready' = usable. None when not installed.
+ it["dep_status"] = status_map.get(iid) if iid in status_map else None
+ it["dep_reason"] = reason_map.get(iid)
+ return items
diff --git a/src/backend/features/marketplace/skill_installation.py b/src/backend/features/marketplace/skill_installation.py
new file mode 100644
index 00000000..52351f53
--- /dev/null
+++ b/src/backend/features/marketplace/skill_installation.py
@@ -0,0 +1,199 @@
+"""skill installation responsibilities."""
+from __future__ import annotations
+from core.capabilities.skills import installation as package_installation
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+from core.capabilities.skills.binary_files import is_binary_value, pack_directory
+from core.capabilities.skills.cache_refresh import refresh_skill_caches
+from core.capabilities.skills.deps.detector import detect_dependencies
+from core.capabilities.skills.registry import load_skill_metadata_from_str
+from core.db.models import AdminSkill
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.time import utc_now
+from core.ontology.build_validator import ensure_ontology_build_valid
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_modified
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import skill_metadata
+from features.marketplace import skills_state
+
+def load_package_files(pkg_dir: Path) -> Tuple[str, Dict[str, str]]:
+ """Read a market skill package: returns (SKILL.md original text, {relative path: stored content}).
+
+ Binary files are stored as base64 (consistent with admin upload), text as
+ original text; the manifest and junk files are skipped, and per-file/total
+ size caps are enforced.
+ """
+ skill_md = pkg_dir / skills_state.SKILL_MD_NAME
+ if not skill_md.is_file():
+ raise BadRequestError(message="市场技能包缺少 SKILL.md")
+ skill_content = skill_md.read_text(encoding="utf-8")
+ extra_files = pack_directory(pkg_dir, skip_names={skills_state.MANIFEST_NAME, skills_state.SKILL_MD_NAME})
+ return skill_content, extra_files
+
+
+def install_marketplace_skill(
+ db: Session,
+ slug: str,
+ *,
+ owner_user_id: Optional[str],
+ secrets: Optional[Dict[str, str]] = None,
+) -> Dict[str, Any]:
+ """Install a market skill as an ``AdminSkill`` (admin = global / user = private), returning a result dict.
+
+ The preloaded directory takes priority; when the slug isn't in the preloaded
+ directory, fall back to the community listing (approved submission), with
+ install content taken from the submission-time snapshot.
+ """
+ # Built-in default skills are globally resident and always available to
+ # everyone — no install needed (the frontend button is already disabled as
+ # "already built-in"); this is a defensive block to avoid duplicating 1MB+
+ # office skill content into the DB as a private copy.
+ if slug in skills_state.DEFAULT_SKILL_MARKET:
+ raise BadRequestError(message="内置技能已全局可用,无需安装")
+ secrets = secrets or {}
+ m = skill_metadata.read_manifest(slug)
+ if m:
+ pkg_dir = skills_state.MARKETPLACE_DIR / slug
+ skill_content, extra_files = load_package_files(pkg_dir)
+ else:
+ sub = skill_metadata.get_approved_submission(db, slug)
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_skill", slug)
+ m = {
+ "slug": sub.slug,
+ "entry_name": sub.slug,
+ "display_name": sub.display_name,
+ "summary": sub.summary or "",
+ "user_intro": sub.user_intro or "",
+ "category": sub.category or "社区共享",
+ "tags": list(sub.tags or []),
+ "version": sub.version or "1.0.0",
+ "required_secrets": skill_metadata.submission_required_secrets(sub),
+ }
+ skill_content = sub.skill_content or ""
+ extra_files = dict(sub.extra_files or {})
+ # The sentinel entry only carries the credential field names; it never lands in the install directory
+ extra_files.pop(skills_state.REQUIRED_SECRETS_SNAPSHOT, None)
+
+ entry_name = str(m["entry_name"]).strip()
+ install_id = package_installation.compute_install_id(entry_name, owner_user_id)
+ # Always rewrite the frontmatter ``name`` to the install id: this both
+ # namespaces private user installs and, along the way, normalizes illegal
+ # names in market skills (containing spaces/Chinese etc.) — otherwise
+ # load_skill_metadata_from_str fails validation with "invalid skill id".
+ # install_id comes from slug/entry_name and is always a legal slug.
+ skill_content = package_installation.rewrite_frontmatter_name(skill_content, install_id)
+
+ required_secrets = list(m.get("required_secrets") or [])
+ if required_secrets:
+ skill_content = package_installation.inject_secrets(
+ skill_content, extra_files, required_secrets, secrets
+ )
+
+ # Validate SKILL.md (description required, etc.); an invalid one is a straight 400 rather than being persisted.
+ try:
+ meta = load_skill_metadata_from_str(skill_content, install_id)
+ except Exception as exc: # noqa: BLE001
+ raise BadRequestError(message=f"市场技能 SKILL.md 不合法:{exc}")
+
+ dependencies = detect_dependencies(
+ {fn: c for fn, c in extra_files.items() if not is_binary_value(c)}
+ )
+
+ display_name = m.get("display_name") or meta.name or install_id
+ description = meta.description or m.get("summary") or ""
+ tags = list(m.get("tags") or meta.tags or [])
+ version = m.get("version") or meta.version or "1.0.0"
+ user_intro = build_user_intro(m) or None
+
+ ensure_ontology_build_valid(
+ db,
+ asset_type="skill",
+ name=display_name or install_id,
+ description=description,
+ instructions=skill_content,
+ tool_names=list(meta.allowed_tools or []),
+ ontology_tags=list(tags),
+ )
+
+ now = utc_now()
+ existing = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id).first()
+ if existing is not None:
+ # Owner conflict: cannot overwrite a public skill or another user's private skill (admin reinstalling a global one is allowed)
+ if existing.owner_user_id != owner_user_id:
+ if existing.owner_user_id is None:
+ raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」与公共技能冲突")
+ raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」已被占用")
+ existing.skill_content = skill_content
+ existing.display_name = display_name
+ existing.description = description
+ existing.user_intro = user_intro
+ existing.version = version
+ existing.tags = tags
+ existing.allowed_tools = list(meta.allowed_tools or [])
+ existing.extra_files = extra_files
+ existing.dependencies = dependencies
+ existing.is_enabled = True
+ existing.updated_at = now
+ flag_modified(existing, "tags")
+ flag_modified(existing, "extra_files")
+ flag_modified(existing, "dependencies")
+ action = "updated"
+ else:
+ db.add(AdminSkill(
+ skill_id=install_id,
+ skill_content=skill_content,
+ display_name=display_name,
+ description=description,
+ user_intro=user_intro,
+ version=version,
+ tags=tags,
+ allowed_tools=list(meta.allowed_tools or []),
+ extra_files=extra_files,
+ dependencies=dependencies,
+ is_enabled=True,
+ owner_user_id=owner_user_id,
+ created_at=now,
+ updated_at=now,
+ ))
+ action = "installed"
+ db.commit()
+ # Strict dependency verdicts happen at the async route layer (gate_skill_deps
+ # needs to actually run probes in the sandbox; this function is synchronous).
+ # Here we only carry the declared dependencies out with the result; the route
+ # awaits the probe → decides installing/ready.
+ # Assign the installed skill a built-in icon by category (users/admins can change it later in skill editing).
+ try:
+ from core.capabilities.skills.icons import get_skill_icon, preset_for_category, set_skill_icon
+ if not get_skill_icon(db, install_id):
+ set_skill_icon(db, install_id, preset_for_category(m.get("category")))
+ except Exception as exc: # noqa: BLE001
+ logger.debug("marketplace set default icon failed: %s", exc)
+ refresh_skill_caches()
+ logger.info(
+ "marketplace_skill_%s: slug=%s id=%s owner=%s files=%d",
+ action, slug, install_id, owner_user_id or "global", len(extra_files),
+ )
+ return {
+ "id": install_id,
+ "slug": slug,
+ "owner": "self" if owner_user_id else "global",
+ "action": action,
+ "dependencies": dependencies,
+ # dep_pending / message are filled in by the route layer after awaiting gate_skill_deps (strict probe result).
+ "dep_pending": False,
+ "message": "技能已安装" if action == "installed" else "技能已更新",
+ }
+
+
+def build_user_intro(m: Dict[str, Any]) -> str:
+ """Return only an explicitly authored user intro from marketplace metadata.
+
+ The marketplace ``summary`` is card copy. Treating it as ``user_intro``
+ suppresses the catalog's SKILL.md-body fallback and reduces rich skill
+ documentation to a single line after installation.
+ """
+ return str(m.get("user_intro") or "").strip()
diff --git a/src/backend/features/marketplace/skill_metadata.py b/src/backend/features/marketplace/skill_metadata.py
new file mode 100644
index 00000000..159b57c9
--- /dev/null
+++ b/src/backend/features/marketplace/skill_metadata.py
@@ -0,0 +1,174 @@
+"""skill metadata responsibilities."""
+from __future__ import annotations
+import json
+from typing import Any, Dict, List, Optional
+from core.capabilities.skills.registry import split_frontmatter
+from features.marketplace.models import MarketplaceSubmission
+from core.infra.exceptions import BadRequestError
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import skills_state
+
+def read_manifest(slug: str) -> Optional[Dict[str, Any]]:
+ """Read and validate a single market skill's ``marketplace.json`` (tolerant; returns None if corrupted)."""
+ if not slug or "/" in slug or ".." in slug:
+ return None
+ path = skills_state.MARKETPLACE_DIR / slug / skills_state.MANIFEST_NAME
+ if not path.is_file():
+ return None
+ try:
+ m = json.loads(path.read_text(encoding="utf-8"))
+ except Exception as exc: # noqa: BLE001
+ logger.warning("marketplace manifest broken: %s (%s)", slug, exc)
+ return None
+ if not isinstance(m, dict) or not m.get("entry_name"):
+ return None
+ m.setdefault("slug", slug)
+ return m
+
+
+def market_skill_requires_secrets(slug: str) -> bool:
+ """Whether a preloaded market skill declares required_secrets (public contract, reusable by other markets/callers)."""
+ m = read_manifest(slug)
+ if not m:
+ return False
+ return bool(m.get("requires_api_key") or m.get("required_secrets"))
+
+
+def public_meta(m: Dict[str, Any]) -> Dict[str, Any]:
+ """Manifest → externally visible metadata (no skill body/file contents)."""
+ return {
+ "slug": m.get("slug"),
+ "entry_name": m.get("entry_name"),
+ "display_name": m.get("display_name") or m.get("slug"),
+ "summary": m.get("summary") or "",
+ "category": m.get("category") or "其他",
+ "tags": list(m.get("tags") or []),
+ "version": m.get("version") or "1.0.0",
+ "author": m.get("author") or "skillhub",
+ "icon_url": m.get("icon_url") or "",
+ "source": m.get("source") or "skillhub",
+ "source_url": m.get("source_url") or "",
+ "downloads": int(m.get("downloads") or 0),
+ "stars": int(m.get("stars") or 0),
+ "featured": bool(m.get("featured")),
+ "requires_api_key": bool(m.get("requires_api_key")),
+ "required_secrets": list(m.get("required_secrets") or []),
+ # Preloaded (filesystem) skills ship with the repo; admins cannot delete them online.
+ "deletable": False,
+ }
+
+
+def submission_public_meta(sub: MarketplaceSubmission) -> Dict[str, Any]:
+ """Approved community submission → external metadata isomorphic to the preloaded manifest form."""
+ return {
+ "slug": sub.slug,
+ "entry_name": sub.slug,
+ "display_name": sub.display_name or sub.slug,
+ "summary": sub.summary or "",
+ "category": sub.category or "社区共享",
+ "tags": list(sub.tags or []),
+ "version": sub.version or "1.0.0",
+ "author": sub.submitter_name or sub.owner_user_id,
+ "icon_url": "",
+ "source": "community",
+ "source_url": "",
+ "downloads": 0,
+ "stars": 0,
+ "featured": False,
+ "requires_api_key": False,
+ "required_secrets": submission_required_secrets(sub),
+ # DB listing records (admin uploads / user community submissions) → admins can delete online (remove from the market).
+ "deletable": True,
+ }
+
+
+def approved_submissions(db: Session) -> List[MarketplaceSubmission]:
+ return (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.status == "approved")
+ .order_by(MarketplaceSubmission.created_at.desc())
+ .all()
+ )
+
+
+def default_skill_public_meta(slug: str) -> Optional[Dict[str, Any]]:
+ """Built-in default skill (extensions/skills/default/) → market-list metadata isomorphic to the preloaded form.
+
+ The Chinese name comes from SKILL.md's ``display_name``; category/summary
+ come from ``DEFAULT_SKILL_MARKET``. Marked ``source='builtin'`` +
+ ``builtin=True`` — these skills are globally resident and always available
+ to everyone; the market shows them as "built-in" (already available) and
+ they do not go through "install as a private copy" (avoiding duplicating
+ 1MB+ office skills into the DB).
+ """
+ cfg = skills_state.DEFAULT_SKILL_MARKET.get(slug)
+ if cfg is None:
+ return None
+ md = skills_state.DEFAULT_SKILLS_DIR / slug / skills_state.SKILL_MD_NAME
+ if not md.is_file():
+ return None
+ try:
+ fm, _ = split_frontmatter(md.read_text(encoding="utf-8", errors="ignore"))
+ except Exception: # noqa: BLE001
+ fm = {}
+ name = str(fm.get("name") or slug).strip()
+ tags = fm.get("tags") or []
+ if isinstance(tags, str):
+ tags = [t.strip() for t in tags.split(",") if t.strip()]
+ return {
+ "slug": slug,
+ "entry_name": name,
+ "display_name": str(fm.get("display_name") or name).strip(),
+ "summary": cfg["summary"],
+ "category": cfg["category"],
+ "tags": list(tags),
+ "version": str(fm.get("version") or "1.0.0"),
+ "author": "内置",
+ "icon_url": "",
+ "source": "builtin",
+ "source_url": "",
+ "downloads": 0,
+ "stars": 0,
+ "featured": False,
+ "requires_api_key": False,
+ "required_secrets": [],
+ "deletable": False, # built-ins ship with the repo; not deletable online
+ "builtin": True, # the frontend uses this to show "built-in" and set the button to "already built-in"
+ }
+
+
+def default_skill_metas() -> List[Dict[str, Any]]:
+ metas = [default_skill_public_meta(s) for s in skills_state.DEFAULT_SKILL_MARKET]
+ return [m for m in metas if m]
+
+
+def validate_category(category: str) -> str:
+ """Validate and return a legal category; anything outside the fixed set is a straight 400."""
+ category = (category or "").strip()
+ if category not in skills_state.MARKETPLACE_CATEGORIES:
+ raise BadRequestError(
+ message=f"请从固定分类中选择:{'、'.join(skills_state.MARKETPLACE_CATEGORIES)}"
+ )
+ return category
+
+
+def get_approved_submission(db: Session, slug: str) -> Optional[MarketplaceSubmission]:
+ return (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.slug == slug, MarketplaceSubmission.status == "approved")
+ .first()
+ )
+
+
+def submission_required_secrets(sub: MarketplaceSubmission) -> List[Dict[str, Any]]:
+ """Recover a community skill's required_secrets from the snapshot sentinel entry (empty list if absent)."""
+ raw = (sub.extra_files or {}).get(skills_state.REQUIRED_SECRETS_SNAPSHOT)
+ if not raw:
+ return []
+ try:
+ fields = json.loads(raw)
+ return fields if isinstance(fields, list) else []
+ except (TypeError, ValueError):
+ return []
diff --git a/src/backend/features/marketplace/skill_publishing.py b/src/backend/features/marketplace/skill_publishing.py
new file mode 100644
index 00000000..504ea6be
--- /dev/null
+++ b/src/backend/features/marketplace/skill_publishing.py
@@ -0,0 +1,231 @@
+"""skill publishing responsibilities."""
+from __future__ import annotations
+from core.capabilities.skills import zip_reader
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+from core.capabilities.skills.registry import load_skill_metadata_from_str
+from features.marketplace.models import MarketplaceSubmission
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.time import utc_now
+from core.ontology.build_validator import ensure_ontology_build_valid
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_modified
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import skill_archives
+from features.marketplace import skill_metadata
+from features.marketplace import skill_submission_data
+from features.marketplace import skills_state
+
+def upsert_admin_submission(
+ db: Session,
+ *,
+ skill_id: str,
+ skill_content: str,
+ display_name: str,
+ summary: str,
+ category: str,
+ tags: List[str],
+ version: str,
+ extra_files: Dict[str, str],
+ note: str,
+ submitter_name: str,
+ user_intro: Optional[str] = None,
+) -> Tuple[str, str]:
+ """Upsert an approved listing record under the ADMIN_UPLOAD_OWNER sentinel, returning (slug, action).
+
+ When the same skill_id already has an admin listing record, update it
+ (keeping the original slug); otherwise create one. Shared by the two admin
+ listing entry points: "upload zip" and "create via form".
+ """
+ category = skill_metadata.validate_category(category)
+ snapshot_files = dict(extra_files)
+ # Credential files are never distributed with the market snapshot (same as community listing).
+ snapshot_files.pop(skills_state.SECRETS_FILENAME, None)
+ now = utc_now()
+ existing = (
+ db.query(MarketplaceSubmission)
+ .filter(
+ MarketplaceSubmission.skill_id == skill_id,
+ MarketplaceSubmission.owner_user_id == skills_state.ADMIN_UPLOAD_OWNER,
+ )
+ .first()
+ )
+ if existing is not None:
+ existing.display_name = display_name
+ existing.summary = summary
+ existing.user_intro = str(user_intro or "").strip() or None
+ existing.category = category
+ existing.tags = list(tags or [])
+ existing.version = version or "1.0.0"
+ existing.skill_content = skill_content
+ existing.extra_files = snapshot_files
+ existing.note = note
+ existing.status = "approved"
+ existing.review_note = ""
+ existing.reviewed_at = now
+ existing.updated_at = now
+ flag_modified(existing, "tags")
+ flag_modified(existing, "extra_files")
+ return existing.slug, "updated"
+
+ slug = skill_submission_data.derive_slug(db, skill_id, skills_state.ADMIN_UPLOAD_OWNER)
+ db.add(MarketplaceSubmission(
+ submission_id=f"mkadm_{uuid.uuid4().hex[:16]}",
+ slug=slug,
+ skill_id=skill_id,
+ owner_user_id=skills_state.ADMIN_UPLOAD_OWNER,
+ submitter_name=(submitter_name or "").strip(),
+ display_name=display_name,
+ summary=summary,
+ user_intro=str(user_intro or "").strip() or None,
+ category=category,
+ tags=list(tags or []),
+ version=version or "1.0.0",
+ note=note,
+ skill_content=skill_content,
+ extra_files=snapshot_files,
+ status="approved",
+ reviewed_at=now,
+ created_at=now,
+ updated_at=now,
+ ))
+ return slug, "published"
+
+
+def publish_skill_zip_to_marketplace(
+ db: Session,
+ data: bytes,
+ *,
+ category: str,
+ display_name: str = "",
+ summary: str = "",
+ submitter_name: str = "管理员上传",
+) -> Dict[str, Any]:
+ """Admin uploads a skill zip → listed directly in the marketplace as approved (not in the catalog, not globally effective).
+
+ When the same skill (by SKILL.md's name = skill_id) is uploaded again, the
+ existing admin listing record is updated (keeping the original slug) rather
+ than adding a new one each time.
+ """
+ parsed = zip_reader.parse_skill_zip(data)
+ meta = parsed["meta"]
+ name = (display_name or "").strip() or meta.name or parsed["skill_id"]
+ ensure_ontology_build_valid(
+ db,
+ asset_type="skill",
+ name=name,
+ description=(summary or "").strip() or meta.description or "",
+ instructions=parsed["skill_content"],
+ tool_names=list(meta.allowed_tools or []),
+ ontology_tags=list(meta.tags or []),
+ )
+ slug, action = upsert_admin_submission(
+ db,
+ skill_id=parsed["skill_id"],
+ skill_content=parsed["skill_content"],
+ display_name=name,
+ summary=(summary or "").strip() or meta.description or "",
+ category=category,
+ tags=list(meta.tags or []),
+ version=meta.version or "1.0.0",
+ extra_files=parsed["extra_files"],
+ note="管理员上传,免审核直接上架",
+ submitter_name=submitter_name,
+ )
+ db.commit()
+ logger.info("marketplace_skill_admin_%s: skill=%s slug=%s", action, parsed["skill_id"], slug)
+ return {
+ "slug": slug,
+ "skill_id": parsed["skill_id"],
+ "display_name": name,
+ "category": skill_metadata.validate_category(category),
+ "stored_files": len([k for k in parsed["extra_files"] if k != skills_state.SECRETS_FILENAME]),
+ "skipped": parsed["skipped"],
+ "action": action,
+ "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新",
+ }
+
+
+def publish_skill_to_marketplace(
+ db: Session,
+ *,
+ skill_id: str,
+ skill_content: str,
+ display_name: str,
+ description: str,
+ version: str,
+ tags: List[str],
+ category: str,
+ user_intro: Optional[str] = None,
+ submitter_name: str = "管理员新建",
+) -> Dict[str, Any]:
+ """Admin "create skill" form → listed in the marketplace as approved (not in the catalog, not globally effective).
+
+ ``skill_content`` is the complete SKILL.md assembled from the form fields
+ (valid frontmatter).
+ """
+ name = (display_name or "").strip() or skill_id
+ try:
+ meta = load_skill_metadata_from_str(skill_content, skill_id)
+ allowed_tools = list(meta.allowed_tools or [])
+ parsed_description = meta.description or ""
+ except Exception: # Some reviewed legacy drafts have minimal frontmatter.
+ allowed_tools = []
+ parsed_description = ""
+ ensure_ontology_build_valid(
+ db,
+ asset_type="skill",
+ name=name,
+ description=description or parsed_description,
+ instructions=skill_content,
+ tool_names=allowed_tools,
+ ontology_tags=list(tags or []),
+ )
+ slug, action = upsert_admin_submission(
+ db,
+ skill_id=skill_id,
+ skill_content=skill_content,
+ display_name=name,
+ summary=(description or "").strip(),
+ category=category,
+ tags=list(tags or []),
+ version=version or "1.0.0",
+ extra_files={},
+ note="管理员新建,免审核直接上架",
+ submitter_name=submitter_name,
+ user_intro=user_intro,
+ )
+ db.commit()
+ logger.info("marketplace_skill_admin_%s: skill=%s slug=%s (form)", action, skill_id, slug)
+ return {
+ "slug": slug,
+ "skill_id": skill_id,
+ "display_name": name,
+ "category": skill_metadata.validate_category(category),
+ "action": action,
+ "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新",
+ }
+
+
+def delete_marketplace_skill(db: Session, slug: str) -> Dict[str, Any]:
+ """Admin deletes a DB listing record from the skill marketplace (admin upload / user community listing).
+
+ Preloaded (filesystem) skills ship with the repo and cannot be deleted
+ online — straight 400. Deletion only removes it from the market; it does not
+ affect any installed skill instances (installs are content snapshots).
+ """
+ if skill_metadata.read_manifest(slug) is not None:
+ raise BadRequestError(message="预置技能随仓库发布,不可在线删除")
+ sub = (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.slug == slug)
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_skill", slug)
+ skill_id, owner = sub.skill_id, sub.owner_user_id
+ db.delete(sub)
+ db.commit()
+ logger.info("marketplace_skill_deleted: slug=%s skill=%s owner=%s", slug, skill_id, owner)
+ return {"slug": slug, "deleted": True}
diff --git a/src/backend/features/marketplace/skill_submission_data.py b/src/backend/features/marketplace/skill_submission_data.py
new file mode 100644
index 00000000..50af4703
--- /dev/null
+++ b/src/backend/features/marketplace/skill_submission_data.py
@@ -0,0 +1,80 @@
+"""skill submission data responsibilities."""
+from __future__ import annotations
+from core.capabilities.skills import installation as package_installation
+import re
+from typing import Any, Dict
+from features.marketplace.models import MarketplaceSubmission
+from sqlalchemy.orm import Session
+from features.marketplace import skills_state
+
+def strip_injected_secrets(skill_content: str) -> str:
+ """Strip the "credentials configuration" section appended by _inject_secrets (including its preceding --- separator).
+
+ Listing snapshots must be in the pre-injection form — the section instructs
+ reading secrets.json, which the snapshot has already excluded; the
+ installer's own credentials get a fresh section re-injected at install time.
+ """
+ idx = skill_content.find(skills_state.SECRETS_SECTION_HEADER)
+ if idx < 0:
+ return skill_content
+ sep = skill_content.rfind("\n---", 0, idx)
+ cut = sep if sep >= 0 else idx
+ return skill_content[:cut].rstrip() + "\n"
+
+
+def existing_slugs(db: Session) -> set:
+ """Currently occupied market slugs: the preloaded directory + all submissions.
+
+ Must include rejected rows — the slug column is table-wide UNIQUE and
+ rejected submissions still occupy their slug; if derivation skipped them,
+ "re-submitting after rejection" would hit the unique constraint and 500
+ outright.
+ """
+ slugs = set()
+ if skills_state.MARKETPLACE_DIR.is_dir():
+ slugs.update(c.name for c in skills_state.MARKETPLACE_DIR.iterdir() if c.is_dir())
+ rows = db.query(MarketplaceSubmission.slug).all()
+ slugs.update(r[0] for r in rows)
+ return slugs
+
+
+def derive_slug(db: Session, skill_id: str, owner_user_id: str) -> str:
+ """Derive the market slug from the source skill id: strip the user's own install-fingerprint suffix, then ensure global uniqueness."""
+ base = skill_id
+ suffix = f"-{package_installation.owner_suffix(owner_user_id)}"
+ if base.endswith(suffix):
+ base = base[: -len(suffix)]
+ base = re.sub(r"[^a-z0-9_-]+", "-", base.lower()).strip("-") or "skill"
+ taken = existing_slugs(db)
+ slug = base
+ n = 2
+ while slug in taken:
+ slug = f"{base}-{n}"
+ n += 1
+ return slug
+
+
+def submission_to_dict(sub: MarketplaceSubmission, *, with_content: bool = False) -> Dict[str, Any]:
+ data = {
+ "submission_id": sub.submission_id,
+ "slug": sub.slug,
+ "skill_id": sub.skill_id,
+ "owner_user_id": sub.owner_user_id,
+ "submitter_name": sub.submitter_name or "",
+ "display_name": sub.display_name,
+ "summary": sub.summary or "",
+ "user_intro": sub.user_intro or "",
+ "category": sub.category or "社区共享",
+ "tags": list(sub.tags or []),
+ "version": sub.version or "1.0.0",
+ "note": sub.note or "",
+ "status": sub.status,
+ "review_note": sub.review_note or "",
+ "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None,
+ "created_at": sub.created_at.isoformat() if sub.created_at else None,
+ "file_count": len(sub.extra_files or {}),
+ }
+ if with_content:
+ data["instructions"] = package_installation.strip_frontmatter(sub.skill_content or "")
+ data["files"] = sorted((sub.extra_files or {}).keys())
+ return data
diff --git a/src/backend/features/marketplace/skill_submissions.py b/src/backend/features/marketplace/skill_submissions.py
new file mode 100644
index 00000000..b64e3cbe
--- /dev/null
+++ b/src/backend/features/marketplace/skill_submissions.py
@@ -0,0 +1,190 @@
+"""skill submissions responsibilities."""
+from __future__ import annotations
+import json
+import uuid
+from typing import Any, Dict, List, Optional
+from core.db.models import AdminSkill
+from features.marketplace.models import MarketplaceSubmission
+from core.infra.exceptions import BadRequestError, ResourceNotFoundError
+from core.infra.time import utc_now
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+import logging
+logger = logging.getLogger(__name__)
+from features.marketplace import skill_metadata
+from features.marketplace import skill_submission_data
+from features.marketplace import skills_state
+
+def submit_to_marketplace(
+ db: Session,
+ skill_id: str,
+ *,
+ owner_user_id: str,
+ submitter_name: str = "",
+ note: str = "",
+ category: str = "",
+ summary: str = "",
+) -> Dict[str, Any]:
+ """User submits their private skill for marketplace listing (content snapshot, pending admin review)."""
+ row = (
+ db.query(AdminSkill)
+ .filter(AdminSkill.skill_id == skill_id, AdminSkill.owner_user_id == owner_user_id)
+ .first()
+ )
+ if row is None:
+ raise ResourceNotFoundError("skill", skill_id)
+
+ active = (
+ db.query(MarketplaceSubmission)
+ .filter(
+ MarketplaceSubmission.skill_id == skill_id,
+ MarketplaceSubmission.owner_user_id == owner_user_id,
+ MarketplaceSubmission.status.in_(["pending", "approved"]),
+ )
+ .first()
+ )
+ if active is not None:
+ state = "待审核" if active.status == "pending" else "已上架"
+ raise HTTPException(status_code=409, detail=f"该技能已有{state}的上架申请")
+
+ # The snapshot excludes the credentials file: when installing a market
+ # skill with secrets, _inject_secrets writes the user's API key in
+ # plaintext into extra_files[SECRETS_FILENAME] — that must never be
+ # distributed to all installers with the listing snapshot. The credential
+ # field names are kept with the snapshot as a sentinel entry (without
+ # values), so installers are prompted to enter their own keys; SKILL.md is
+ # correspondingly stripped of the injected section.
+ snapshot_files = dict(row.extra_files or {})
+ submitter_secrets = snapshot_files.pop(skills_state.SECRETS_FILENAME, None)
+ snapshot_content = row.skill_content
+ if submitter_secrets:
+ snapshot_content = skill_submission_data.strip_injected_secrets(row.skill_content or "")
+ try:
+ secret_keys = sorted(json.loads(submitter_secrets).keys())
+ except (TypeError, ValueError, AttributeError):
+ secret_keys = []
+ if secret_keys:
+ # The original required/label metadata is unavailable in the installed artifact; approximate as "all required"
+ snapshot_files[skills_state.REQUIRED_SECRETS_SNAPSHOT] = json.dumps(
+ [{"key": k, "label": k, "required": True} for k in secret_keys],
+ ensure_ascii=False,
+ )
+
+ now = utc_now()
+ sub = MarketplaceSubmission(
+ submission_id=f"mksub_{uuid.uuid4().hex[:16]}",
+ slug=skill_submission_data.derive_slug(db, skill_id, owner_user_id),
+ skill_id=skill_id,
+ owner_user_id=owner_user_id,
+ submitter_name=(submitter_name or "").strip(),
+ display_name=row.display_name or skill_id,
+ summary=(summary or "").strip() or (row.description or ""),
+ user_intro=str(row.user_intro or "").strip() or None,
+ category=skill_metadata.validate_category(category),
+ tags=list(row.tags or []),
+ version=row.version or "1.0.0",
+ note=(note or "").strip(),
+ skill_content=snapshot_content,
+ extra_files=snapshot_files,
+ status="pending",
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(sub)
+ db.commit()
+ logger.info(
+ "marketplace_submission_created: id=%s skill=%s owner=%s slug=%s",
+ sub.submission_id, skill_id, owner_user_id, sub.slug,
+ )
+ return skill_submission_data.submission_to_dict(sub)
+
+
+def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]:
+ """All of the current user's listing submissions (newest → oldest)."""
+ rows = (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.owner_user_id == owner_user_id)
+ .order_by(MarketplaceSubmission.created_at.desc())
+ .all()
+ )
+ return [skill_submission_data.submission_to_dict(r) for r in rows]
+
+
+def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None:
+ """User withdraws their own submission: pending is deleted directly; approved requires an admin to delist; rejected can be deleted and resubmitted."""
+ sub = (
+ db.query(MarketplaceSubmission)
+ .filter(
+ MarketplaceSubmission.submission_id == submission_id,
+ MarketplaceSubmission.owner_user_id == owner_user_id,
+ )
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_submission", submission_id)
+ if sub.status == "approved":
+ raise BadRequestError(message="技能已上架,如需下架请联系管理员")
+ db.delete(sub)
+ db.commit()
+
+
+def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]:
+ """Admin view: all listing submissions, filterable by status (pending first, newest → oldest)."""
+ q = db.query(MarketplaceSubmission)
+ if status:
+ q = q.filter(MarketplaceSubmission.status == status)
+ rows = q.order_by(MarketplaceSubmission.created_at.desc()).all()
+ order = {"pending": 0, "approved": 1, "rejected": 2}
+ rows.sort(key=lambda r: order.get(r.status, 3))
+ return [skill_submission_data.submission_to_dict(r) for r in rows]
+
+
+def get_submission(db: Session, submission_id: str) -> Dict[str, Any]:
+ """Submission detail for admin review (including a SKILL.md body preview and the attachment list)."""
+ sub = (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.submission_id == submission_id)
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_submission", submission_id)
+ return skill_submission_data.submission_to_dict(sub, with_content=True)
+
+
+def review_submission(
+ db: Session,
+ submission_id: str,
+ *,
+ approve: bool,
+ review_note: str = "",
+ category: str = "",
+) -> Dict[str, Any]:
+ """Admin review: approval means listing (appears in the market list); rejecting pending = decline, rejecting approved = delist."""
+ sub = (
+ db.query(MarketplaceSubmission)
+ .filter(MarketplaceSubmission.submission_id == submission_id)
+ .first()
+ )
+ if sub is None:
+ raise ResourceNotFoundError("marketplace_submission", submission_id)
+ # State-machine guard: approve only allows pending→approved (prevents two
+ # concurrent admins / a stale list mis-click from listing a just-rejected
+ # submission directly); reject allows pending (decline) and approved
+ # (delist).
+ if approve and sub.status != "pending":
+ state = "已上架" if sub.status == "approved" else "已驳回"
+ raise HTTPException(status_code=409, detail=f"该申请{state},不能重复审核通过(如需重新上架请让用户重新提交)")
+ if not approve and sub.status == "rejected":
+ raise HTTPException(status_code=409, detail="该申请已是驳回状态")
+ sub.status = "approved" if approve else "rejected"
+ sub.review_note = (review_note or "").strip()
+ if approve and (category or "").strip():
+ sub.category = skill_metadata.validate_category(category)
+ sub.reviewed_at = utc_now()
+ sub.updated_at = utc_now()
+ db.commit()
+ logger.info(
+ "marketplace_submission_reviewed: id=%s slug=%s status=%s",
+ sub.submission_id, sub.slug, sub.status,
+ )
+ return skill_submission_data.submission_to_dict(sub)
diff --git a/src/backend/core/services/marketplace_skill_versions.py b/src/backend/features/marketplace/skill_versions.py
similarity index 72%
rename from src/backend/core/services/marketplace_skill_versions.py
rename to src/backend/features/marketplace/skill_versions.py
index f624a3bc..0776627f 100644
--- a/src/backend/core/services/marketplace_skill_versions.py
+++ b/src/backend/features/marketplace/skill_versions.py
@@ -1,36 +1,38 @@
"""Version-aware sources for the existing skill marketplace API."""
from __future__ import annotations
+from typing import Any, Dict, Optional
+from sqlalchemy.orm import Session
+from core.capabilities.skills.binary_files import is_binary_value
+from core.capabilities.skills.cache_refresh import refresh_skill_caches
+from core.capabilities.skills.deps.detector import detect_dependencies
+from core.capabilities.skills.registry import load_skill_metadata_from_str
+from core.db.models import AdminSkill
+from core.infra.exceptions import BadRequestError
+from core.infra.time import utc_now
+from core.ontology.build_validator import ensure_ontology_build_valid
+from fastapi import HTTPException
+import features.marketplace.skill_catalog as market_owner_skill_catalog
+from features.marketplace.skill_installation import build_user_intro as _build_user_intro
+import features.marketplace.skill_installation as market_owner_skill_installation
+from features.marketplace.skills_state import DEFAULT_SKILL_MARKET
+from features.marketplace.skills_state import MANIFEST_NAME
+import features.marketplace.skills_state as market_owner_skills_state
+from sqlalchemy.orm.attributes import flag_modified
+import logging
+logger = logging.getLogger(__name__)
+
+
+import features.marketplace.versions.store as store
+from core.capabilities.skills.installation import inject_secrets as _inject_secrets, rewrite_frontmatter_name as _rewrite_frontmatter_name, compute_install_id as compute_install_id
+
+
+
-from . import marketplace_service as original
-from . import marketplace_version_store as store
-from .marketplace_service import (
- DEFAULT_SKILL_MARKET,
- MANIFEST_NAME,
- AdminSkill,
- BadRequestError,
- HTTPException,
- _build_user_intro,
- _inject_secrets,
- _load_skill_metadata_from_str,
- _rewrite_frontmatter_name,
- compute_install_id,
- detect_dependencies,
- ensure_ontology_build_valid,
- flag_modified,
- is_binary_value,
- logger,
- refresh_skill_caches,
- utc_now,
-)
-
-
-def __getattr__(name):
- return getattr(original, name)
def get_marketplace_skill(slug, db=None):
- native = original.get_marketplace_skill(slug, db)
+ native = market_owner_skill_catalog.get_marketplace_skill(slug, db)
return store.detail(db, "skill", slug, native)
@@ -47,16 +49,16 @@ def install_marketplace_skill(
directory, fall back to the community listing (approved submission), with
install content taken from the submission-time snapshot.
"""
- current = original.get_marketplace_skill(slug, db)
+ current = market_owner_skill_catalog.get_marketplace_skill(slug, db)
snapshot = store.active(db, "skill", slug)
if not snapshot:
- return original.install_marketplace_skill(
+ return market_owner_skill_installation.install_marketplace_skill(
db, slug, owner_user_id=owner_user_id, secrets=secrets
)
if current.get("builtin"):
raise BadRequestError(message="内置技能已全局可用,无需安装")
secrets = secrets or {}
- from core.agent_skills.binary_files import encode_upload
+ from core.capabilities.skills.binary_files import encode_upload
files = store.decode(snapshot)
m = store.detail(db, "skill", slug, current)
@@ -64,7 +66,7 @@ def install_marketplace_skill(
extra_files = {
n: encode_upload(n, v)
for n, v in files.items()
- if n not in {"SKILL.md", MANIFEST_NAME, original.REQUIRED_SECRETS_SNAPSHOT}
+ if n not in {"SKILL.md", MANIFEST_NAME, market_owner_skills_state.REQUIRED_SECRETS_SNAPSHOT}
}
entry_name = str(m["entry_name"]).strip()
@@ -76,7 +78,7 @@ def install_marketplace_skill(
skill_content = _inject_secrets(skill_content, extra_files, required_secrets, secrets)
try:
- meta = _load_skill_metadata_from_str(skill_content, install_id)
+ meta = load_skill_metadata_from_str(skill_content, install_id)
except Exception as exc: # noqa: BLE001
raise BadRequestError(message=f"市场技能 SKILL.md 不合法:{exc}")
@@ -146,11 +148,7 @@ def install_marketplace_skill(
action = "installed"
db.commit()
try:
- from core.services.skill_icon_service import (
- get_skill_icon,
- preset_for_category,
- set_skill_icon,
- )
+ from core.capabilities.skills.icons import get_skill_icon, preset_for_category, set_skill_icon
if not get_skill_icon(db, install_id):
set_skill_icon(db, install_id, preset_for_category(m.get("category")))
diff --git a/src/backend/features/marketplace/skills_state.py b/src/backend/features/marketplace/skills_state.py
new file mode 100644
index 00000000..ed1ce54a
--- /dev/null
+++ b/src/backend/features/marketplace/skills_state.py
@@ -0,0 +1,56 @@
+"""skills state responsibilities."""
+from __future__ import annotations
+from extensions.paths import SKILLS_DIR
+
+MARKETPLACE_DIR = SKILLS_DIR / "marketplace"
+
+
+DEFAULT_SKILLS_DIR = SKILLS_DIR / "default"
+
+
+MANIFEST_NAME = "marketplace.json"
+
+
+SKILL_MD_NAME = "SKILL.md"
+
+
+SECRETS_FILENAME = "secrets.json"
+
+
+DEFAULT_SKILL_MARKET = {
+ "pdf-editing": {"category": "文档处理", "summary": "生成 / 合并 / 拆分 / 抽页 / 填表单 / 读取 PDF 文档"},
+ "capability-guide-brief": {"category": "办公效率", "summary": "速答系统能力清单与使用指引"},
+ # OfficeCLI 系列(上游 iOfficeAI/OfficeCLI,Apache 2.0):靠沙盒预装的
+ # ``officecli`` 单文件二进制驱动,是 Word / Excel / PPT 的**唯一**入口——
+ # 自研的 word-editing / excel-editing / ppt-design 三个技能已于 2026-08 下线,
+ # 能力全部由这一系列承接。三个基座(docx / xlsx / pptx)之上叠场景层,
+ # 分工写在各自 SKILL.md 的 description 里,由技能选择器路由。
+ "officecli-docx": {"category": "文档处理", "summary": "OfficeCLI 深度编辑 .docx:TOC / 域 / 脚注 / 公式 / 原生图表 / 批注修订 / schema 校验"},
+ "officecli-xlsx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .xlsx:透视表 / 迷你图 / 条件格式 / 数据验证 / 切片器"},
+ "officecli-pptx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .pptx:母版版式 / 连接线 / 形状级控制 / 逐页截图质检"},
+ "officecli-word-form": {"category": "文档处理", "summary": "制作可填写 Word 表单:内容控件(SDT) + 复选框 + 邮件合并域 + 文档保护"},
+ "officecli-academic-paper": {"category": "文档处理", "summary": "学术论文排版:APA / Chicago / IEEE / MLA 引用、编号公式、交叉引用、参考文献"},
+ "officecli-data-dashboard": {"category": "数据分析", "summary": "CSV / 表格数据做成 Excel 数据看板:KPI 卡片 + 多图表 + 迷你图 + 条件格式"},
+ "officecli-financial-model": {"category": "数据分析", "summary": "Excel 财务模型:三表 / DCF / LBO / 单位经济 / 敏感性分析,全活公式驱动"},
+ "officecli-pitch-deck": {"category": "办公效率", "summary": "融资路演 PPT:种子轮 / A-C 轮 / SAFE / 可转债的投资人 deck"},
+ "morph-ppt": {"category": "办公效率", "summary": "带 Morph 平滑转场动效的 PPT,自带 40 套成品设计风格包"},
+ "morph-ppt-3d": {"category": "办公效率", "summary": "带 .glb 三维模型与电影级镜头运动的 Morph 动效 PPT"},
+}
+
+
+MARKETPLACE_CATEGORIES = [
+ "写作助手", "文档处理", "数据分析", "政策产业",
+ "营销创意", "法务合规", "办公效率", "研发效率",
+]
+
+
+SECRETS_SECTION_HEADER = "## 🔑 凭据配置(由安装者提供)"
+
+
+REQUIRED_SECRETS_SNAPSHOT = "_required_secrets.json"
+
+
+ADMIN_UPLOAD_OWNER = "__admin_upload__"
+
+
+ZIP_SKIP_DIR_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"}
diff --git a/src/backend/features/marketplace/versions/__init__.py b/src/backend/features/marketplace/versions/__init__.py
new file mode 100644
index 00000000..b3281e46
--- /dev/null
+++ b/src/backend/features/marketplace/versions/__init__.py
@@ -0,0 +1 @@
+"""versions package."""
diff --git a/src/backend/core/services/marketplace_version_archive.py b/src/backend/features/marketplace/versions/archive.py
similarity index 100%
rename from src/backend/core/services/marketplace_version_archive.py
rename to src/backend/features/marketplace/versions/archive.py
diff --git a/src/backend/core/services/marketplace_version_config.py b/src/backend/features/marketplace/versions/config.py
similarity index 95%
rename from src/backend/core/services/marketplace_version_config.py
rename to src/backend/features/marketplace/versions/config.py
index b2014fb0..42d48abf 100644
--- a/src/backend/core/services/marketplace_version_config.py
+++ b/src/backend/features/marketplace/versions/config.py
@@ -4,8 +4,8 @@
from pydantic import BaseModel, ConfigDict, Field
-from .marketplace_version_archive import check_url, fail
-from .marketplace_version_store import decode
+from features.marketplace.versions.archive import check_url, fail
+from features.marketplace.versions.store import decode
class AgentPackage(BaseModel):
diff --git a/src/backend/core/services/marketplace_version_sources.py b/src/backend/features/marketplace/versions/sources.py
similarity index 84%
rename from src/backend/core/services/marketplace_version_sources.py
rename to src/backend/features/marketplace/versions/sources.py
index 8ab138d1..ef972d8a 100644
--- a/src/backend/core/services/marketplace_version_sources.py
+++ b/src/backend/features/marketplace/versions/sources.py
@@ -1,24 +1,28 @@
+import features.marketplace.mcp_listing as market_owner_mcp_listing
+
+import features.marketplace.mcp_versions as market_owner_mcp_versions
"""Read and activate versioned marketplace sources, never installed copies."""
import json
import tempfile
from pathlib import Path
-from core.agent_skills.binary_files import encode_upload
-from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter
-from core.db.models import AdminSkill, McpMarketItem, McpMarketVersion
+from core.capabilities.skills.binary_files import encode_upload
+from core.capabilities.skills.registry import load_skill_metadata_from_str, split_frontmatter
+from core.db.models import AdminSkill
+from features.marketplace.models import McpMarketItem, McpMarketVersion
from fastapi import HTTPException
-from . import marketplace_version_store as store
-from .marketplace_version_archive import fail, read_zip, skill_version, strip_wrapper
+import features.marketplace.versions.store as store
+from features.marketplace.versions.archive import fail, read_zip, skill_version, strip_wrapper
def source_detail(db, kind, slug):
- from core.plugins.management.details import get_plugin_detail
+ from core.capabilities.plugins.management.details import get_plugin_detail
+
+ import features.marketplace.agent_versions as agents
+ import features.marketplace.skill_versions as skills
- from . import marketplace_agent_versions as agents
- from . import marketplace_skill_versions as skills
- from . import mcp_marketplace_service as mcps
if kind == "skill":
return skills.get_marketplace_skill(slug, db)
@@ -28,13 +32,13 @@ def source_detail(db, kind, slug):
detail = get_plugin_detail(slug, db)
detail["native_version"], detail["source_revision"] = store.native_plugin_source(db, slug)
return detail
- return mcps.get_market_item(db, slug, owner_user_id=None, viewer_user_id=None, admin=True)
+ return market_owner_mcp_listing.get_market_item(db, slug, owner_user_id=None, viewer_user_id=None, admin=True)
def capture(db, kind, slug):
- from . import marketplace_agent_versions as agents
- from . import marketplace_export as export
- from . import mcp_marketplace_service as mcps
+ import features.marketplace.agent_versions as agents
+ import features.marketplace.export as export
+
detail = source_detail(db, kind, slug)
existing = store.active(db, kind, slug)
@@ -52,7 +56,7 @@ def capture(db, kind, slug):
}
else:
item = db.get(McpMarketItem, slug)
- version = mcps._version_for_item(db, item)
+ version = market_owner_mcp_versions.version_for_item(db, item)
config = {
k: getattr(version, k)
for k in (
@@ -75,7 +79,7 @@ def capture(db, kind, slug):
def normalized_plugin(files):
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
with tempfile.TemporaryDirectory(prefix="market-version-") as directory:
root = Path(directory)
@@ -95,7 +99,7 @@ def prepare(db, kind, slug, raw, number):
if kind == "skill":
try:
content = skill_version(files["SKILL.md"].decode(), number)
- meta = _load_skill_metadata_from_str(content, detail["entry_name"])
+ meta = load_skill_metadata_from_str(content, detail["entry_name"])
except (ValueError, KeyError, UnicodeError):
fail("SKILL.md 格式无效")
if meta.id != detail["entry_name"]:
@@ -121,7 +125,7 @@ def prepare(db, kind, slug, raw, number):
manifest["version"] = number
files["marketplace.json"] = json.dumps(manifest, ensure_ascii=False).encode()
detail.update(
- instructions=_split_frontmatter(content)[1].strip(),
+ instructions=split_frontmatter(content)[1].strip(),
summary=meta.description,
tags=meta.tags,
files=[
@@ -152,7 +156,7 @@ def prepare(db, kind, slug, raw, number):
np = normalized_plugin(files)
if np.slug != slug or np.dropped:
fail("插件 ID 不一致或包内包含无效组件")
- from core.plugins.management.details import _normalized_to_detail
+ from core.capabilities.plugins.management.details import normalized_to_detail as _normalized_to_detail
detail.update(_normalized_to_detail(np))
detail["skills_count"] = len(np.skills)
@@ -166,7 +170,7 @@ def prepare(db, kind, slug, raw, number):
fail("配置 JSON 无效")
if not isinstance(config, dict) or config.get("slug") != slug:
fail("配置 slug 与当前市场资源不一致")
- from .marketplace_version_config import validate
+ from features.marketplace.versions.config import validate
config = validate(db, kind, config, old)
config["version"] = number
@@ -223,7 +227,7 @@ def activate(db, kind, slug, snapshot):
if row and (row.owner_user_id is not None or row.source_plugin):
raise HTTPException(409, "内置技能 ID 与其他资源冲突")
content = store.decode(snapshot)
- metadata = _load_skill_metadata_from_str(content["SKILL.md"].decode(), identity)
+ metadata = load_skill_metadata_from_str(content["SKILL.md"].decode(), identity)
if row is None:
row = AdminSkill(skill_id=identity, owner_user_id=None, is_enabled=True)
db.add(row)
@@ -248,13 +252,13 @@ def activate(db, kind, slug, snapshot):
def connector_snapshot(db, slug, version_id):
- from . import mcp_marketplace_service as mcps
+
item = db.get(McpMarketItem, slug)
version = db.get(McpMarketVersion, version_id)
if not item or not version or version.slug != slug:
fail("所选历史版本不存在")
- detail = mcps._item_dict(db, item, version, installed=False)
+ detail = market_owner_mcp_listing.item_dict(db, item, version, installed=False)
config = {
k: getattr(version, k)
for k in (
@@ -281,7 +285,7 @@ def connector_snapshot(db, slug, version_id):
def lock_source(db, kind, slug):
- from core.db.models import AgentMarketSubmission, MarketplaceSubmission, PluginMarketPackage
+ from features.marketplace.models import AgentMarketSubmission, MarketplaceSubmission, PluginMarketPackage
models = {
"skill": MarketplaceSubmission,
diff --git a/src/backend/core/services/marketplace_version_store.py b/src/backend/features/marketplace/versions/store.py
similarity index 90%
rename from src/backend/core/services/marketplace_version_store.py
rename to src/backend/features/marketplace/versions/store.py
index 2d784959..6abaf6e7 100644
--- a/src/backend/core/services/marketplace_version_store.py
+++ b/src/backend/features/marketplace/versions/store.py
@@ -1,3 +1,5 @@
+import features.marketplace.agent_catalog as market_owner_agent_catalog
+import features.marketplace.skill_catalog as market_owner_skill_catalog
"""Version snapshots belonging to existing marketplace entries."""
import base64
@@ -31,13 +33,13 @@ def active(db, kind, slug):
if not value:
return None
if kind in {"skill", "agent"}:
- from . import agent_market_service as agents
- from . import marketplace_service as skills
+
+
native = (
- skills.get_marketplace_skill(slug, db)
+ market_owner_skill_catalog.get_marketplace_skill(slug, db)
if kind == "skill"
- else agents.get_agent_detail(db, slug)
+ else market_owner_agent_catalog.get_agent_detail(db, slug)
)
if value.get("source_version", native["version"]) != native["version"]:
return None
@@ -49,7 +51,7 @@ def active(db, kind, slug):
return None
row = db.get(ContentBlock, PREFIX + value["active"])
if kind == "connector" and row:
- from core.db.models import McpMarketItem, McpMarketVersion
+ from features.marketplace.models import McpMarketItem, McpMarketVersion
item = db.get(McpMarketItem, slug)
version = db.get(McpMarketVersion, item.latest_version_id) if item else None
@@ -99,18 +101,18 @@ def revision_token(db, kind, slug, current):
def native_plugin_source(db, slug):
- from core.plugins.management.packages import _market_row
- from core.plugins.packaging import sources
+ from core.capabilities.plugins.management.packages import market_row
+ from core.capabilities.plugins.packaging import sources
- root = sources._resolve_plugin_dir(slug)
+ root = sources.resolve_plugin_dir(slug)
if root is not None:
- meta = sources._scan_native_manifest(root)
+ meta = sources.scan_native_manifest(root)
if not meta:
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
return normalize_plugin_dir(root).version, None
return meta.get("version") or "1.0.0", None
- row = _market_row(db, slug)
+ row = market_row(db, slug)
if row is None:
from core.infra.exceptions import ResourceNotFoundError
@@ -121,7 +123,7 @@ def native_plugin_source(db, slug):
def versions(db, kind, slug, current):
value = state(db, kind, slug)
if kind == "connector":
- from core.db.models import McpMarketItem, McpMarketVersion
+ from features.marketplace.models import McpMarketItem, McpMarketVersion
rows = (
db.query(McpMarketVersion)
@@ -181,13 +183,8 @@ def versions(db, kind, slug, current):
def save(db, kind, slug, snapshot, current, expected, target=None):
"""Atomically compare source + pointer; immutable snapshots and active content."""
- from .marketplace_version_archive import fail
- from .marketplace_version_sources import (
- activate,
- connector_snapshot,
- lock_source,
- source_detail,
- )
+ from features.marketplace.versions.archive import fail
+ from features.marketplace.versions.sources import activate, connector_snapshot, lock_source, source_detail
# Refresh ORM state after waiting for another publication to finish.
pointer_id = POINTER + key(kind, slug)
@@ -211,7 +208,7 @@ def save(db, kind, slug, snapshot, current, expected, target=None):
raise HTTPException(409, "市场版本已变化,请刷新详情后重试")
# Native publisher updates must also be retained before selecting an older snapshot.
if kind != "connector" and (not old or old.get("source_version") != native_number):
- from .marketplace_version_sources import capture
+ from features.marketplace.versions.sources import capture
baseline = str(uuid.uuid4())
db.add(
diff --git a/src/backend/features/sites/README.md b/src/backend/features/sites/README.md
new file mode 100644
index 00000000..e072ea41
--- /dev/null
+++ b/src/backend/features/sites/README.md
@@ -0,0 +1,11 @@
+# 站点托管与发布
+
+负责站点模型与仓储、静态内容服务、访问密码与速率限制、站点打包、本机来源发布,以及内置站点插件升级。设计选择工具由 extension.register_tools 注册;模型由 FULL 和 CE 安装清单加载,不依赖运行开关。
+
+HTTP 路由保持原有路径、顺序和权限策略;sites_serve 仍在 API 应用的原位置挂载,其余路由通过中央路由表注册。Site 表结构与约束不变,CE 通过 edition.py 选择权限字段和访问策略。
+
+公开入口包括 service、management、listing、packaging、repository 及各 API 路由。调用方直接依赖实际模块,不保留旧路径转发。依赖内核数据库、文件、项目权限和插件基础设施。
+
+站点通过 extension.register_gateway_contributions 向桌面网关注册上传格式、接收实现和回执本地化;桌面网关不导入站点实现。sites → desktop 是单向依赖,用于桥接账号校验、能力修订和发布传输。HTTP 执行位置策略来自宿主接口,注册在 API/CLI 初始化阶段完成。
+
+版本差异统一由 edition.py 选择 ORM 字段、访问策略及请求字段;CE 只覆盖此入口。
diff --git a/src/backend/features/sites/__init__.py b/src/backend/features/sites/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/sites/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/features/sites/api/__init__.py b/src/backend/features/sites/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/sites/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/internal_site_auth.py b/src/backend/features/sites/api/internal_site_auth.py
similarity index 100%
rename from src/backend/api/routes/v1/internal_site_auth.py
rename to src/backend/features/sites/api/internal_site_auth.py
diff --git a/src/backend/api/routes/v1/internal_sites.py b/src/backend/features/sites/api/internal_sites.py
similarity index 94%
rename from src/backend/api/routes/v1/internal_sites.py
rename to src/backend/features/sites/api/internal_sites.py
index 17e75689..d3f7e21e 100644
--- a/src/backend/api/routes/v1/internal_sites.py
+++ b/src/backend/features/sites/api/internal_sites.py
@@ -23,16 +23,14 @@
from typing import List, Optional, Tuple
from core.infra.responses import success_response
-from core.services.artifact_edition import personal_artifact_predicates
-from core.services.site_access_policy import SitePublishScopeFields, site_scope_ref
-from core.services.site_packaging import (
- BUILD_SOURCE_EXCLUDES, SITE_SOURCE_EXCLUDES, pack_and_fetch_dir, resolve_project_context,
-)
+from core.db.artifact_scope import personal_artifact_predicates
+from features.sites.edition import SitePublishScopeFields, site_scope_ref
+from features.sites.packaging import BUILD_SOURCE_EXCLUDES, SITE_SOURCE_EXCLUDES, pack_and_fetch_dir, resolve_project_context
from fastapi import APIRouter, Header, HTTPException
from pydantic import BaseModel, Field
from starlette.concurrency import run_in_threadpool
-from .internal_site_auth import _check_internal_token
+from features.sites.api.internal_site_auth import _check_internal_token
logger = logging.getLogger(__name__)
@@ -74,8 +72,8 @@ def _resolve_target_site_id(project_id: str, user_id: str) -> str:
if not project_id:
return ""
from core.db.engine import SessionLocal
- from core.db.models import Site
- from core.services.site_service import SiteService
+ from features.sites.models import Site
+ from features.sites.service import SiteService
with SessionLocal() as db:
rows = db.query(Site.site_id).filter(
Site.project_id == project_id, Site.deleted_at.is_(None),
@@ -95,7 +93,7 @@ def _mirror_files_to_project_folder(
if not project_id or not files:
return
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
with SessionLocal() as db:
ProjectSourceService(db).write_files(project_id, user_id, files)
@@ -114,12 +112,13 @@ def _ensure_project_for_site(site_id: str, user_id: str, title: str) -> Optional
return None
try:
from core.db.engine import SessionLocal
- from core.db.models import Project, Site
- from core.services.project_service import ProjectService
+ from core.db.models import Project
+ from features.sites.models import Site
+ from core.projects.edition import ProjectService
with SessionLocal() as db:
site = db.query(Site).filter(Site.site_id == site_id).first()
- from core.services.site_access_policy import site_management_permission
+ from features.sites.edition import site_management_permission
if site is None or site_management_permission(db, site, user_id) not in ("edit", "admin"):
return None
# Already linked and the project still exists → use it directly
@@ -182,7 +181,7 @@ def _project_root_has_package_json(project_id: str, user_id: str) -> bool:
if not project_id:
return False
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
with SessionLocal() as db:
try:
ProjectSourceService(db).read_bytes(project_id, user_id, "package.json")
@@ -204,7 +203,7 @@ async def list_sites(
if not user_id:
return success_response(data={"error": "当前会话缺少用户身份,无法查询站点"})
- from core.services.site_listing import list_sites as collect
+ from features.sites.listing import list_sites as collect
try:
items = await run_in_threadpool(collect, user_id, body.chat_id or "", body.limit)
@@ -219,10 +218,10 @@ async def publish(
x_internal_token: Optional[str] = Header(None, alias="X-Internal-Token"),
):
_check_internal_token(x_internal_token)
- from core.services.desktop_cloud_bridge import bridge_enabled
+ from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled
if bridge_enabled():
- from core.services.desktop_site_publish import forward_local_site_tool
+ from features.sites.desktop_publish import forward_local_site_tool
return success_response(
data=await forward_local_site_tool(
@@ -234,8 +233,8 @@ async def publish(
)
from core.llm.tools._common import resolve_sandbox_session
- from core.llm.tools._paths import to_physical_path
- from core.llm.tools._tool_helpers import _validate_workspace_path
+ from core.sandbox.workspace.resolution import to_physical_path
+ from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path
user_id = (body.user_id or "").strip()
if not user_id:
@@ -292,11 +291,11 @@ async def publish(
from core.db.engine import SessionLocal
from core.infra.exceptions import AppException
- from core.services.site_service import SiteService
+ from features.sites.service import SiteService
# Resolve authorization before touching a sandbox or storing any new bytes.
from core.db.models import Project
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
team_source_excludes = frozenset(
SITE_SOURCE_EXCLUDES + (BUILD_SOURCE_EXCLUDES if source_dir else ())
diff --git a/src/backend/api/routes/v1/local_site_sources.py b/src/backend/features/sites/api/local_site_sources.py
similarity index 94%
rename from src/backend/api/routes/v1/local_site_sources.py
rename to src/backend/features/sites/api/local_site_sources.py
index f0fbcbcf..70b0a90b 100644
--- a/src/backend/api/routes/v1/local_site_sources.py
+++ b/src/backend/features/sites/api/local_site_sources.py
@@ -4,7 +4,7 @@
from pydantic import BaseModel, Field
from core.auth.backend import UserContext, get_current_user
from core.infra.responses import success_response
-from core.services import local_site_sources as sources
+from features.sites import local_sources as sources
router = APIRouter(prefix="/v1/local/site-sources", tags=["local"])
@@ -29,4 +29,3 @@ def list_sources(user: UserContext = Depends(get_current_user)):
@router.post("/{site_id}/edit")
def edit(site_id: str, user: UserContext = Depends(get_current_user)):
return success_response(data=sources.open_editor(str(user.user_id), site_id))
-
diff --git a/src/backend/api/routes/site_gate.py b/src/backend/features/sites/api/site_gate.py
similarity index 100%
rename from src/backend/api/routes/site_gate.py
rename to src/backend/features/sites/api/site_gate.py
diff --git a/src/backend/api/routes/v1/sites.py b/src/backend/features/sites/api/sites.py
similarity index 95%
rename from src/backend/api/routes/v1/sites.py
rename to src/backend/features/sites/api/sites.py
index bd18dc0f..0af4d784 100644
--- a/src/backend/api/routes/v1/sites.py
+++ b/src/backend/features/sites/api/sites.py
@@ -10,14 +10,9 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
from core.infra.responses import paginated_response, success_response
-from core.services.site_access_policy import (
- SiteUpdateScopeFields,
- serialize_site_scope,
- site_management_permission,
- site_scope_ref,
-)
-from core.services.site_password import MAX_PASSWORD_LENGTH
-from core.services.site_service import SiteService
+from features.sites.edition import SiteUpdateScopeFields, serialize_site_scope, site_management_permission, site_scope_ref
+from features.sites.password import MAX_PASSWORD_LENGTH
+from features.sites.service import SiteService
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
diff --git a/src/backend/api/routes/sites_serve.py b/src/backend/features/sites/api/sites_serve.py
similarity index 82%
rename from src/backend/api/routes/sites_serve.py
rename to src/backend/features/sites/api/sites_serve.py
index 08ccbdb9..d2808ee5 100644
--- a/src/backend/api/routes/sites_serve.py
+++ b/src/backend/features/sites/api/sites_serve.py
@@ -24,23 +24,17 @@
import logging
from typing import Optional
-from api.routes.site_gate import render_site_gate
-from core.db.engine import get_db
-from core.db.repository import SiteRepository
-from core.services.site_password import (
- ACCESS_COOKIE_NAME,
- access_cookie_params,
- check_access_password,
- issue_access_token,
- site_base_path,
- verify_access_token,
-)
-from core.services.site_service import SiteService
-from fastapi import APIRouter, Depends, HTTPException, Request
+from features.sites.api.site_gate import render_site_gate
+from features.sites.repository import SiteRepository
+from features.sites.password import ACCESS_COOKIE_NAME, access_cookie_params, check_access_password, issue_access_token, site_base_path, verify_access_token
+from features.sites.service import SiteService
+from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
+from core.db.threads import run_in_session
+
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/site", tags=["sites-serve"])
@@ -100,31 +94,48 @@ def _common_headers(content_type: str, *, sandbox: bool, cache: Optional[str] =
return headers
-async def _load_authorized_site(
- slug: str, request: Request, db: Session, *, require_unlock: bool = True
-):
- """取站点并鉴权,返回 ``(site, gate)``。
+def _site(db: Session, slug: str):
+ site = SiteRepository(db).get_by_slug(slug)
+ if site is not None:
+ db.expunge(site)
+ return site
+
+
+def _can_view(db: Session, site_id: str, user_id: Optional[str]) -> bool:
+ from features.sites.models import Site
+
+ return SiteService(db).authorize_view(db.get(Site, site_id), user_id)
+
+
+def _can_unlock(db: Session, site_id: str, user_id: str) -> bool:
+ from features.sites.models import Site
+
+ return SiteService(db).authorize_access(db.get(Site, site_id), user_id, None)
+
+
+async def _load_authorized_site(slug: str, request: Request, *, require_unlock: bool = True):
+ """取站点并鉴权,返回 ``(site, gate)``;``site`` 已脱离会话,只读不改。
两道闸:可见性(不通过一律 404,不泄露站点是否存在)与访问密码(不通过返回
``gate`` 响应,由调用方直接回给访客)。默认两道都过——``require_unlock=False``
是显式豁免,只给 ``__api/*`` 用:站内脚本跑在沙箱的不透明源上、请求不带凭据,
仅解锁入口 ``__api/access`` 可豁免;应用表单仍执行密码验证。
"""
- site = await run_in_threadpool(SiteRepository(db).get_by_slug, slug)
+ site = await run_in_session(_site, slug)
if not site:
raise HTTPException(status_code=404, detail="Site not found")
if site.visibility != "public":
from api.deps import _resolve_session_user_id
user_id = await _resolve_session_user_id(request)
- if not await run_in_threadpool(SiteService(db).authorize_view, site, user_id):
+ if not await run_in_session(_can_view, site.site_id, user_id):
raise HTTPException(status_code=404, detail="Site not found")
if not require_unlock or not site.access_password_hash:
return site, None
- return site, await _unlock_gate(site, request, db)
+ return site, await _unlock_gate(site, request)
-async def _unlock_gate(site, request: Request, db: Session) -> Optional[Response]:
+async def _unlock_gate(site, request: Request) -> Optional[Response]:
"""已解锁返回 None;否则返回验证页。凭据校验是纯 HMAC,不值得丢进线程池。"""
if verify_access_token(site, request.cookies.get(ACCESS_COOKIE_NAME)):
return None
@@ -132,7 +143,7 @@ async def _unlock_gate(site, request: Request, db: Session) -> Optional[Response
from api.deps import _resolve_session_user_id
user_id = await _resolve_session_user_id(request)
- if user_id and await run_in_threadpool(SiteService(db).authorize_access, site, user_id, None):
+ if user_id and await run_in_session(_can_unlock, site.site_id, user_id):
# 给管理者也发一枚凭据,否则同一个页面的每个资源都要重跑一遍会话 + 权限查询。
request.state.site_access_grant = site
return None
@@ -153,12 +164,12 @@ async def site_api_preflight(slug: str, rest: str):
@router.post("/{slug}/__api/access", summary="站点访问密码校验")
-async def site_unlock(slug: str, request: Request, db: Session = Depends(get_db)):
+async def site_unlock(slug: str, request: Request):
"""校验访问密码并下发解锁凭据(统一验证页提交到这里)。这是解锁入口,必须免闸。"""
# 限流放在取站点之前:被挡下的尝试连一次 get_by_slug 都不该花。
- from core.services.site_rate_limit import count_attempt
+ from features.sites.rate_limit import count_attempt
await count_attempt(_client_ip(request), slug, unlock=True)
- site, _ = await _load_authorized_site(slug, request, db, require_unlock=False)
+ site, _ = await _load_authorized_site(slug, request, require_unlock=False)
if not site.access_password_hash:
return _api_json({"ok": True})
try:
@@ -187,14 +198,13 @@ async def serve_site_file(
slug: str,
path: str,
request: Request,
- db: Session = Depends(get_db),
):
if path == "__api" or path.startswith("__api/"):
raise HTTPException(404, "API route not found")
- site, gate = await _load_authorized_site(slug, request, db)
+ site, gate = await _load_authorized_site(slug, request)
if gate is not None:
return gate
- response = await run_in_threadpool(_site_file_response, db, site, path)
+ response = await run_in_session(_site_file_response, site.site_id, path)
granted = getattr(request.state, "site_access_grant", None)
if granted is not None:
response.set_cookie(value=issue_access_token(granted), **access_cookie_params(slug))
@@ -218,8 +228,11 @@ def _gate_response(site, request: Request) -> Response:
)
-def _site_file_response(db: Session, site, path: str) -> Response:
+def _site_file_response(db: Session, site_id: str, path: str) -> Response:
# Storage I/O, row updates and ORM refreshes all run off the event loop.
+ from features.sites.models import Site
+
+ site = db.get(Site, site_id)
resolved = SiteService(db).resolve_site_file(site, path)
if resolved is None:
raise HTTPException(status_code=404, detail="File not found")
diff --git a/src/backend/core/plugins/local/site_upgrade.py b/src/backend/features/sites/builtin_upgrade.py
similarity index 83%
rename from src/backend/core/plugins/local/site_upgrade.py
rename to src/backend/features/sites/builtin_upgrade.py
index 73979d25..7e3e3e84 100644
--- a/src/backend/core/plugins/local/site_upgrade.py
+++ b/src/backend/features/sites/builtin_upgrade.py
@@ -8,12 +8,12 @@
def upgrade_builtin_sites(db) -> int:
from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin
- from core.plugins.management import components as plugin_components
- from core.plugins.management import projection as plugin_projection
- from core.plugins.packaging import sources as plugin_sources
- from core.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins.management import components as plugin_components
+ from core.capabilities.plugins.management import projection as plugin_projection
+ from core.capabilities.plugins.packaging import sources as plugin_sources
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
- bundle = plugin_sources._resolve_plugin_dir("sites")
+ bundle = plugin_sources.resolve_plugin_dir("sites")
if bundle is None:
return 0
normalized = normalize_plugin_dir(bundle)
@@ -38,7 +38,7 @@ def upgrade_builtin_sites(db) -> int:
ids = {key: list(value) for key, value in (install.component_ids or {}).items()}
added_skills = []
siblings = {
- sk.name: plugin_sources._make_skill_id("sites", sk.name, owner)
+ sk.name: plugin_sources.make_skill_id("sites", sk.name, owner)
for sk in normalized.skills
}
for sk in normalized.skills:
@@ -59,7 +59,7 @@ def upgrade_builtin_sites(db) -> int:
enabled = (
row.is_enabled if row else any(item and item.is_enabled for item in existing_skills)
)
- plugin_components._apply_skill(
+ plugin_components.apply_skill(
db,
sk,
slug="sites",
@@ -71,7 +71,7 @@ def upgrade_builtin_sites(db) -> int:
sibling_ids=siblings,
)
for mc in normalized.mcp:
- sid = plugin_sources._make_server_id("sites", mc.name, owner)
+ sid = plugin_sources.make_server_id("sites", mc.name, owner)
row = db.get(AdminMcpServer, sid)
if (
sid not in ids.get("mcp", [])
@@ -82,7 +82,7 @@ def upgrade_builtin_sites(db) -> int:
continue
# Only shipped tool instructions change: URLs, credentials, runtime
# settings and enabled states remain exactly as the user configured.
- row.tools_json = plugin_components._merge_tool_metadata(
+ row.tools_json = plugin_components.merge_tool_metadata(
row.tools_json, list(mc.tools or [])
)
# Retired shipped KV tools must not remain in the persisted catalog.
@@ -107,18 +107,19 @@ def upgrade_builtin_sites(db) -> int:
if changed_owners:
db.commit()
for owner in set(changed_owners):
- plugin_projection._refresh_after_change(owner)
+ plugin_projection.refresh_after_change(owner)
return len(changed_owners)
def _refresh_existing_projection(db, install, added_skills):
"""Update only the already-owned device snapshot, preserving enablement and edges."""
- from core.capabilities.paths import capabilities_enabled, revision_for_hash
+ from core.harness.registry.run_capabilities import has_run_capabilities
+ from features.desktop.capabilities.paths import revision_for_hash
- if not capabilities_enabled():
+ if not has_run_capabilities():
return
- from core.capabilities import plugins, registry, store
- from core.services.desktop_capability_protocol import entity_content_hash
+ from features.desktop.capabilities import plugins, registry, store
+ from core.capabilities.manifest_protocol import entity_content_hash
current = registry.get("plugin:local:sites", db=db)
if (
diff --git a/src/backend/core/services/desktop_site_publish.py b/src/backend/features/sites/desktop_publish.py
similarity index 83%
rename from src/backend/core/services/desktop_site_publish.py
rename to src/backend/features/sites/desktop_publish.py
index 82913cef..4b537f77 100644
--- a/src/backend/core/services/desktop_site_publish.py
+++ b/src/backend/features/sites/desktop_publish.py
@@ -6,7 +6,7 @@
import json
import tarfile
-from core.services.site_access_policy import SitePublishScopeFields, site_scope_ref
+from features.sites.edition import SitePublishScopeFields, site_scope_ref
from pydantic import Field
MAX_ARCHIVE_BYTES = 40 * 1024 * 1024
@@ -32,10 +32,10 @@ def _session_workspace_root(chat_id: str | None) -> str | None:
async def package_local_site(arguments, headers):
"""Read the current local sandbox, never a path on the remote backend."""
from core.config.local_mode import local_mode_enabled
- from core.llm.tools._paths import to_physical_path
- from core.llm.tools._tool_helpers import _validate_workspace_path
- from core.services.site_packaging import pack_and_fetch_dir, resolve_project_context
- from core.services.site_service import SiteService
+ from core.sandbox.workspace.resolution import to_physical_path
+ from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path
+ from features.sites.packaging import pack_and_fetch_dir, resolve_project_context
+ from features.sites.service import SiteService
headers = {k.lower(): v for k, v in headers.items()}
user_id = headers.get("x-current-user-id", "")
@@ -84,7 +84,7 @@ async def package_local_site(arguments, headers):
if source and to_physical_path(source, user_id, session_id=chat_id) == src:
raise ValueError("src_dir 必须指向构建产物,不能与 source_dir 相同")
if local_mode_enabled() and chat_id:
- from core.services.local_site_sources import validate_source
+ from features.sites.local_sources import validate_source
source_path = to_physical_path(source, user_id, session_id=chat_id) if source else src
arguments["_desktop_source"] = validate_source(user_id, chat_id, source_path, src)
@@ -114,7 +114,7 @@ def localize_site_result(data, cloud_base, arguments=None, headers=None):
published = json.loads(block["text"])
context = (arguments or {}).get("_desktop_source")
if context and published.get("ok") and published.get("site_id"):
- from core.services.local_site_sources import save_receipt
+ from features.sites.local_sources import save_receipt
normalized = {k.lower(): v for k, v in (headers or {}).items()}
from fastapi import HTTPException
@@ -149,8 +149,8 @@ def publish_uploaded_site(user_id, data, options):
Source files remain in the desktop project; only build output is uploaded.
"""
from core.db.engine import SessionLocal
- from core.services.site_packaging import safe_extract_tar
- from core.services.site_service import SiteService
+ from features.sites.packaging import safe_extract_tar
+ from features.sites.service import SiteService
files = safe_extract_tar(data)
if SiteService.looks_like_source_tree(name for name, _ in files):
@@ -188,14 +188,11 @@ def publish_uploaded_site(user_id, data, options):
async def forward_local_site_tool(tool_name, arguments, *, user_id, chat_id=""):
"""Sites are cloud-hosted: a local site MCP's call goes to the cloud gateway plugin."""
- from core.llm.factory.tools.mcp_config import _inject_runtime_headers
- from core.llm.mcp_pool import make_client
- from core.services.desktop_cloud_bridge import (
- cloud_gateway_mcp_configs,
- get_state,
- require_current_account,
- )
- from core.services.desktop_gateway_uploads import endpoint_plugin
+ from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers
+ from core.llm.tools.cloud_manifest import manifest_tools
+ from features.desktop.bridge.gateway import cloud_gateway_mcp_configs
+ from features.desktop.bridge.identity import get_state, require_current_account
+ from core.harness.registry.transfer_channels import endpoint_plugin
state = get_state()
if state is None:
@@ -213,23 +210,22 @@ async def forward_local_site_tool(tool_name, arguments, *, user_id, chat_id=""):
channel_origin=None,
reranker_enabled=False,
)
- client = make_client(sid, configs[sid], is_stateful=False)
- try:
- tool = await client.get_tool(tool_name)
- result = await tool(**arguments)
- require_current_account(state)
- if tool_name in {"manage_application", "publish_mcp"}:
- data = {"content": [block.model_dump() for block in result.content]}
- localize_application_result(data, state["cloud_base"])
- return json.loads(
- next(block["text"] for block in data["content"] if block["type"] == "text")
- )
- for block in result.content:
- if getattr(block, "type", None) == "text":
- return json.loads(block.text)
- return {"error": f"云端 {tool_name} 结果未知,请在云端站点列表核对"}
- finally:
- await client.close()
+ tools = manifest_tools(sid, configs[sid])
+ tool = next((t for t in tools if t.name == tool_name), None)
+ if tool is None:
+ return {"error": "云端站点工具不可用"}
+ result = await tool(**arguments)
+ require_current_account(state)
+ if tool_name in {"manage_application", "publish_mcp"}:
+ data = {"content": [block.model_dump() for block in result.content]}
+ localize_application_result(data, state["cloud_base"])
+ return json.loads(
+ next(block["text"] for block in data["content"] if block["type"] == "text")
+ )
+ for block in result.content:
+ if getattr(block, "type", None) == "text":
+ return json.loads(block.text)
+ return {"error": f"云端 {tool_name} 结果未知,请在云端站点列表核对"}
return {"error": "云端站点能力不可用,请启用云端站点插件后重试"}
diff --git a/src/backend/features/sites/edition.py b/src/backend/features/sites/edition.py
new file mode 100644
index 00000000..e097dcf1
--- /dev/null
+++ b/src/backend/features/sites/edition.py
@@ -0,0 +1,10 @@
+"""Edition policy selected by the installed distribution."""
+
+def __getattr__(name):
+ if name in ('SiteUpdateScopeFields', 'SitePublishScopeFields', 'resolve_site_scope', 'site_scope_write_fields', 'site_scope_ref', 'serialize_site_scope', 'can_view_site', 'site_management_permission', 'site_listing_predicate'):
+ import features.sites.personal_access as provider
+ return getattr(provider, name)
+ if name in ('SiteScopeMixin', 'site_scope_table_args'):
+ import features.sites.personal_scope as provider
+ return getattr(provider, name)
+ raise AttributeError(name)
diff --git a/src/backend/features/sites/extension.py b/src/backend/features/sites/extension.py
new file mode 100644
index 00000000..3ec506e8
--- /dev/null
+++ b/src/backend/features/sites/extension.py
@@ -0,0 +1,34 @@
+"""Site design tool contribution; hosting routes retain central ordering."""
+NAME = "sites"
+
+
+def register_tools():
+ from features.sites.tools.design_picker import register_native_contribution
+ register_native_contribution()
+
+
+def setup():
+ register_tools()
+ return True
+
+
+def register_gateway_contributions():
+ from core.harness.registry.transfer_channels import UploadChannel, UploadReceiver, register_upload_channel, register_result_localizer
+ from features.sites import desktop_publish
+ register_upload_channel(
+ "sites", "publish_site",
+ lambda: UploadChannel("site-publish", "application/gzip", desktop_publish.package_local_site),
+ receiver=UploadReceiver(
+ desktop_publish.SiteUploadOptions, desktop_publish.MAX_ARCHIVE_BYTES,
+ lambda *args: desktop_publish.publish_uploaded_site(*args),
+ ),
+ )
+ for name, method in {
+ "publish_site": "localize_site_result",
+ "manage_application": "localize_application_result",
+ "publish_mcp": "localize_application_result",
+ }.items():
+ register_result_localizer(
+ "sites", name,
+ lambda *args, _method=method, **kwargs: getattr(desktop_publish, _method)(*args, **kwargs),
+ )
diff --git a/src/backend/core/services/site_listing.py b/src/backend/features/sites/listing.py
similarity index 94%
rename from src/backend/core/services/site_listing.py
rename to src/backend/features/sites/listing.py
index f81784e6..7827badf 100644
--- a/src/backend/core/services/site_listing.py
+++ b/src/backend/features/sites/listing.py
@@ -46,7 +46,7 @@ def _chat_project_id(user_id: str, chat_id: str) -> str:
def _local_entries(user_id: str) -> List[dict]:
- from core.services.local_site_sources import current_cloud, list_sources
+ from features.sites.local_sources import current_cloud, list_sources
from fastapi import HTTPException
cloud, subject = current_cloud()
@@ -78,8 +78,8 @@ def _local_entries(user_id: str) -> List[dict]:
def _cloud_entries(user_id: str) -> List[dict]:
from core.db.engine import SessionLocal
- from core.services.site_access_policy import site_management_permission
- from core.services.site_service import SiteService
+ from features.sites.edition import site_management_permission
+ from features.sites.service import SiteService
with SessionLocal() as db:
sites, _ = SiteService(db).list_sites(user_id, page=1, page_size=MAX_LIMIT)
@@ -129,7 +129,7 @@ def _cloud_source_dir(db, project_id: Any) -> str:
project = _live_project(db, project_id)
if project is None:
return ""
- from core.services.edition_workspace import source_directory
+ from core.files.edition import source_directory
directory = source_directory(project)
if directory:
return directory
diff --git a/src/backend/core/services/local_site_sources.py b/src/backend/features/sites/local_sources.py
similarity index 93%
rename from src/backend/core/services/local_site_sources.py
rename to src/backend/features/sites/local_sources.py
index a0a29ad5..b8076329 100644
--- a/src/backend/core/services/local_site_sources.py
+++ b/src/backend/features/sites/local_sources.py
@@ -21,9 +21,9 @@ def require_local():
def prepare_project(user_id: str, title: str, project_id: str = "") -> dict:
require_local()
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
- from core.services.project_service import ProjectService
- from core.llm.tools._paths import WORKSPACE_ROOT
+ from core.projects.source import ProjectSourceService
+ from core.projects.edition import ProjectService
+ from core.sandbox.workspace.resolution import WORKSPACE_ROOT
with SessionLocal() as db:
if project_id:
@@ -49,8 +49,8 @@ def prepare_project(user_id: str, title: str, project_id: str = "") -> dict:
def current_cloud() -> tuple[str, str]:
- from core.services.desktop_cloud_bridge import get_state
- from core.services.desktop_capability_protocol import token_subject
+ from features.desktop.bridge.identity import get_state
+ from core.capabilities.manifest_protocol import token_subject
state = get_state() or {}
return (
@@ -61,7 +61,7 @@ def current_cloud() -> tuple[str, str]:
def _entries(db, user_id):
from core.db.models import Project
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
cloud, subject = current_cloud()
if not cloud or not subject:
@@ -92,7 +92,7 @@ def _entries(db, user_id):
continue
result.append({**saved, "project_id": project.project_id, "project_name": project.name})
from core.db.models import ChatSession
- from core.services.desktop_site_publish import _session_workspace_root
+ from features.sites.desktop_publish import _session_workspace_root
chats = db.query(ChatSession).filter(
ChatSession.user_id == user_id,
@@ -130,7 +130,7 @@ def project_sources(user_id: str, project_id: str) -> list[dict]:
"""Read this project's current-account receipts without selecting an edit target."""
require_local()
from core.db.engine import SessionLocal
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
with SessionLocal() as db:
project = ProjectSourceService(db).authorized_project(project_id, user_id, write=False)
@@ -153,7 +153,7 @@ def save_receipt(user_id: str, chat_id: str, context: dict, published: dict, clo
require_local()
from core.db.engine import SessionLocal
from core.db.models import ChatSession
- from core.services.project_source import ProjectSourceService
+ from core.projects.source import ProjectSourceService
cloud, subject = current_cloud()
if cloud != cloud_base.rstrip("/") or not subject:
@@ -163,7 +163,7 @@ def save_receipt(user_id: str, chat_id: str, context: dict, published: dict, clo
if (validated.get("project_id") or "") != (context.get("project_id") or ""):
raise ValueError("建站会话的项目已变化")
with SessionLocal() as db:
- from core.services.project_source import reserve_sqlite_writer
+ from core.projects.source import reserve_sqlite_writer
reserve_sqlite_writer(db)
project_id = context.get("project_id") or ""
@@ -239,7 +239,7 @@ def open_editor(user_id: str, site_id: str) -> dict:
require_local()
from core.db.engine import SessionLocal
from core.db.models import ChatSession
- from core.services.project_source import reserve_sqlite_writer
+ from core.projects.source import reserve_sqlite_writer
with SessionLocal() as db:
reserve_sqlite_writer(db)
@@ -295,8 +295,8 @@ def open_editor(user_id: str, site_id: str) -> dict:
def validate_source(user_id: str, chat_id: str, source: str, publish_dir: str) -> dict:
"""Require existing source and output inside the local project or session workspace."""
require_local()
- from core.services.site_packaging import resolve_project_context
- from core.services.project_source import ProjectSourceService
+ from features.sites.packaging import resolve_project_context
+ from core.projects.source import ProjectSourceService
from core.db.engine import SessionLocal
from core.db.models import ChatSession
@@ -313,7 +313,7 @@ def validate_source(user_id: str, chat_id: str, source: str, publish_dir: str) -
raise ValueError("桌面站点必须使用本地项目")
root = Path(project_dir).resolve()
else:
- from core.services.desktop_site_publish import _session_workspace_root
+ from features.sites.desktop_publish import _session_workspace_root
root = Path(_session_workspace_root(chat_id) or "").resolve()
paths = [Path(source), Path(publish_dir)]
diff --git a/src/backend/core/services/site_management.py b/src/backend/features/sites/management.py
similarity index 95%
rename from src/backend/core/services/site_management.py
rename to src/backend/features/sites/management.py
index 8d0f4544..ef206b0c 100644
--- a/src/backend/core/services/site_management.py
+++ b/src/backend/features/sites/management.py
@@ -5,18 +5,12 @@
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-from core.db.models import Site
+from features.sites.models import Site
from core.infra.exceptions import BadRequestError, ResourceNotFoundError
from core.infra.time import utc_now
-from core.services.site_access_policy import (
- can_view_site,
- resolve_site_scope,
- site_listing_predicate,
- site_management_permission,
- site_scope_write_fields,
-)
-from core.services.site_password import hash_access_password, verify_access_token
-from core.storage import get_storage
+from features.sites.edition import can_view_site, resolve_site_scope, site_listing_predicate, site_management_permission, site_scope_write_fields
+from features.sites.password import hash_access_password, verify_access_token
+from core.files.storage import get_storage
class SiteManagementMixin:
diff --git a/src/backend/core/db/models/site.py b/src/backend/features/sites/models.py
similarity index 97%
rename from src/backend/core/db/models/site.py
rename to src/backend/features/sites/models.py
index 7d4245a9..62c01abd 100644
--- a/src/backend/core/db/models/site.py
+++ b/src/backend/features/sites/models.py
@@ -3,7 +3,7 @@
from datetime import datetime
from core.db.engine import Base
-from core.db.models.site_scope import SiteScopeMixin, site_scope_table_args
+from features.sites.edition import SiteScopeMixin, site_scope_table_args
from core.db.utc_datetime import UTCDateTime
from core.infra.time import utc_now
from sqlalchemy import (
diff --git a/src/backend/core/services/site_packaging.py b/src/backend/features/sites/packaging.py
similarity index 92%
rename from src/backend/core/services/site_packaging.py
rename to src/backend/features/sites/packaging.py
index d711815f..02b043b6 100644
--- a/src/backend/core/services/site_packaging.py
+++ b/src/backend/features/sites/packaging.py
@@ -6,6 +6,7 @@
"""
from __future__ import annotations
+import inspect
import io
import json
@@ -30,9 +31,9 @@ def resolve_project_context(chat_id: str, user_id: str):
from fastapi import HTTPException
from core.db.engine import SessionLocal
from core.db.models import ChatSession, Project
- from core.auth.permissions_iface import resolve_project_permission
- from core.services.project_scope import build_project_ctx
- from core.llm.tools._paths import to_physical_path
+ from core.auth.edition import resolve_project_permission
+ from core.projects.edition import build_project_ctx
+ from core.sandbox.workspace.resolution import to_physical_path
with SessionLocal() as db:
chat = db.get(ChatSession, chat_id)
@@ -66,7 +67,7 @@ def resolve_project_context(chat_id: str, user_id: str):
folder = (ctx or {}).get("project_folder_name")
if not folder:
raise HTTPException(409, "源码项目未绑定有效空间文件夹")
- from core.services.edition_workspace import source_directory
+ from core.files.edition import source_directory
directory = source_directory(project)
if directory:
return project.project_id, directory
@@ -85,11 +86,11 @@ async def pack_and_fetch_dir(
from core.sandbox import ProcessRequest, get_sandbox_provider
from core.sandbox import SandboxError, SandboxConnectError
import core.sandbox.workspace.archive as directory_archive
- from core.services.site_service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES
+ from features.sites.service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES
provider = get_sandbox_provider()
archive_name = f".__site_pack_{uuid.uuid4().hex}.tgz"
- from core.llm.tools._paths import workspace_directory
+ from core.sandbox.workspace.resolution import workspace_directory
import os
working_directory = workspace_directory(_sess)
@@ -104,7 +105,7 @@ async def pack_and_fetch_dir(
"max_total_bytes": MAX_SITE_TOTAL_BYTES,
"max_archive_bytes": MAX_PACK_BYTES,
}
- program = Path(directory_archive.__file__).read_text(encoding="utf-8")
+ program = inspect.getsource(directory_archive)
request = ProcessRequest(
script_content=program,
params=options,
@@ -153,7 +154,7 @@ def safe_extract_tar(data: bytes) -> List[Tuple[str, bytes]]:
(guarding against symlink escape). Absolute paths and ``..`` traversal are re-checked by
the service layer's normalize_rel_path.
"""
- from core.services.site_service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES
+ from features.sites.service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES
if len(data) > MAX_PACK_BYTES:
raise ValueError("站点发布包过大")
diff --git a/src/backend/core/services/site_password.py b/src/backend/features/sites/password.py
similarity index 100%
rename from src/backend/core/services/site_password.py
rename to src/backend/features/sites/password.py
diff --git a/src/backend/core/services/site_access_policy.py b/src/backend/features/sites/personal_access.py
similarity index 97%
rename from src/backend/core/services/site_access_policy.py
rename to src/backend/features/sites/personal_access.py
index 47ef7cc6..e1217ed1 100644
--- a/src/backend/core/services/site_access_policy.py
+++ b/src/backend/features/sites/personal_access.py
@@ -57,5 +57,5 @@ def site_management_permission(_db, site, user_id: str) -> str:
def site_listing_predicate(user_id: str):
- from core.db.models import Site
+ from features.sites.models import Site
return Site.user_id == user_id
diff --git a/src/backend/core/db/models/site_scope.py b/src/backend/features/sites/personal_scope.py
similarity index 100%
rename from src/backend/core/db/models/site_scope.py
rename to src/backend/features/sites/personal_scope.py
diff --git a/src/backend/core/services/site_rate_limit.py b/src/backend/features/sites/rate_limit.py
similarity index 100%
rename from src/backend/core/services/site_rate_limit.py
rename to src/backend/features/sites/rate_limit.py
diff --git a/src/backend/core/db/repository/site.py b/src/backend/features/sites/repository.py
similarity index 98%
rename from src/backend/core/db/repository/site.py
rename to src/backend/features/sites/repository.py
index ba262c6a..1333991d 100644
--- a/src/backend/core/db/repository/site.py
+++ b/src/backend/features/sites/repository.py
@@ -3,7 +3,7 @@
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-from core.db.models import Site
+from features.sites.models import Site
from core.infra.time import utc_now
from sqlalchemy import desc
from sqlalchemy.orm import Session
diff --git a/src/backend/core/services/site_service.py b/src/backend/features/sites/service.py
similarity index 97%
rename from src/backend/core/services/site_service.py
rename to src/backend/features/sites/service.py
index 68657e9d..9db6337b 100644
--- a/src/backend/core/services/site_service.py
+++ b/src/backend/features/sites/service.py
@@ -20,18 +20,13 @@
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
-from core.db.models import Site
-from core.db.repository import SiteRepository
+from features.sites.models import Site
+from features.sites.repository import SiteRepository
from core.infra.exceptions import BadRequestError
from core.infra.time import utc_now
-from core.services.site_access_policy import (
- resolve_site_scope,
- site_management_permission,
- site_scope_ref,
- site_scope_write_fields,
-)
-from core.services.site_management import SiteManagementMixin
-from core.storage import get_storage
+from features.sites.edition import resolve_site_scope, site_management_permission, site_scope_ref, site_scope_write_fields
+from features.sites.management import SiteManagementMixin
+from core.files.storage import get_storage
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
@@ -225,7 +220,7 @@ def publish(
)
if project_id:
- from core.auth.permissions_iface import resolve_project_permission
+ from core.auth.edition import resolve_project_permission
from core.db.models import Project
project = self.db.get(Project, project_id)
diff --git a/src/backend/features/sites/tools/__init__.py b/src/backend/features/sites/tools/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/sites/tools/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/core/llm/tools/design_picker_tool.py b/src/backend/features/sites/tools/design_picker.py
similarity index 86%
rename from src/backend/core/llm/tools/design_picker_tool.py
rename to src/backend/features/sites/tools/design_picker.py
index 3d4cc175..e4519a83 100644
--- a/src/backend/core/llm/tools/design_picker_tool.py
+++ b/src/backend/features/sites/tools/design_picker.py
@@ -5,14 +5,15 @@
resumes in place once chosen. Conditionally registered by agent_factory only in
site-building chats (site-builder skill enabled).
"""
+from core.llm.factory.tools.definitions import function_tool
import logging
from typing import Any, Dict, List, Optional
from agentscope.tool import Toolkit
-from agentscope.tool._response import ToolChunk as ToolResponse
+from agentscope.tool._response import ToolChunk
-from core.llm.tools import _tool_confirmation as _mc
+from core.llm.tools import confirmation as _mc
from core.llm.tools._common import resp_json
logger = logging.getLogger(__name__)
@@ -40,7 +41,7 @@ def skill_uses_choose_design(skill_id: str) -> bool:
)
-def _err(msg: str) -> ToolResponse:
+def _err(msg: str) -> ToolChunk:
return resp_json({"ok": False, "error": msg})
@@ -61,7 +62,7 @@ def register_choose_design(
instead of erroring out on a missing tool.
"""
- async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolResponse:
+ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolChunk:
"""让用户从多个设计方案预览图中选择一个(**会挂起等待用户点选**)。
仅在建站流程中使用:先为同一站点做出 2-4 个(推荐 3 个)风格迥异的
@@ -97,7 +98,7 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes
if not isinstance(options, list) or not (_MIN_OPTIONS <= len(options) <= _MAX_OPTIONS):
return _err(f"options 必须是 {_MIN_OPTIONS}-{_MAX_OPTIONS} 个方案的列表(推荐 3 个)")
- from core.artifacts.store import get_artifact
+ from core.files.artifacts.store import get_artifact
normalized: List[Dict[str, Any]] = []
seen_ids: set[str] = set()
@@ -114,8 +115,8 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes
import asyncio
from fastapi import HTTPException
from core.infra.logging import user_id_var
- from core.artifacts.local_reference import project_file_path
- from core.artifacts.local_reference import reference_local_file
+ from core.files.artifacts.local_reference import project_file_path
+ from core.files.artifacts.local_reference import reference_local_file
if not isinstance(image_path, str):
return _err(f"options[{i}].image_path 必须为字符串")
@@ -199,10 +200,11 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes
}
return resp_json(payload)
- from core.llm.tool_permissions import ToolPermissionSpec, local_path_tool, READ
+ from core.sandbox.permissions.dispatch_models import ToolPermissionSpec, READ
+ from core.sandbox.permissions.tool_specs import local_path_tool
def resolve_image_paths(args, runtime):
- from core.artifacts.local_reference import project_file_path
+ from core.files.artifacts.local_reference import project_file_path
intents = []
for option in args.get("options") or []:
@@ -217,11 +219,29 @@ def resolve_image_paths(args, runtime):
))
return intents
- toolkit.register_tool_function(
- choose_design, namesake_strategy="override",
- permission=ToolPermissionSpec("design-preview-paths", resolve_image_paths),
- )
+ toolkit.append(function_tool(choose_design, dispatch_spec=ToolPermissionSpec("design-preview-paths", resolve_image_paths)))
logger.info("[factory] Registered choose_design tool (site-builder session)")
__all__ = ["register_choose_design", "skill_uses_choose_design"]
+
+
+def design_tool_enabled(context):
+ return bool(context.skill_ids) and any(
+ skill_uses_choose_design(str(sid)) for sid in context.skill_ids
+ )
+
+
+async def install_design_tool(toolkit, context):
+ request = context.request
+ register_choose_design(
+ toolkit, chat_id=request.chat_id,
+ interactive=context.interactive and not context.is_channel_run and not request.automation_run,
+ sandbox_session_id=context.sandbox_session_id,
+ user_id=request.current_user_id, scope=context.project_scope,
+ )
+
+
+def register_native_contribution():
+ from core.harness.registry import register_tool
+ register_tool("choose_design", install_design_tool, design_tool_enabled, stage="after_sandbox")
diff --git a/src/backend/features/workbench/README.md b/src/backend/features/workbench/README.md
new file mode 100644
index 00000000..ff79b980
--- /dev/null
+++ b/src/backend/features/workbench/README.md
@@ -0,0 +1,9 @@
+# 工作台
+
+拥有应用目录、外部应用边界、插件界面声明、安装修订、授权租约、资源归属、后台观察及工作台配置。托管业务应用仍属于 hosted_apps;工作台配置继续存入 ContentBlock,不增加数据库表。
+
+FULL 与 CE 均安装。API 在中央有序路由表注册,工作台内部路由组与配置保护保持原顺序。CE 的 migration_snapshot 位于相同功能路径,通过已有 overlay 保留版型差异。
+
+依赖 core 的认证、通用插件及技能运行、数据存储和扩展接口;工作台通过宿主注册 PackageUpgradePolicy 与 ResourceOwner,core 不保留工作台实现转发。站点元数据依赖 features.sites。工作台依赖桌面安装登记、版本修订和能力绑定;界面声明投影由宿主注入,桌面不反向依赖工作台。
+
+专属测试与 HTTP/进程观察夹具位于 tests/features/workbench。测试使用独立临时库和 .scratch 数据目录,不连接真实账号或用户数据。
diff --git a/src/backend/core/applications/__init__.py b/src/backend/features/workbench/__init__.py
similarity index 100%
rename from src/backend/core/applications/__init__.py
rename to src/backend/features/workbench/__init__.py
diff --git a/src/backend/core/applications/activity.py b/src/backend/features/workbench/activity.py
similarity index 98%
rename from src/backend/core/applications/activity.py
rename to src/backend/features/workbench/activity.py
index 35b412d0..702a49ce 100644
--- a/src/backend/core/applications/activity.py
+++ b/src/backend/features/workbench/activity.py
@@ -1,6 +1,6 @@
"""Bounded, durable activity projection; reads never load a page or start a worker."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
import re
@@ -9,7 +9,7 @@
from hashlib import sha256
from uuid import uuid4
-from core.applications import catalog, external
+from features.workbench import catalog, external
from core.db.models import ContentBlock
from core.infra.time import utc_now
@@ -96,7 +96,7 @@ def register(db, user_id, key, instance_id, expected, module_id=None):
def declared(item, db, user_id, module_id=None):
if item["declaration"]["target"]["kind"] != "plugin":
return set()
- from core.plugins.resources.installation import resolve
+ from core.capabilities.plugins.resources.installation import resolve
selected = resolve(db, item["binding"]["slug"], user_id, item["binding"]["installId"])
module = next(
diff --git a/src/backend/features/workbench/api/__init__.py b/src/backend/features/workbench/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/features/workbench/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/plugin_ui.py b/src/backend/features/workbench/api/plugin_ui.py
similarity index 91%
rename from src/backend/api/routes/v1/plugin_ui.py
rename to src/backend/features/workbench/api/plugin_ui.py
index 88ce9993..fe25415c 100644
--- a/src/backend/api/routes/v1/plugin_ui.py
+++ b/src/backend/features/workbench/api/plugin_ui.py
@@ -25,11 +25,12 @@
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.exceptions import ResourceNotFoundError
from core.infra.responses import success_response
-from core.plugins.ui import data_proxy as proxy
-from core.plugins.ui import contributions as plugin_ui
-from core.plugins.ui.contract import find_data_source, find_module
+from features.workbench.ui import data_proxy as proxy
+from features.workbench.ui import contributions as plugin_ui
+from features.workbench.ui.contract import find_data_source, find_module
router = APIRouter(prefix="/v1/plugins", tags=["Plugin UI"])
logger = logging.getLogger(__name__)
@@ -63,7 +64,7 @@ def list_ui_contributions(
definitions are projected down to id + parameter schema; the upstream URL
and auth header stay on the server.
"""
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
# 同上:云端账号的插件不投影进来,混合模式下这些插件的面板就整片消失。
items = device_catalog.merge_items(
@@ -87,14 +88,13 @@ async def call_plugin_data_source(
slug: str = Path(..., min_length=1, max_length=100),
source_id: str = Path(..., min_length=1, max_length=64),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
"""Run one manifest-declared upstream call on the caller's behalf.
Params are validated against the declaration's ``params_schema`` and
credentials are interpolated server-side, so the browser never sees them.
"""
- ui = _require_ui(db, slug, str(user.user_id))
+ ui = await run_in_session(_require_ui, slug, str(user.user_id))
source = find_data_source(ui, source_id)
if source is None:
raise ResourceNotFoundError("plugin_data_source", f"{slug}/{source_id}")
@@ -127,9 +127,9 @@ def get_plugin_web_asset(
def web_asset_response(db, slug, user_id, asset_path, install_id=None, *, _installation=None):
"""Serve authenticated or ticket-authorized static files with identical isolation."""
- from core.services.site_service import guess_site_mime
+ from features.sites.service import guess_site_mime
- from core.plugins.resources.installation import resolve, safe_path
+ from core.capabilities.plugins.resources.installation import resolve, safe_path
selected = _installation or resolve(db, slug, user_id, install_id)
target = safe_path(selected.package / "web", asset_path)
if not target.is_file():
diff --git a/src/backend/api/routes/v1/workbench.py b/src/backend/features/workbench/api/workbench.py
similarity index 82%
rename from src/backend/api/routes/v1/workbench.py
rename to src/backend/features/workbench/api/workbench.py
index e59267fd..59ece49c 100644
--- a/src/backend/api/routes/v1/workbench.py
+++ b/src/backend/features/workbench/api/workbench.py
@@ -1,18 +1,18 @@
"""Workbench directory and ordinary external links; managed SQL/MCP apps stay separate."""
-from core.applications.coordination import configuration
+from features.workbench.coordination import configuration
from typing import Any
-from api.deps import _resolve_session_user_id, require_config
-from api.routes.v1.workbench_auth import browser_write
-from api.routes.v1.workbench_auth import router as auth_router
-from core.applications.diagnostics import observed
-from core.applications import catalog, external, external_auth, policy, origin
-from core.applications.directory_revision import matches, snapshot
-from core.applications.external_embed import EmbeddingPolicy, launch_policy, validate_policy
-from core.applications.external_boundary import validate_cookie_scope
+from api.deps import console_actor, require_config
+from features.workbench.api.workbench_auth import browser_write
+from features.workbench.api.workbench_auth import router as auth_router
+from features.workbench.diagnostics import observed
+from features.workbench import catalog, external, external_auth, policy, origin
+from features.workbench.directory_revision import matches, snapshot
+from features.workbench.external_embed import EmbeddingPolicy, launch_policy, validate_policy
+from features.workbench.external_boundary import validate_cookie_scope
from core.auth.backend import UserContext, require_auth
from core.config.settings import settings
from core.db.engine import get_db
@@ -106,11 +106,14 @@ def admin_audit(response: Response, db: Session = Depends(get_db)):
"/admin/external-applications",
dependencies=[Depends(require_config), Depends(require_json_write)],
)
-async def create_external(
- body: ExternalWrite, request: Request, response: Response, db: Session = Depends(get_db)
+def create_external(
+ body: ExternalWrite,
+ request: Request,
+ response: Response,
+ actor: str = Depends(console_actor),
+ db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "no-store"
- actor = await _resolve_session_user_id(request) or "config_token"
external.validate_definition(db, body.declaration)
embedding = validate_policy(
body.declaration["target"],
@@ -128,15 +131,15 @@ async def create_external(
"/admin/external-applications/{key}",
dependencies=[Depends(require_config), Depends(require_json_write)],
)
-async def update_external(
+def update_external(
key: str,
body: ExternalUpdate,
request: Request,
response: Response,
+ actor: str = Depends(console_actor),
db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "no-store"
- actor = await _resolve_session_user_id(request) or "config_token"
external.validate_definition(db, body.declaration)
embedding = validate_policy(
body.declaration["target"],
@@ -205,7 +208,7 @@ def launch(
if not item["availability"]["available"]:
external.fail(409, item["availability"]["reason"])
if item["declaration"]["target"]["kind"] == "plugin":
- from core.applications.plugin_catalog import launch as plugin_launch
+ from features.workbench.plugin_catalog import launch as plugin_launch
return success_response(data=plugin_launch(db, user.user_id, item, moduleId))
if item["declaration"]["target"]["kind"] == "external":
@@ -269,7 +272,7 @@ def catalog_directory(
@router.get("/admin/catalog", dependencies=[Depends(require_config)])
def admin_catalog(response: Response, db: Session = Depends(get_db)):
response.headers["Cache-Control"] = "no-store"
- from core.applications.plugin_catalog import administrative_items
+ from features.workbench.plugin_catalog import administrative_items
return success_response(data={"items": catalog.definitions(db) + administrative_items(db)})
@@ -281,17 +284,19 @@ def admin_policy(response: Response, db: Session = Depends(get_db)):
@router.put("/admin/policy", dependencies=[Depends(require_config), Depends(require_json_write)])
-async def update_policy(
- body: PolicyWrite, request: Request, response: Response, db: Session = Depends(get_db)
+def update_policy(
+ body: PolicyWrite,
+ response: Response,
+ actor: str = Depends(console_actor),
+ db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "no-store"
- actor = await _resolve_session_user_id(request) or "config_token"
return success_response(data=policy.save(db, body.model_dump(), actor, trace_id_var.get()))
@router.get("/admin/legacy-config", dependencies=[Depends(require_config)])
def admin_legacy_config(response: Response, db: Session = Depends(get_db)):
- from core.applications import migration_compat
+ from features.workbench import migration_compat
response.headers["Cache-Control"] = "no-store"
return success_response(
@@ -304,39 +309,39 @@ def admin_legacy_config(response: Response, db: Session = Depends(get_db)):
router.include_router(auth_router)
-from api.routes.v1.workbench_preferences import routes as preference_routes
+from features.workbench.api.workbench_preferences import routes as preference_routes
router.include_router(preference_routes(WorkbenchRoute, require_json_write))
-from api.routes.v1.workbench_workspace import router as workspace_router
+from features.workbench.api.workbench_workspace import router as workspace_router
router.include_router(workspace_router)
-from api.routes.v1.workbench_resources import routes as resource_routes
+from features.workbench.api.workbench_resources import routes as resource_routes
router.include_router(resource_routes(WorkbenchRoute, require_json_write))
-from api.routes.v1.workbench_state import routes as state_routes
+from features.workbench.api.workbench_state import routes as state_routes
router.include_router(state_routes(WorkbenchRoute, require_json_write))
-from api.routes.v1.workbench_capabilities import router as capability_router
+from features.workbench.api.workbench_capabilities import router as capability_router
router.include_router(capability_router)
-from api.routes.v1.workbench_events import routes as event_routes
+from features.workbench.api.workbench_events import routes as event_routes
router.include_router(event_routes(WorkbenchRoute))
-from api.routes.v1.workbench_activity import routes as activity_routes
-from api.routes.v1.workbench_diagnostics import routes as diagnostic_routes
-from api.routes.v1.workbench_upgrades import routes as upgrade_routes
-from api.routes.v1.workbench_revision_leases import routes as revision_routes
+from features.workbench.api.workbench_activity import routes as activity_routes
+from features.workbench.api.workbench_diagnostics import routes as diagnostic_routes
+from features.workbench.api.workbench_upgrades import routes as upgrade_routes
+from features.workbench.api.workbench_revision_leases import routes as revision_routes
router.include_router(activity_routes(WorkbenchRoute, require_json_write))
router.include_router(diagnostic_routes(WorkbenchRoute, require_json_write))
router.include_router(upgrade_routes(WorkbenchRoute, require_json_write))
router.include_router(revision_routes(WorkbenchRoute, require_json_write))
-from api.routes.v1.workbench_application_config import router as application_config_router
+from features.workbench.api.workbench_application_config import router as application_config_router
router.include_router(application_config_router)
diff --git a/src/backend/api/routes/v1/workbench_activity.py b/src/backend/features/workbench/api/workbench_activity.py
similarity index 82%
rename from src/backend/api/routes/v1/workbench_activity.py
rename to src/backend/features/workbench/api/workbench_activity.py
index f3c7409b..1547ea06 100644
--- a/src/backend/api/routes/v1/workbench_activity.py
+++ b/src/backend/features/workbench/api/workbench_activity.py
@@ -6,9 +6,10 @@
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy.orm import Session
-from core.applications import activity, background
+from features.workbench import activity, background
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
@@ -44,6 +45,14 @@ def private(response: Response):
response.headers["Cache-Control"] = "private, no-store"
+def _check_declared(db, user_id, key, instance_id, capabilities):
+ _, row, item = activity.authorize(db, user_id, key, instance_id)
+ if set(capabilities) - activity.declared(item, db, user_id, row.get("moduleId")):
+ from features.workbench.external import fail
+
+ fail(403, "capability_not_declared")
+
+
def routes(route_class, json_write):
router = APIRouter(
route_class=route_class, dependencies=[Depends(require_auth()), Depends(private)]
@@ -54,11 +63,10 @@ def routes(route_class, json_write):
async def projection(
response: Response,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "private, no-store"
- await background.reconcile(db, user.user_id)
- return success_response(data=activity.projection(db, user.user_id))
+ await background.reconcile(user.user_id)
+ return success_response(data=await run_in_session(activity.projection, user.user_id))
@router.post(path, dependencies=[Depends(json_write)])
def register(
@@ -97,17 +105,14 @@ async def consent(
instance_id: str,
body: Consent,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
- _, row, item = activity.authorize(db, user.user_id, key, instance_id)
- if set(body.capabilities) - activity.declared(item, db, user.user_id, row.get("moduleId")):
- from core.applications.external import fail
-
- fail(403, "capability_not_declared")
+ await run_in_session(_check_declared, user.user_id, key, instance_id, body.capabilities)
if "background" not in body.capabilities:
- await background.stop_all(db, user.user_id, application_key=key)
+ await background.stop_all(user.user_id, application_key=key)
return success_response(
- data=activity.consent(db, user.user_id, key, instance_id, body.capabilities)
+ data=await run_in_session(
+ activity.consent, user.user_id, key, instance_id, body.capabilities
+ )
)
@router.put(path + "/badge", dependencies=[Depends(json_write)])
@@ -161,10 +166,9 @@ async def start(
instance_id: str,
body: Background,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
return success_response(
- data=await background.start(db, user.user_id, key, instance_id, body.resourceId)
+ data=await background.start(user.user_id, key, instance_id, body.resourceId)
)
@router.post(path + "/background/{resource_id}/lease", dependencies=[Depends(json_write)])
@@ -173,10 +177,9 @@ async def renew(
instance_id: str,
resource_id: str,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
return success_response(
- data=await background.renew(db, user.user_id, key, instance_id, resource_id)
+ data=await background.renew(user.user_id, key, instance_id, resource_id)
)
@router.delete(path + "/background/{resource_id}")
@@ -185,9 +188,8 @@ async def stop(
instance_id: str,
resource_id: str,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
- await background.stop(db, user.user_id, key, instance_id, resource_id)
+ await background.stop(user.user_id, key, instance_id, resource_id)
return success_response(data={})
return router
diff --git a/src/backend/api/routes/v1/workbench_application_config.py b/src/backend/features/workbench/api/workbench_application_config.py
similarity index 88%
rename from src/backend/api/routes/v1/workbench_application_config.py
rename to src/backend/features/workbench/api/workbench_application_config.py
index 44f6bef7..2e2499f4 100644
--- a/src/backend/api/routes/v1/workbench_application_config.py
+++ b/src/backend/features/workbench/api/workbench_application_config.py
@@ -1,9 +1,9 @@
"""Protected administration of exact installed Workbench configuration."""
from api.deps import require_system_settings, require_admin
-from api.routes.v1.workbench_capabilities import CapabilityRoute
-from core.applications import application_config as config
-from core.applications.application_config_bindings import selections, resolve_admin
-from core.applications.external import fail
+from features.workbench.api.workbench_capabilities import CapabilityRoute
+from features.workbench import application_config as config
+from features.workbench.application_config_bindings import selections, resolve_admin
+from features.workbench.external import fail
from core.db.engine import get_db
from core.auth.backend import UserContext, require_auth
from core.infra.responses import success_response
@@ -15,14 +15,14 @@
tags=["workbench installed configuration"])
-async def configuration_admin(request: Request, db: Session = Depends(get_db),
+async def configuration_admin(request: Request,
user: UserContext | None = Depends(require_auth(False))):
if user is not None and user.api_key_id:
fail(403, "configuration_session_required")
try:
- return await require_system_settings(request, db, user)
+ return await require_system_settings(request, user)
except HTTPException:
- await require_admin(request, request.headers.get("authorization"), db)
+ await require_admin(request, request.headers.get("authorization"))
return user.user_id if user is not None else "admin_token"
diff --git a/src/backend/api/routes/v1/workbench_auth.py b/src/backend/features/workbench/api/workbench_auth.py
similarity index 92%
rename from src/backend/api/routes/v1/workbench_auth.py
rename to src/backend/features/workbench/api/workbench_auth.py
index 4f0d5430..66a47b3b 100644
--- a/src/backend/api/routes/v1/workbench_auth.py
+++ b/src/backend/features/workbench/api/workbench_auth.py
@@ -4,9 +4,10 @@
from typing import Literal
from urllib.parse import urlencode
-from api.deps import _resolve_session_user_id, require_config
-from core.applications import external_auth
-from core.applications.external_boundary import validate_cookie_scope
+from api.deps import console_actor, require_config
+from core.db.threads import run_in_session
+from features.workbench import external_auth
+from features.workbench.external_boundary import validate_cookie_scope
from core.auth.backend import UserContext, require_auth
from core.auth.session import find_session_by_hash
from core.config.settings import settings
@@ -18,7 +19,6 @@
from fastapi.routing import APIRoute
from pydantic import BaseModel, ConfigDict, Field, model_validator
from sqlalchemy.orm import Session
-from starlette.concurrency import run_in_threadpool
class PrivateRoute(APIRoute):
@@ -107,19 +107,18 @@ def auth_config(key: str, db: Session = Depends(get_db)):
@router.put(
"/admin/external-auth/{key}", dependencies=[Depends(require_config), Depends(browser_write)]
)
-async def update_auth(key: str, body: AuthWrite, request: Request, db: Session = Depends(get_db)):
- actor = await _resolve_session_user_id(request) or "config_token"
- data = await run_in_threadpool(
- external_auth.save, db, key, body.model_dump(), actor, trace_id_var.get()
- )
+def update_auth(
+ key: str, body: AuthWrite, actor: str = Depends(console_actor), db: Session = Depends(get_db)
+):
+ data = external_auth.save(db, key, body.model_dump(), actor, trace_id_var.get())
return success_response(data=data)
@router.post("/external-auth/exchange", dependencies=[Depends(browser_write)])
-async def exchange(body: Exchange, request: Request, db: Session = Depends(get_db)):
+async def exchange(body: Exchange, request: Request):
auth = request.headers.get("authorization", "")
credential = auth[len("Bearer ") :] if auth.startswith("Bearer ") else ""
- data = await run_in_threadpool(external_auth.redeem, db, body.model_dump(), credential)
+ data = await run_in_session(external_auth.redeem, body.model_dump(), credential)
session_hash = data.pop("_sessionHash", None)
if session_hash:
session = await find_session_by_hash(session_hash)
diff --git a/src/backend/api/routes/v1/workbench_capabilities.py b/src/backend/features/workbench/api/workbench_capabilities.py
similarity index 81%
rename from src/backend/api/routes/v1/workbench_capabilities.py
rename to src/backend/features/workbench/api/workbench_capabilities.py
index af29dd85..1b384353 100644
--- a/src/backend/api/routes/v1/workbench_capabilities.py
+++ b/src/backend/features/workbench/api/workbench_capabilities.py
@@ -1,6 +1,6 @@
"""Authenticated host broker. Opaque plugin frames never receive approval credentials."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
import asyncio
@@ -8,13 +8,14 @@
from typing import Any
from urllib.parse import quote, urlencode
-from core.applications import capabilities as grants
-from core.applications import external, application_config
+from features.workbench import capabilities as grants
+from features.workbench import external, application_config
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.plugins.ui.contract import find_data_source
-from core.plugins.ui.data_proxy import call_data_source
+from features.workbench.ui.contract import find_data_source
+from features.workbench.ui.data_proxy import call_data_source
from fastapi import (
APIRouter,
Depends,
@@ -165,47 +166,58 @@ def revoke(
return success_response(data={"revoked": True})
+def _reserve_query(db, user_id, key, body):
+ lease, selected, effective = grants.reserve(
+ db, user_id, key, body.revision, body.approvalId, "data.query", body.instanceId
+ )
+ try:
+ if "data_source:" + body.sourceId not in effective:
+ external.fail(403, "data_source_denied")
+ source = find_data_source(selected.ui, body.sourceId)
+ if source is None:
+ external.fail(404, "data_source_not_found")
+ return lease, selected, source, {
+ "config_values": application_config.data_values(db, key, selected),
+ "configured_origins": application_config.trusted_origins(db, key, selected),
+ }
+ except BaseException:
+ grants.release(db, body.approvalId, lease)
+ raise
+
+
+def _confirm(db, user_id, key, revision, approval_id, capability, instance_id):
+ grants.authorized(db, user_id, key, revision, approval_id, capability, instance_id)
+
+
@router.post("/data/query")
async def query(
key: str,
body: Query,
response: Response,
user: UserContext = Depends(host_identity),
- db: Session = Depends(get_db),
):
if len(json.dumps(body.params).encode()) > 64 * 1024:
external.fail(413, "request_too_large")
- lease, selected, effective = grants.reserve(
- db, user.user_id, key, body.revision, body.approvalId, "data.query", body.instanceId
+ lease, selected, source, configuration = await run_in_session(
+ _reserve_query, user.user_id, key, body
)
try:
- if "data_source:" + body.sourceId not in effective:
- external.fail(403, "data_source_denied")
- source = find_data_source(selected.ui, body.sourceId)
- if source is None:
- external.fail(404, "data_source_not_found")
try:
result = await asyncio.wait_for(
- call_data_source(
- source,
- body.params,
- slug=selected.slug,
- config_values=application_config.data_values(db, key, selected),
- configured_origins=application_config.trusted_origins(db, key, selected),
- ),
+ call_data_source(source, body.params, slug=selected.slug, **configuration),
timeout=30,
)
except TimeoutError:
external.fail(504, "capability_timeout")
- grants.authorized(
- db, user.user_id, key, body.revision, body.approvalId, "data.query", body.instanceId
+ await run_in_session(
+ _confirm, user.user_id, key, body.revision, body.approvalId, "data.query", body.instanceId
)
if len(json.dumps(result).encode()) > 256 * 1024:
external.fail(413, "response_too_large")
response.headers["Cache-Control"] = "no-store"
return success_response(data=result)
finally:
- grants.release(db, body.approvalId, lease)
+ await run_in_session(grants.release, body.approvalId, lease)
def file_info(artifact):
@@ -314,7 +326,7 @@ def download(
import tempfile
from pathlib import Path
- from core.storage import get_storage
+ from core.files.storage import get_storage
from fastapi.responses import FileResponse
from starlette.background import BackgroundTask
@@ -343,6 +355,27 @@ def download(
)
+def _reserve_save(db, user_id, key, revision, approval_id, instance_id):
+ lease, _, _ = grants.reserve(db, user_id, key, revision, approval_id, "file.save", instance_id)
+ return lease
+
+
+def _store_saved(db, user_id, key, revision, approval_id, instance_id, content, filename, mime_type):
+ runtime(db, user_id)
+ grants.authorized(db, user_id, key, revision, approval_id, "file.save", instance_id)
+ from core.files.artifacts.service import store_bytes_as_artifact
+
+ artifact = store_bytes_as_artifact(
+ db,
+ user_id=user_id,
+ content=content,
+ filename=filename,
+ mime_type=mime_type,
+ source="workbench",
+ )
+ return file_info(artifact)
+
+
@router.post("/files/save")
async def save(
key: str,
@@ -351,11 +384,8 @@ async def save(
instanceId: str | None = Form(None),
file: UploadFile = File(...),
user: UserContext = Depends(host_identity),
- db: Session = Depends(get_db),
):
- lease, _, _ = grants.reserve(
- db, user.user_id, key, revision, approvalId, "file.save", instanceId
- )
+ lease = await run_in_session(_reserve_save, user.user_id, key, revision, approvalId, instanceId)
try:
content = await asyncio.wait_for(file.read(50 * 1024 * 1024 + 1), timeout=30)
if not content or len(content) > 50 * 1024 * 1024:
@@ -363,18 +393,10 @@ async def save(
filename = file.filename or ""
if not filename or len(filename) > 200 or "/" in filename or "\\" in filename:
external.fail(422, "invalid_filename")
- runtime(db, user.user_id)
- grants.authorized(db, user.user_id, key, revision, approvalId, "file.save", instanceId)
- from core.services.artifact_service import store_bytes_as_artifact
-
- artifact = store_bytes_as_artifact(
- db,
- user_id=user.user_id,
- content=content,
- filename=filename,
- mime_type=file.content_type,
- source="workbench",
+ info = await run_in_session(
+ _store_saved, user.user_id, key, revision, approvalId, instanceId,
+ content, filename, file.content_type,
)
- return success_response(data=file_info(artifact))
+ return success_response(data=info)
finally:
- grants.release(db, approvalId, lease)
+ await run_in_session(grants.release, approvalId, lease)
diff --git a/src/backend/api/routes/v1/workbench_diagnostics.py b/src/backend/features/workbench/api/workbench_diagnostics.py
similarity index 97%
rename from src/backend/api/routes/v1/workbench_diagnostics.py
rename to src/backend/features/workbench/api/workbench_diagnostics.py
index bf021098..d130e30a 100644
--- a/src/backend/api/routes/v1/workbench_diagnostics.py
+++ b/src/backend/features/workbench/api/workbench_diagnostics.py
@@ -3,7 +3,7 @@
from typing import Any, Literal
from api.deps import require_config
-from core.applications import catalog, diagnostics, external, plugin_catalog, origin
+from features.workbench import catalog, diagnostics, external, plugin_catalog, origin
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.db.models import ContentBlock, UserShadow
@@ -119,7 +119,7 @@ def own_resources(
user: UserContext = Depends(require_auth()),
db: Session = Depends(get_db),
):
- from core.applications.diagnostic_resources import snapshot
+ from features.workbench.diagnostic_resources import snapshot
response.headers["Cache-Control"] = "private, no-store"
return success_response(data=snapshot(db, user.user_id))
@@ -166,7 +166,7 @@ def preview(body: Preview, response: Response, db: Session = Depends(get_db)):
@router.post("/admin/diagnostics/probe/{key}", dependencies=write)
def check(key: str, response: Response, db: Session = Depends(get_db)):
- from core.applications.diagnostic_probe import probe
+ from features.workbench.diagnostic_probe import probe
response.headers["Cache-Control"] = "no-store"
item = external.record(db, key)
diff --git a/src/backend/api/routes/v1/workbench_events.py b/src/backend/features/workbench/api/workbench_events.py
similarity index 85%
rename from src/backend/api/routes/v1/workbench_events.py
rename to src/backend/features/workbench/api/workbench_events.py
index a0b3f3d7..7ba5c0f8 100644
--- a/src/backend/api/routes/v1/workbench_events.py
+++ b/src/backend/features/workbench/api/workbench_events.py
@@ -3,7 +3,7 @@
import asyncio
import time
-from core.applications.directory_revision import snapshot
+from features.workbench.directory_revision import snapshot
from core.auth.backend import UserContext, get_current_user, require_auth
from core.db.engine import SessionLocal
from core.infra.responses import success_response
@@ -29,10 +29,9 @@ async def directory_events(
while True:
# No request-long transaction or cached user capabilities. Recheck session and
# projection even if Last-Event-ID/revision comes from an authenticated client.
- with SessionLocal() as db:
- current = await get_current_user(request, credentials=None, db=db)
- if current.user_id != user.user_id:
- raise HTTPException(401, detail={"code": "session_revoked"})
+ current = await get_current_user(request, None)
+ if current.user_id != user.user_id:
+ raise HTTPException(401, detail={"code": "session_revoked"})
data, scope = await run_in_threadpool(current_snapshot, current.user_id)
if data["revision"] != revision:
response.headers["Cache-Control"] = "private, no-store"
diff --git a/src/backend/api/routes/v1/workbench_preferences.py b/src/backend/features/workbench/api/workbench_preferences.py
similarity index 97%
rename from src/backend/api/routes/v1/workbench_preferences.py
rename to src/backend/features/workbench/api/workbench_preferences.py
index 32b962e6..0cad1efa 100644
--- a/src/backend/api/routes/v1/workbench_preferences.py
+++ b/src/backend/features/workbench/api/workbench_preferences.py
@@ -2,7 +2,7 @@
from typing import Literal
-from core.applications import preferences
+from features.workbench import preferences
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.infra.responses import success_response
diff --git a/src/backend/api/routes/v1/workbench_resources.py b/src/backend/features/workbench/api/workbench_resources.py
similarity index 69%
rename from src/backend/api/routes/v1/workbench_resources.py
rename to src/backend/features/workbench/api/workbench_resources.py
index 74e7bc40..f1ee2311 100644
--- a/src/backend/api/routes/v1/workbench_resources.py
+++ b/src/backend/features/workbench/api/workbench_resources.py
@@ -1,12 +1,12 @@
"""Authenticated application resource routes; shared tickets/stream remain reusable."""
from core.auth.backend import UserContext, require_auth
-from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.plugins.resources import application, service
+from core.harness.registry.resource_owners import resource_owner
+from core.capabilities.plugins.resources import service
from fastapi import APIRouter, Depends, Header, HTTPException, Response
from pydantic import BaseModel, ConfigDict, Field
-from sqlalchemy.orm import Session
class ResourceOpen(BaseModel):
@@ -24,6 +24,23 @@ class ResourceCommand(BaseModel):
params: dict = Field(default_factory=dict)
+def _owned(db, user_id, key, instance_id, resource_id, *, allow_closed=False):
+ row = resource_owner().owned(db, user_id, key, instance_id, resource_id, allow_closed=allow_closed)
+ return service.detached(db, row)
+
+
+def _commandable(db, user_id, key, instance_id, resource_id):
+ from core.capabilities.plugins.resources.owners import resolve_installation
+ from features.workbench.ui.contract import find_module
+
+ row = resource_owner().owned(db, user_id, key, instance_id, resource_id)
+ selected = resolve_installation(db, row, row.user_id)
+ module = find_module(selected.ui, row.module_id)
+ if not module or "resource.command" not in module.get("grants", []):
+ raise HTTPException(403, "resource_command_not_granted")
+ return service.detached(db, row)
+
+
def routes(route_class, json_write):
router = APIRouter(
prefix="/applications/{key}/instances/{instance_id}/resources",
@@ -39,15 +56,13 @@ async def create(
response: Response,
idempotency_key: str = Header(alias="Idempotency-Key"),
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
from fastapi import HTTPException
if not instance_id or len(instance_id) > 128:
raise HTTPException(422, "invalid_instance_id")
response.headers["Cache-Control"] = "no-store"
- value = await application.create(
- db,
+ value = await resource_owner().create(
str(user.user_id),
key,
instance_id,
@@ -63,12 +78,11 @@ async def lease(
resource_id: str,
response: Response,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "no-store"
return success_response(
data={
- "resource": await application.renew(db, str(user.user_id), key, instance_id, resource_id)
+ "resource": await resource_owner().lease(str(user.user_id), key, instance_id, resource_id)
}
)
@@ -79,14 +93,13 @@ async def close(
resource_id: str,
response: Response,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
response.headers["Cache-Control"] = "no-store"
- row = application.owned(
- db, str(user.user_id), key, instance_id, resource_id, allow_closed=True
+ row = await run_in_session(
+ _owned, str(user.user_id), key, instance_id, resource_id, allow_closed=True
)
if row.status != "closed":
- await service.close(db, row)
+ await service.close(row)
return success_response(data={"closed": True})
@router.post("/{resource_id}/command")
@@ -97,18 +110,10 @@ async def command(
body: ResourceCommand,
response: Response,
user: UserContext = Depends(require_auth()),
- db: Session = Depends(get_db),
):
import json
- from core.plugins.resources.owners import resolve_installation
- from core.plugins.ui.contract import find_module
-
- row = application.owned(db, str(user.user_id), key, instance_id, resource_id)
- selected = resolve_installation(db, row, row.user_id)
- module = find_module(selected.ui, row.module_id)
- if not module or "resource.command" not in module.get("grants", []):
- raise HTTPException(403, "resource_command_not_granted")
+ row = await run_in_session(_commandable, str(user.user_id), key, instance_id, resource_id)
if len(json.dumps(body.model_dump()).encode()) > 256 * 1024:
raise HTTPException(413, "resource_command_too_large")
response.headers["Cache-Control"] = "no-store"
diff --git a/src/backend/api/routes/v1/workbench_revision_leases.py b/src/backend/features/workbench/api/workbench_revision_leases.py
similarity index 97%
rename from src/backend/api/routes/v1/workbench_revision_leases.py
rename to src/backend/features/workbench/api/workbench_revision_leases.py
index 1cd0dcf5..3a8d20f5 100644
--- a/src/backend/api/routes/v1/workbench_revision_leases.py
+++ b/src/backend/features/workbench/api/workbench_revision_leases.py
@@ -1,6 +1,6 @@
"""Revision retention is bounded and tied to an authenticated running instance."""
-from core.applications import plugin_revisions
+from features.workbench import plugin_revisions
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.infra.responses import success_response
diff --git a/src/backend/api/routes/v1/workbench_state.py b/src/backend/features/workbench/api/workbench_state.py
similarity index 98%
rename from src/backend/api/routes/v1/workbench_state.py
rename to src/backend/features/workbench/api/workbench_state.py
index a6d59894..47b6e6e3 100644
--- a/src/backend/api/routes/v1/workbench_state.py
+++ b/src/backend/features/workbench/api/workbench_state.py
@@ -1,7 +1,7 @@
"""Authenticated state boundary, mounted under /v1/workbench."""
from typing import Any, Annotated
-from core.applications import state
+from features.workbench import state
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.infra.responses import success_response
diff --git a/src/backend/api/routes/v1/workbench_upgrades.py b/src/backend/features/workbench/api/workbench_upgrades.py
similarity index 98%
rename from src/backend/api/routes/v1/workbench_upgrades.py
rename to src/backend/features/workbench/api/workbench_upgrades.py
index d4725876..8a4ace7b 100644
--- a/src/backend/api/routes/v1/workbench_upgrades.py
+++ b/src/backend/features/workbench/api/workbench_upgrades.py
@@ -1,7 +1,7 @@
"""Private owner and administrator upgrade decisions; candidate data never grants access."""
from api.deps import require_admin
-from core.applications import plugin_upgrades
+from features.workbench import plugin_upgrades
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
from core.infra.responses import success_response
diff --git a/src/backend/api/routes/v1/workbench_workspace.py b/src/backend/features/workbench/api/workbench_workspace.py
similarity index 83%
rename from src/backend/api/routes/v1/workbench_workspace.py
rename to src/backend/features/workbench/api/workbench_workspace.py
index 57be0d51..7a3c33c5 100644
--- a/src/backend/api/routes/v1/workbench_workspace.py
+++ b/src/backend/features/workbench/api/workbench_workspace.py
@@ -1,13 +1,13 @@
"""Authenticated workspace package assets. Every fetch rechecks the exact binding."""
-from api.routes.v1.plugin_ui import _MODULE_CSP
-from core.applications import catalog
-from core.applications.plugin_revisions import resolve_revision
-from core.applications.workspace_page import render
+from features.workbench.api.plugin_ui import _MODULE_CSP
+from features.workbench import catalog
+from features.workbench.plugin_revisions import resolve_revision
+from features.workbench.workspace_page import render
from core.auth.backend import UserContext, require_auth
from core.db.engine import get_db
-from core.plugins.resources.installation import safe_path
-from core.services.site_service import guess_site_mime
+from core.capabilities.plugins.resources.installation import safe_path
+from features.sites.service import guess_site_mime
from fastapi.responses import FileResponse, HTMLResponse
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
diff --git a/src/backend/core/applications/application_config.py b/src/backend/features/workbench/application_config.py
similarity index 93%
rename from src/backend/core/applications/application_config.py
rename to src/backend/features/workbench/application_config.py
index 62b6dc14..becd4629 100644
--- a/src/backend/core/applications/application_config.py
+++ b/src/backend/features/workbench/application_config.py
@@ -9,13 +9,13 @@
from hashlib import sha256
from urllib.parse import urlsplit
-from core.applications import origin
-from core.applications.external import fail
-from core.applications.coordination import configuration
+from features.workbench import origin
+from features.workbench.external import fail
+from features.workbench.coordination import configuration
from core.db.models import ContentBlock, SystemConfig
from core.infra.time import utc_now
-from core.plugins.packaging.importer import _normalize_admin_config, manifest_extensions
-from core.plugins.resources.crypto import encrypt_secret, decrypt_secret
+from core.capabilities.plugins.packaging.importer import normalize_admin_config as _normalize_admin_config, manifest_extensions
+from core.capabilities.plugins.resources.crypto import encrypt_secret, decrypt_secret
PREFIX = "workbench_app_config:"
GROUP = "workbench_app_config"
@@ -112,7 +112,7 @@ def view(db, item, selected, with_values=False):
def write(db, actor, key, revision, values, clear_keys):
- from core.applications.application_config_bindings import resolve_admin
+ from features.workbench.application_config_bindings import resolve_admin
configuration(db)
db.expire_all()
@@ -171,7 +171,7 @@ def write(db, actor, key, revision, values, clear_keys):
)
)
db.commit()
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
SystemConfigService.get_instance().invalidate_cache()
return view(db, item, selected, with_values=True)
diff --git a/src/backend/core/applications/application_config_bindings.py b/src/backend/features/workbench/application_config_bindings.py
similarity index 82%
rename from src/backend/core/applications/application_config_bindings.py
rename to src/backend/features/workbench/application_config_bindings.py
index 90bf9613..a3ac2a22 100644
--- a/src/backend/core/applications/application_config_bindings.py
+++ b/src/backend/features/workbench/application_config_bindings.py
@@ -1,25 +1,25 @@
"""Admin discovery resolves current installations without granting execution."""
-from core.applications import origin
-from core.applications.external import fail
-from core.applications.plugin_catalog import application_key
-from core.applications.protocol import validate_contributions
+from features.workbench import origin
+from features.workbench.external import fail
+from features.workbench.plugin_catalog import application_key
+from features.workbench.protocol import validate_contributions
from core.db.models import InstalledPlugin, UserShadow
-from core.plugins.resources.installation import Installation
-from core.plugins.packaging.runtime_assets import materialize
+from core.capabilities.plugins.resources.installation import Installation
+from core.capabilities.plugins.packaging.runtime_assets import materialize
def selections(db, actor):
source = origin.current(db)
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
represented = set()
if source["kind"] == "local":
- from core.capabilities import plugins, store
- from core.capabilities.dependency import component_hash
- from core.capabilities.local_plugin_runtime import enabled_for
- from core.applications.installation_binding import DeviceRecord
+ from features.desktop.capabilities import plugins, store
+ from features.desktop.capabilities.dependency import component_hash
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
+ from features.workbench.installation_binding import DeviceRecord
owners = {actor} | {r.user_id for r in db.query(UserShadow).all()}
for owner in owners:
- for inst in device_catalog._installations("plugin", user_id=owner):
+ for inst in device_catalog.installations("plugin", user_id=owner):
if not inst.ready or inst.created_at is None:
continue
component = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision)
@@ -83,7 +83,7 @@ def resolve_admin(db, actor, key, revision):
def _enabled(db, row, actor):
- from core.applications.installation_binding import exact
+ from features.workbench.installation_binding import exact
from fastapi import HTTPException
try:
exact(db, row.slug, row.owner_user_id or actor, row.install_id)
diff --git a/src/backend/core/applications/background.py b/src/backend/features/workbench/background.py
similarity index 63%
rename from src/backend/core/applications/background.py
rename to src/backend/features/workbench/background.py
index 66486fe2..02495bcd 100644
--- a/src/backend/core/applications/background.py
+++ b/src/backend/features/workbench/background.py
@@ -1,12 +1,13 @@
"""Separate background permission and budget for existing application resources."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
import time
import logging
-from core.applications import activity, external
-from core.plugins.resources import service, store
+from features.workbench import activity, external
+from core.capabilities.plugins.resources import service, store
+from core.db.threads import run_in_session
MAX_PER_USER = 2
MAX_PER_APP = 1
@@ -51,13 +52,15 @@ def admission(db, user_id, key, instance_id, resource_id):
return value, instance, row
-async def start(db, user_id, key, instance_id, resource_id):
- from core.plugins.resources.deadlines import arm, provider_for
+def _probe(db, user_id, key, instance_id, resource_id):
+ from core.capabilities.plugins.resources.deadlines import provider_for
- # Probe the runner without holding a database write/coordination lock.
_, _, probe = admission(db, user_id, key, instance_id, resource_id)
provider_for(probe)
- await arm(db, probe, require_background=True)
+
+
+def _reserve(db, user_id, key, instance_id, resource_id):
+ """Charge a background reservation under the owner lock; None when one exists."""
runtime(db, user_id)
db.expire_all()
# Authorization and quotas are evaluated again after the probe and owner lock.
@@ -67,7 +70,8 @@ async def start(db, user_id, key, instance_id, resource_id):
if existing:
if existing["expiresAt"] <= now or existing["deadline"] <= now:
external.fail(410, "background_expired")
- return {"resourceId": resource_id, "expiresAt": existing["expiresAt"]}
+ db.commit()
+ return existing["expiresAt"], False
# Expired reservations remain charged until actual stop/reconcile.
if (
len(value["background"]) >= MAX_PER_USER
@@ -84,7 +88,18 @@ async def start(db, user_id, key, instance_id, resource_id):
}
row.expires_at = min(row.expires_at, expires)
activity.save(db, user_id, value)
- await arm(db, row, require_background=True)
+ return expires, True
+
+
+async def start(user_id, key, instance_id, resource_id):
+ from core.capabilities.plugins.resources.deadlines import arm
+
+ # Probe the runner without holding a database write/coordination lock.
+ await run_in_session(_probe, user_id, key, instance_id, resource_id)
+ await arm(resource_id, require_background=True)
+ expires, reserved = await run_in_session(_reserve, user_id, key, instance_id, resource_id)
+ if reserved:
+ await arm(resource_id, require_background=True)
return {"resourceId": resource_id, "expiresAt": expires}
@@ -115,9 +130,7 @@ def authorized_resource(db, row):
return True
-async def renew(db, user_id, key, instance_id, resource_id):
- from core.plugins.resources.deadlines import arm
-
+def _extend(db, user_id, key, instance_id, resource_id):
runtime(db, user_id)
db.expire_all()
value = activity.read(db, user_id)
@@ -130,11 +143,18 @@ async def renew(db, user_id, key, instance_id, resource_id):
record["expiresAt"] = expires
row.expires_at = expires
activity.save(db, user_id, value)
- await arm(db, row, require_background=True)
+ return expires
+
+
+async def renew(user_id, key, instance_id, resource_id):
+ from core.capabilities.plugins.resources.deadlines import arm
+
+ expires = await run_in_session(_extend, user_id, key, instance_id, resource_id)
+ await arm(resource_id, require_background=True)
return {"resourceId": resource_id, "expiresAt": expires}
-async def stop(db, user_id, key, instance_id, resource_id):
+def _owned_row(db, user_id, key, instance_id, resource_id):
# Ownership, not availability, gates cleanup so revoked users can still stop.
row = store.get(db, resource_id, user_id)
if (
@@ -142,7 +162,11 @@ async def stop(db, user_id, key, instance_id, resource_id):
or getattr(row, "instance_id", None) != instance_id
):
external.fail(404, "background_not_found")
- await service.close(db, row)
+ db.expunge(row)
+ return row
+
+
+def _release(db, user_id, resource_id):
runtime(db, user_id)
db.expire_all()
value = activity.read(db, user_id)
@@ -150,37 +174,53 @@ async def stop(db, user_id, key, instance_id, resource_id):
activity.save(db, user_id, value)
-async def stop_all(db, user_id, application_key=None, install_id=None):
- value = activity.read(db, user_id)
- for rid, record in list(value["background"].items()):
+async def stop(user_id, key, instance_id, resource_id):
+ row = await run_in_session(_owned_row, user_id, key, instance_id, resource_id)
+ await service.close(row)
+ await run_in_session(_release, user_id, resource_id)
+
+
+def _records(db, user_id):
+ return list(activity.read(db, user_id)["background"].items())
+
+
+async def stop_all(user_id, application_key=None, install_id=None):
+ for rid, record in await run_in_session(_records, user_id):
if (application_key is None or record["applicationKey"] == application_key) and (
install_id is None or record["binding"].get("installId") == install_id
):
- await stop(db, user_id, record["applicationKey"], record["instanceId"], rid)
+ await stop(user_id, record["applicationKey"], record["instanceId"], rid)
-async def reconcile(db, user_id):
+def _revoked_row(db, user_id, rid, record):
+ """None while the reservation is still authorized; else the row to stop (or False if gone)."""
from fastapi import HTTPException
- value = activity.read(db, user_id)
- for rid, record in list(value["background"].items()):
- try:
- row = resource(db, rid, user_id, record["applicationKey"], record["instanceId"])
- authorized_resource(db, row)
- except HTTPException:
- # Metadata is retained until real shutdown succeeds.
- row = db.get(store.PluginResource, rid)
- if row is not None and row.user_id == user_id:
- try:
- await service.close(db, row)
- except Exception as error:
- db.rollback()
- logging.getLogger(__name__).warning(
- "Background cleanup deferred: %s", type(error).__name__
- )
- continue
- runtime(db, user_id)
- db.expire_all()
- current = activity.read(db, user_id)
- current["background"].pop(rid, None)
- activity.save(db, user_id, current)
+ try:
+ row = resource(db, rid, user_id, record["applicationKey"], record["instanceId"])
+ authorized_resource(db, row)
+ return None
+ except HTTPException:
+ db.rollback()
+ # Metadata is retained until real shutdown succeeds.
+ row = db.get(store.PluginResource, rid)
+ if row is None or row.user_id != user_id:
+ return False
+ db.expunge(row)
+ return row
+
+
+async def reconcile(user_id):
+ for rid, record in await run_in_session(_records, user_id):
+ row = await run_in_session(_revoked_row, user_id, rid, record)
+ if row is None:
+ continue
+ if row is not False:
+ try:
+ await service.close(row)
+ except Exception as error:
+ logging.getLogger(__name__).warning(
+ "Background cleanup deferred: %s", type(error).__name__
+ )
+ continue
+ await run_in_session(_release, user_id, rid)
diff --git a/src/backend/core/applications/capabilities.py b/src/backend/features/workbench/capabilities.py
similarity index 95%
rename from src/backend/core/applications/capabilities.py
rename to src/backend/features/workbench/capabilities.py
index 9631d37e..47374af0 100644
--- a/src/backend/core/applications/capabilities.py
+++ b/src/backend/features/workbench/capabilities.py
@@ -1,6 +1,6 @@
"""Server-owned approvals for exact installations, users and environments."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
from copy import deepcopy
@@ -8,10 +8,10 @@
from hashlib import sha256
from secrets import token_urlsafe
-from core.applications import catalog, external
+from features.workbench import catalog, external
from core.db.models import ContentBlock
from core.infra.time import as_utc, utc_now
-from core.applications.installation_binding import exact as resolve
+from features.workbench.installation_binding import exact as resolve
SUPPORTED = {
"data.query",
@@ -35,7 +35,7 @@ def binding(db, user_id, key, revision, module_id=None, instance_id=None):
if selected.ui.get("version") == 3 and not instance_id:
external.fail(403, "instance_binding_mismatch")
if instance_id or revision != current["packageRevision"]:
- from core.applications.plugin_revisions import resolve_revision
+ from features.workbench.plugin_revisions import resolve_revision
selected = resolve_revision(db, user_id, key, revision, instance_id)
current = {**current, "packageRevision": revision}
@@ -223,9 +223,9 @@ def select_file(
def readable_file(db, user_id, file_id):
- from core.auth.permissions_iface import resolve_artifact_access
+ from core.auth.edition import resolve_artifact_access
from core.db.models import Artifact
- from core.services.artifact_edition import artifact_access_metadata
+ from core.files.edition import artifact_access_metadata
artifact = (
db.query(Artifact)
@@ -248,6 +248,6 @@ def configured_hosts(db, selected, key=None):
"""No global manifest-key trust. Legacy callers without exact identity get none."""
if key is None:
return set()
- from core.applications.application_config import trusted_origins
+ from features.workbench.application_config import trusted_origins
return {host for _, host, _ in trusted_origins(db, key, selected)}
diff --git a/src/backend/core/applications/catalog.py b/src/backend/features/workbench/catalog.py
similarity index 96%
rename from src/backend/core/applications/catalog.py
rename to src/backend/features/workbench/catalog.py
index c3dd5acc..f2361eaf 100644
--- a/src/backend/core/applications/catalog.py
+++ b/src/backend/features/workbench/catalog.py
@@ -7,7 +7,7 @@
from copy import deepcopy
-from core.applications import external, policy, origin
+from features.workbench import external, policy, origin
from core.auth.capabilities import resolve_user_capabilities
from core.config.local_mode import local_mode_enabled
from core.db.models import ContentBlock
@@ -54,7 +54,7 @@ def definitions(db, user_id=None, include_disabled=False):
**item["declaration"], "icon": {"light": item["legacyIcon"]}
}
values.extend(external_values)
- from core.applications.migration_compat import active, app_config
+ from features.workbench.migration_compat import active, app_config
config = {"apps": []} if active(db) else app_config(db)
for app in config["apps"]:
@@ -79,7 +79,7 @@ def definitions(db, user_id=None, include_disabled=False):
}
)
if user_id is not None:
- from core.applications import plugin_catalog
+ from features.workbench import plugin_catalog
values.extend(plugin_catalog.items(db, user_id, include_disabled=include_disabled))
return values
diff --git a/src/backend/core/applications/coordination.py b/src/backend/features/workbench/coordination.py
similarity index 100%
rename from src/backend/core/applications/coordination.py
rename to src/backend/features/workbench/coordination.py
diff --git a/src/backend/core/applications/diagnostic_probe.py b/src/backend/features/workbench/diagnostic_probe.py
similarity index 98%
rename from src/backend/core/applications/diagnostic_probe.py
rename to src/backend/features/workbench/diagnostic_probe.py
index c6f5ddb7..e4d03735 100644
--- a/src/backend/core/applications/diagnostic_probe.py
+++ b/src/backend/features/workbench/diagnostic_probe.py
@@ -12,7 +12,7 @@
from threading import BoundedSemaphore
from urllib.parse import urlsplit
-from core.applications.external import validate_url
+from features.workbench.external import validate_url
from fastapi import HTTPException
_slots = BoundedSemaphore(2)
diff --git a/src/backend/core/applications/diagnostic_resources.py b/src/backend/features/workbench/diagnostic_resources.py
similarity index 100%
rename from src/backend/core/applications/diagnostic_resources.py
rename to src/backend/features/workbench/diagnostic_resources.py
diff --git a/src/backend/core/applications/diagnostics.py b/src/backend/features/workbench/diagnostics.py
similarity index 99%
rename from src/backend/core/applications/diagnostics.py
rename to src/backend/features/workbench/diagnostics.py
index 5243e20d..3bb745bd 100644
--- a/src/backend/core/applications/diagnostics.py
+++ b/src/backend/features/workbench/diagnostics.py
@@ -209,7 +209,7 @@ def run(*args, **kwargs):
db, user = kwargs.get("db"), kwargs.get("user")
item = None
if event == "launch":
- from core.applications import catalog
+ from features.workbench import catalog
try:
item = catalog.readable(db, user.user_id, kwargs["key"])
diff --git a/src/backend/core/applications/directory_revision.py b/src/backend/features/workbench/directory_revision.py
similarity index 96%
rename from src/backend/core/applications/directory_revision.py
rename to src/backend/features/workbench/directory_revision.py
index e952d205..7fdbab3d 100644
--- a/src/backend/core/applications/directory_revision.py
+++ b/src/backend/features/workbench/directory_revision.py
@@ -3,7 +3,7 @@
import hashlib
import json
-from core.applications import catalog
+from features.workbench import catalog
from core.config.local_mode import local_mode_enabled
from core.db.models import ContentBlock
diff --git a/src/backend/features/workbench/edition.py b/src/backend/features/workbench/edition.py
new file mode 100644
index 00000000..659a0388
--- /dev/null
+++ b/src/backend/features/workbench/edition.py
@@ -0,0 +1,10 @@
+"""Community operators must reject unknown authorization tables."""
+from sqlalchemy import inspect
+from core.db.engine import Base
+
+
+def snapshot_permissions(db):
+ names = set(inspect(db.get_bind()).get_table_names())
+ if names - set(Base.metadata.tables) - {"alembic_version"}:
+ raise ValueError("ce_identity_database_requires_full_edition_operator")
+ return {}
diff --git a/src/backend/features/workbench/extension.py b/src/backend/features/workbench/extension.py
new file mode 100644
index 00000000..14a9a647
--- /dev/null
+++ b/src/backend/features/workbench/extension.py
@@ -0,0 +1,4 @@
+"""Installed workbench extension."""
+NAME = "workbench"
+def setup():
+ return True
diff --git a/src/backend/core/applications/external.py b/src/backend/features/workbench/external.py
similarity index 94%
rename from src/backend/core/applications/external.py
rename to src/backend/features/workbench/external.py
index 78aaec59..d2365ae4 100644
--- a/src/backend/core/applications/external.py
+++ b/src/backend/features/workbench/external.py
@@ -11,9 +11,9 @@
from urllib.parse import parse_qsl, urlsplit
from uuid import uuid4
-from core.applications.coordination import configuration
-from core.applications.protocol import validate_application
-from core.content.content_blocks import normalize_app_config
+from features.workbench.coordination import configuration
+from features.workbench.protocol import validate_application
+from core.files.content.branding import normalize_app_config
from core.db.models import ContentBlock
from core.infra.time import utc_now
from fastapi import HTTPException
@@ -29,7 +29,7 @@ def fail(status, code):
def legacy_ids(db):
- from core.applications.migration_compat import app_config
+ from features.workbench.migration_compat import app_config
return {app["id"] for app in app_config(db)["apps"]} | {
"plan_mode",
@@ -112,7 +112,7 @@ def record(db, key):
def save(db, declaration, enabled, actor, trace, revision=None, key=None, embedding=None):
configuration(db)
validate_definition(db, declaration)
- from core.applications.external_embed import validate_policy
+ from features.workbench.external_embed import validate_policy
embedding = validate_policy(declaration["target"], embedding)
expected_key = "external:" + declaration["id"]
@@ -169,7 +169,7 @@ def save(db, declaration, enabled, actor, trace, revision=None, key=None, embedd
def reject_legacy_conflicts(db, payload):
- from core.applications.migration_compat import active
+ from features.workbench.migration_compat import active
if active(db):
return # The versioned adapter validates and writes canonical records.
diff --git a/src/backend/core/applications/external_auth.py b/src/backend/features/workbench/external_auth.py
similarity index 99%
rename from src/backend/core/applications/external_auth.py
rename to src/backend/features/workbench/external_auth.py
index 85404de0..c0cc3d8e 100644
--- a/src/backend/core/applications/external_auth.py
+++ b/src/backend/features/workbench/external_auth.py
@@ -5,7 +5,7 @@
ContentBlock provides portable transactions without adding a second app source.
"""
-from core.applications.coordination import configuration
+from features.workbench.coordination import configuration
import hashlib
@@ -16,7 +16,7 @@
from urllib.parse import urlencode, urlsplit, urlunsplit
from uuid import uuid4
-from core.applications import catalog, external
+from features.workbench import catalog, external
from core.db.models import ContentBlock
from core.infra.time import utc_now
from sqlalchemy import delete
diff --git a/src/backend/core/applications/external_boundary.py b/src/backend/features/workbench/external_boundary.py
similarity index 97%
rename from src/backend/core/applications/external_boundary.py
rename to src/backend/features/workbench/external_boundary.py
index 52458491..1f5cdf2e 100644
--- a/src/backend/core/applications/external_boundary.py
+++ b/src/backend/features/workbench/external_boundary.py
@@ -7,7 +7,7 @@
import ipaddress
from urllib.parse import urlsplit
-from core.applications import external
+from features.workbench import external
def validate_cookie_scope(request, target, config, cookie_domain):
diff --git a/src/backend/core/applications/external_embed.py b/src/backend/features/workbench/external_embed.py
similarity index 97%
rename from src/backend/core/applications/external_embed.py
rename to src/backend/features/workbench/external_embed.py
index 4bfdc860..4461cca4 100644
--- a/src/backend/core/applications/external_embed.py
+++ b/src/backend/features/workbench/external_embed.py
@@ -36,7 +36,7 @@ def exact_origin(value):
def validate_policy(target, embedding, host_origin=None):
- from core.applications.external import validate_url
+ from features.workbench.external import validate_url
if target["mode"] != "embedded":
if embedding is not None:
diff --git a/src/backend/core/applications/installation_binding.py b/src/backend/features/workbench/installation_binding.py
similarity index 77%
rename from src/backend/core/applications/installation_binding.py
rename to src/backend/features/workbench/installation_binding.py
index eed5b3ef..f878d91b 100644
--- a/src/backend/core/applications/installation_binding.py
+++ b/src/backend/features/workbench/installation_binding.py
@@ -3,7 +3,7 @@
from dataclasses import dataclass
from datetime import datetime
-from core.plugins.resources.installation import Installation, resolve
+from core.capabilities.plugins.resources.installation import Installation, resolve
from fastapi import HTTPException
@@ -17,14 +17,14 @@ class DeviceRecord:
def device_records(db, user_id):
- from core.capabilities import device_catalog, plugins, store
- from core.capabilities.local_plugin_runtime import enabled_for
- from core.capabilities.paths import capabilities_enabled
+ from features.desktop.capabilities import device_catalog, plugins, store
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
return []
result = []
- for inst in device_catalog._installations("plugin", user_id=user_id):
+ for inst in device_catalog.installations("plugin", user_id=user_id):
if not inst.ready or not enabled_for(inst, user_id) or inst.created_at is None:
continue
component = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision)
@@ -52,16 +52,16 @@ def device_records(db, user_id):
def exact(db, slug, user_id, install_id):
- from core.capabilities import device_catalog, plugins, store
- from core.capabilities.dependency import component_hash
- from core.capabilities.local_plugin_runtime import enabled_for
- from core.capabilities.paths import capabilities_enabled
+ from features.desktop.capabilities import device_catalog, plugins, store
+ from features.desktop.capabilities.dependency import component_hash
+ from features.desktop.capabilities.local_plugin_runtime import enabled_for
+ from core.harness.registry.run_capabilities import has_run_capabilities
# Legacy resolver also accepts aliases. Workbench has no alias authority.
- if capabilities_enabled():
+ if has_run_capabilities():
candidates = [
inst
- for inst in device_catalog._installations("plugin", user_id=user_id)
+ for inst in device_catalog.installations("plugin", user_id=user_id)
if inst.install_id == install_id
]
if candidates:
diff --git a/src/backend/core/applications/local_plugin_upgrades.py b/src/backend/features/workbench/local_plugin_upgrades.py
similarity index 86%
rename from src/backend/core/applications/local_plugin_upgrades.py
rename to src/backend/features/workbench/local_plugin_upgrades.py
index add3ff8c..aa2e989f 100644
--- a/src/backend/core/applications/local_plugin_upgrades.py
+++ b/src/backend/features/workbench/local_plugin_upgrades.py
@@ -2,27 +2,21 @@
from copy import deepcopy
-from core.applications.external import fail
-from core.applications.plugin_upgrades import (
- PREFIX,
- history_id,
- binding_token,
- permission_set,
- schema_changes,
- verify_package,
-)
-from core.capabilities import registry, store
-from core.capabilities.paths import LOCAL_PROFILE, capabilities_enabled
-from core.capabilities.plugins import load_manifest
+from features.workbench.external import fail
+from features.workbench.plugin_upgrades import PREFIX, history_id, binding_token, permission_set, schema_changes, verify_package
+from features.desktop.capabilities import registry, store
+from core.harness.registry.run_capabilities import has_run_capabilities
+from core.capabilities.identity_rules import LOCAL_PROFILE
+from features.desktop.capabilities.plugins import load_manifest
from core.db.models import ContentBlock
from core.infra.time import utc_now
-from core.services.management_package import serialized
+from core.capabilities.plugins.management.package import serialized
def owned(db, user_id, install_id):
- from core.plugins.local.service import _owned
+ from features.desktop.local.plugins.service import _owned
- if not capabilities_enabled():
+ if not has_run_capabilities():
fail(404, "plugin_upgrade_not_found")
try:
return _owned(user_id, install_id, db)
@@ -48,7 +42,7 @@ def public(inst, record):
def read(db, user_id, install_id):
inst = owned(db, user_id, install_id)
- from core.applications.plugin_candidate_storage import summary
+ from features.workbench.plugin_candidate_storage import summary
return {
**public(inst, db.get(ContentBlock, identifier(inst))),
@@ -67,7 +61,7 @@ def prepare(db, inst, prepared, approved_revision=None):
verify_package(np)
added = sorted(permission_set(np.ui) - permission_set(old_ui))
changes = schema_changes(old_ui, np.ui)
- from core.applications import plugin_candidate_storage as storage
+ from features.workbench import plugin_candidate_storage as storage
record = db.get(ContentBlock, identifier(inst))
if (added or changes) and comp.revision != approved_revision:
@@ -124,7 +118,7 @@ def prepare(db, inst, prepared, approved_revision=None):
@serialized
def decide(user_id, install_id, expected, candidate, decision, reset_state):
- from core.plugins.local import service
+ from features.desktop.local.plugins import service as service
with registry._session() as db:
inst = owned(db, user_id, install_id)
@@ -139,7 +133,7 @@ def decide(user_id, install_id, expected, candidate, decision, reset_state):
):
fail(409, "revision_conflict")
if decision == "reject":
- from core.applications import plugin_candidate_storage as storage
+ from features.workbench import plugin_candidate_storage as storage
storage.change(db, binding_token(inst), candidate, "discarded")
record.payload = {**value, "status": "rejected"}
diff --git a/src/backend/core/applications/migration.py b/src/backend/features/workbench/migration.py
similarity index 97%
rename from src/backend/core/applications/migration.py
rename to src/backend/features/workbench/migration.py
index 72a9635d..69d4d857 100644
--- a/src/backend/core/applications/migration.py
+++ b/src/backend/features/workbench/migration.py
@@ -1,21 +1,15 @@
"""WB-20 operator boundary. Explicit freeze, protected snapshot, atomic cutover."""
-from core.applications.coordination import configuration
+from features.workbench.coordination import configuration
from copy import deepcopy
from pathlib import Path
from uuid import uuid4
-from core.applications import external
-from core.applications.migration_plan import plan
-from core.applications.migration_snapshot import (
- RECEIPT,
- digest,
- read_snapshot,
- state,
- write_snapshot,
-)
+from features.workbench import external
+from features.workbench.migration_plan import plan
+from features.workbench.migration_snapshot import RECEIPT, digest, read_snapshot, state, write_snapshot
from core.db.models import ContentBlock, UserShadow
from core.infra.time import utc_now
diff --git a/src/backend/core/applications/migration_compat.py b/src/backend/features/workbench/migration_compat.py
similarity index 88%
rename from src/backend/core/applications/migration_compat.py
rename to src/backend/features/workbench/migration_compat.py
index d354a362..e9b3a094 100644
--- a/src/backend/core/applications/migration_compat.py
+++ b/src/backend/features/workbench/migration_compat.py
@@ -1,12 +1,12 @@
"""Versioned content-v1 adapter after WB-20 cutover; canonical rows are the only source."""
-from core.applications.coordination import configuration
+from features.workbench.coordination import configuration
from copy import deepcopy
-from core.applications import external
-from core.applications.migration_snapshot import RECEIPT
+from features.workbench import external
+from features.workbench.migration_snapshot import RECEIPT
from core.db.models import ContentBlock
from core.infra.time import utc_now
@@ -18,7 +18,8 @@ def active(db):
def app_config(db):
if not active(db):
- from core.content.content_blocks import DEFAULT_APP_CONFIG, normalize_app_config
+ from core.files.content.content_blocks_constants import DEFAULT_APP_CONFIG
+ from core.files.content.branding import normalize_app_config
row = db.get(ContentBlock, "app_config")
return normalize_app_config(row.payload if row else DEFAULT_APP_CONFIG)
@@ -56,7 +57,7 @@ def page_config(db, payload):
result = deepcopy(payload)
if not active(db):
return result
- from core.applications import policy
+ from features.workbench import policy
layout = policy.read(db)
nav = result.setdefault("navigation", {})
@@ -77,8 +78,8 @@ def stage_write(db, alias, payload):
if alias == "page_config":
navigation = payload.get("navigation", {})
if isinstance(navigation, dict) and ({"sidebar_items", "menu_items"} & set(navigation)):
- from core.applications import policy
- from core.applications.catalog import BUILTINS
+ from features.workbench import policy
+ from features.workbench.catalog import BUILTINS
layout = policy.read(db)
for field in ("sidebar_items", "menu_items"):
@@ -104,7 +105,7 @@ def stage_write(db, alias, payload):
return False
if alias != "app_config":
return False
- from core.applications.migration_plan import plan
+ from features.workbench.migration_plan import plan
if (
not isinstance(payload, dict)
@@ -141,7 +142,7 @@ def _put(db, key, value):
def build_docs_snapshot(db):
- from core.content.content_blocks import build_docs_snapshot as original
+ from core.files.content.content_blocks import build_docs_snapshot as original
result = original(db)
if active(db):
@@ -154,9 +155,9 @@ def build_docs_snapshot(db):
def import_docs_snapshot(db, snapshot, *, overwrite=True, default_updated_by=None):
- from core.content.content_blocks import DOCS_BLOCK_MAP
- from core.content.content_blocks import import_docs_snapshot as original
- from core.content.content_blocks import normalize_docs_snapshot
+ from core.files.content.content_blocks_constants import DOCS_BLOCK_MAP
+ from core.files.content.content_blocks import import_docs_snapshot as original
+ from core.files.content.content_blocks import normalize_docs_snapshot
if not active(db):
return original(db, snapshot, overwrite=overwrite, default_updated_by=default_updated_by)
diff --git a/src/backend/core/applications/migration_plan.py b/src/backend/features/workbench/migration_plan.py
similarity index 96%
rename from src/backend/core/applications/migration_plan.py
rename to src/backend/features/workbench/migration_plan.py
index b7e4fe7a..61251481 100644
--- a/src/backend/core/applications/migration_plan.py
+++ b/src/backend/features/workbench/migration_plan.py
@@ -4,11 +4,11 @@
from collections import Counter
from copy import deepcopy
-from core.applications import external, preferences
-from core.applications.catalog import BUILTINS
-from core.applications.migration_snapshot import digest
-from core.applications.protocol import validate_application
-from core.content.content_blocks import DEFAULT_PAGE_CONFIG
+from features.workbench import external, preferences
+from features.workbench.catalog import BUILTINS
+from features.workbench.migration_snapshot import digest
+from features.workbench.protocol import validate_application
+from core.files.content.content_blocks_constants import DEFAULT_PAGE_CONFIG
from fastapi import HTTPException
FIXED = {"chat", "settings", "docs", "plan_mode"}
@@ -29,7 +29,7 @@ def configured_auth(auth, target):
mode = auth.get("mode")
if mode == "independent":
return set(auth) <= {"revision", "mode"}
- from core.applications.external_auth import destination
+ from features.workbench.external_auth import destination
try:
if mode == "shared_identity":
diff --git a/src/backend/core/applications/migration_snapshot.py b/src/backend/features/workbench/migration_snapshot.py
similarity index 91%
rename from src/backend/core/applications/migration_snapshot.py
rename to src/backend/features/workbench/migration_snapshot.py
index addc466f..a624ad5c 100644
--- a/src/backend/core/applications/migration_snapshot.py
+++ b/src/backend/features/workbench/migration_snapshot.py
@@ -15,7 +15,6 @@
UserShadow,
)
from sqlalchemy import inspect, or_
-from core.db.engine import Base
RECEIPT = "workbench_migration:v1"
@@ -58,12 +57,9 @@ def state(db):
}
for row in db.query(InstalledPlugin).order_by(InstalledPlugin.install_id)
}
- # Refuse unrecognized tables: community operators cannot promise to preserve their authorization state.
- names = set(inspect(db.get_bind()).get_table_names())
- unsupported = names - set(Base.metadata.tables) - {"alembic_version"}
- if unsupported:
- raise ValueError("ce_identity_database_requires_full_edition_operator")
- permissions = {}
+ from features.workbench.edition import snapshot_permissions
+
+ permissions = snapshot_permissions(db)
permissions["pluginSkills"] = {
row.skill_id: {
"enabled": row.is_enabled,
diff --git a/src/backend/features/workbench/models.py b/src/backend/features/workbench/models.py
new file mode 100644
index 00000000..49d08342
--- /dev/null
+++ b/src/backend/features/workbench/models.py
@@ -0,0 +1 @@
+"""Workbench persists configuration in core ContentBlock; no additional ORM tables."""
diff --git a/src/backend/core/applications/origin.py b/src/backend/features/workbench/origin.py
similarity index 100%
rename from src/backend/core/applications/origin.py
rename to src/backend/features/workbench/origin.py
diff --git a/src/backend/features/workbench/package_source.py b/src/backend/features/workbench/package_source.py
new file mode 100644
index 00000000..56099fbd
--- /dev/null
+++ b/src/backend/features/workbench/package_source.py
@@ -0,0 +1,19 @@
+"""Workbench owns plugin upgrade staging and candidate retention."""
+
+
+class WorkbenchPackagePolicy:
+ def lock_configuration(self, db):
+ from features.workbench.coordination import configuration
+ configuration(db)
+
+ def prepare(self, db, existing, package, source, approved_revision):
+ from features.workbench.plugin_upgrades import prepare
+ return prepare(db, existing, package, source, approved_revision)
+
+ def reserve_local(self, existing, revision, files):
+ from features.workbench.plugin_candidate_storage import local_reserve
+ return local_reserve(existing, revision, files)
+
+ def prepare_local(self, db, current, prepared):
+ from features.workbench.local_plugin_upgrades import prepare
+ return prepare(db, current, prepared)
diff --git a/src/backend/core/applications/plugin_candidate_storage.py b/src/backend/features/workbench/plugin_candidate_storage.py
similarity index 88%
rename from src/backend/core/applications/plugin_candidate_storage.py
rename to src/backend/features/workbench/plugin_candidate_storage.py
index 60b155b3..c8d1956c 100644
--- a/src/backend/core/applications/plugin_candidate_storage.py
+++ b/src/backend/features/workbench/plugin_candidate_storage.py
@@ -4,12 +4,13 @@
import json
import time
-from core.applications.external import fail
-from core.applications.coordination import configuration
-from core.db.models import ContentBlock, InstalledPlugin, DeviceCapabilityInstallation
+from features.workbench.external import fail
+from features.workbench.coordination import configuration
+from core.db.models import ContentBlock, InstalledPlugin
+from features.desktop.models import DeviceCapabilityInstallation
from core.db.models.plugin_resource import PluginResource, PluginResourcePackage
from core.infra.time import utc_now
-from core.capabilities.errors import IntegrityFailed
+from core.harness.capability_errors import IntegrityFailed
PREFIX = "workbench_plugin_candidate:"
MAX_CANDIDATES = 2
@@ -33,7 +34,7 @@ def owner_key(user_id):
def live_bindings(db):
- from core.applications.plugin_upgrades import binding_token
+ from features.workbench.plugin_upgrades import binding_token
rows = list(db.query(InstalledPlugin).all())
rows.extend(
@@ -49,8 +50,8 @@ def live_bindings(db):
def protected(db):
"""Existing authoritative references win over candidate cleanup."""
- from core.applications.plugin_upgrades import PREFIX as UPGRADES, HISTORY
- from core.applications.plugin_revisions import LEASE
+ from features.workbench.plugin_upgrades import PREFIX as UPGRADES, HISTORY
+ from features.workbench.plugin_revisions import LEASE
active_bindings = live_bindings(db)
keep = {
@@ -102,7 +103,7 @@ def collect(db, binding=None, owner_scope=None):
if value["binding"] not in active_bindings:
value = {**value, "status": "discarded"}
row.payload = value
- from core.applications.plugin_upgrades import PREFIX as UPGRADES
+ from features.workbench.plugin_upgrades import PREFIX as UPGRADES
orphan = db.get(ContentBlock, UPGRADES + value["binding"])
if orphan is not None:
@@ -119,11 +120,11 @@ def collect(db, binding=None, owner_scope=None):
continue
try:
if value["scope"] == "local":
- from core.capabilities import store
+ from features.desktop.capabilities import store
store.remove_revision("plugin", "local", value["key"], value["revision"])
else:
- from core.plugins.packaging.runtime_assets import forget_cache
+ from core.capabilities.plugins.packaging.runtime_assets import forget_cache
forget_cache(value["revision"])
package = db.get(PluginResourcePackage, value["revision"])
@@ -199,8 +200,8 @@ def replace(db, binding, previous, revision):
def local_reserve(inst, revision, files):
"""Reserve before any candidate/skill directory is written; existing file lock is held."""
- from core.capabilities import registry
- from core.applications.plugin_upgrades import binding_token
+ from features.desktop.capabilities import registry
+ from features.workbench.plugin_upgrades import binding_token
size = sum(len(v if isinstance(v, bytes) else v.encode()) for v in files.values())
if size > 50 * 1024 * 1024:
@@ -221,13 +222,13 @@ def local_reserve(inst, revision, files):
def janitor(db):
"""Avoid deleting local prepared files while a management mutation holds the file lock."""
- from core.capabilities.paths import capabilities_enabled
+ from core.harness.registry.run_capabilities import has_run_capabilities
- if not capabilities_enabled():
+ if not has_run_capabilities():
configuration(db)
return collect(db)
- from core.capabilities.paths import meta_root
- from core.capabilities.lockfile import locked, LockTimeout
+ from features.desktop.capabilities.paths import meta_root
+ from core.infra.file_lock import locked, LockTimeout
try:
with locked(meta_root() / "management.lock", timeout=0.1):
diff --git a/src/backend/core/applications/plugin_catalog.py b/src/backend/features/workbench/plugin_catalog.py
similarity index 95%
rename from src/backend/core/applications/plugin_catalog.py
rename to src/backend/features/workbench/plugin_catalog.py
index d9db27de..b8b778a7 100644
--- a/src/backend/core/applications/plugin_catalog.py
+++ b/src/backend/features/workbench/plugin_catalog.py
@@ -4,11 +4,11 @@
from hashlib import sha256
from urllib.parse import quote
-from core.applications.protocol import validate_contributions
-from core.applications import origin
+from features.workbench.protocol import validate_contributions
+from features.workbench import origin
from core.db.models import InstalledPlugin
from core.infra.time import as_utc
-from core.applications.installation_binding import exact as resolve, device_records
+from features.workbench.installation_binding import exact as resolve, device_records
from fastapi import HTTPException
from sqlalchemy import or_
@@ -55,7 +55,7 @@ def items(db, user_id, include_disabled=False):
)
.all()
)
- from core.capabilities import device_catalog
+ from features.desktop.capabilities import device_catalog
if device_catalog.active():
records = []
@@ -137,7 +137,7 @@ def launch(db, user_id, item, module_id=None):
if module_id not in allowed:
raise HTTPException(404, detail={"code": "app_view_not_found"})
module = next(m for m in checked["modules"] if m["id"] == module_id)
- from core.plugins.resources.installation import safe_path
+ from core.capabilities.plugins.resources.installation import safe_path
target = safe_path(selected.package, module["entry"])
if not target.is_file():
diff --git a/src/backend/core/applications/plugin_revisions.py b/src/backend/features/workbench/plugin_revisions.py
similarity index 89%
rename from src/backend/core/applications/plugin_revisions.py
rename to src/backend/features/workbench/plugin_revisions.py
index 2d723abb..54d0cb6e 100644
--- a/src/backend/core/applications/plugin_revisions.py
+++ b/src/backend/features/workbench/plugin_revisions.py
@@ -4,23 +4,17 @@
from hashlib import sha256
import time
-from core.applications import catalog
-from core.applications.external import fail
-from core.applications.coordination import configuration, runtime
-from core.applications.plugin_upgrades import (
- HISTORY,
- history_id,
- binding_token,
- current_revision,
- permission_set,
-)
+from features.workbench import catalog
+from features.workbench.external import fail
+from features.workbench.coordination import configuration, runtime
+from features.workbench.plugin_upgrades import HISTORY, history_id, binding_token, current_revision, permission_set
from core.db.models import ContentBlock, InstalledPlugin
from core.infra.time import utc_now
-from core.plugins.packaging.runtime_assets import materialize
+from core.capabilities.plugins.packaging.runtime_assets import materialize
from sqlalchemy import or_
-from core.plugins.resources.installation import Installation
-from core.applications.installation_binding import exact as resolve
+from core.capabilities.plugins.resources.installation import Installation
+from features.workbench.installation_binding import exact as resolve
LEASE = "workbench_plugin_lease:"
LEASE_SECONDS = 90
@@ -43,7 +37,7 @@ def current(db, user_id, key):
selected = resolve(db, binding["slug"], user_id, binding["installId"])
row = db.get(InstalledPlugin, selected.install_id)
if row is None:
- from core.capabilities import registry
+ from features.desktop.capabilities import registry
row = registry.get(selected.install_id, db=db)
if row is None or row.profile_id != "local" or row.state == "removed":
@@ -144,7 +138,7 @@ def resolve_revision(db, user_id, key, revision, instance_id=None):
):
fail(404, "app_not_found")
if previous.get("executionScope") == "local":
- from core.capabilities import store
+ from features.desktop.capabilities import store
component = store.get("plugin", "local", row.key, revision)
if component is None:
@@ -182,7 +176,7 @@ def close(db, user_id, key, revision, instance_id):
if record:
# Keep expiresAt: closed instances do not force early asset collection.
record.payload = {**record.payload, "closed": True}
- from core.applications.capabilities import revoke_matching
+ from features.workbench.capabilities import revoke_matching
revoke_matching(db, user_id, {key}, revision, instance_id)
db.commit()
@@ -197,7 +191,7 @@ def collect(db):
leases = db.query(ContentBlock).filter(ContentBlock.id.startswith(LEASE, autoescape=True)).all()
retained = {r.payload["revision"] for r in leases if r.payload["expiresAt"] > time.time()}
retained.update(current_revision(r) for r in db.query(InstalledPlugin).all())
- from core.db.models import DeviceCapabilityInstallation
+ from features.desktop.models import DeviceCapabilityInstallation
retained.update(
r.resolved_revision
@@ -211,7 +205,7 @@ def collect(db):
.filter(or_(PluginResource.expires_at > time.time(), PluginResource.status != "closed"))
.all()
)
- from core.applications.plugin_upgrades import PREFIX
+ from features.workbench.plugin_upgrades import PREFIX
pending = (
db.query(ContentBlock).filter(ContentBlock.id.startswith(PREFIX, autoescape=True)).all()
@@ -219,7 +213,7 @@ def collect(db):
retained.update(
r.payload.get("candidateRevision") for r in pending if r.payload.get("status") == "pending"
)
- from core.applications.plugin_candidate_storage import records
+ from features.workbench.plugin_candidate_storage import records
retained.update(
r.payload["revision"]
@@ -234,7 +228,7 @@ def collect(db):
for history in histories:
if history.payload["revision"] in candidates:
if history.payload.get("executionScope") == "local":
- from core.capabilities import store
+ from features.desktop.capabilities import store
store.remove_revision(
"plugin", "local", history.payload["slug"], history.payload["revision"]
@@ -248,7 +242,7 @@ def collect(db):
if lease.payload["expiresAt"] <= time.time():
db.delete(lease)
db.commit()
- from core.plugins.packaging.runtime_assets import forget_cache
+ from core.capabilities.plugins.packaging.runtime_assets import forget_cache
for revision in candidates:
forget_cache(revision)
diff --git a/src/backend/core/applications/plugin_upgrades.py b/src/backend/features/workbench/plugin_upgrades.py
similarity index 90%
rename from src/backend/core/applications/plugin_upgrades.py
rename to src/backend/features/workbench/plugin_upgrades.py
index eb2f6516..6a776b86 100644
--- a/src/backend/core/applications/plugin_upgrades.py
+++ b/src/backend/features/workbench/plugin_upgrades.py
@@ -7,11 +7,11 @@
import zipfile
from pathlib import Path
-from core.applications.external import fail
-from core.applications.coordination import configuration
+from features.workbench.external import fail
+from features.workbench.coordination import configuration
from core.db.models import ContentBlock, InstalledPlugin
from core.infra.time import as_utc, utc_now
-from core.plugins.packaging.runtime_assets import materialize, retain, pack
+from core.capabilities.plugins.packaging.runtime_assets import materialize, retain, pack
PREFIX = "workbench_plugin_upgrade:"
HISTORY = "workbench_plugin_revision:"
@@ -83,8 +83,8 @@ def verify_package(np):
fail(409, "host_incompatible")
if ui.get("version") != 3:
return
- from core.applications.plugin_catalog import WORKSPACE_CAPABILITIES
- from core.plugins.resources.installation import safe_path
+ from features.workbench.plugin_catalog import WORKSPACE_CAPABILITIES
+ from core.capabilities.plugins.resources.installation import safe_path
for app in ui.get("contributes", {}).get("applications", []):
required = set(app.get("compatibility", {}).get("requiredCapabilities", []))
@@ -95,7 +95,7 @@ def verify_package(np):
if not np.package_dir or not safe_path(Path(np.package_dir), module["entry"]).is_file():
fail(409, "package_asset_missing")
# Verify the exact HTML and all bundled subassets before activation.
- from core.applications.workspace_page import render
+ from features.workbench.workspace_page import render
render(Path(np.package_dir) / "web", module["entry"][4:])
@@ -115,11 +115,11 @@ def public(row, pending):
def read(db, install_id, owner_user_id):
if install_id.startswith("plugin:local:") and owner_user_id is not None:
- from core.applications import local_plugin_upgrades
+ from features.workbench import local_plugin_upgrades
return local_plugin_upgrades.read(db, owner_user_id, install_id)
row = owned(db, install_id, owner_user_id)
- from core.applications.plugin_candidate_storage import summary
+ from features.workbench.plugin_candidate_storage import summary
return {
**public(row, db.get(ContentBlock, record_id(row))),
@@ -134,7 +134,7 @@ def prepare(db, row, np, source, approved_revision=None):
if old_ui.get("version") != 3 and (np.ui or {}).get("version") != 3:
return None
verify_package(np)
- from core.applications import plugin_candidate_storage as storage
+ from features.workbench import plugin_candidate_storage as storage
packed = pack(Path(np.package_dir)) if np.package_dir and np.ui else None
revision = packed[0] if packed else None
@@ -211,7 +211,7 @@ def decide(
db, install_id, owner_user_id, expected, candidate, decision, reset_state=False, secrets=None
):
if install_id.startswith("plugin:local:") and owner_user_id is not None:
- from core.applications import local_plugin_upgrades
+ from features.workbench import local_plugin_upgrades
return local_plugin_upgrades.decide(
owner_user_id, install_id, expected, candidate, decision, reset_state
@@ -232,7 +232,7 @@ def decide(
db.rollback()
fail(409, "revision_conflict")
if decision == "reject":
- from core.applications import plugin_candidate_storage as storage
+ from features.workbench import plugin_candidate_storage as storage
storage.change(db, binding_token(row), candidate, "discarded")
pending.payload = {**value, "status": "rejected"}
@@ -242,8 +242,8 @@ def decide(
if value["stateSchemaChanges"] and not reset_state:
db.rollback()
fail(409, "state_reset_confirmation_required")
- from core.plugins.packaging.importer import normalize_plugin_dir
- from core.plugins.management.installation import _apply_normalized
+ from core.capabilities.plugins.packaging.importer import normalize_plugin_dir
+ from core.capabilities.plugins.management.installation import apply_normalized as _apply_normalized
try:
np = normalize_plugin_dir(materialize(db, candidate))
diff --git a/src/backend/core/applications/policy.py b/src/backend/features/workbench/policy.py
similarity index 94%
rename from src/backend/core/applications/policy.py
rename to src/backend/features/workbench/policy.py
index a09ca437..718e1bd1 100644
--- a/src/backend/core/applications/policy.py
+++ b/src/backend/features/workbench/policy.py
@@ -1,12 +1,12 @@
"""Administrator layout policy. It narrows discovery; it never grants execution."""
-from core.applications.coordination import configuration
+from features.workbench.coordination import configuration
from copy import deepcopy
from uuid import uuid4
-from core.applications import external
+from features.workbench import external
from core.db.models import ContentBlock
from core.infra.time import utc_now
from sqlalchemy import update
@@ -24,7 +24,7 @@ def read(db):
def save(db, body, actor, trace):
- from core.applications.catalog import definitions
+ from features.workbench.catalog import definitions
configuration(db)
known = {item["applicationKey"] for item in definitions(db)}
diff --git a/src/backend/core/applications/preferences.py b/src/backend/features/workbench/preferences.py
similarity index 96%
rename from src/backend/core/applications/preferences.py
rename to src/backend/features/workbench/preferences.py
index 3a5ee697..cae7098f 100644
--- a/src/backend/core/applications/preferences.py
+++ b/src/backend/features/workbench/preferences.py
@@ -1,13 +1,13 @@
"""User and environment scoped navigation. Independent of installs and permissions."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
import re
from copy import deepcopy
from hashlib import sha256
-from core.applications import external
+from features.workbench import external
from core.config.local_mode import local_mode_enabled
from core.db.models import ContentBlock
from core.infra.time import utc_now
diff --git a/src/backend/core/applications/protocol.py b/src/backend/features/workbench/protocol.py
similarity index 100%
rename from src/backend/core/applications/protocol.py
rename to src/backend/features/workbench/protocol.py
diff --git a/src/backend/features/workbench/resource_deadlines.py b/src/backend/features/workbench/resource_deadlines.py
new file mode 100644
index 00000000..08888695
--- /dev/null
+++ b/src/backend/features/workbench/resource_deadlines.py
@@ -0,0 +1,122 @@
+"""Workbench leases bind process deadlines to application ownership."""
+import hashlib
+import json
+from fastapi import HTTPException
+from features.workbench.coordination import runtime
+from features.workbench import activity, origin
+from core.capabilities.plugins.resources.deadlines import provider_for
+from core.capabilities.plugins.resources.owners import runtime_session
+
+def invalidate(db, row):
+ from features.workbench import external
+
+ rid, user = row.resource_id, row.user_id
+ db.commit()
+ runtime(db, user)
+ db.expire_all()
+ if row.status != "closed":
+ row.status = "closing"
+ row.expires_at = 0
+ current = activity.read(db, user)
+ record = current["background"].get(rid)
+ if record:
+ record["expiresAt"] = 0
+ record["revoked"] = True
+ activity.save(db, user, current) # Retain the exact charged cleanup reservation.
+
+
+def _arming(db, resource_id, require_background):
+ """Validate the lease record and resolve the runner timer; None when nothing to arm."""
+ from core.capabilities.plugins.resources import store
+
+ row = db.get(store.PluginResource, resource_id)
+ if row is None or row.application_key is None or row.chat_id is not None:
+ return None # Ordinary chat resources retain their existing lifetime.
+ value = activity.read(db, row.user_id)
+ record = value["background"].get(row.resource_id)
+ source = origin.current(db)
+ expires = row.expires_at
+ if record:
+ binding = record["binding"]
+ if (
+ record["applicationKey"] != row.application_key
+ or record["instanceId"] != row.instance_id
+ or binding.get("installId") != row.install_id
+ or binding.get("packageRevision") != row.revision
+ or binding.get("executionScope") != row.scope
+ or binding.get("origin") != source
+ ):
+ raise HTTPException(403, detail={"code": "resource_binding_mismatch"})
+ expires = min(expires, record["expiresAt"], record["deadline"])
+ try:
+ method, process_id = provider_for(row)
+ except HTTPException as error:
+ if not (require_background or record) and error.status_code == 403:
+ return None # Remote providers have no independent persistent deadline contract.
+ raise
+ digest = hashlib.sha256(
+ json.dumps(
+ [
+ row.resource_id,
+ row.user_id,
+ row.application_key,
+ row.instance_id,
+ row.install_id,
+ row.revision,
+ row.scope,
+ source,
+ ],
+ sort_keys=True,
+ ).encode()
+ ).hexdigest()
+ return {
+ "method": method,
+ "process_id": process_id,
+ "session": runtime_session(row),
+ "user_id": row.user_id,
+ "digest": digest,
+ "expires": expires,
+ "deadline": record["deadline"] if record else None,
+ "background": bool(record),
+ }
+
+
+def _invalidate(db, resource_id):
+ from core.capabilities.plugins.resources import store
+
+ invalidate(db, db.get(store.PluginResource, resource_id))
+
+
+async def arm(resource_id, *, require_background=False):
+ """Hand the lease deadline to the runner; revoke the lease if delivery is uncertain.
+
+ The lease is read and, on failure, invalidated on sessions of their own; the
+ runner call in between holds no connection.
+ """
+ from core.db.threads import run_in_session
+
+ timer = await run_in_session(_arming, resource_id, require_background)
+ if timer is None:
+ return
+ try:
+ result = await timer["method"](
+ timer["process_id"],
+ sandbox_session_id=timer["session"],
+ user_id=timer["user_id"],
+ binding=timer["digest"],
+ expires_at=timer["expires"],
+ absolute_deadline=timer["deadline"],
+ )
+ except Exception:
+ # Do not leave an authorized extended lease after uncertain timer delivery.
+ await run_in_session(_invalidate, resource_id)
+ raise HTTPException(503, detail={"code": "release_failed"}) from None
+ if result.get("supported") is False:
+ if timer["background"]:
+ await run_in_session(_invalidate, resource_id)
+ if require_background or timer["background"]:
+ raise HTTPException(403, detail={"code": "unsupported_capability"})
+ return
+ if result.get("armed") is not True:
+ await run_in_session(_invalidate, resource_id)
+ raise HTTPException(503, detail={"code": "release_failed"})
diff --git a/src/backend/features/workbench/resource_owner.py b/src/backend/features/workbench/resource_owner.py
new file mode 100644
index 00000000..4e53d8aa
--- /dev/null
+++ b/src/backend/features/workbench/resource_owner.py
@@ -0,0 +1,65 @@
+"""Workbench-specific binding and revision policy for generic plugin resources."""
+from fastapi import HTTPException
+from core.capabilities.plugins.resources.owners import execution_scope
+
+def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None):
+ from features.workbench import catalog
+
+ item = catalog.readable(db, user_id, key)
+ if not item["availability"]["available"]:
+ raise HTTPException(403, "application_unavailable")
+ binding = item.get("binding", {})
+ if (
+ item["declaration"]["target"]["kind"] != "plugin"
+ or binding.get("installId") != install_id
+ or (
+ binding.get("packageRevision") != revision
+ and (selected is None or selected.revision != revision)
+ )
+ or scope != execution_scope()
+ or (binding.get("origin") is not None and binding["origin"].get("kind") != scope)
+ ):
+ raise HTTPException(409, "resource_binding_mismatch")
+ target = item["declaration"]["target"]
+ if selected is not None and binding.get("packageRevision") != revision:
+ previous = next(
+ (
+ a
+ for a in (selected.ui.get("contributes") or {}).get("applications", [])
+ if a["id"] == item["declaration"]["id"]
+ ),
+ None,
+ )
+ if previous is None:
+ raise HTTPException(403, "resource_application_removed")
+ target = previous["target"]
+ allowed = {target.get("moduleId")} | {
+ view.get("moduleId") for view in item["declaration"].get("views", [])
+ }
+ if module_id not in allowed:
+ raise HTTPException(403, "resource_module_not_granted")
+ return item
+
+
+
+def resolve_installation(db, row, user_id):
+ from features.workbench.installation_binding import exact
+ selected = exact(db, row.slug, user_id, row.install_id)
+ if selected.install_id != row.install_id:
+ raise HTTPException(403, "resource_installation_mismatch")
+ if selected.revision == row.revision:
+ return selected
+ try:
+ from features.workbench.plugin_revisions import resolve_revision
+ except ModuleNotFoundError as error:
+ if error.name != "features.workbench.plugin_revisions":
+ raise
+ raise HTTPException(409, "revision_retention_unavailable") from None
+ return resolve_revision(db, user_id, row.application_key, row.revision, row.instance_id)
+
+
+def collect(db):
+ from features.workbench.plugin_revisions import collect
+ collect(db)
+ from features.workbench.plugin_candidate_storage import janitor
+ janitor(db)
diff --git a/src/backend/core/applications/revocation.py b/src/backend/features/workbench/revocation.py
similarity index 87%
rename from src/backend/core/applications/revocation.py
rename to src/backend/features/workbench/revocation.py
index ea294506..d2f60785 100644
--- a/src/backend/core/applications/revocation.py
+++ b/src/backend/features/workbench/revocation.py
@@ -1,8 +1,9 @@
"""Persist credential revocation before best-effort worker cleanup."""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
+from core.db.threads import run_in_session
import asyncio
import logging
from copy import deepcopy
@@ -13,7 +14,7 @@
PluginResourceAssetTicket,
PluginResourceTicket,
)
-from core.plugins.resources import service
+from core.capabilities.plugins.resources import service
from sqlalchemy import delete, or_
EXTERNAL_PREFIXES = (
@@ -31,7 +32,7 @@ def _lock(db, user_id):
def _revoke_activity(db, user_id, keys=None):
"""Stage revoked activity credentials in the caller's transaction."""
- from core.applications import activity
+ from features.workbench import activity
row = db.get(ContentBlock, activity.identity(user_id))
if row is None:
@@ -62,7 +63,7 @@ def matches(key):
def _revoke_worker_credentials(db, user_id, rows, keys=None):
_revoke_activity(db, user_id, keys)
- from core.applications.capabilities import revoke_matching
+ from features.workbench.capabilities import revoke_matching
revoke_matching(db, user_id, keys)
leases = (
@@ -106,13 +107,8 @@ def _external_tickets(db, user_id):
)
-async def revoke_user_workbench(db, user_id, session_hash=None):
- """Revoke all user-owned credentials atomically; then attempt every worker.
-
- Resources and grants are user-owned, not browser-session-owned. Logout
- therefore revokes all matching host tickets, including other tab tickets.
- Third-party sessions remain outside this host boundary.
- """
+def _revoke_credentials(db, user_id):
+ """Revoke every user-owned credential in one transaction; return the live rows."""
_lock(db, user_id)
rows = (
db.query(PluginResource)
@@ -128,16 +124,29 @@ async def revoke_user_workbench(db, user_id, session_hash=None):
_revoke_worker_credentials(db, user_id, rows)
# No external shutdown or independently committing helper before this barrier.
db.commit()
- return await _close_workers(db, rows)
+ for row in rows:
+ db.refresh(row)
+ db.expunge(row)
+ return rows
+
+
+async def revoke_user_workbench(user_id, session_hash=None):
+ """Revoke all user-owned credentials atomically; then attempt every worker.
+
+ Resources and grants are user-owned, not browser-session-owned. Logout
+ therefore revokes all matching host tickets, including other tab tickets.
+ Third-party sessions remain outside this host boundary.
+ """
+ return await _close_workers(await run_in_session(_revoke_credentials, user_id))
def stage_unavailable(db, user_id):
"""Revoke stale credentials before attempting each affected worker."""
- from core.applications import activity
- from core.applications.plugin_revisions import resolve_revision
+ from features.workbench import activity
+ from features.workbench.plugin_revisions import resolve_revision
- from core.applications.catalog import directory
- from core.applications import origin
+ from features.workbench.catalog import directory
+ from features.workbench import origin
origin.current(db)
_lock(db, user_id)
@@ -218,7 +227,7 @@ def stage_unavailable(db, user_id):
def _release_background_reservation(db, row):
- from core.applications import activity
+ from features.workbench import activity
_lock(db, row.user_id)
stored = db.get(ContentBlock, activity.identity(row.user_id))
@@ -231,19 +240,18 @@ def _release_background_reservation(db, row):
db.commit()
-async def _close_workers(db, rows):
+async def _close_workers(rows):
+ """Stop each worker, then release its reservation; no session spans a shutdown."""
closed = 0
failures = []
- # Never share a database session across concurrent close operations.
for row in rows:
try:
- await asyncio.wait_for(service.close(db, row), timeout=10)
- _release_background_reservation(db, row)
+ await asyncio.wait_for(service.close(row), timeout=10)
+ await run_in_session(_release_background_reservation, row)
closed += 1
except Exception as exc:
- db.rollback()
failures.append(type(exc).__name__)
if failures:
logging.getLogger(__name__).warning("Workbench worker cleanup deferred: %s", len(failures))
# Authorization is already revoked; deferred does not mean OS-process gone.
- return {"closed": closed, "deferred": len(failures)}
+ return {"closed": closed, "deferred": len(failures)}
\ No newline at end of file
diff --git a/src/backend/core/applications/revocation_sweep.py b/src/backend/features/workbench/revocation_sweep.py
similarity index 78%
rename from src/backend/core/applications/revocation_sweep.py
rename to src/backend/features/workbench/revocation_sweep.py
index 080e4e75..dd1bf337 100644
--- a/src/backend/core/applications/revocation_sweep.py
+++ b/src/backend/features/workbench/revocation_sweep.py
@@ -5,8 +5,9 @@
from sqlalchemy import select, union
from starlette.concurrency import run_in_threadpool
-from core.applications.revocation import EXTERNAL_PREFIXES, stage_unavailable, _close_workers
+from features.workbench.revocation import EXTERNAL_PREFIXES, stage_unavailable, _close_workers
from core.db.engine import SessionLocal
+from core.db.threads import run_in_session
from core.db.models import ContentBlock
from core.db.models.plugin_resource import PluginResource
@@ -46,19 +47,24 @@ def stage(user_id):
return stage_unavailable(db, user_id)
+def _owner_rows(db, user_id, resource_ids):
+ rows = (
+ db.query(PluginResource)
+ .filter(
+ PluginResource.user_id == user_id,
+ PluginResource.resource_id.in_(resource_ids),
+ )
+ .order_by(PluginResource.resource_id)
+ .all()
+ )
+ for row in rows:
+ db.expunge(row)
+ return rows
+
+
async def reconcile_owner(user_id):
resource_ids = await run_in_threadpool(stage, user_id)
- with SessionLocal() as db:
- rows = (
- db.query(PluginResource)
- .filter(
- PluginResource.user_id == user_id,
- PluginResource.resource_id.in_(resource_ids),
- )
- .order_by(PluginResource.resource_id)
- .all()
- )
- return await _close_workers(db, rows)
+ return await _close_workers(await run_in_session(_owner_rows, user_id, resource_ids))
async def sweep(after="", limit=100):
diff --git a/src/backend/core/applications/schemas/application.schema.json b/src/backend/features/workbench/schemas/application.schema.json
similarity index 100%
rename from src/backend/core/applications/schemas/application.schema.json
rename to src/backend/features/workbench/schemas/application.schema.json
diff --git a/src/backend/core/applications/schemas/host.json b/src/backend/features/workbench/schemas/host.json
similarity index 100%
rename from src/backend/core/applications/schemas/host.json
rename to src/backend/features/workbench/schemas/host.json
diff --git a/src/backend/core/applications/schemas/module.schema.json b/src/backend/features/workbench/schemas/module.schema.json
similarity index 100%
rename from src/backend/core/applications/schemas/module.schema.json
rename to src/backend/features/workbench/schemas/module.schema.json
diff --git a/src/backend/core/applications/state.py b/src/backend/features/workbench/state.py
similarity index 96%
rename from src/backend/core/applications/state.py
rename to src/backend/features/workbench/state.py
index 248f8587..0135b0ca 100644
--- a/src/backend/core/applications/state.py
+++ b/src/backend/features/workbench/state.py
@@ -5,7 +5,7 @@
serializes CAS and aggregate quotas in one transaction on SQLite and PostgreSQL.
"""
-from core.applications.coordination import runtime
+from features.workbench.coordination import runtime
import json
@@ -13,7 +13,7 @@
import re
from copy import deepcopy
from hashlib import sha256
-from core.applications import catalog, external
+from features.workbench import catalog, external
from core.db.models import ContentBlock
from core.infra.time import utc_now
from sqlalchemy import update
@@ -136,7 +136,7 @@ def save(
raw = encode(body["payload"])
# Bootstrap the durable service identity before acquiring the write barrier.
# origin.current may commit on first use; it must never release our CAS lock.
- from core.applications import origin as service_origin
+ from features.workbench import origin as service_origin
service_origin.current(db)
try:
@@ -178,7 +178,7 @@ def save(
"sizeBytes": size,
}
if row and body["reset"] and body["schemaVersion"] != current["schemaVersion"]:
- from core.applications.state_recovery import preserve_before_reset
+ from features.workbench.state_recovery import preserve_before_reset
preserve_before_reset(db, row)
now = utc_now()
diff --git a/src/backend/core/applications/state_recovery.py b/src/backend/features/workbench/state_recovery.py
similarity index 100%
rename from src/backend/core/applications/state_recovery.py
rename to src/backend/features/workbench/state_recovery.py
diff --git a/src/backend/core/plugins/ui/__init__.py b/src/backend/features/workbench/ui/__init__.py
similarity index 100%
rename from src/backend/core/plugins/ui/__init__.py
rename to src/backend/features/workbench/ui/__init__.py
diff --git a/src/backend/core/plugins/ui/contract.py b/src/backend/features/workbench/ui/contract.py
similarity index 99%
rename from src/backend/core/plugins/ui/contract.py
rename to src/backend/features/workbench/ui/contract.py
index d877e1c6..34b618d3 100644
--- a/src/backend/core/plugins/ui/contract.py
+++ b/src/backend/features/workbench/ui/contract.py
@@ -25,7 +25,7 @@
from typing import Any, Dict, List, Optional, Tuple
-from core.applications.protocol import UI_VERSION, validate_contributions
+from features.workbench.protocol import UI_VERSION, validate_contributions
from .declarations import (
_canvas_views,
diff --git a/src/backend/core/plugins/ui/contributions.py b/src/backend/features/workbench/ui/contributions.py
similarity index 87%
rename from src/backend/core/plugins/ui/contributions.py
rename to src/backend/features/workbench/ui/contributions.py
index 4adc5dbd..7f705be7 100644
--- a/src/backend/core/plugins/ui/contributions.py
+++ b/src/backend/features/workbench/ui/contributions.py
@@ -6,9 +6,9 @@
from typing import Any, Dict, List, Optional
from core.db.models import InstalledPlugin
-from core.plugins.management import queries as plugin_queries
-from core.plugins.packaging import sources as plugin_sources
-from core.plugins.ui.contract import public_contributions
+from core.capabilities.plugins.management import queries as plugin_queries
+from core.capabilities.plugins.packaging import sources as plugin_sources
+from features.workbench.ui.contract import public_contributions
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
@@ -30,8 +30,8 @@ def refresh_builtin_ui_contributions(db: Session) -> int:
"""
import json
- from core.plugins.packaging.importer import _ext_or_top, manifest_extensions
- from core.plugins.ui.contract import normalize_ui
+ from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions
+ from features.workbench.ui.contract import normalize_ui
# One manifest read per distinct slug, not per install row.
ui_by_slug: Dict[str, Optional[Dict[str, Any]]] = {}
@@ -39,7 +39,7 @@ def refresh_builtin_ui_contributions(db: Session) -> int:
def _bundle_ui(slug: str) -> Optional[Dict[str, Any]]:
if slug not in ui_by_slug:
ui: Optional[Dict[str, Any]] = None
- plugin_dir = plugin_sources._resolve_plugin_dir(slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(slug)
if plugin_dir is not None:
try:
manifest = json.loads((plugin_dir / "plugin.json").read_text(encoding="utf-8"))
@@ -54,7 +54,7 @@ def _bundle_ui(slug: str) -> Optional[Dict[str, Any]]:
updated = 0
rows = db.query(InstalledPlugin).filter(InstalledPlugin.source == "builtin").all()
for row in rows:
- plugin_dir = plugin_sources._resolve_plugin_dir(row.slug)
+ plugin_dir = plugin_sources.resolve_plugin_dir(row.slug)
if plugin_dir is None:
# Bundle removed from this build (e.g. CE tree): leave the row alone.
continue
@@ -115,7 +115,7 @@ def resolve_installed_ui(db: Session, slug: str, owner_user_id: str) -> Optional
one slug directly instead of enumerating every installed plugin (which also
re-reads each bundle's plugin.json for display metadata).
"""
- from core.config.catalog_resolver import resolve_all_runtime_enabled
+ from core.capabilities.catalog.resolver import resolve_all_runtime_enabled
from sqlalchemy import or_
rows = (
@@ -144,6 +144,6 @@ def resolve_installed_ui(db: Session, slug: str, owner_user_id: str) -> Optional
enabled_skills = set(eff_skills or [])
enabled_mcps = set(eff_mcps or [])
enabled = any(
- s in enabled_skills for s in (plugin_sources._component_keys(cids, "skills"))
- ) or any(m in enabled_mcps for m in (plugin_sources._component_keys(cids, "mcp")))
+ s in enabled_skills for s in (plugin_sources.component_keys(cids, "skills"))
+ ) or any(m in enabled_mcps for m in (plugin_sources.component_keys(cids, "mcp")))
return row.ui_contributions if enabled else None
diff --git a/src/backend/core/plugins/ui/data_proxy.py b/src/backend/features/workbench/ui/data_proxy.py
similarity index 97%
rename from src/backend/core/plugins/ui/data_proxy.py
rename to src/backend/features/workbench/ui/data_proxy.py
index 40534504..a2b8c3e0 100644
--- a/src/backend/core/plugins/ui/data_proxy.py
+++ b/src/backend/features/workbench/ui/data_proxy.py
@@ -63,7 +63,7 @@ def __init__(self, message: str, *, status_code: int = 502, retryable: bool = Fa
def _config_value(key: str) -> str:
- from core.services.system_config import SystemConfigService
+ from core.config.store.system_config import SystemConfigService
try:
return str(SystemConfigService.get_instance().get(key) or "").strip()
@@ -93,10 +93,10 @@ def _admin_config_field_keys(slug: str) -> List[str]:
skills directory) — far too heavy for a per-request path. This mirrors the
lightweight pattern of ``plugin_admin_config._has_admin_config_for_slug``.
"""
- from core.plugins.packaging.importer import _ext_or_top, manifest_extensions
- from core.plugins.packaging.sources import _resolve_plugin_dir
+ from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions
+ from core.capabilities.plugins.packaging.sources import resolve_plugin_dir
- plugin_dir = _resolve_plugin_dir(slug)
+ plugin_dir = resolve_plugin_dir(slug)
if plugin_dir is None:
return []
try:
@@ -350,9 +350,9 @@ def module_asset_path(slug: str, entry_or_path: str) -> Optional["object"]:
frontend tree, so this is the only door to them.
"""
- from core.plugins.packaging.sources import _resolve_plugin_dir
+ from core.capabilities.plugins.packaging.sources import resolve_plugin_dir
- plugin_dir = _resolve_plugin_dir(slug)
+ plugin_dir = resolve_plugin_dir(slug)
if plugin_dir is None:
return None
web_root = (plugin_dir / "web").resolve()
diff --git a/src/backend/core/plugins/ui/declarations.py b/src/backend/features/workbench/ui/declarations.py
similarity index 99%
rename from src/backend/core/plugins/ui/declarations.py
rename to src/backend/features/workbench/ui/declarations.py
index e38e50a5..6c338f26 100644
--- a/src/backend/core/plugins/ui/declarations.py
+++ b/src/backend/features/workbench/ui/declarations.py
@@ -289,7 +289,7 @@ def _modules(
if surface not in MODULE_SURFACES and not (allow_workspace and surface == "workspace"):
dropper.drop("modules", mid, f"未知 surface:{surface}")
continue
- from core.applications.protocol import HOST_CAPABILITIES
+ from features.workbench.protocol import HOST_CAPABILITIES
grants = [
g
diff --git a/src/backend/core/plugins/ui/primitives.py b/src/backend/features/workbench/ui/primitives.py
similarity index 99%
rename from src/backend/core/plugins/ui/primitives.py
rename to src/backend/features/workbench/ui/primitives.py
index 90aeb1ec..7bb8fe9d 100644
--- a/src/backend/core/plugins/ui/primitives.py
+++ b/src/backend/features/workbench/ui/primitives.py
@@ -6,7 +6,7 @@
import re
from typing import Any, Dict, List, Optional, Tuple
-from core.applications.protocol import UI_VERSION
+from features.workbench.protocol import UI_VERSION
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/plugins/ui/workspace_enabled.py b/src/backend/features/workbench/ui/workspace_enabled.py
similarity index 100%
rename from src/backend/core/plugins/ui/workspace_enabled.py
rename to src/backend/features/workbench/ui/workspace_enabled.py
diff --git a/src/backend/core/applications/workspace_page.py b/src/backend/features/workbench/workspace_page.py
similarity index 98%
rename from src/backend/core/applications/workspace_page.py
rename to src/backend/features/workbench/workspace_page.py
index 4b361b7a..6c4824a2 100644
--- a/src/backend/core/applications/workspace_page.py
+++ b/src/backend/features/workbench/workspace_page.py
@@ -11,8 +11,8 @@
from pathlib import Path
from urllib.parse import unquote, urlsplit
from fastapi import HTTPException
-from core.plugins.resources.installation import safe_path
-from core.services.site_service import guess_site_mime
+from core.capabilities.plugins.resources.installation import safe_path
+from features.sites.service import guess_site_mime
MAX_PAGE_BYTES = 8 * 1024 * 1024
diff --git a/src/backend/hugagent_cli/schema.py b/src/backend/hugagent_cli/schema.py
new file mode 100644
index 00000000..44f0a8d5
--- /dev/null
+++ b/src/backend/hugagent_cli/schema.py
@@ -0,0 +1,20 @@
+"""Schema and catalog initialization for the local CLI."""
+
+def ensure_schema_and_seed():
+ """create_all + built-in catalog seed (mirrors the app's startup hooks)."""
+ from core.db.engine import SessionLocal, init_db
+ from core.capabilities.mcp.service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty
+
+ from features.installed import INSTALLED_FEATURES
+ from core.harness.registry import load_models
+
+ load_models(INSTALLED_FEATURES)
+ from core.harness.registry.schema_lifecycle import load_schema_lifecycle
+ load_schema_lifecycle(INSTALLED_FEATURES)
+ init_db()
+ db = SessionLocal()
+ try:
+ prune_removed_builtin_mcp_servers(db)
+ seed_builtin_mcp_servers_if_empty(db)
+ finally:
+ db.close()
diff --git a/src/backend/integrations/__init__.py b/src/backend/integrations/__init__.py
new file mode 100644
index 00000000..479f29da
--- /dev/null
+++ b/src/backend/integrations/__init__.py
@@ -0,0 +1,8 @@
+"""Host composition for installed external-service integrations."""
+def register_credentials():
+ from importlib import import_module
+ from features.installed import INSTALLED_FEATURES
+ for package in INSTALLED_FEATURES:
+ register = getattr(import_module(f"{package}.extension"), "register_credentials", None)
+ if register:
+ register()
diff --git a/src/backend/integrations/api/__init__.py b/src/backend/integrations/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/integrations/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/integrations.py b/src/backend/integrations/api/integrations.py
similarity index 86%
rename from src/backend/api/routes/v1/integrations.py
rename to src/backend/integrations/api/integrations.py
index 48018f0b..51724b67 100644
--- a/src/backend/api/routes/v1/integrations.py
+++ b/src/backend/integrations/api/integrations.py
@@ -13,15 +13,13 @@
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
-from sqlalchemy.orm import Session
from core.auth.backend import UserContext, get_current_user
-from core.db.engine import get_db
from core.infra.responses import success_response
-from core.services.dingtalk_service import DingTalkService
-from core.services.email_service import EmailService
-from core.services.lark_service import LarkService
-from core.services.yida_service import YidaService
+from integrations.dingtalk.service import DingTalkService
+from integrations.email.service import EmailService
+from integrations.lark.service import LarkService
+from integrations.yida.service import YidaService
router = APIRouter(prefix="/v1/integrations", tags=["Integrations"])
@@ -30,19 +28,17 @@
async def dingtalk_status(
probe: bool = Query(False, description="true 时到沙箱实时核对登录态并对账"),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = DingTalkService(db)
- data = await svc.probe_status(str(user.user_id)) if probe else svc.get_status(str(user.user_id))
+ svc = DingTalkService()
+ data = await svc.probe_status(str(user.user_id)) if probe else await svc.get_status(str(user.user_id))
return success_response(data=data)
@router.post("/dingtalk/login", summary="发起钉钉设备流登录")
async def dingtalk_login(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = DingTalkService(db)
+ svc = DingTalkService()
data = await svc.start_device_login(str(user.user_id))
return success_response(data=data)
@@ -50,9 +46,8 @@ async def dingtalk_login(
@router.post("/dingtalk/login/poll", summary="轮询钉钉登录是否完成")
async def dingtalk_login_poll(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = DingTalkService(db)
+ svc = DingTalkService()
data = await svc.poll_login(str(user.user_id))
return success_response(data=data)
@@ -60,9 +55,8 @@ async def dingtalk_login_poll(
@router.post("/dingtalk/disconnect", summary="断开钉钉连接并清除凭据")
async def dingtalk_disconnect(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = DingTalkService(db)
+ svc = DingTalkService()
data = await svc.disconnect(str(user.user_id))
return success_response(data=data)
@@ -74,19 +68,17 @@ async def dingtalk_disconnect(
async def lark_status(
probe: bool = Query(False, description="true 时真实 API 探活并对账"),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = LarkService(db)
- data = await svc.probe_status(str(user.user_id)) if probe else svc.get_status(str(user.user_id))
+ svc = LarkService()
+ data = await svc.probe_status(str(user.user_id)) if probe else await svc.get_status(str(user.user_id))
return success_response(data=data)
@router.post("/lark/login", summary="发起飞书设备流登录(扫码)")
async def lark_login(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = LarkService(db)
+ svc = LarkService()
data = await svc.start_device_login(str(user.user_id))
return success_response(data=data)
@@ -94,9 +86,8 @@ async def lark_login(
@router.post("/lark/login/poll", summary="轮询飞书扫码登录是否完成")
async def lark_login_poll(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = LarkService(db)
+ svc = LarkService()
data = await svc.poll_login(str(user.user_id))
return success_response(data=data)
@@ -104,9 +95,8 @@ async def lark_login_poll(
@router.post("/lark/disconnect", summary="断开飞书连接并清除凭据")
async def lark_disconnect(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = LarkService(db)
+ svc = LarkService()
data = await svc.disconnect(str(user.user_id))
return success_response(data=data)
@@ -188,10 +178,9 @@ class _EmailConnectRequest(BaseModel):
async def email_status(
probe: bool = Query(False, description="true 时真实连通性探活并对账"),
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = EmailService(db)
- data = await svc.probe_status(str(user.user_id)) if probe else svc.get_status(str(user.user_id))
+ svc = EmailService()
+ data = await svc.probe_status(str(user.user_id)) if probe else await svc.get_status(str(user.user_id))
return success_response(data=data)
@@ -199,9 +188,8 @@ async def email_status(
async def email_connect(
body: _EmailConnectRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = EmailService(db)
+ svc = EmailService()
data = await svc.connect(
str(user.user_id),
email_address=body.email_address,
@@ -215,8 +203,7 @@ async def email_connect(
@router.post("/email/disconnect", summary="断开电子邮箱连接并清除凭据")
async def email_disconnect(
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = EmailService(db)
+ svc = EmailService()
data = await svc.disconnect(str(user.user_id))
return success_response(data=data)
diff --git a/src/backend/integrations/channels/README.md b/src/backend/integrations/channels/README.md
new file mode 100644
index 00000000..2dfbe2d3
--- /dev/null
+++ b/src/backend/integrations/channels/README.md
@@ -0,0 +1,9 @@
+# 消息通道
+
+本模块拥有通道连接、设备中继数据、适配器、入站运行、出站投递和附件工具。FULL 与 CE 均安装模型,运行开关不控制建表。API 注册仍由中央有序路由表负责,路由及权限策略保持原样。
+
+入站处理按会话、附件、消息观察、历史拉取及回复投递拆分。inbound_state 是串行锁、去重缓存和限额的唯一所有者。适配器的历史解析、消息解析和发送方法按职责分组;具体适配器保留能力声明及令牌缓存,飞书连接入口保留线程事件循环隔离状态。
+
+依赖 core 的认证、数据库、聊天运行、存储和工具注册接口;桌面中继暂时引用 core 中的桌面服务,P5 随归属迁移。外部调用方直接引用所属模块,不提供旧路径转发。通道账号与个人 OAuth 账号相互独立,入站始终采用机器人所有者身份与原资源范围。
+
+测试位于 tests/integrations/channels。bot_cases 模块按组存放用例,test_channel_bots 显式收集这些函数,以保留原测试节点身份及失败基线映射;不减少用例或断言。
diff --git a/src/backend/core/channels/__init__.py b/src/backend/integrations/channels/__init__.py
similarity index 75%
rename from src/backend/core/channels/__init__.py
rename to src/backend/integrations/channels/__init__.py
index 0a26d72e..c48904c2 100644
--- a/src/backend/core/channels/__init__.py
+++ b/src/backend/integrations/channels/__init__.py
@@ -8,13 +8,8 @@
See internal design docs.
"""
-from core.channels.protocol import (
- ChannelAdapter,
- ChannelCaps,
- InboundMsg,
- SendResult,
-)
-from core.channels.registry import get_adapter, register_adapter, list_adapters
+from integrations.channels.protocol import ChannelAdapter, ChannelCaps, InboundMsg, SendResult
+from integrations.channels.registry import get_adapter, register_adapter, list_adapters
__all__ = [
"ChannelAdapter",
diff --git a/src/backend/core/channels/adapters/__init__.py b/src/backend/integrations/channels/adapters/__init__.py
similarity index 100%
rename from src/backend/core/channels/adapters/__init__.py
rename to src/backend/integrations/channels/adapters/__init__.py
diff --git a/src/backend/integrations/channels/adapters/dingtalk.py b/src/backend/integrations/channels/adapters/dingtalk.py
new file mode 100644
index 00000000..1f830d7b
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/dingtalk.py
@@ -0,0 +1,325 @@
+"""DingTalk inbound channel adapter.
+
+- **Binding validation**: exchange AppKey/AppSecret for an access_token
+ (DingTalk v1.0 OAuth) — obtaining one is treated as valid credentials.
+- **Inbound (Stream long connection)**: WebSocket via the official
+ ``dingtalk-stream`` SDK, listening for bot message callbacks
+ (``ChatbotMessage.TOPIC``). Zero public IP, no callback configuration. If the
+ SDK is missing, this channel is unavailable but the process is unaffected.
+- **Outbound replies**: use the ``sessionWebhook`` carried by each inbound
+ message (no access_token needed), unified for p2p / group.
+ Agent replies go through ``send_markdown`` as ``msgtype=markdown`` (DingTalk
+ natively renders headings/bold/links/lists, etc.; tables/code fences do not
+ render on mobile, so content is downgraded via
+ ``core.channels.markdown.downgrade_for_dingtalk`` before sending).
+ Short system messages (placeholders/receipts) still use ``send_text`` (plain text).
+ DingTalk bots **do not support editing messages** → ``edit_message`` returns
+ failure; but they do support **silent recall** of messages sent via the robot
+ API (group ``groupMessages/recall`` / one-to-one ``otoMessages/batchRecall``,
+ using the processQueryKey returned at send time). So placeholder messages go
+ through ``send_placeholder`` via the robot API (obtaining a recallable key),
+ and before the final reply the upper layer calls ``recall_message`` to remove
+ the placeholder — visually equivalent to "replace". When the robot API is
+ unavailable (no permission, etc.), placeholders automatically fall back to
+ sessionWebhook plain text (not recallable, same behavior as the old version).
+- **Outbound proactive delivery (automation scheduled tasks, etc., no inbound
+ message)**: synthetic messages have no sessionWebhook → ``send_text`` /
+ ``send_markdown`` automatically fall back to the robot API (``sampleText`` /
+ ``sampleMarkdown``).
+- **Outbound files**: sessionWebhook only accepts text-type messages, cannot
+ send files → ``push_file`` uses the robot API: first ``media/upload`` to get
+ a mediaId, then send a robot message. The robot message picks its endpoint by
+ conversation type: group ``groupMessages/send`` (openConversationId);
+ one-to-one with a staffId uses ``oToMessages/batchSend``, without one
+ (proactive delivery) uses the human-bot conversation's openConversationId via
+ ``privateChatMessages/send``.
+ The app must be granted the "internal enterprise robot send message"
+ permission on the DingTalk open platform, otherwise the send endpoints return 403.
+
+Conversation keying (``external_conversation_id``) always uses DingTalk ``conversationId``:
+ - one-to-one (conversationType==``1``) conversationId maps one-to-one to a user → naturally one conversation per person;
+ - group (conversationType==``2``) conversationId is unique per group → the whole group shares one conversation.
+
+Isomorphic to the [[lark]] adapter. See internal design docs.
+"""
+
+from __future__ import annotations
+from integrations.channels.adapters.dingtalk_history import DingTalkHistory
+from integrations.channels.adapters.dingtalk_delivery import DingTalkDelivery
+
+import asyncio
+import json
+import logging
+import os
+import re
+import shutil
+import tempfile
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+import httpx
+
+from datetime import datetime, timezone
+
+from integrations.channels.markdown import derive_title, downgrade_for_dingtalk
+from integrations.channels.protocol import ChannelCaps, ChannelResourceError, HistoryItem, InboundMsg, SendResult, chunk_text
+from core.infra.crypto import decrypt_secret
+
+logger = logging.getLogger(__name__)
+
+from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE
+
+
+class DingTalkAdapter(DingTalkHistory, DingTalkDelivery):
+ caps = ChannelCaps(
+ channel_type="dingtalk",
+ max_message_len=4000,
+ supports_markdown=True,
+ splits_long_messages=False,
+ supports_long_conn=True,
+ # DingTalk exposes ``isInAtList`` on every bot callback, so the gate is already
+ # wired; whether non-@ group messages ever get delivered depends on the tenant
+ # holding DingTalk's group-message read permission (applied for and reviewed on the
+ # open platform). Without it, group listening stays inert rather than broken.
+ supports_group_observe=True,
+ bind_mode="credentials",
+ credential_fields=("app_id", "app_secret"),
+ )
+
+ # access_token cache: {app_id: (token, expire_epoch)} (only for validate/testing; replies use sessionWebhook)
+ _token_cache: Dict[str, tuple] = {}
+
+ # ── Credentials ─────────────────────────────────────────────────────
+ @staticmethod
+ def _app_secret(conn: Any) -> str:
+ cfg = conn.config if isinstance(conn.config, dict) else {}
+ return decrypt_secret(cfg.get("app_secret_enc")) or ""
+
+ async def _access_token(self, app_id: str, app_secret: str) -> str:
+ cached = self._token_cache.get(app_id)
+ if cached and cached[1] > time.time() + 60:
+ return cached[0]
+ url = f"{DINGTALK_API_BASE}/v1.0/oauth2/accessToken"
+ async with httpx.AsyncClient(timeout=10) as client:
+ resp = await client.post(url, json={"appKey": app_id, "appSecret": app_secret})
+ data = resp.json()
+ token = data.get("accessToken")
+ if not token:
+ raise RuntimeError(f"钉钉 access_token 获取失败: {data}")
+ self._token_cache[app_id] = (token, time.time() + int(data.get("expireIn", 7000)))
+ return token
+
+ async def validate_credentials(self, conn: Any) -> Dict[str, Any]:
+ secret = self._app_secret(conn)
+ if not conn.app_id or not secret:
+ raise RuntimeError("缺少 AppKey / AppSecret")
+ await self._access_token(conn.app_id, secret)
+ return {"app_id": conn.app_id}
+
+ # ── Webhook verification (DingTalk uses the Stream long connection; no webhook entry point) ──
+ def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool:
+ return True
+
+ # ── Event → InboundMsg (the dict from the Stream callback) ──────────
+ def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]:
+ """DingTalk bot callback data (dict) → InboundMsg. Returns None for non-text/unsupported messages."""
+ msgtype = payload.get("msgtype")
+ text = ""
+ if msgtype == "text":
+ text = ((payload.get("text") or {}).get("content") or "").strip()
+ elif msgtype == "richText":
+ # Rich text: concatenate the plain-text nodes inside
+ parts = (payload.get("content") or {}).get("richText") or []
+ text = " ".join(p.get("text", "") for p in parts if isinstance(p, dict) and p.get("text")).strip()
+ attachments = self._extract_attachments(msgtype, payload)
+ if not text and not attachments:
+ return None
+ conv_type = str(payload.get("conversationType") or "1")
+ chat_type = "group" if conv_type == "2" else "p2p"
+ conv_id = payload.get("conversationId") or ""
+ return InboundMsg(
+ channel_id=conn.channel_id,
+ channel_type="dingtalk",
+ text=text,
+ chat_type=chat_type,
+ external_conversation_id=conv_id,
+ sender_id=payload.get("senderStaffId") or payload.get("senderId") or "",
+ sender_name=payload.get("senderNick") or "",
+ message_id=payload.get("msgId") or "",
+ attachments=attachments,
+ # ``isInAtList`` is DingTalk's own "was the bot in the @ list" flag. With only the
+ # default permission DingTalk delivers @bot messages exclusively, so this is
+ # always true and the gate is a no-op; once the tenant is granted group-message
+ # read, non-@ messages start arriving with it false and silent observation kicks
+ # in with no further code change. Absent field → treat as addressed.
+ addressed_to_bot=(
+ True if chat_type == "p2p" else bool(payload.get("isInAtList", True))
+ ),
+ raw={
+ "dingtalk_session_webhook": payload.get("sessionWebhook") or "",
+ "dingtalk_conversation_id": conv_id,
+ # Needed by download_resource: messageFiles/download requires robotCode, and
+ # per DingTalk's docs it is **not** the AppKey — the only reliable source is
+ # this field on the callback payload (present in Stream mode, which is what
+ # the long connection uses).
+ "dingtalk_robot_code": payload.get("robotCode") or "",
+ },
+ )
+
+ # ── Group history pull (via the official dws CLI) ────────────────────
+ # DingTalk has **no** REST OpenAPI for reading a group's message history — the
+ # server-side API list only covers sending, recalling and read receipts. The capability
+ # is reachable two ways, and this takes the second:
+ # 1. DingTalk's MCP gateway (`list_conversation_message_v2` on the group-chat MCP
+ # server). Works, but needs credentials provisioned per app — extra setup.
+ # 2. The official `dws` CLI, which the backend **already** runs as a subprocess for
+ # this user's DingTalk connection and which is **already logged in**. No new
+ # credentials, no extra configuration: it simply reuses the connection the owner
+ # has set up. That is why this is the one implemented.
+ #
+ # Runs as the connection's owner (per-user $HOME is the credential isolation boundary,
+ # same as every other backend dws call). The 8s cap keeps a slow CLI from eating into
+ # the caller's own history budget; any failure returns [] and history is skipped.
+ _DWS_HISTORY_TIMEOUT_S = 8
+
+
+ _DWS_DOWNLOAD_TIMEOUT_S = 45
+
+
+
+
+
+ # dws renders a file message as **plain text**, not a structured attachment:
+ # "[文件] 周报.docx fileId: X6GRez... 注意:如需下载使用dws drive download命令下载"
+ # Parsed back out here, otherwise a pulled file is just an unusable sentence — the agent
+ # can see a file was shared but has no handle to open it. Images use the same shape.
+ _DWS_FILE_RE = re.compile(
+ r"\[(?P文件|图片)\]\s*(?P.+?)\s+fileId:\s*(?P[A-Za-z0-9_\-]+)"
+ r"(?:\s*注意[::][^\n]*)?"
+ )
+
+ # A DingTalk **online document** shared into a group arrives as a link card, not as a
+ # file: a title line, an icon, and an alidocs URL. It is a different resource from a Drive
+ # file — read with `dws doc read`, not downloaded by fileId — and without recognising it
+ # the agent sees only an unusable link.
+ _DWS_DOC_RE = re.compile(r"https?://alidocs\.dingtalk\.com/i/nodes/(?P[A-Za-z0-9_\-]+)")
+
+
+
+
+ # ── Inbound attachments ─────────────────────────────────────────────
+
+
+ # ── Outbound push (via sessionWebhook, no access_token needed) ───────
+
+
+
+ # Lets the upper layer do a whole-content downgrade **before** chunking
+ # (avoids a table getting bisected by a chunk boundary and becoming unrecognizable for conversion).
+ prepare_markdown = staticmethod(downgrade_for_dingtalk)
+
+
+
+
+ # ── Placeholder messages: sent via robot API (recallable) → recall as an equivalent of "replace" ──
+
+
+
+ # ── Outbound file delivery (sessionWebhook cannot send files → use the robot API) ──
+
+
+
+ # ── Long connection (Stream, requires the dingtalk-stream SDK) ──────
+ def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any:
+ """Build the DingTalk Stream long-connection runner. ``on_message`` is invoked synchronously on the SDK thread (dispatched to the main loop).
+
+ dingtalk_stream is lazily imported; if not installed a RuntimeError is
+ raised, which the manager records as an error without affecting the
+ process. The SDK's ``start()`` is async and ``start_forever()`` cannot
+ be stopped → wrap it in a controllable runner so the manager can start
+ and stop cleanly (actually disconnecting when a bot is disabled).
+ """
+ try:
+ import dingtalk_stream
+ except Exception as exc: # noqa: BLE001
+ raise RuntimeError("dingtalk-stream SDK 未安装,长连接不可用") from exc
+
+ adapter = self
+ app_id = conn.app_id
+ secret = self._app_secret(conn)
+
+ class _Handler(dingtalk_stream.ChatbotHandler):
+ async def process(self, callback): # type: ignore[override]
+ try:
+ inbound = adapter.parse_inbound(conn, callback.data or {})
+ if inbound is not None:
+ on_message(inbound)
+ except Exception: # noqa: BLE001
+ logger.exception("[dingtalk] Stream 事件处理失败 channel_id=%s", conn.channel_id)
+ return dingtalk_stream.AckMessage.STATUS_OK, "OK"
+
+ def _build_client():
+ sdk_logger = logging.getLogger("dingtalk_stream")
+ sdk_logger.setLevel(logging.WARNING)
+ credential = dingtalk_stream.Credential(app_id, secret)
+ client = dingtalk_stream.DingTalkStreamClient(credential, logger=sdk_logger)
+ client.register_callback_handler(
+ dingtalk_stream.chatbot.ChatbotMessage.TOPIC, _Handler()
+ )
+ return client
+
+ return _DingTalkStreamRunner(_build_client)
+
+
+class _DingTalkStreamRunner:
+ """Wraps dingtalk-stream's async ``client.start()`` into the "blocking start() + stoppable stop()" the manager expects.
+
+ Creates a dedicated event loop in the worker thread to run ``client.start()``
+ (the SDK reconnects on its own); ``stop()`` closes the websocket and cancels
+ the task from the main thread via ``call_soon_threadsafe``, making
+ ``start()`` return → the manager's thread body sees stop_flag set and exits
+ cleanly.
+ """
+
+ def __init__(self, build_client: Callable[[], Any]):
+ self._build_client = build_client
+ self._loop: Optional[asyncio.AbstractEventLoop] = None
+ self._client: Any = None
+ self._task: Optional[asyncio.Task] = None
+
+ def start(self) -> None:
+ loop = asyncio.new_event_loop()
+ self._loop = loop
+ asyncio.set_event_loop(loop)
+ try:
+ self._client = self._build_client()
+ self._task = loop.create_task(self._client.start())
+ loop.run_until_complete(self._task)
+ except asyncio.CancelledError:
+ pass
+ finally:
+ try:
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ except Exception: # noqa: BLE001
+ pass
+ loop.close()
+
+ def stop(self) -> None:
+ loop, task, client = self._loop, self._task, self._client
+ if loop is None or loop.is_closed():
+ return
+
+ def _cancel() -> None:
+ ws = getattr(client, "websocket", None)
+ if ws is not None:
+ try:
+ asyncio.ensure_future(ws.close())
+ except Exception: # noqa: BLE001
+ pass
+ if task is not None:
+ task.cancel()
+
+ try:
+ loop.call_soon_threadsafe(_cancel)
+ except Exception: # noqa: BLE001
+ pass
diff --git a/src/backend/integrations/channels/adapters/dingtalk_delivery.py b/src/backend/integrations/channels/adapters/dingtalk_delivery.py
new file mode 100644
index 00000000..faca8bc4
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/dingtalk_delivery.py
@@ -0,0 +1,213 @@
+"""dingtalk delivery behavior."""
+from __future__ import annotations
+import json
+import logging
+from typing import Any, Dict, Optional
+import httpx
+from integrations.channels.markdown import derive_title, downgrade_for_dingtalk
+from integrations.channels.protocol import InboundMsg, SendResult, chunk_text
+from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE
+logger = logging.getLogger(__name__)
+
+class DingTalkDelivery:
+ @staticmethod
+ def _session_webhook(inbound: InboundMsg) -> str:
+ return (inbound.raw or {}).get("dingtalk_session_webhook") or ""
+
+ async def _post_webhook(self, webhook: str, payload: Dict[str, Any]) -> SendResult:
+ if not webhook:
+ return SendResult.fail("forbidden", "缺少 sessionWebhook(可能已过期)")
+ try:
+ async with httpx.AsyncClient(timeout=15) as client:
+ resp = await client.post(webhook, json=payload)
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ if data.get("errcode", 0) == 0:
+ return SendResult.ok()
+ kind = "rate_limited" if data.get("errcode") in (130101, 88) else "unknown"
+ return SendResult.fail(kind, f"errcode={data.get('errcode')} {data.get('errmsg')}")
+
+ async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
+ webhook = self._session_webhook(inbound)
+ if webhook:
+ return await self._post_webhook(
+ webhook, {"msgtype": "text", "text": {"content": text}}
+ )
+ # Proactive delivery (automation scheduled tasks, etc.) has no inbound sessionWebhook → use the robot API
+ try:
+ token = await self._access_token(conn.app_id, self._app_secret(conn))
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ return await self._robot_send(token, conn, inbound, "sampleText", {"content": text})
+
+ async def send_markdown(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
+ """Send a DingTalk markdown message (natively rendered). The downgrade is idempotent; calling directly or via prepare_markdown is equally safe.
+
+ title is required for DingTalk's markdown type; it shows in the
+ conversation-list summary / push notification and is derived from the
+ first line of the body.
+ """
+ md = downgrade_for_dingtalk(text)
+ title = derive_title(md)
+ webhook = self._session_webhook(inbound)
+ if webhook:
+ return await self._post_webhook(
+ webhook, {"msgtype": "markdown", "markdown": {"title": title, "text": md}}
+ )
+ # Proactive delivery → robot-API markdown message
+ try:
+ token = await self._access_token(conn.app_id, self._app_secret(conn))
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ return await self._robot_send(
+ token, conn, inbound, "sampleMarkdown", {"title": title, "text": md}
+ )
+
+ async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult:
+ md = downgrade_for_dingtalk(content)
+ chunks = chunk_text(md, self.caps.max_message_len) or [md]
+ first: Optional[SendResult] = None
+ for i, c in enumerate(chunks):
+ r = await self.send_markdown(conn, inbound, c)
+ if i == 0:
+ first = r
+ if not r.success:
+ return r
+ return first or SendResult.fail("unknown", "空内容")
+
+ async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult:
+ # DingTalk bots cannot edit messages — the upper-layer placeholder logic
+ # therefore switches to "recall + resend" (see recall_message).
+ return SendResult.fail("bad_format", "钉钉不支持编辑消息")
+
+ async def send_placeholder(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
+ """Send a recallable placeholder message via the robot API; message_id is the processQueryKey (recall credential).
+
+ If the robot API is unavailable (no permission / token failure) → fall
+ back to sessionWebhook plain text; then there is no message_id and the
+ upper layer will not attempt recall — behavior degrades to the old
+ version (placeholder stays, reply is sent fresh).
+ """
+ try:
+ token = await self._access_token(conn.app_id, self._app_secret(conn))
+ r = await self._robot_send(token, conn, inbound, "sampleText", {"content": text})
+ if r.success:
+ return r
+ except Exception: # noqa: BLE001
+ logger.debug("[dingtalk] 机器人占位发送失败,回退 webhook", exc_info=True)
+ return await self.send_text(conn, inbound, text)
+
+ def _recall_url_body(
+ self, conn: Any, inbound: InboundMsg, message_id: str
+ ) -> Optional[tuple]:
+ """Pick the recall endpoint by conversation type. Group chats need openConversationId; one-to-one batchRecall needs only the key."""
+ body: Dict[str, Any] = {"robotCode": conn.app_id, "processQueryKeys": [message_id]}
+ if inbound.chat_type == "group":
+ conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
+ if not conv_id:
+ return None
+ body["openConversationId"] = conv_id
+ return f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/recall", body
+ return f"{DINGTALK_API_BASE}/v1.0/robot/otoMessages/batchRecall", body
+
+ async def recall_message(self, conn: Any, inbound: InboundMsg, message_id: str) -> SendResult:
+ """Silently recall a message sent via the robot API (the client shows no recall notice)."""
+ if not message_id:
+ return SendResult.fail("bad_format", "缺少 processQueryKey")
+ target = self._recall_url_body(conn, inbound, message_id)
+ if target is None:
+ return SendResult.fail("bad_format", "缺少 openConversationId")
+ url, body = target
+ try:
+ token = await self._access_token(conn.app_id, self._app_secret(conn))
+ async with httpx.AsyncClient(timeout=15) as client:
+ resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body)
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ if resp.status_code == 200:
+ return SendResult.ok(message_id)
+ return SendResult.fail("unknown", f"撤回失败: {resp.status_code} {str(data)[:200]}")
+
+ async def push_file(
+ self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str
+ ) -> SendResult:
+ secret = self._app_secret(conn)
+ if not conn.app_id or not secret:
+ return SendResult.fail("forbidden", "缺少 AppKey / AppSecret")
+ try:
+ token = await self._access_token(conn.app_id, secret)
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+
+ is_image = (mime_type or "").startswith("image/")
+ media_id = await self._upload_media(token, content, filename, "image" if is_image else "file")
+ if isinstance(media_id, SendResult):
+ return media_id
+
+ if is_image:
+ # sampleImageMsg's photoURL accepts a mediaId directly
+ msg_key, msg_param = "sampleImageMsg", {"photoURL": media_id}
+ else:
+ ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() or "file"
+ msg_key = "sampleFile"
+ msg_param = {"mediaId": media_id, "fileName": filename, "fileType": ext}
+ return await self._robot_send(token, conn, inbound, msg_key, msg_param)
+
+ async def _upload_media(self, token: str, content: bytes, filename: str, media_type: str):
+ """Upload media in exchange for a mediaId; returns SendResult on failure (str on success)."""
+ url = f"https://oapi.dingtalk.com/media/upload?access_token={token}&type={media_type}"
+ try:
+ async with httpx.AsyncClient(timeout=60) as client:
+ resp = await client.post(url, files={"media": (filename, content)})
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", f"媒体上传失败: {exc}")
+ media_id = data.get("media_id")
+ if data.get("errcode", 0) != 0 or not media_id:
+ return SendResult.fail("unknown", f"媒体上传失败: errcode={data.get('errcode')} {data.get('errmsg')}")
+ return media_id
+
+ async def _robot_send(
+ self, token: str, conn: Any, inbound: InboundMsg, msg_key: str, msg_param: Dict[str, Any]
+ ) -> SendResult:
+ """Robot proactive message: group chats use openConversationId, one-to-one uses the recipient's staffId."""
+ body: Dict[str, Any] = {
+ "robotCode": conn.app_id, # for internal enterprise apps, robotCode == appKey
+ "msgKey": msg_key,
+ "msgParam": json.dumps(msg_param, ensure_ascii=False),
+ }
+ if inbound.chat_type == "group":
+ conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
+ if not conv_id:
+ return SendResult.fail("bad_format", "缺少 openConversationId")
+ url = f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send"
+ body["openConversationId"] = conv_id
+ else:
+ staff_id = inbound.sender_id or ""
+ if staff_id:
+ url = f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend"
+ body["userIds"] = [staff_id]
+ else:
+ # In proactive-delivery scenarios no staffId is available → send directly via the human-bot one-to-one conversation's openConversationId
+ conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or ""
+ if not conv_id:
+ return SendResult.fail("bad_format", "缺少接收人 staffId / openConversationId")
+ url = f"{DINGTALK_API_BASE}/v1.0/robot/privateChatMessages/send"
+ body["openConversationId"] = conv_id
+ try:
+ async with httpx.AsyncClient(timeout=30) as client:
+ resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body)
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ key = data.get("processQueryKey") or data.get("processQueryKeys")
+ if resp.status_code == 200 and key:
+ # The processQueryKey is the recall credential, returned to the upper layer as message_id (used by recall_message)
+ if isinstance(key, list):
+ key = key[0] if key else None
+ return SendResult.ok(key if isinstance(key, str) else None)
+ code = data.get("code") or resp.status_code
+ kind = "forbidden" if resp.status_code == 403 else "unknown"
+ return SendResult.fail(kind, f"机器人消息发送失败: {code} {str(data)[:200]}")
diff --git a/src/backend/integrations/channels/adapters/dingtalk_history.py b/src/backend/integrations/channels/adapters/dingtalk_history.py
new file mode 100644
index 00000000..56398c59
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/dingtalk_history.py
@@ -0,0 +1,361 @@
+"""dingtalk history behavior."""
+from __future__ import annotations
+import json
+import logging
+import os
+import shutil
+import tempfile
+from typing import Any, Dict, List, Optional
+import httpx
+from datetime import datetime, timezone
+from integrations.channels.protocol import ChannelResourceError, HistoryItem, InboundMsg
+from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE
+logger = logging.getLogger(__name__)
+
+class DingTalkHistory:
+ async def fetch_history(
+ self,
+ conn: Any,
+ conversation_id: str,
+ *,
+ since_ms: int,
+ limit: int,
+ newest_first: bool = False,
+ ) -> List[HistoryItem]:
+ """`dws chat message list --group --time ` → normalized history items."""
+ conv = (conversation_id or "").strip()
+ owner = getattr(conn, "owner_user_id", "") or ""
+ if not conv or not owner:
+ return []
+ from integrations.dingtalk.service import _run_dws
+
+ # dws takes a local-time string, not epoch ms. `--forward` picks the direction:
+ # true = messages after --time (incremental from the cursor); false = messages
+ # before --time, which anchored at *now* is how the cold start gets the most recent
+ # ones instead of the oldest of a week-long window.
+ anchor_ms = int(datetime.now(timezone.utc).timestamp() * 1000) if newest_first else since_ms
+ anchor = datetime.fromtimestamp(max(anchor_ms, 0) / 1000, tz=timezone.utc).astimezone()
+ args = [
+ "chat", "message", "list",
+ "--group", conv,
+ "--time", anchor.strftime("%Y-%m-%d %H:%M:%S"),
+ "--limit", str(max(1, min(int(limit or 50), 100))),
+ "--forward=false" if newest_first else "--forward=true",
+ "--format", "json",
+ ]
+ try:
+ out, err, code = await _run_dws(owner, args, timeout=self._DWS_HISTORY_TIMEOUT_S)
+ except Exception: # noqa: BLE001
+ logger.warning("[dingtalk] dws 群历史拉取异常 conv=%s", conv[:24], exc_info=True)
+ return []
+ if code != 0 or not out.strip():
+ # Not logged in / no permission / timeout all land here. Debug, not warning:
+ # a tenant without the DingTalk connection set up would otherwise log on every
+ # single group message.
+ logger.debug(
+ "[dingtalk] dws 群历史拉取未成功 code=%s err=%s", code, (err or "")[:160]
+ )
+ return []
+ try:
+ return self._parse_dws_history(json.loads(out))
+ except Exception: # noqa: BLE001
+ logger.debug("[dingtalk] dws 输出解析失败: %s", out[:200], exc_info=True)
+ return []
+
+ @staticmethod
+ def _dws_business_error(raw_out: str) -> Optional[str]:
+ """Pull the human-readable reason out of a dws error envelope, if that's what this is.
+
+ Worth surfacing rather than collapsing to "download failed": the messages are
+ specific and actionable ("this data belongs to organization X while the tool is
+ configured for organization Y"), and reading that as an expiry sends the user
+ looking in entirely the wrong place.
+ """
+ try:
+ err = (json.loads(raw_out or "{}") or {}).get("error")
+ except Exception: # noqa: BLE001
+ return None
+ if not isinstance(err, dict):
+ return None
+ msg = str(err.get("message") or "").strip()
+ return msg or None
+
+ async def _read_doc_via_dws(self, conn: Any, node_id: str) -> Optional[bytes]:
+ """`dws doc read --node --content-format markdown` → markdown bytes.
+
+ A DingTalk online document has no file to download; it is read as content. Returning
+ markdown keeps it in the same artifact pipeline as everything else.
+ """
+ owner = getattr(conn, "owner_user_id", "") or ""
+ if not owner:
+ return None
+ from integrations.dingtalk.service import _run_dws
+
+ out, err, code = await _run_dws(
+ owner,
+ ["doc", "read", "--node", node_id, "--content-format", "markdown", "--format", "json"],
+ timeout=self._DWS_DOWNLOAD_TIMEOUT_S,
+ )
+ reason = self._dws_business_error(out) or self._dws_business_error(err)
+ if reason:
+ logger.warning("[dingtalk] 钉钉文档读取被拒 node=%s: %s", node_id[:16], reason[:200])
+ raise ChannelResourceError(reason)
+ if code != 0 or not out.strip():
+ logger.debug("[dingtalk] 钉钉文档读取失败 code=%s err=%s", code, (err or "")[:160])
+ return None
+ try:
+ payload = json.loads(out)
+ except Exception: # noqa: BLE001
+ return out.encode("utf-8")
+ # Content sits under a few possible keys depending on CLI version; fall back to the
+ # whole payload rather than silently returning nothing.
+ for key in ("content", "markdown", "data", "result"):
+ val = payload.get(key) if isinstance(payload, dict) else None
+ if isinstance(val, str) and val.strip():
+ return val.encode("utf-8")
+ if isinstance(val, dict):
+ for sub in ("content", "markdown", "text"):
+ inner = val.get(sub)
+ if isinstance(inner, str) and inner.strip():
+ return inner.encode("utf-8")
+ return json.dumps(payload, ensure_ascii=False).encode("utf-8")
+
+ async def _download_via_dws(self, conn: Any, file_id: str) -> Optional[bytes]:
+ """`dws drive download --node ` → bytes.
+
+ The CLI only writes to a local path (it does the signed-URL dance internally), so the
+ bytes are read back from a temp file that is always removed, including on failure —
+ this runs on the backend host, not in a disposable sandbox.
+ """
+ owner = getattr(conn, "owner_user_id", "") or ""
+ if not owner:
+ return None
+ from integrations.dingtalk.service import _run_dws
+
+ tmpdir = tempfile.mkdtemp(prefix="dws_dl_")
+ target = os.path.join(tmpdir, "download.bin")
+ try:
+ out, err, code = await _run_dws(
+ owner,
+ ["drive", "download", "--node", file_id, "--output", target, "--format", "json"],
+ timeout=self._DWS_DOWNLOAD_TIMEOUT_S,
+ )
+ if code != 0 or not os.path.exists(target):
+ logger.warning(
+ "[dingtalk] dws 文件下载失败 file_id=%s code=%s err=%s",
+ file_id[:16], code, (err or out or "")[:160],
+ )
+ return None
+ with open(target, "rb") as fh:
+ return fh.read()
+ except Exception: # noqa: BLE001
+ logger.warning("[dingtalk] dws 文件下载异常 file_id=%s", file_id[:16], exc_info=True)
+ return None
+ finally:
+ shutil.rmtree(tmpdir, ignore_errors=True)
+
+ @staticmethod
+ def _dws_ts_ms(value: Any) -> int:
+ """dws ``createTime`` → epoch ms.
+
+ The CLI returns a **local-time string** (``"2026-08-08 21:57:33"``), not epoch
+ milliseconds. Parsing it as an int silently yielded 0 for every message, which left
+ the pull cursor pinned at the cold-start window — so every @ re-fetched the same 24
+ hours forever. Numeric forms are still accepted in case a CLI version returns them.
+ """
+ if isinstance(value, (int, float)) and value > 0:
+ return int(value)
+ text = str(value or "").strip()
+ if not text:
+ return 0
+ if text.isdigit():
+ return int(text)
+ try:
+ naive = datetime.strptime(text, "%Y-%m-%d %H:%M:%S")
+ except ValueError:
+ return 0
+ # Local time, as the CLI prints it; astimezone() attaches the process zone.
+ return int(naive.astimezone().timestamp() * 1000)
+
+ @classmethod
+ def _extract_dws_docs(cls, content: str) -> List[Dict[str, Any]]:
+ """DingTalk Docs link cards → attachment refs (deduped by node id)."""
+ nodes = {m.group("node") for m in cls._DWS_DOC_RE.finditer(content or "")}
+ if not nodes:
+ return []
+ # The card's first non-empty line is the document title; skip the markdown image line
+ # and the "创建者:" byline, neither of which names the document.
+ title = ""
+ for line in (content or "").splitlines():
+ line = line.strip()
+ if line and not line.startswith("![") and not line.startswith("[http"):
+ if line.startswith("创建者"):
+ continue
+ title = line
+ break
+ return [
+ {"kind": "doc", "key": node, "name": title or "钉钉文档"}
+ for node in sorted(nodes)
+ ]
+
+ @classmethod
+ def _extract_dws_files(cls, content: str) -> tuple:
+ """(cleaned_text, attachment_refs) for one history message body."""
+ refs: List[Dict[str, Any]] = []
+ for m in cls._DWS_FILE_RE.finditer(content or ""):
+ refs.append({
+ "kind": "image" if m.group("kind") == "图片" else "file",
+ "key": m.group("fid"),
+ "name": m.group("name").strip(),
+ })
+ doc_refs = cls._extract_dws_docs(content)
+ if doc_refs:
+ # The card body is title + icon + a long URL repeated twice; the attachment entry
+ # carries title and handle, so keeping the raw card would just burn prompt space.
+ return "", refs + doc_refs
+ if not refs:
+ return (content or "").strip(), []
+ # Drop the matched sentences: the attachment ref already renders name + handle, and
+ # the "use dws drive download" tail is instruction noise aimed at a CLI user.
+ return cls._DWS_FILE_RE.sub("", content or "").strip(), refs
+
+ @staticmethod
+ def _parse_dws_history(body: Dict[str, Any]) -> List[HistoryItem]:
+ """dws JSON output → HistoryItem list.
+
+ Field names verified against live CLI output — they are **not** the ones the bot
+ callback uses, which is the trap here:
+ {"result": {"messages": [{"openMessageId", "sender", "content", "createTime",
+ "openConversationId", "senderOpenDingTalkId"}], ...}}
+ A bare ``{"messages": [...]}`` is accepted too, so a CLI version that drops the
+ envelope does not silently yield nothing.
+ """
+ # Preserve the concrete adapter binding used by history parsing.
+ from integrations.channels.adapters.dingtalk import DingTalkAdapter
+
+ if not isinstance(body, dict):
+ return []
+ if body.get("success") is False or body.get("errorCode"):
+ return []
+ result = body.get("result") if isinstance(body.get("result"), dict) else body
+ out: List[HistoryItem] = []
+ for m in result.get("messages") or []:
+ if not isinstance(m, dict):
+ continue
+ # Written out rather than chained: `a or b if cond else ""` binds the conditional
+ # looser than the `or`, which would silently make every message look empty.
+ text = m.get("content") or ""
+ if not text and isinstance(m.get("text"), dict):
+ text = m["text"].get("content") or ""
+ if not isinstance(text, str):
+ text = str(text)
+ text, refs = DingTalkAdapter._extract_dws_files(text)
+ if not text and not refs:
+ continue
+ out.append(HistoryItem(
+ # openMessageId is the real id; msgId/messageId kept as fallbacks only.
+ message_id=str(
+ m.get("openMessageId") or m.get("msgId") or m.get("messageId") or ""
+ ),
+ # `sender` is already the display name — no directory lookup needed.
+ sender_name=str(
+ m.get("sender") or m.get("senderNick") or m.get("senderName") or "群成员"
+ ),
+ text=text,
+ ts_ms=DingTalkAdapter._dws_ts_ms(m.get("createTime") or m.get("createAt")),
+ attachments=refs,
+ # Marks which download route this file needs: history files live on DingTalk
+ # Drive and are fetched by fileId, not by the bot callback's downloadCode.
+ raw={"dws_drive": True} if refs else {},
+ ))
+ return out
+
+ @staticmethod
+ def _extract_attachments(msgtype: str, payload: Dict[str, Any]) -> list:
+ """Message → attachment list. ``key`` carries DingTalk's ``downloadCode``.
+
+ Content layout per DingTalk's "received message types" reference:
+ picture → {pictureDownloadCode, downloadCode}
+ file → {spaceId, fileName, downloadCode, fileId}
+ audio → {downloadCode, recognition}
+ video → {duration, videoType, downloadCode}
+ richText → {richText: [ ...nodes, image nodes carry downloadCode... ]}
+ Everything is read defensively (``downloadCode`` looked up in more than one place)
+ because these payloads are only loosely specified and vary by client version — a
+ missing field must degrade to "no attachment", never raise and kill the connection.
+ """
+ content = payload.get("content") if isinstance(payload.get("content"), dict) else {}
+ out: list = []
+
+ def _add(kind: str, code: Any, name: str) -> None:
+ if code:
+ out.append({"kind": kind, "key": str(code), "name": name})
+
+ if msgtype == "picture":
+ code = content.get("downloadCode") or content.get("pictureDownloadCode")
+ _add("image", code, f"dingtalk_image_{payload.get('msgId', '')[:8] or 'x'}.png")
+ elif msgtype == "file":
+ _add("file", content.get("downloadCode"), content.get("fileName") or "file.bin")
+ elif msgtype == "audio":
+ _add("file", content.get("downloadCode"), "dingtalk_audio.amr")
+ elif msgtype == "video":
+ _add("file", content.get("downloadCode"), "dingtalk_video.mp4")
+ elif msgtype == "richText":
+ for i, node in enumerate(content.get("richText") or []):
+ if not isinstance(node, dict):
+ continue
+ code = node.get("downloadCode") or node.get("pictureDownloadCode")
+ _add("image", code, f"dingtalk_richtext_{i}.png")
+ return out
+
+ async def download_resource(
+ self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any]
+ ) -> Optional[bytes]:
+ """Download an inbound attachment: downloadCode → temporary downloadUrl → bytes.
+
+ Two hops, per DingTalk's design:
+ POST /v1.0/robot/messageFiles/download {downloadCode, robotCode} → {downloadUrl}
+ GET downloadUrl → bytes
+ The temporary URL serves the file with a ``.file`` extension; we ignore that and keep
+ the filename taken from the message, so the artifact keeps its real extension.
+ Download codes expire, so a deferred fetch can legitimately fail — returning None
+ (rather than raising) lets the caller degrade to "attachment unavailable".
+ """
+ code = attachment.get("key")
+ if not code:
+ return None
+ # Files seen through *history* are DingTalk Drive entries addressed by fileId, not
+ # bot-callback attachments addressed by a short-lived downloadCode — different
+ # storage, different API. The marker is set when the history item is parsed.
+ if attachment.get("kind") == "doc":
+ return await self._read_doc_via_dws(conn, str(code))
+ if (inbound.raw or {}).get("dws_drive"):
+ return await self._download_via_dws(conn, str(code))
+ robot_code = (inbound.raw or {}).get("dingtalk_robot_code") or ""
+ if not robot_code:
+ logger.warning("[dingtalk] 缺少 robotCode,无法下载附件 code=%s", str(code)[:12])
+ return None
+ try:
+ token = await self._access_token(conn.app_id, self._app_secret(conn))
+ async with httpx.AsyncClient(timeout=30) as client:
+ resp = await client.post(
+ f"{DINGTALK_API_BASE}/v1.0/robot/messageFiles/download",
+ headers={"x-acs-dingtalk-access-token": token},
+ json={"downloadCode": code, "robotCode": robot_code},
+ )
+ if resp.status_code != 200:
+ logger.warning(
+ "[dingtalk] 下载地址获取失败 status=%s body=%s",
+ resp.status_code, resp.text[:200],
+ )
+ return None
+ url = (resp.json() or {}).get("downloadUrl")
+ if not url:
+ return None
+ fr = await client.get(url)
+ if fr.status_code == 200:
+ return fr.content
+ logger.warning("[dingtalk] 附件下载失败 status=%s", fr.status_code)
+ except Exception: # noqa: BLE001
+ logger.exception("[dingtalk] 附件下载异常 code=%s", str(code)[:12])
+ return None
diff --git a/src/backend/integrations/channels/adapters/dingtalk_settings.py b/src/backend/integrations/channels/adapters/dingtalk_settings.py
new file mode 100644
index 00000000..56b7348f
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/dingtalk_settings.py
@@ -0,0 +1,3 @@
+"""dingtalk API configuration."""
+import os
+DINGTALK_API_BASE = "https://api.dingtalk.com"
diff --git a/src/backend/integrations/channels/adapters/lark.py b/src/backend/integrations/channels/adapters/lark.py
new file mode 100644
index 00000000..ae5a4b83
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/lark.py
@@ -0,0 +1,261 @@
+"""Feishu (Lark) inbound channel adapter.
+
+- **Binding validation / outbound push**: Feishu OpenAPI REST (httpx) with a
+ tenant_access_token; no SDK needed.
+- **Webhook decryption/signature verification**: AES-256-CBC (Encrypt Key) +
+ signature check, pure cryptography; no SDK needed.
+- **Long connection (WebSocket)**: uses the official ``lark_oapi`` SDK's
+ ws.Client (lazy import; if the SDK is missing only the long connection is
+ unavailable, webhook/push are unaffected). Long connection is recommended in
+ production: the user only needs App ID/Secret, no public callback URL.
+
+Conversation keying (``external_conversation_id``) always uses Feishu
+``message.chat_id``:
+ - a p2p chat_id maps one-to-one to a user → naturally one conversation per DM peer;
+ - a group chat_id is unique per group → the whole group shares one conversation.
+Outbound push also sends by chat_id (``receive_id_type=chat_id``), unified for p2p/group.
+
+See internal design docs.
+"""
+
+from __future__ import annotations
+from integrations.channels.adapters.lark_messages import LarkMessages
+from integrations.channels.adapters.lark_delivery import LarkDelivery
+
+import asyncio
+import base64
+import hashlib
+import json
+import logging
+import os
+import re
+import threading
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+import httpx
+
+from integrations.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult
+from core.infra.crypto import decrypt_secret
+
+logger = logging.getLogger(__name__)
+
+from integrations.channels.adapters.lark_settings import LARK_API_BASE
+_MENTION_RE = re.compile(r"@_user_\d+")
+
+
+# ── lark_oapi ws event-loop isolation (supports multiple coexisting bots) ────
+# Root cause: ``lark_oapi/ws/client.py`` runs everything on a single
+# **module-level global** ``loop`` (``loop.run_until_complete`` in ``start()``,
+# ``loop.create_task`` in ``_connect`` ...). Each bot's long connection runs in
+# its own worker thread, but they all share that one global → when the second
+# bot starts it overwrites the global loop with its own thread's loop, and the
+# first bot's already-running coroutines then call ``loop.create_task`` on
+# someone else's loop → "This event loop is already running" / cross-thread
+# crashes. Single-bot setups hit the same cause sporadically (the global gets
+# rewritten on reconnect/retry).
+# Fix: replace that global ``loop`` with a **per-thread-resolving proxy** — each
+# worker thread always resolves to its own thread's loop (prepare_ws_thread
+# writes it into thread-local storage), fully eliminating cross-thread sharing
+# and overwriting.
+_ws_loop_tls = threading.local()
+_ws_proxy_installed = False
+_ws_proxy_lock = threading.Lock()
+
+
+class _PerThreadLoopProxy:
+ """Masquerades as an event loop: every attribute/method access is forwarded to the real loop bound to the **current thread**."""
+
+ def __getattr__(self, name: str) -> Any:
+ loop = getattr(_ws_loop_tls, "loop", None)
+ if loop is None:
+ loop = asyncio.get_event_loop()
+ return getattr(loop, name)
+
+
+def _install_ws_loop_proxy() -> None:
+ """One-time replacement of lark_oapi.ws.client's module-level global loop with the per-thread proxy (idempotent)."""
+ global _ws_proxy_installed
+ if _ws_proxy_installed:
+ return
+ with _ws_proxy_lock:
+ if _ws_proxy_installed:
+ return
+ try:
+ import lark_oapi.ws.client as _wsc
+ _wsc.loop = _PerThreadLoopProxy()
+ _ws_proxy_installed = True
+ except Exception: # noqa: BLE001 — if the SDK is missing, the long connection is unavailable anyway
+ pass
+
+
+class LarkAdapter(LarkMessages, LarkDelivery):
+ caps = ChannelCaps(
+ channel_type="lark",
+ max_message_len=4000, # Feishu's per-message text cap is fairly high; leave headroom for chunking
+ supports_markdown=False, # uses text messages; rich text (post) left for later
+ splits_long_messages=False,
+ supports_long_conn=True,
+ # Feishu gates group delivery with three mutually distinct scopes:
+ # im:message.group_at_msg → only messages @-ing this bot
+ # im:message.group_at_msg.include_bot:readonly → messages @-ing any bot
+ # im:message.group_msg (sensitive) → every group message
+ # Group listening needs the last one, plus republishing the app version; with the
+ # others Feishu never pushes a non-@ message and observe_all stays inert. Note the
+ # @-gate is worth running under the middle scope too: it stops us answering a message
+ # that @'d somebody else's bot.
+ supports_group_observe=True,
+ )
+
+ # tenant_access_token cache: {app_id: (token, expire_epoch)}
+ _token_cache: Dict[str, tuple] = {}
+ # The bot's own open_id: {app_id: open_id}. Stable per app → cached without expiry.
+ _bot_id_cache: Dict[str, str] = {}
+
+ # ── Credentials ─────────────────────────────────────────────────────
+ @staticmethod
+ def _app_secret(conn: Any) -> str:
+ cfg = conn.config if isinstance(conn.config, dict) else {}
+ return decrypt_secret(cfg.get("app_secret_enc")) or ""
+
+ @staticmethod
+ def _encrypt_key(conn: Any) -> str:
+ cfg = conn.config if isinstance(conn.config, dict) else {}
+ return decrypt_secret(cfg.get("encrypt_key_enc")) or ""
+
+ @staticmethod
+ def _verification_token(conn: Any) -> str:
+ cfg = conn.config if isinstance(conn.config, dict) else {}
+ return decrypt_secret(cfg.get("verification_token_enc")) or ""
+
+ async def _tenant_token(self, app_id: str, app_secret: str) -> str:
+ cached = self._token_cache.get(app_id)
+ if cached and cached[1] > time.time() + 60:
+ return cached[0]
+ url = f"{LARK_API_BASE}/open-apis/auth/v3/tenant_access_token/internal"
+ async with httpx.AsyncClient(timeout=10) as client:
+ resp = await client.post(url, json={"app_id": app_id, "app_secret": app_secret})
+ data = resp.json()
+ if data.get("code") != 0:
+ raise RuntimeError(f"飞书 tenant_access_token 获取失败: {data.get('msg') or data}")
+ token = data["tenant_access_token"]
+ self._token_cache[app_id] = (token, time.time() + int(data.get("expire", 7000)))
+ return token
+
+ async def validate_credentials(self, conn: Any) -> Dict[str, Any]:
+ """Successfully obtaining a token means the credentials are valid; returns an identity summary. Raises on failure."""
+ secret = self._app_secret(conn)
+ if not conn.app_id or not secret:
+ raise RuntimeError("缺少 App ID / App Secret")
+ await self._tenant_token(conn.app_id, secret)
+ return {"app_id": conn.app_id}
+
+ # ── Webhook decryption / signature verification ─────────────────────
+
+
+
+ # ── Event → InboundMsg ──────────────────────────────────────────────
+
+
+
+
+
+
+
+ # ── Group history pull (app credentials, independent of group listening) ──
+
+
+ # ── Group listening: is this group message @'ing the bot? ───────────
+ async def resolve_addressed(self, conn: Any, inbound: InboundMsg) -> bool:
+ """True when the bot's own open_id appears in the message's mention list.
+
+ Fails open (True) whenever the bot identity cannot be fetched, so a transient
+ ``bot/v3/info`` failure degrades to today's behavior (answer everything delivered)
+ rather than making the bot go mute in the group.
+ """
+ try:
+ bot_open_id = await self._bot_open_id(conn)
+ except Exception: # noqa: BLE001
+ logger.debug("[lark] 机器人 open_id 获取失败,按「已被 @」处理", exc_info=True)
+ return True
+ if not bot_open_id:
+ return True
+ return bot_open_id in (inbound.mentioned_ids or [])
+
+ async def _bot_open_id(self, conn: Any) -> str:
+ """The bot's own open_id (``/open-apis/bot/v3/info``), cached per app_id.
+
+ Feishu never puts the bot identity in the event payload, so it has to be fetched once
+ and kept. The value is stable for the lifetime of the app, hence a plain process cache
+ with no expiry (a restart just re-fetches).
+ """
+ cached = self._bot_id_cache.get(conn.app_id)
+ if cached:
+ return cached
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ async with httpx.AsyncClient(timeout=10) as client:
+ resp = await client.get(
+ f"{LARK_API_BASE}/open-apis/bot/v3/info",
+ headers={"Authorization": f"Bearer {token}"},
+ )
+ open_id = ((resp.json() or {}).get("bot") or {}).get("open_id") or ""
+ if open_id:
+ self._bot_id_cache[conn.app_id] = open_id
+ return open_id
+
+ # ── Outbound push ───────────────────────────────────────────────────
+
+
+
+
+
+ # ── Inbound file download ───────────────────────────────────────────
+
+ # ── Outbound file delivery ──────────────────────────────────────────
+
+
+
+ # ── Long connection (WebSocket, requires the lark_oapi SDK) ─────────
+ def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any:
+ """Build the Feishu long-connection client. ``on_message`` is invoked synchronously on the SDK thread; the caller dispatches it to the main loop.
+
+ lark_oapi is lazily imported; if not installed a RuntimeError is raised,
+ which the manager records as an error state without affecting the process.
+ """
+ try:
+ import lark_oapi as lark
+ from lark_oapi.api.im.v1 import P2ImMessageReceiveV1
+ except Exception as exc: # noqa: BLE001
+ raise RuntimeError("lark_oapi SDK 未安装,长连接不可用") from exc
+
+ secret = self._app_secret(conn)
+
+ def _do_message(data: "P2ImMessageReceiveV1") -> None:
+ try:
+ inbound = self._inbound_from_sdk_event(conn, data)
+ if inbound is not None:
+ on_message(inbound)
+ except Exception: # noqa: BLE001
+ logger.exception("[lark] 长连接事件处理失败 channel_id=%s", conn.channel_id)
+
+ handler = (
+ lark.EventDispatcherHandler.builder("", "")
+ .register_p2_im_message_receive_v1(_do_message)
+ .build()
+ )
+ return lark.ws.Client(
+ conn.app_id, secret, event_handler=handler, log_level=lark.LogLevel.WARNING
+ )
+
+ def prepare_ws_thread(self, loop: Any) -> None:
+ """Called inside the worker thread: install the per-thread loop proxy + write this thread's loop into thread-local storage.
+
+ From then on, every access to the global ``loop`` inside
+ lark_oapi.ws.client is forwarded by the proxy to **this thread's** loop —
+ multiple Feishu bots each get their own thread and their own loop with
+ no mutual overwriting (fixes "This event loop is already running"). The
+ proxy is installed only once (process-level idempotent); each thread
+ writes its own thread-local value.
+ """
+ _install_ws_loop_proxy()
+ _ws_loop_tls.loop = loop
diff --git a/src/backend/integrations/channels/adapters/lark_delivery.py b/src/backend/integrations/channels/adapters/lark_delivery.py
new file mode 100644
index 00000000..cc178df6
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/lark_delivery.py
@@ -0,0 +1,152 @@
+"""lark delivery behavior."""
+from __future__ import annotations
+import json
+import logging
+from typing import Any, Dict, Optional
+import httpx
+from integrations.channels.protocol import InboundMsg, SendResult
+from integrations.channels.adapters.lark_settings import LARK_API_BASE
+logger = logging.getLogger(__name__)
+
+class LarkDelivery:
+ def _chat_id(self, inbound: InboundMsg) -> str:
+ return (inbound.raw or {}).get("lark_chat_id") or inbound.external_conversation_id
+
+ async def _post_message(self, conn: Any, chat_id: str, msg_type: str, content: Dict[str, Any]) -> SendResult:
+ if not chat_id:
+ return SendResult.fail("bad_format", "缺少 chat_id")
+ try:
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ url = f"{LARK_API_BASE}/open-apis/im/v1/messages?receive_id_type=chat_id"
+ payload = {"receive_id": chat_id, "msg_type": msg_type,
+ "content": json.dumps(content, ensure_ascii=False)}
+ try:
+ async with httpx.AsyncClient(timeout=15) as client:
+ resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ if data.get("code") == 0:
+ return SendResult.ok((data.get("data") or {}).get("message_id"))
+ code = data.get("code")
+ kind = "rate_limited" if code in (99991400, 99991401) else "forbidden" if code in (230002, 230013) else "unknown"
+ return SendResult.fail(kind, f"code={code} msg={data.get('msg')}")
+
+ async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult:
+ return await self._post_message(conn, self._chat_id(inbound), "text", {"text": text})
+
+ async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult:
+ """Over-length content is auto-chunked by caps.max_message_len and sent piece by piece."""
+ from integrations.channels.protocol import chunk_text
+
+ chunks = chunk_text(content, self.caps.max_message_len) or [content]
+ first: Optional[SendResult] = None
+ cont: list = []
+ for i, c in enumerate(chunks):
+ r = await self.send_text(conn, inbound, c)
+ if i == 0:
+ first = r
+ if not r.success:
+ return r
+ elif r.message_id:
+ cont.append(r.message_id)
+ if first is not None:
+ first.continuation_ids = cont
+ return first
+ return SendResult.fail("unknown", "空内容")
+
+ async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult:
+ if not message_id:
+ return SendResult.fail("bad_format", "缺少 message_id")
+ try:
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{message_id}"
+ payload = {"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)}
+ try:
+ async with httpx.AsyncClient(timeout=15) as client:
+ resp = await client.put(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
+ data = resp.json()
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+ if data.get("code") == 0:
+ return SendResult.ok(message_id)
+ return SendResult.fail("unknown", f"code={data.get('code')} msg={data.get('msg')}")
+
+ async def download_resource(
+ self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any]
+ ) -> Optional[bytes]:
+ """Download a resource file from a message (requires the im:resource permission).
+
+ GET /open-apis/im/v1/messages/{message_id}/resources/{file_key}?type=file|image
+ """
+ msg_id = (inbound.raw or {}).get("lark_message_id") or inbound.message_id
+ key = attachment.get("key")
+ res_type = "image" if attachment.get("kind") == "image" else "file"
+ if not msg_id or not key:
+ return None
+ try:
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{msg_id}/resources/{key}?type={res_type}"
+ async with httpx.AsyncClient(timeout=30) as client:
+ resp = await client.get(url, headers={"Authorization": f"Bearer {token}"})
+ if resp.status_code == 200 and not resp.headers.get("content-type", "").startswith("application/json"):
+ return resp.content
+ logger.warning("[lark] 资源下载失败 msg=%s key=%s body=%s", msg_id, key, resp.text[:200])
+ except Exception: # noqa: BLE001
+ logger.exception("[lark] 资源下载异常 msg=%s key=%s", msg_id, key)
+ return None
+
+ async def push_file(
+ self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str
+ ) -> SendResult:
+ chat_id = self._chat_id(inbound)
+ if not chat_id:
+ return SendResult.fail("bad_format", "缺少 chat_id")
+ try:
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", str(exc))
+
+ is_image = (mime_type or "").startswith("image/")
+ try:
+ if is_image:
+ key = await self._upload_image(token, content)
+ msg_type, msg_content = "image", {"image_key": key}
+ else:
+ key = await self._upload_file(token, content, filename)
+ msg_type, msg_content = "file", {"file_key": key}
+ except Exception as exc: # noqa: BLE001
+ return SendResult.fail("transient", f"上传失败: {exc}")
+ if not key:
+ return SendResult.fail("unknown", "未拿到资源 key")
+ return await self._post_message(conn, chat_id, msg_type, msg_content)
+
+ async def _upload_image(self, token: str, content: bytes) -> Optional[str]:
+ url = f"{LARK_API_BASE}/open-apis/im/v1/images"
+ async with httpx.AsyncClient(timeout=30) as client:
+ resp = await client.post(
+ url, headers={"Authorization": f"Bearer {token}"},
+ data={"image_type": "message"},
+ files={"image": ("img.png", content)},
+ )
+ data = resp.json()
+ return (data.get("data") or {}).get("image_key") if data.get("code") == 0 else None
+
+ async def _upload_file(self, token: str, content: bytes, filename: str) -> Optional[str]:
+ # file_type: Feishu distinguishes by extension; anything not in the known set is sent as stream
+ ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower()
+ known = {"opus", "mp4", "pdf", "doc", "xls", "ppt"}
+ file_type = ext if ext in known else "stream"
+ url = f"{LARK_API_BASE}/open-apis/im/v1/files"
+ async with httpx.AsyncClient(timeout=60) as client:
+ resp = await client.post(
+ url, headers={"Authorization": f"Bearer {token}"},
+ data={"file_type": file_type, "file_name": filename},
+ files={"file": (filename, content)},
+ )
+ data = resp.json()
+ return (data.get("data") or {}).get("file_key") if data.get("code") == 0 else None
diff --git a/src/backend/integrations/channels/adapters/lark_messages.py b/src/backend/integrations/channels/adapters/lark_messages.py
new file mode 100644
index 00000000..a2ce1954
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/lark_messages.py
@@ -0,0 +1,317 @@
+"""lark messages behavior."""
+from __future__ import annotations
+import base64
+import hashlib
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional
+import httpx
+from integrations.channels.protocol import HistoryItem, InboundMsg
+from integrations.channels.adapters.lark_settings import LARK_API_BASE
+logger = logging.getLogger(__name__)
+_MENTION_RE = re.compile(r"@_user_\d+")
+
+class LarkMessages:
+ def decrypt_webhook(self, conn: Any, body: bytes) -> Dict[str, Any]:
+ """If an Encrypt Key is configured, the body looks like {"encrypt": "..."} and is AES-256-CBC decrypted; otherwise plain JSON."""
+ try:
+ raw = json.loads(body.decode("utf-8"))
+ except Exception:
+ return {}
+ if "encrypt" not in raw:
+ return raw
+ key = self._encrypt_key(conn)
+ if not key:
+ logger.warning("[lark] 收到加密事件但未配 Encrypt Key")
+ return {}
+ return self._aes_decrypt(raw["encrypt"], key)
+
+ @staticmethod
+ def _aes_decrypt(encrypt_b64: str, encrypt_key: str) -> Dict[str, Any]:
+ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+ key = hashlib.sha256(encrypt_key.encode("utf-8")).digest()
+ data = base64.b64decode(encrypt_b64)
+ iv, ct = data[:16], data[16:]
+ decryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor()
+ plain = decryptor.update(ct) + decryptor.finalize()
+ plain = plain[: -plain[-1]] # strip PKCS7 padding
+ return json.loads(plain.decode("utf-8"))
+
+ def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool:
+ """Feishu signature: sha256(timestamp + nonce + encrypt_key + body). Without an Encrypt Key, falls back to token verification."""
+ key = self._encrypt_key(conn)
+ sig = headers.get("x-lark-signature") or headers.get("X-Lark-Signature")
+ ts = headers.get("x-lark-request-timestamp") or headers.get("X-Lark-Request-Timestamp")
+ nonce = headers.get("x-lark-request-nonce") or headers.get("X-Lark-Request-Nonce")
+ if key and sig and ts is not None and nonce is not None:
+ h = hashlib.sha256((ts + nonce + key).encode("utf-8") + body).hexdigest()
+ return h == sig
+ # No encryption configured → fall back to verification_token (re-checked at the parse stage after decryption)
+ return True
+
+ def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]:
+ """Decrypted event dict → InboundMsg. Returns None for non-message events."""
+ # schema 2.0: header.event_type; legacy schema: payload["event"]["type"]
+ header = payload.get("header") or {}
+ event_type = header.get("event_type") or payload.get("type")
+ if event_type != "im.message.receive_v1":
+ return None
+ event = payload.get("event") or {}
+ message = event.get("message") or {}
+ sender = event.get("sender") or {}
+ mtype = message.get("message_type")
+ text, attachments = self._extract_payload(mtype, message.get("content"))
+ if not text and not attachments:
+ return None # unsupported message type (sticker/location, etc.)
+ sender_open_id = (sender.get("sender_id") or {}).get("open_id", "")
+ return self._make_inbound(
+ conn=conn,
+ chat_id=message.get("chat_id", ""),
+ chat_type=message.get("chat_type", "p2p"),
+ message_id=message.get("message_id", ""),
+ text=text,
+ sender_open_id=sender_open_id,
+ sender_name=(sender.get("sender_id") or {}).get("user_id", "") or "",
+ attachments=attachments,
+ mentioned_ids=self._mentioned_ids(message),
+ raw=payload,
+ )
+
+ @staticmethod
+ def _mentioned_ids(message: Dict[str, Any]) -> List[str]:
+ """open_ids of everyone @'d in this message (``message.mentions``).
+
+ Only relevant once the app holds ``im:message.group_msg`` and Feishu starts pushing
+ non-@ group messages too — until then every delivered group message has the bot in
+ here anyway. The bot's own open_id is not known synchronously, so matching happens
+ later in ``resolve_addressed``.
+ """
+ out: List[str] = []
+ for m in message.get("mentions") or []:
+ if not isinstance(m, dict):
+ continue
+ oid = (m.get("id") or {}).get("open_id") if isinstance(m.get("id"), dict) else None
+ if oid:
+ out.append(oid)
+ return out
+
+ @staticmethod
+ def _strip_mentions(text: str) -> str:
+ return _MENTION_RE.sub("", text or "").strip()
+
+ @classmethod
+ def _extract_text(cls, content: Any) -> str:
+ if not content:
+ return ""
+ try:
+ obj = json.loads(content) if isinstance(content, str) else content
+ text = obj.get("text", "") if isinstance(obj, dict) else ""
+ except Exception:
+ text = str(content)
+ return cls._strip_mentions(text)
+
+ @classmethod
+ def _extract_payload(cls, mtype: str, content: Any):
+ """Extract (text, attachments) by message type.
+
+ - text: plain text body (@ mentions stripped)
+ - image: image_key → attachment(kind=image)
+ - file/media/audio: file_key + file_name → attachment(kind=file)
+ - post (rich text): extract text + embedded image_key
+ """
+ try:
+ obj = json.loads(content) if isinstance(content, str) else (content or {})
+ except Exception:
+ obj = {}
+ if not isinstance(obj, dict):
+ obj = {}
+ attachments = []
+ if mtype == "text":
+ return cls._strip_mentions(obj.get("text", "")), attachments
+ if mtype == "image":
+ key = obj.get("image_key")
+ if key:
+ attachments.append({"kind": "image", "key": key, "name": f"{key}.png"})
+ return "", attachments
+ if mtype in ("file", "media", "audio"):
+ key = obj.get("file_key")
+ if key:
+ name = obj.get("file_name") or f"{key}.bin"
+ attachments.append({"kind": "file", "key": key, "name": name})
+ return "", attachments
+ if mtype == "post": # rich text: extract text + embedded images
+ texts, img_keys = [], []
+ for node in cls._post_nodes(obj):
+ if node.get("tag") == "text":
+ texts.append(node.get("text", ""))
+ elif node.get("tag") == "img" and node.get("image_key"):
+ img_keys.append(node["image_key"])
+ title = obj.get("title", "")
+ body = " ".join(texts)
+ text = (title + "\n" + body) if title else body
+ attachments = [{"kind": "image", "key": k, "name": f"{k}.png"} for k in img_keys]
+ return text.strip(), attachments
+ return "", attachments
+
+ @staticmethod
+ def _post_nodes(obj: Dict[str, Any]):
+ """Flatten all nodes of a post rich-text message (structure may be {content:[[node...]]} or wrapped in a language layer)."""
+ content = obj.get("content") or []
+ if isinstance(content, dict): # language layer wrapper {"zh_cn": {"content": [...]}}
+ content = next(iter(content.values()), {}).get("content", []) if content else []
+ for line in content or []:
+ for node in line or []:
+ if isinstance(node, dict):
+ yield node
+
+ @staticmethod
+ def _make_inbound(
+ *, conn: Any, chat_id: str, chat_type: str, message_id: str,
+ text: str, sender_open_id: str, sender_name: str,
+ raw: Dict[str, Any], attachments=None, mentioned_ids=None,
+ ) -> InboundMsg:
+ return InboundMsg(
+ channel_id=conn.channel_id,
+ channel_type="lark",
+ text=text,
+ chat_type=chat_type,
+ external_conversation_id=chat_id, # always keyed by chat_id
+ sender_id=sender_open_id,
+ sender_name=sender_name,
+ message_id=message_id,
+ attachments=attachments or [],
+ # p2p is always aimed at the bot; in a group it depends on whether the bot's own
+ # open_id is in the mention list — resolved asynchronously in resolve_addressed.
+ addressed_to_bot=True if chat_type == "p2p" else None,
+ mentioned_ids=list(mentioned_ids or []),
+ raw={"lark_chat_id": chat_id, "lark_message_id": message_id},
+ )
+
+ async def fetch_history(
+ self,
+ conn: Any,
+ conversation_id: str,
+ *,
+ since_ms: int,
+ limit: int,
+ newest_first: bool = False,
+ ) -> List[HistoryItem]:
+ """GET /open-apis/im/v1/messages — this chat's own messages, oldest-first.
+
+ Uses the tenant token (app credentials), so it is a property of the bound app rather
+ than of any user's session. Note this sits behind a *different* permission from event
+ push: an app that is not allowed to receive non-@ group messages may still be allowed
+ to read the chat's history, which is precisely why pulling is worth having.
+
+ Feishu takes ``start_time`` in **seconds**; the caller works in milliseconds.
+ Any failure returns [] — the caller treats "unsupported" and "failed" alike.
+ """
+ chat_id = (conversation_id or "").strip()
+ if not chat_id:
+ return []
+ try:
+ token = await self._tenant_token(conn.app_id, self._app_secret(conn))
+ params = {
+ "container_id_type": "chat",
+ "container_id": chat_id,
+ # Desc for the cold start so the first page holds the *newest* messages of
+ # the window; Asc when walking forward from the cursor so nothing is skipped.
+ "sort_type": "ByCreateTimeDesc" if newest_first else "ByCreateTimeAsc",
+ "page_size": str(max(1, min(int(limit or 50), 50))), # Feishu caps page_size at 50
+ }
+ if since_ms > 0:
+ params["start_time"] = str(int(since_ms // 1000))
+ async with httpx.AsyncClient(timeout=15) as client:
+ resp = await client.get(
+ f"{LARK_API_BASE}/open-apis/im/v1/messages",
+ params=params,
+ headers={"Authorization": f"Bearer {token}"},
+ )
+ body = resp.json() or {}
+ if resp.status_code != 200 or body.get("code") not in (0, None):
+ logger.warning(
+ "[lark] 群历史拉取失败 chat=%s code=%s msg=%s",
+ chat_id, body.get("code"), str(body.get("msg"))[:120],
+ )
+ return []
+ return [
+ item
+ for item in (
+ self._history_item(raw) for raw in (body.get("data") or {}).get("items") or []
+ )
+ if item is not None
+ ]
+ except Exception: # noqa: BLE001
+ logger.warning("[lark] 群历史拉取异常 chat=%s", chat_id, exc_info=True)
+ return []
+
+ @classmethod
+ def _history_item(cls, raw: Dict[str, Any]) -> Optional[HistoryItem]:
+ """One ``im/v1/messages`` entry → HistoryItem, reusing the inbound content parser."""
+ if not isinstance(raw, dict) or raw.get("deleted"):
+ return None
+ body = raw.get("body") or {}
+ text, attachments = cls._extract_payload(raw.get("msg_type"), body.get("content"))
+ if not text and not attachments:
+ return None
+ sender = raw.get("sender") or {}
+ message_id = raw.get("message_id") or ""
+ try:
+ ts_ms = int(raw.get("create_time") or 0)
+ except (TypeError, ValueError):
+ ts_ms = 0
+ return HistoryItem(
+ message_id=message_id,
+ # Feishu returns only ids here; a display name would need a directory lookup per
+ # sender, which is not worth an extra API call per message for background context.
+ sender_name=(sender.get("id") or "")[-6:] or "群成员",
+ text=text,
+ ts_ms=ts_ms,
+ attachments=attachments,
+ # download_resource resolves resources by message_id — carry it so a file pulled
+ # from history stays fetchable later via channel_read_attachment.
+ raw={"lark_message_id": message_id},
+ )
+
+ def _inbound_from_sdk_event(self, conn: Any, data: Any) -> Optional[InboundMsg]:
+ """lark_oapi's typed P2ImMessageReceiveV1 event → InboundMsg."""
+ event = getattr(data, "event", None)
+ message = getattr(event, "message", None)
+ sender = getattr(event, "sender", None)
+ if message is None:
+ return None
+ mtype = getattr(message, "message_type", None)
+ text, attachments = self._extract_payload(mtype, getattr(message, "content", None))
+ if not text and not attachments:
+ return None
+ sender_id_obj = getattr(sender, "sender_id", None)
+ open_id = getattr(sender_id_obj, "open_id", "") if sender_id_obj else ""
+ return self._make_inbound(
+ conn=conn,
+ chat_id=getattr(message, "chat_id", "") or "",
+ chat_type=getattr(message, "chat_type", "p2p") or "p2p",
+ message_id=getattr(message, "message_id", "") or "",
+ text=text,
+ sender_open_id=open_id or "",
+ sender_name="",
+ attachments=attachments,
+ mentioned_ids=self._sdk_mentioned_ids(message),
+ raw={},
+ )
+
+ @staticmethod
+ def _sdk_mentioned_ids(message: Any) -> List[str]:
+ """SDK-typed counterpart of ``_mentioned_ids`` (``MentionEvent.id.open_id``).
+
+ The long connection is the production path, so mention extraction must exist on both
+ sides; missing it here would make every group message look un-@'d.
+ """
+ out: List[str] = []
+ for m in getattr(message, "mentions", None) or []:
+ ident = getattr(m, "id", None)
+ oid = getattr(ident, "open_id", "") if ident is not None else ""
+ if oid:
+ out.append(oid)
+ return out
diff --git a/src/backend/integrations/channels/adapters/lark_settings.py b/src/backend/integrations/channels/adapters/lark_settings.py
new file mode 100644
index 00000000..f27648d1
--- /dev/null
+++ b/src/backend/integrations/channels/adapters/lark_settings.py
@@ -0,0 +1,3 @@
+"""lark API configuration."""
+import os
+LARK_API_BASE = os.getenv("LARK_API_BASE", "https://open.feishu.cn").rstrip("/")
diff --git a/src/backend/core/channels/adapters/wecom.py b/src/backend/integrations/channels/adapters/wecom.py
similarity index 99%
rename from src/backend/core/channels/adapters/wecom.py
rename to src/backend/integrations/channels/adapters/wecom.py
index ba590795..f963ef69 100644
--- a/src/backend/core/channels/adapters/wecom.py
+++ b/src/backend/integrations/channels/adapters/wecom.py
@@ -24,7 +24,7 @@
import httpx
-from core.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text
+from integrations.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text
from core.infra.crypto import decrypt_secret
logger = logging.getLogger(__name__)
diff --git a/src/backend/core/channels/adapters/weixin.py b/src/backend/integrations/channels/adapters/weixin.py
similarity index 99%
rename from src/backend/core/channels/adapters/weixin.py
rename to src/backend/integrations/channels/adapters/weixin.py
index 3e8a1eaa..d57591db 100644
--- a/src/backend/core/channels/adapters/weixin.py
+++ b/src/backend/integrations/channels/adapters/weixin.py
@@ -45,7 +45,7 @@
import httpx
-from core.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text
+from integrations.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text
from core.infra.crypto import decrypt_secret
logger = logging.getLogger(__name__)
diff --git a/src/backend/integrations/channels/api/__init__.py b/src/backend/integrations/channels/api/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/backend/integrations/channels/api/__init__.py
@@ -0,0 +1 @@
+
diff --git a/src/backend/api/routes/v1/channel_desktop.py b/src/backend/integrations/channels/api/channel_desktop.py
similarity index 75%
rename from src/backend/api/routes/v1/channel_desktop.py
rename to src/backend/integrations/channels/api/channel_desktop.py
index 6d82adb2..498c16d3 100644
--- a/src/backend/api/routes/v1/channel_desktop.py
+++ b/src/backend/integrations/channels/api/channel_desktop.py
@@ -1,10 +1,11 @@
"""Desktop channel registration and leased execution; device capabilities only."""
-from api.routes.v1.desktop_capability import _require_capability_user
+from features.desktop.api.capability_dependencies import require_capability_user as _require_capability_user
from core.auth.backend import UserContext, get_current_user
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import success_response
-from core.services.channel_relay import ChannelRelayService
+from integrations.channels.relay import ChannelRelayService
from fastapi import APIRouter, Depends, Request
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -56,7 +57,7 @@ def claim(
):
from dataclasses import asdict
- from core.channels.registry import get_adapter
+ from integrations.channels.registry import get_adapter
item = ChannelRelayService(db).claim(owner, device(request), body.binding_ids)
if item:
@@ -77,24 +78,30 @@ def renew(
return success_response(data={"ok": True})
+def _complete(db, owner, device_id, delivery_id, body):
+ svc = ChannelRelayService(db)
+ row, conn = svc.authorized(owner, device_id, delivery_id, body.lease)
+ payload = row.payload
+ db.expunge(conn)
+ svc.complete(owner, device_id, delivery_id, body.lease, body.error)
+ return payload, conn
+
+
@router.post("/{delivery_id}/complete")
async def complete(
delivery_id: str,
body: Lease,
request: Request,
owner: str = Depends(_require_capability_user),
- db: Session = Depends(get_db),
):
- svc = ChannelRelayService(db)
- row, conn = svc.authorized(owner, device(request), delivery_id, body.lease)
- svc.complete(owner, device(request), delivery_id, body.lease, body.error)
+ payload, conn = await run_in_session(_complete, owner, device(request), delivery_id, body)
if body.error:
import logging
- from core.channels.protocol import InboundMsg
- from core.channels.registry import get_adapter
+ from integrations.channels.protocol import InboundMsg
+ from integrations.channels.registry import get_adapter
- msg = InboundMsg(**row.payload["message"])
+ msg = InboundMsg(**payload["message"])
if msg.addressed_to_bot is not False:
try:
await get_adapter(conn.channel_type).send_text(
@@ -113,12 +120,10 @@ async def operation(
body: Operation,
request: Request,
owner: str = Depends(_require_capability_user),
- db: Session = Depends(get_db),
):
- from core.services.channel_relay_rpc import channel_operation
+ from integrations.channels.relay_rpc import channel_operation
data = await channel_operation(
- db,
owner,
device(request),
delivery_id,
@@ -131,9 +136,7 @@ async def operation(
@router.post("/local-binding")
-async def local_binding(
- user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)
-):
- from core.services.channel_desktop_worker import prepare_binding
+async def local_binding(user: UserContext = Depends(get_current_user)):
+ from integrations.channels.desktop_worker import prepare_binding
- return success_response(data=await prepare_binding(db, str(user.user_id)))
+ return success_response(data=await prepare_binding(str(user.user_id)))
diff --git a/src/backend/api/routes/v1/channels.py b/src/backend/integrations/channels/api/channels.py
similarity index 89%
rename from src/backend/api/routes/v1/channels.py
rename to src/backend/integrations/channels/api/channels.py
index fdafaffc..c65bedf8 100644
--- a/src/backend/api/routes/v1/channels.py
+++ b/src/backend/integrations/channels/api/channels.py
@@ -18,11 +18,12 @@
from sqlalchemy.orm import Session
from core.auth.backend import UserContext, get_current_user
-from core.channels.protocol import ChannelCaps
-from core.channels.registry import get_adapter, list_adapters
+from integrations.channels.protocol import ChannelCaps
+from integrations.channels.registry import get_adapter, list_adapters
from core.db.engine import get_db
+from core.db.threads import run_in_session
from core.infra.responses import created_response, success_response
-from core.services.channel_service import ChannelService, bot_to_dict
+from integrations.channels.service import ChannelService, bot_to_dict
router = APIRouter(prefix="/v1/channels", tags=["Channels"])
@@ -103,7 +104,7 @@ def list_my_bots(
bot(设置「我的机器人」用);都不传 → 全部。"""
svc = ChannelService(db)
bots = svc.list_bots(str(user.user_id), agent_id=agent_id, main_only=main_only)
- from core.services.channel_relay import ChannelRelayService, execution
+ from integrations.channels.relay import ChannelRelayService, execution
relay = ChannelRelayService(db)
return success_response(data={"bots": [
{
@@ -122,7 +123,7 @@ def list_my_conversations(
db: Session = Depends(get_db),
):
"""列出本人渠道 bot 已产生的会话(群/单聊),供 Web 后台配置定时投递目标。"""
- from core.services.channel_service import list_owner_conversations
+ from integrations.channels.service import list_owner_conversations
return success_response(data={"conversations": list_owner_conversations(db, str(user.user_id))})
@@ -130,13 +131,13 @@ def list_my_conversations(
async def create_bot(
body: CreateBotRequest,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
from core.infra.exceptions import BadRequestError
+ from integrations.channels.service import create_bot as register
+
if (body.execution_location == "local") != bool(body.local_binding_id):
raise BadRequestError("选择本机需要当前桌面设备授权")
- svc = ChannelService(db)
- conn = await svc.create_bot(
+ conn = await register(
str(user.user_id),
channel_type=body.channel_type,
app_id=body.app_id,
@@ -195,10 +196,10 @@ def delete_bot(
async def test_bot(
channel_id: str,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = ChannelService(db)
- data = await svc.test_bot(str(user.user_id), channel_id)
+ from integrations.channels.service import test_bot as check
+
+ data = await check(str(user.user_id), channel_id)
return success_response(data=data)
@@ -209,13 +210,13 @@ async def weixin_bind_start(
execution_location: Literal["cloud", "local"] = "cloud",
local_binding_id: Optional[str] = None,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = ChannelService(db)
from core.infra.exceptions import BadRequestError
+ from integrations.channels.service import start_weixin_bind
+
if (execution_location == "local") != bool(local_binding_id):
raise BadRequestError("选择本机需要当前桌面设备授权")
- data = await svc.start_weixin_bind(str(user.user_id), agent_id=agent_id, local_binding_id=local_binding_id)
+ data = await start_weixin_bind(str(user.user_id), agent_id=agent_id, local_binding_id=local_binding_id)
return success_response(data=data)
@@ -223,10 +224,10 @@ async def weixin_bind_start(
async def weixin_bind_status(
bind_id: str,
user: UserContext = Depends(get_current_user),
- db: Session = Depends(get_db),
):
- svc = ChannelService(db)
- data = await svc.poll_weixin_bind(str(user.user_id), bind_id)
+ from integrations.channels.service import poll_weixin_bind
+
+ data = await poll_weixin_bind(str(user.user_id), bind_id)
return success_response(data=data)
@@ -248,7 +249,6 @@ def channel_webhook_verify(
async def channel_webhook(
channel_id: str,
request: Request,
- db: Session = Depends(get_db),
):
body = await request.body()
# Signature params: Lark uses headers (x-lark-*), WeCom uses the query string
@@ -257,8 +257,14 @@ async def channel_webhook(
headers = {k.lower(): v for k, v in request.headers.items()}
for k, v in request.query_params.items():
headers[k.lower()] = v
- svc = ChannelService(db)
- result = svc.handle_webhook(channel_id, headers, body)
+ result, inbound = await run_in_session(
+ lambda db: ChannelService(db).handle_webhook(channel_id, headers, body)
+ )
+ if inbound is not None:
+ from core.infra.background import spawn
+ from integrations.channels.inbound import handle_inbound
+
+ spawn(handle_inbound(inbound), name="channel.handle_inbound")
# The webhook response must be the bare JSON the channel expects (e.g. Lark url_verification's
# challenge), not wrapped in an envelope.
return JSONResponse(content=result)
diff --git a/src/backend/core/services/channel_desktop_worker.py b/src/backend/integrations/channels/desktop_worker.py
similarity index 81%
rename from src/backend/core/services/channel_desktop_worker.py
rename to src/backend/integrations/channels/desktop_worker.py
index 5e94273e..d2f26306 100644
--- a/src/backend/core/services/channel_desktop_worker.py
+++ b/src/backend/integrations/channels/desktop_worker.py
@@ -1,6 +1,7 @@
"""Desktop worker: authorized cloud channel I/O with the existing local chat runtime."""
from __future__ import annotations
+from features.desktop.bridge import identity as bridge_identity
import asyncio
import hashlib
@@ -8,11 +9,13 @@
import platform
import httpx
-from core.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult
-from core.db.models import ChannelConnection, ContentBlock, UserShadow
+from integrations.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult
+from integrations.channels.models import ChannelConnection
+from core.db.models import ContentBlock, UserShadow
+from core.db.threads import run_db, run_in_session
from core.infra.exceptions import AccessDeniedError, BadRequestError
-from core.services import desktop_cloud_bridge as bridge
-from core.services.channel_relay_rpc import pack_bytes, unpack_bytes
+from features.desktop.auth import bridge as bridge_auth
+from integrations.channels.relay_rpc import pack_bytes, unpack_bytes
from sqlalchemy import select
logger = logging.getLogger(__name__)
@@ -21,20 +24,20 @@
async def cloud_post(captured, path, payload):
- bridge.require_current_account(captured)
- current = bridge.get_state()
+ bridge_identity.require_current_account(captured)
+ current = bridge_identity.get_state()
if not current:
raise AccessDeniedError("桌面云端会话已失效")
try:
async with httpx.AsyncClient(timeout=25) as client:
response = await client.post(
current["cloud_base"] + "/api/v1/channels/desktop/" + path,
- headers=bridge.cloud_headers(current),
+ headers=bridge_identity.cloud_headers(current),
json=payload,
)
except httpx.HTTPError as exc:
raise BadRequestError("渠道执行服务连接失败,请检查网络后重试") from exc
- bridge.require_current_account(captured)
+ bridge_identity.require_current_account(captured)
try:
data = response.json()
except ValueError as exc:
@@ -52,8 +55,8 @@ async def cloud_post(captured, path, payload):
def identity_owner(db):
- identity = bridge.get_identity_state()
- if not bridge.bridge_enabled() or not bridge.get_state() or not identity:
+ identity = bridge_identity.get_identity_state()
+ if not bridge_auth.bridge_enabled() or not bridge_identity.get_state() or not identity:
raise AccessDeniedError("请在已登录的混合模式桌面端使用本机机器人")
owner = db.scalar(
select(UserShadow.user_id).where(
@@ -65,11 +68,21 @@ def identity_owner(db):
return identity, owner
-async def prepare_binding(db, owner):
- identity, actual_owner = identity_owner(db)
+def _store_grant(db, captured, grant, identity, owner):
+ with bridge_identity.account_scope(captured):
+ db.add(
+ ContentBlock(
+ id=GRANT_PREFIX + grant["binding_id"], payload={**identity, "local_owner": owner}
+ )
+ )
+ db.commit()
+
+
+async def prepare_binding(owner):
+ identity, actual_owner = await run_in_session(identity_owner)
if owner != actual_owner:
raise AccessDeniedError("本机账号不匹配")
- captured = bridge.get_state()
+ captured = bridge_identity.get_state()
grant = await cloud_post(captured, "register", {"device_name": platform.node() or "本机"})
if (
not isinstance(grant, dict)
@@ -79,13 +92,7 @@ async def prepare_binding(db, owner):
or not isinstance(grant.get("device_name"), str)
):
raise BadRequestError("本机授权响应格式无效,请重试")
- with bridge.account_scope(captured):
- db.add(
- ContentBlock(
- id=GRANT_PREFIX + grant["binding_id"], payload={**identity, "local_owner": owner}
- )
- )
- db.commit()
+ await run_in_session(_store_grant, captured, grant, identity, owner)
return grant
@@ -123,7 +130,7 @@ def record_run(self, run_id, owner_id):
async def cancel(self):
if self.run_id and self.owner_id:
- from orchestration.chat_run_executor import cancel_run
+ from core.harness.registry.run_cancellation import cancel_run
await cancel_run(self.run_id, user_id=self.owner_id)
@@ -195,13 +202,9 @@ async def recall_message(self, conn, msg, message_id):
return SendResult(**await self.call("recall_message", {"message_id": message_id}))
-async def run_delivery(factory, captured, identity, owner, item):
- from core.channels.inbound import handle_inbound
- from core.channels.registry import scoped_adapter
- from core.services.user_agent_service import UserAgentService
+def _accept_delivery(factory, identity, owner, item, receipt_id):
+ from core.capabilities.agents.edition import UserAgentService
- bridge.require_current_account(captured)
- receipt_id = RECEIPT_PREFIX + item["delivery_id"]
with factory() as db:
if item["binding_id"] not in local_grants(db, identity, owner):
raise AccessDeniedError("机器人尚未在这台电脑授权")
@@ -243,6 +246,23 @@ async def run_delivery(factory, captured, identity, owner, item):
)
)
db.commit()
+ return local_id
+
+
+def _finish_receipt(factory, receipt_id, binding_id, error):
+ with factory() as db:
+ receipt = db.get(ContentBlock, receipt_id)
+ receipt.payload = {"state": "failed" if error else "completed", "binding_id": binding_id}
+ db.commit()
+
+
+async def run_delivery(factory, captured, identity, owner, item):
+ from integrations.channels.inbound import handle_inbound
+ from integrations.channels.registry import scoped_adapter
+
+ bridge_identity.require_current_account(captured)
+ receipt_id = RECEIPT_PREFIX + item["delivery_id"]
+ local_id = await run_db(_accept_delivery, factory, identity, owner, item, receipt_id)
msg = InboundMsg(**{**item["message"], "channel_id": local_id})
adapter = DesktopChannelAdapter(captured, item, local_id)
with scoped_adapter(adapter):
@@ -250,7 +270,7 @@ async def run_delivery(factory, captured, identity, owner, item):
try:
while not task.done():
done, _ = await asyncio.wait({task}, timeout=8)
- bridge.require_current_account(captured)
+ bridge_identity.require_current_account(captured)
await cloud_post(captured, item["delivery_id"] + "/renew", {"lease": item["lease"]})
if done:
break
@@ -260,14 +280,23 @@ async def run_delivery(factory, captured, identity, owner, item):
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await adapter.cancel()
+ await run_db(_finish_receipt, factory, receipt_id, item["binding_id"], adapter.error)
+ return adapter.error
+
+
+def _claim_inputs(factory):
with factory() as db:
- receipt = db.get(ContentBlock, receipt_id)
- receipt.payload = {
- "state": "failed" if adapter.error else "completed",
- "binding_id": item["binding_id"],
- }
+ identity, owner = identity_owner(db)
+ return identity, owner, local_grants(db, identity, owner)
+
+
+def _forget_grants(factory, captured, binding_ids):
+ with bridge_identity.account_scope(captured), factory() as db:
+ for binding_id in binding_ids:
+ row = db.get(ContentBlock, GRANT_PREFIX + binding_id)
+ if row:
+ db.delete(row)
db.commit()
- return adapter.error
class ChannelDesktopWorker:
@@ -288,20 +317,13 @@ async def loop(self):
captured = None
item = None
try:
- with self.factory() as db:
- identity, owner = identity_owner(db)
- bindings = local_grants(db, identity, owner)
- captured = bridge.get_state()
+ identity, owner, bindings = await run_db(_claim_inputs, self.factory)
+ captured = bridge_identity.get_state()
for offset in range(0, len(bindings), 200):
batch = bindings[offset : offset + 200]
reply = await cloud_post(captured, "claim", {"binding_ids": batch})
retained = set(reply["retained_binding_ids"])
- with bridge.account_scope(captured), self.factory() as db:
- for binding_id in set(batch) - retained:
- row = db.get(ContentBlock, GRANT_PREFIX + binding_id)
- if row:
- db.delete(row)
- db.commit()
+ await run_db(_forget_grants, self.factory, captured, set(batch) - retained)
item = reply["delivery"]
if item:
error = await run_delivery(self.factory, captured, identity, owner, item)
diff --git a/src/backend/integrations/channels/extension.py b/src/backend/integrations/channels/extension.py
new file mode 100644
index 00000000..df46323e
--- /dev/null
+++ b/src/backend/integrations/channels/extension.py
@@ -0,0 +1,7 @@
+"""Installed channel tool contribution; receivers retain their existing lifecycle."""
+NAME = "channels"
+def register_tools():
+ from integrations.channels.tools.attachment import register_native_contribution
+ register_native_contribution()
+def setup():
+ return True
diff --git a/src/backend/integrations/channels/inbound.py b/src/backend/integrations/channels/inbound.py
new file mode 100644
index 00000000..60e9bf1e
--- /dev/null
+++ b/src/backend/integrations/channels/inbound.py
@@ -0,0 +1,368 @@
+"""Channel inbound responsibilities."""
+from __future__ import annotations
+from typing import Optional
+import core.chat.runtime.admission as admission
+from integrations.channels.protocol import InboundMsg
+from integrations.channels.registry import get_adapter
+from core.db.engine import SessionLocal
+from integrations.channels.models import ChannelConnection
+from integrations.channels.repository import ChannelConnectionRepository
+import logging
+from core.db.threads import run_db, run_in_session
+logger = logging.getLogger(__name__)
+from integrations.channels import inbound_attachments
+from integrations.channels import inbound_delivery
+from integrations.channels import inbound_history
+from integrations.channels import inbound_observation
+from integrations.channels import inbound_sessions
+from integrations.channels import inbound_state
+
+def module_route(db, conn: ChannelConnection):
+ """The admit hook of the first enabled module that claims this bot, if any."""
+ from core.harness.registry import channel_routes
+
+ for handles, admit in channel_routes():
+ if handles(db, conn):
+ return admit
+ return None
+
+
+async def handle_inbound(msg: InboundMsg) -> None:
+ """Main entry point for inbound messages. Scheduled on the main event loop (long-connection threads submit via run_coroutine_threadsafe).
+
+ All exceptions are swallowed and logged — one message's failure must not take down the long connection / process.
+ """
+ from integrations.channels.relay_dispatch import dispatch_to_desktop
+ if await dispatch_to_desktop(msg, SessionLocal):
+ return
+ if inbound_sessions.already_handled(f"{msg.channel_id}:{msg.message_id}" if msg.message_id else ""):
+ return
+ if not (msg.text or "").strip() and not msg.attachments:
+ return
+ # #1 Per-conversation serialization: until an in-progress run finishes, later messages in the same conversation queue up, avoiding history races/corruption.
+ async with inbound_state.conv_locks[inbound_sessions.conv_key(msg)]:
+ await process_inbound(msg)
+
+
+def _inbound_target(db, channel_id: str) -> Optional[ChannelConnection]:
+ conn = ChannelConnectionRepository(db).get_by_id(channel_id)
+ if conn is None or not conn.enabled:
+ return None
+ _ = conn.config # force-load the credential column so it remains usable after detaching
+ db.expunge(conn)
+ return conn
+
+
+def _observe(db, channel_id: str, msg: InboundMsg) -> None:
+ repo = ChannelConnectionRepository(db)
+ conn = repo.get_by_id(channel_id)
+ inbound_observation.observe_message(db, conn, msg)
+ repo.touch_event(conn.channel_id) # bystander traffic still proves the connection is alive
+
+
+def _reset(db, channel_id: str, msg: InboundMsg) -> None:
+ conn = ChannelConnectionRepository(db).get_by_id(channel_id)
+ inbound_sessions.reset_session(db, conn, msg)
+
+
+def _session_for(db, channel_id: str, msg: InboundMsg) -> str:
+ conn = ChannelConnectionRepository(db).get_by_id(channel_id)
+ return inbound_sessions.find_or_create_session(db, conn, msg).chat_id
+
+
+def _drain_observed(db, chat_id: str):
+ from core.db.models import ChatSession
+
+ return inbound_observation.drain_observed(db, db.get(ChatSession, chat_id))
+
+
+def _admit_run(db, msg: InboundMsg, chat_id: str, user_text: str, uploaded_files):
+ """Build the run context and accept the run; returns everything the launch needs, detached."""
+ from core.chat.context import build_runtime_context, collect_historical_attachments
+ from core.chat.sequencer import ChatSequencer
+ from core.ontology.runtime_policy import build_user_ontology_runtime
+
+ repo = ChannelConnectionRepository(db)
+ conn = repo.get_by_id(msg.channel_id)
+ owner_id = conn.owner_user_id
+ enabled = inbound_sessions.resolve_enabled(db, conn, owner_id)
+ # Enable thinking: same as the web client's default in non-fast mode. With thinking off,
+ # models (Qwen family especially) tend to emit shallow filler like "I'll get right on X"
+ # without actually landing on tool calls, idling repeatedly across turns. Thinking never
+ # leaks into the channel reply: structured-reasoning models emit separate thinking events
+ # (ignored by _collect_reply, which only accumulates content/meta), and inline-thinking
+ # models'